Skip to content

Commit db85292

Browse files
jeremyfowersjeremyfowers
committed
Move logged_subprocess() from build to plugin_helers
Signed-off-by: Jeremy Fowers <[email protected]>
1 parent 908f3d8 commit db85292

File tree

4 files changed

+82
-70
lines changed

4 files changed

+82
-70
lines changed

‎src/turnkeyml/common/build.py

-60
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
import sklearn.base
1919
import turnkeyml.common.exceptions as exp
2020
import turnkeyml.common.tf_helpers as tf_helpers
21-
import turnkeyml.run.plugin_helpers as plugin_helpers
2221
from turnkeyml.version import __version__ as turnkey_version
2322

2423

@@ -445,65 +444,6 @@ def flush(self):
445444
pass
446445

447446

448-
def logged_subprocess(
449-
cmd: List[str],
450-
cwd: str = os.getcwd(),
451-
env: Optional[Dict] = None,
452-
log_file_path: Optional[str] = None,
453-
log_to_std_streams: bool = True,
454-
log_to_file: bool = True,
455-
) -> None:
456-
"""
457-
This function calls a subprocess and sends the logs to either a file, stdout/stderr, or both.
458-
459-
cmd Command that will run o a sbprocess
460-
cwd Working directory from where the subprocess should run
461-
env Evironment to be used by the subprocess (useful for passing env vars)
462-
log_file_path Where logs will be stored
463-
log_to_file Whether or not to store the subprocess's stdout/stderr into a file
464-
log_to_std Whether or not to print subprocess's stdout/stderr to the screen
465-
"""
466-
if env is None:
467-
env = os.environ.copy()
468-
if log_to_file and log_file_path is None:
469-
raise ValueError("log_file_path must be set when log_to_file is True")
470-
471-
log_stdout = ""
472-
log_stderr = ""
473-
try:
474-
proc = subprocess.run(
475-
cmd,
476-
check=True,
477-
env=env,
478-
capture_output=True,
479-
cwd=cwd,
480-
)
481-
except Exception as e: # pylint: disable=broad-except
482-
log_stdout = e.stdout.decode("utf-8") # pylint: disable=no-member
483-
log_stderr = e.stderr.decode("utf-8") # pylint: disable=no-member
484-
raise plugin_helpers.CondaError(
485-
f"Exception {e} encountered, \n\nstdout was: "
486-
f"\n{log_stdout}\n\n and stderr was: \n{log_stderr}"
487-
)
488-
else:
489-
log_stdout = proc.stdout.decode("utf-8")
490-
log_stderr = proc.stderr.decode("utf-8")
491-
finally:
492-
if log_to_std_streams:
493-
# Print log to stdout
494-
# This might be useful when this subprocess is being logged externally
495-
print(log_stdout, file=sys.stdout)
496-
print(log_stderr, file=sys.stdout)
497-
if log_to_file:
498-
log = f"{log_stdout}\n{log_stderr}"
499-
with open(
500-
log_file_path,
501-
"w",
502-
encoding="utf-8",
503-
) as f:
504-
f.write(log)
505-
506-
507447
def get_system_info():
508448
os_type = platform.system()
509449
info_dict = {}

‎src/turnkeyml/run/onnxrt/execute.py

+1-2
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
import json
99
from statistics import mean
1010
import platform
11-
import turnkeyml.common.build as build
1211
import turnkeyml.run.plugin_helpers as plugin_helpers
1312

1413
ORT_VERSION = "1.15.1"
@@ -84,7 +83,7 @@ def execute_benchmark(
8483
]
8584

8685
# Execute command and log stdout/stderr
87-
build.logged_subprocess(
86+
plugin_helpers.logged_subprocess(
8887
cmd=cmd,
8988
cwd=os.path.dirname(output_dir),
9089
log_to_std_streams=False,

‎src/turnkeyml/run/plugin_helpers.py

+61
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import subprocess
22
import logging
33
import os
4+
import sys
5+
from typing import List, Optional, Dict
46

57
TIMEOUT = 900
68

@@ -72,3 +74,62 @@ def get_python_path(conda_env_name):
7274
f"An error occurred while getting Python path for {conda_env_name} environment"
7375
f"{e.stderr.decode()}"
7476
)
77+
78+
79+
def logged_subprocess(
80+
cmd: List[str],
81+
cwd: str = os.getcwd(),
82+
env: Optional[Dict] = None,
83+
log_file_path: Optional[str] = None,
84+
log_to_std_streams: bool = True,
85+
log_to_file: bool = True,
86+
) -> None:
87+
"""
88+
This function calls a subprocess and sends the logs to either a file, stdout/stderr, or both.
89+
90+
cmd Command that will run o a sbprocess
91+
cwd Working directory from where the subprocess should run
92+
env Evironment to be used by the subprocess (useful for passing env vars)
93+
log_file_path Where logs will be stored
94+
log_to_file Whether or not to store the subprocess's stdout/stderr into a file
95+
log_to_std Whether or not to print subprocess's stdout/stderr to the screen
96+
"""
97+
if env is None:
98+
env = os.environ.copy()
99+
if log_to_file and log_file_path is None:
100+
raise ValueError("log_file_path must be set when log_to_file is True")
101+
102+
log_stdout = ""
103+
log_stderr = ""
104+
try:
105+
proc = subprocess.run(
106+
cmd,
107+
check=True,
108+
env=env,
109+
capture_output=True,
110+
cwd=cwd,
111+
)
112+
except Exception as e: # pylint: disable=broad-except
113+
log_stdout = e.stdout.decode("utf-8") # pylint: disable=no-member
114+
log_stderr = e.stderr.decode("utf-8") # pylint: disable=no-member
115+
raise CondaError(
116+
f"Exception {e} encountered, \n\nstdout was: "
117+
f"\n{log_stdout}\n\n and stderr was: \n{log_stderr}"
118+
)
119+
else:
120+
log_stdout = proc.stdout.decode("utf-8")
121+
log_stderr = proc.stderr.decode("utf-8")
122+
finally:
123+
if log_to_std_streams:
124+
# Print log to stdout
125+
# This might be useful when this subprocess is being logged externally
126+
print(log_stdout, file=sys.stdout)
127+
print(log_stderr, file=sys.stdout)
128+
if log_to_file:
129+
log = f"{log_stdout}\n{log_stderr}"
130+
with open(
131+
log_file_path,
132+
"w",
133+
encoding="utf-8",
134+
) as f:
135+
f.write(log)

‎test/unit.py

+20-8
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,8 @@ def test_003_subprocess_logger(self):
101101
traceback_error_cmd = f"raise ValueError('{traceback_error_msg}')"
102102

103103
# Perform basic test (no exceptions inside logger)
104-
cmd = ["python","-c",f"import sys\n{inside_stdout_cmd}\n{inside_sterr_cmd}"]
105-
build.logged_subprocess(cmd=cmd, log_file_path=logfile_path)
104+
cmd = ["python", "-c", f"import sys\n{inside_stdout_cmd}\n{inside_sterr_cmd}"]
105+
plugin_helpers.logged_subprocess(cmd=cmd, log_file_path=logfile_path)
106106

107107
# Make sure we captured everything we intended to capture
108108
with open(logfile_path, "r", encoding="utf-8") as file:
@@ -111,9 +111,13 @@ def test_003_subprocess_logger(self):
111111
assert inside_sterr_msg in log_contents
112112

113113
# Perform test with exceptions inside the logger
114-
cmd = ["python","-c",f"import sys\n{inside_stdout_cmd}\n{inside_sterr_cmd}\n{traceback_error_cmd}"]
114+
cmd = [
115+
"python",
116+
"-c",
117+
f"import sys\n{inside_stdout_cmd}\n{inside_sterr_cmd}\n{traceback_error_cmd}",
118+
]
115119
with self.assertRaises(plugin_helpers.CondaError):
116-
build.logged_subprocess(cmd=cmd, log_file_path=logfile_path)
120+
plugin_helpers.logged_subprocess(cmd=cmd, log_file_path=logfile_path)
117121

118122
# Make sure we captured everything we intended to capture
119123
with open(logfile_path, "r", encoding="utf-8") as file:
@@ -126,16 +130,24 @@ def test_003_subprocess_logger(self):
126130
subprocess_env = os.environ.copy()
127131
expected_env_var_value = "Expected Value"
128132
subprocess_env["TEST_ENV_VAR"] = expected_env_var_value
129-
cmd = ["python","-c",f'import os\nprint(os.environ["TEST_ENV_VAR"])']
130-
build.logged_subprocess(cmd=cmd, log_file_path=logfile_path, env=subprocess_env)
133+
cmd = ["python", "-c", f'import os\nprint(os.environ["TEST_ENV_VAR"])']
134+
plugin_helpers.logged_subprocess(
135+
cmd=cmd, log_file_path=logfile_path, env=subprocess_env
136+
)
131137
with open(logfile_path, "r", encoding="utf-8") as file:
132138
log_contents = file.read()
133139
assert expected_env_var_value in log_contents
134140

135141
# Test log_to_std_streams
136-
cmd = ["python","-c",f'print("{outside_stdout_msg}")\nprint("{outside_stderr_msg}")']
142+
cmd = [
143+
"python",
144+
"-c",
145+
f'print("{outside_stdout_msg}")\nprint("{outside_stderr_msg}")',
146+
]
137147
with build.Logger("", logfile_path):
138-
build.logged_subprocess(cmd=cmd, log_to_std_streams=True, log_to_file=False)
148+
plugin_helpers.logged_subprocess(
149+
cmd=cmd, log_to_std_streams=True, log_to_file=False
150+
)
139151
with open(logfile_path, "r", encoding="utf-8") as file:
140152
log_contents = file.read()
141153
assert outside_stdout_msg in log_contents

0 commit comments

Comments
 (0)