From 9f3e75bbcb50f6da71b63d1e0eaaaf19fa5aefe7 Mon Sep 17 00:00:00 2001 From: Wonjae Park Date: Tue, 11 Feb 2025 09:52:09 +0900 Subject: [PATCH] Update SCANOSS version and redirect log (#204) * change minimum SCANOSS version to 1.18.0 Signed-off-by: Wonjae Park * Redirect SCANOSS output to log file Signed-off-by: Wonjae Park * Add comment on info sheet for limit exceed Signed-off-by: Wonjae Park * Fix flake8 fail Signed-off-by: Wonjae Park --------- Signed-off-by: Wonjae Park --- requirements.txt | 2 +- src/fosslight_source/cli.py | 12 ++++++++---- src/fosslight_source/run_scanoss.py | 12 ++++++++++-- tests/cli_test.py | 2 +- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2928b52..896725d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ pyparsing -scanoss<=1.14.0 +scanoss>=1.18.0 XlsxWriter fosslight_util>=2.1.10 PyYAML diff --git a/src/fosslight_source/cli.py b/src/fosslight_source/cli.py index eb775ef..f83ddbd 100755 --- a/src/fosslight_source/cli.py +++ b/src/fosslight_source/cli.py @@ -148,7 +148,7 @@ def create_report_file( output_path: str = "", output_files: list = [], output_extensions: list = [], correct_mode: bool = True, correct_filepath: str = "", path_to_scan: str = "", path_to_exclude: list = [], - formats: list = [], excluded_file_list: list = [] + formats: list = [], excluded_file_list: list = [], api_limit_exceed: bool = False ) -> 'ScannerItem': """ Create report files for given scanned result. @@ -212,6 +212,9 @@ def create_report_file( scan_item.set_cover_comment(f"Total number of files : {files_count}") scan_item.set_cover_comment(f"Removed files : {removed_files_count}") + if api_limit_exceed: + scan_item.set_cover_comment("(Some of) SCANOSS scan was skipped. (API limits being exceeded)") + if not merged_result: if files_count < 1: scan_item.set_cover_comment("(No file detected.)") @@ -350,14 +353,15 @@ def run_scanners( print_matched_text, formats, called_by_cli, time_out, correct_mode, correct_filepath) if selected_scanner == 'scanoss' or selected_scanner == 'all' or selected_scanner == '': - scanoss_result = run_scanoss_py(path_to_scan, output_file_name, formats, True, write_json_file, num_cores, - path_to_exclude) + scanoss_result, api_limit_exceed = run_scanoss_py(path_to_scan, output_file_name, formats, True, write_json_file, + num_cores, path_to_exclude) if selected_scanner in SCANNER_TYPE: spdx_downloads = get_spdx_downloads(path_to_scan, path_to_exclude) merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads) scan_item = create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner, print_matched_text, output_path, output_files, output_extensions, correct_mode, - correct_filepath, path_to_scan, path_to_exclude, formats, excluded_file_list) + correct_filepath, path_to_scan, path_to_exclude, formats, excluded_file_list, + api_limit_exceed) else: print_help_msg_source_scanner() result_log[RESULT_KEY] = "Unsupported scanner" diff --git a/src/fosslight_source/run_scanoss.py b/src/fosslight_source/run_scanoss.py index 32eab8a..676d7cc 100755 --- a/src/fosslight_source/run_scanoss.py +++ b/src/fosslight_source/run_scanoss.py @@ -17,6 +17,8 @@ import shutil from pathlib import Path from scanoss.scanner import Scanner, ScanType +import io +import contextlib logger = logging.getLogger(constant.LOGGER_NAME) warnings.filterwarnings("ignore", category=FutureWarning) @@ -75,7 +77,13 @@ def run_scanoss_py(path_to_scan: str, output_file_name: str = "", format: list = scan_options=ScanType.SCAN_SNIPPETS.value, nb_threads=num_threads if num_threads > 0 else 10 ) - scanner.scan_folder_with_options(scan_dir=path_to_scan) + + output_buffer = io.StringIO() + with contextlib.redirect_stdout(output_buffer), contextlib.redirect_stderr(output_buffer): + scanner.scan_folder_with_options(scan_dir=path_to_scan) + captured_output = output_buffer.getvalue() + api_limit_exceed = "due to service limits being exceeded" in captured_output + logger.debug(f"{captured_output}") if os.path.isfile(output_json_file): total_files_to_excluded = [] @@ -117,4 +125,4 @@ def run_scanoss_py(path_to_scan: str, output_file_name: str = "", format: list = except Exception as error: logger.debug(f"Moving scanoss raw files failed.: {error}") - return scanoss_file_list + return scanoss_file_list, api_limit_exceed diff --git a/tests/cli_test.py b/tests/cli_test.py index 5d68af7..b5f28bd 100755 --- a/tests/cli_test.py +++ b/tests/cli_test.py @@ -26,7 +26,7 @@ def main(): logger, result_item = init_log(os.path.join(output_dir, "fosslight_log_"+_start_time+".txt")) ret = run_scan(path_to_find_bin, fosslight_report_name, True, -1, True, True, [], False) - ret_scanoss = run_scanoss_py(path_to_find_bin, fosslight_report_name, [], False, True, -1) + ret_scanoss, api_limit_exceed = run_scanoss_py(path_to_find_bin, fosslight_report_name, [], False, True, -1) logger.warning("[Scan] Result: %s" % (ret[0])) logger.warning("[Scan] Result_msg: %s" % (ret[1]))