From 1ee6caf1b9a2fd8d291d9a91d6e12ba0d5e1cb8b Mon Sep 17 00:00:00 2001 From: AllyWang Date: Thu, 18 May 2023 19:01:54 +0800 Subject: [PATCH 1/3] add version diff --- azdev/commands.py | 1 + azdev/operations/command_change/__init__.py | 40 +++++++++++++++++++-- azdev/operations/command_change/util.py | 37 +++++++++++++++++++ azdev/params.py | 5 +++ 4 files changed, 81 insertions(+), 2 deletions(-) diff --git a/azdev/commands.py b/azdev/commands.py index 5dc15a25..ab873600 100644 --- a/azdev/commands.py +++ b/azdev/commands.py @@ -37,6 +37,7 @@ def operation_group(name): with CommandGroup(self, 'command-change', operation_group('command_change')) as g: g.command('meta-export', 'export_command_meta') g.command('meta-diff', 'cmp_command_meta') + g.command('version-diff', 'cmp_command_meta_of_versions') with CommandGroup(self, 'cli', operation_group('pypi')) as g: g.command('check-versions', 'verify_versions') diff --git a/azdev/operations/command_change/__init__.py b/azdev/operations/command_change/__init__.py index 993ccdd4..6ed5a5fc 100644 --- a/azdev/operations/command_change/__init__.py +++ b/azdev/operations/command_change/__init__.py @@ -13,9 +13,10 @@ from deepdiff import DeepDiff from knack.log import get_logger -from azdev.utilities import display, require_azure_cli, heading, get_path_table, filter_by_git_diff +from azdev.utilities import display, require_azure_cli, heading, get_path_table, filter_by_git_diff, get_cli_repo_path from .custom import MetaChangeDetects, DiffExportFormat -from .util import export_meta_changes_to_json, gen_commands_meta, get_commands_meta +from .util import export_meta_changes_to_json, gen_commands_meta, get_commands_meta, \ + extrct_module_name_from_meta_file, export_meta_changes_to_csv from ..statistics import _create_invoker_and_load_cmds, _get_command_source, \ _command_codegen_info # pylint: disable=protected-access from ..statistics.util import filter_modules @@ -140,3 +141,38 @@ def cmp_command_meta(base_meta_file, diff_meta_file, only_break=False, output_ty detected_changes.check_deep_diffs() result = detected_changes.export_meta_changes(only_break, output_type) return export_meta_changes_to_json(result, output_file) + + +def cmp_command_meta_of_versions(base_version, diff_version, only_break=False, version_diff_file=None): + cli_repo_path = get_cli_repo_path() + base_version_meta_path = os.path.join(cli_repo_path, 'cmd_meta', "azure-cli-" + base_version) + diff_version_meta_path = os.path.join(cli_repo_path, 'cmd_meta', "azure-cli-" + diff_version) + if not os.path.exists(base_version_meta_path) or not os.path.exists(diff_version_meta_path): + return + version_diffs = [] + for base_meta_file in os.listdir(base_version_meta_path): + module_name = extrct_module_name_from_meta_file(base_meta_file) + if not module_name: + continue + diff_meta_file = os.path.join(diff_version_meta_path, base_meta_file) + if not os.path.exists(diff_meta_file): + display(f"Module {module_name} removed for {diff_version}") + continue + with open(os.path.join(base_version_meta_path, base_meta_file), "r") as g: + command_tree_before = json.load(g) + with open(diff_meta_file, "r") as g: + command_tree_after = json.load(g) + diff = DeepDiff(command_tree_before, command_tree_after) + if not diff: + display(f"No meta diffs from version: {diff_version}/{base_meta_file} for module: {module_name}") + continue + else: + detected_changes = MetaChangeDetects(diff, command_tree_before, command_tree_after) + detected_changes.check_deep_diffs() + diff_objs = detected_changes.export_meta_changes(only_break, "dict") + mod_obj = {"module": module_name} + for obj in diff_objs: + obj.update(mod_obj) + version_diffs.append(obj) + time.sleep(1) + return export_meta_changes_to_csv(version_diffs, version_diff_file) diff --git a/azdev/operations/command_change/util.py b/azdev/operations/command_change/util.py index 62797a07..9001625a 100644 --- a/azdev/operations/command_change/util.py +++ b/azdev/operations/command_change/util.py @@ -7,6 +7,7 @@ import json import os import re +import csv from enum import Enum import jsbeautifier from knack.log import get_logger @@ -16,6 +17,10 @@ SUBGROUP_NAME_PATTERN = re.compile(r"\[\'sub_groups\'\]\[\'([a-zA-Z0-9\-\s]+)\'\]") CMD_NAME_PATTERN = re.compile(r"\[\'commands\'\]\[\'([a-zA-Z0-9\-\s]+)\'\]") CMD_PARAMETER_PROPERTY_PATTERN = re.compile(r"\[(.*?)\]") +MODULE_NAME_PATTERN = re.compile("az_([a-zA-Z0-9\-\_]+)_meta.json") + +EXPORTED_CSV_META_HEADER = ["module", "cmd_name", "rule_id", "rule_name", "is_break", + "rule_message", "suggest_message"] class ChangeType(int, Enum): @@ -215,6 +220,13 @@ def extract_para_info(key): return property_res +def extrct_module_name_from_meta_file(file_name): + name_res = re.findall(MODULE_NAME_PATTERN, file_name) + if not name_res or len(name_res) == 0: + return None + return name_res[0] + + def export_meta_changes_to_json(output, output_file): if not output_file: return output @@ -225,3 +237,28 @@ def export_meta_changes_to_json(output, output_file): if output: f_out.write(json.dumps(output, indent=4)) return None + + +def format_module_diff_csv(module_diffs): + csv_res = [EXPORTED_CSV_META_HEADER] + for diff_obj in module_diffs: + _row = [] + for attr in EXPORTED_CSV_META_HEADER: + if attr == "cmd_name": + _row.append(diff_obj.get(attr, None) or diff_obj.get("subgroup_name", "-")) + else: + _row.append(diff_obj.get(attr, None)) + csv_res.append(_row) + return csv_res + + +def export_meta_changes_to_csv(module_diffs, version_diff_file): + csv_res = format_module_diff_csv(module_diffs) + if not version_diff_file: + return csv_res + diff_file_folder = os.path.dirname(version_diff_file) + if diff_file_folder and not os.path.exists(diff_file_folder): + os.makedirs(diff_file_folder) + with open(version_diff_file, "w", newline='') as f: + writer = csv.writer(f) + writer.writerows(csv_res) diff --git a/azdev/params.py b/azdev/params.py index da207abc..eb11ddb7 100644 --- a/azdev/params.py +++ b/azdev/params.py @@ -130,6 +130,11 @@ def load_arguments(self, _): help='format to print diff and suggest message') c.argument('output_file', help='command meta diff json file path to store') + with ArgumentsContext(self, 'command-change version-diff') as c: + c.argument('base_version', required=True, help='azure cli version as base') + c.argument('diff_version', required=True, help='azure cli version to diff') + c.argument('version_diff_file', help='command meta version diff file path to store') + with ArgumentsContext(self, 'perf') as c: c.argument('runs', type=int, help='Number of runs to average performance over.') From 53c629d5187815b9373f74bd4c9e54378566041f Mon Sep 17 00:00:00 2001 From: AllyWang Date: Mon, 29 May 2023 17:32:55 +0800 Subject: [PATCH 2/3] use blob for meta data --- azdev/operations/command_change/__init__.py | 28 ++++++++++++++------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/azdev/operations/command_change/__init__.py b/azdev/operations/command_change/__init__.py index 6ed5a5fc..944e073f 100644 --- a/azdev/operations/command_change/__init__.py +++ b/azdev/operations/command_change/__init__.py @@ -13,7 +13,7 @@ from deepdiff import DeepDiff from knack.log import get_logger -from azdev.utilities import display, require_azure_cli, heading, get_path_table, filter_by_git_diff, get_cli_repo_path +from azdev.utilities import display, require_azure_cli, heading, get_path_table, filter_by_git_diff, cmd from .custom import MetaChangeDetects, DiffExportFormat from .util import export_meta_changes_to_json, gen_commands_meta, get_commands_meta, \ extrct_module_name_from_meta_file, export_meta_changes_to_csv @@ -144,17 +144,28 @@ def cmp_command_meta(base_meta_file, diff_meta_file, only_break=False, output_ty def cmp_command_meta_of_versions(base_version, diff_version, only_break=False, version_diff_file=None): - cli_repo_path = get_cli_repo_path() - base_version_meta_path = os.path.join(cli_repo_path, 'cmd_meta', "azure-cli-" + base_version) - diff_version_meta_path = os.path.join(cli_repo_path, 'cmd_meta', "azure-cli-" + diff_version) - if not os.path.exists(base_version_meta_path) or not os.path.exists(diff_version_meta_path): - return version_diffs = [] - for base_meta_file in os.listdir(base_version_meta_path): + base_version_meta_path = "azure-cli-" + base_version + base_meta_download_cmd = """az storage blob download-batch --account-name versionmeta -s $web --pattern """ \ + + base_version_meta_path + """/* -d . """ + display(f"Downloading {base_version} meta data using '{base_meta_download_cmd}'") + cmd(base_meta_download_cmd, show_stderr=True) + if not os.path.exists(os.getcwd() + "/" + base_version_meta_path): + display(f"No meta downloaded from blob for {base_version}, please check blob data") + return export_meta_changes_to_csv(version_diffs, version_diff_file) + diff_version_meta_path = "azure-cli-" + diff_version + diff_meta_download_cmd = """az storage blob download-batch --account-name versionmeta -s $web --pattern """ \ + + diff_version_meta_path + """/* -d . """ + display(f"Downloading {diff_version} meta data using '{diff_meta_download_cmd}'") + cmd(diff_meta_download_cmd, show_stderr=True) + if not os.path.exists(os.getcwd() + "/" + diff_version_meta_path): + display(f"No meta downloaded from blob for {diff_version}, please check blob data") + return export_meta_changes_to_csv(version_diffs, version_diff_file) + for base_meta_file in os.listdir(os.getcwd() + "/" + base_version_meta_path): module_name = extrct_module_name_from_meta_file(base_meta_file) if not module_name: continue - diff_meta_file = os.path.join(diff_version_meta_path, base_meta_file) + diff_meta_file = os.path.join(os.getcwd(), diff_version_meta_path, base_meta_file) if not os.path.exists(diff_meta_file): display(f"Module {module_name} removed for {diff_version}") continue @@ -174,5 +185,4 @@ def cmp_command_meta_of_versions(base_version, diff_version, only_break=False, v for obj in diff_objs: obj.update(mod_obj) version_diffs.append(obj) - time.sleep(1) return export_meta_changes_to_csv(version_diffs, version_diff_file) From 64c91be25bbd68b8e5d1dff3a5aeae5c50345758 Mon Sep 17 00:00:00 2001 From: AllyWang Date: Mon, 29 May 2023 19:18:36 +0800 Subject: [PATCH 3/3] adjust style --- azdev/operations/command_change/__init__.py | 15 +++++++-------- azdev/operations/command_change/util.py | 3 ++- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/azdev/operations/command_change/__init__.py b/azdev/operations/command_change/__init__.py index 944e073f..8508a9b5 100644 --- a/azdev/operations/command_change/__init__.py +++ b/azdev/operations/command_change/__init__.py @@ -177,12 +177,11 @@ def cmp_command_meta_of_versions(base_version, diff_version, only_break=False, v if not diff: display(f"No meta diffs from version: {diff_version}/{base_meta_file} for module: {module_name}") continue - else: - detected_changes = MetaChangeDetects(diff, command_tree_before, command_tree_after) - detected_changes.check_deep_diffs() - diff_objs = detected_changes.export_meta_changes(only_break, "dict") - mod_obj = {"module": module_name} - for obj in diff_objs: - obj.update(mod_obj) - version_diffs.append(obj) + detected_changes = MetaChangeDetects(diff, command_tree_before, command_tree_after) + detected_changes.check_deep_diffs() + diff_objs = detected_changes.export_meta_changes(only_break, "dict") + mod_obj = {"module": module_name} + for obj in diff_objs: + obj.update(mod_obj) + version_diffs.append(obj) return export_meta_changes_to_csv(version_diffs, version_diff_file) diff --git a/azdev/operations/command_change/util.py b/azdev/operations/command_change/util.py index 9001625a..0502e575 100644 --- a/azdev/operations/command_change/util.py +++ b/azdev/operations/command_change/util.py @@ -17,7 +17,7 @@ SUBGROUP_NAME_PATTERN = re.compile(r"\[\'sub_groups\'\]\[\'([a-zA-Z0-9\-\s]+)\'\]") CMD_NAME_PATTERN = re.compile(r"\[\'commands\'\]\[\'([a-zA-Z0-9\-\s]+)\'\]") CMD_PARAMETER_PROPERTY_PATTERN = re.compile(r"\[(.*?)\]") -MODULE_NAME_PATTERN = re.compile("az_([a-zA-Z0-9\-\_]+)_meta.json") +MODULE_NAME_PATTERN = re.compile(r"az_([a-zA-Z0-9\-\_]+)_meta.json") EXPORTED_CSV_META_HEADER = ["module", "cmd_name", "rule_id", "rule_name", "is_break", "rule_message", "suggest_message"] @@ -262,3 +262,4 @@ def export_meta_changes_to_csv(module_diffs, version_diff_file): with open(version_diff_file, "w", newline='') as f: writer = csv.writer(f) writer.writerows(csv_res) + return None