diff --git a/examples/change_item_topics.py b/examples/change_item_topics.py index ad6cb25..aeb1f5b 100644 --- a/examples/change_item_topics.py +++ b/examples/change_item_topics.py @@ -1112,11 +1112,8 @@ def validate_ddi_implementing_topic_reassignments(input_file_name, if len(items_found_in_source_topics)==0 and len(items_found_in_destination_topics)==len(data): print("The creation of DDI items that implement all the topic reassignments has been successful") else: - raise ValueError("There were issues with the creation of DDI items that implement all the topic reassignments." - " Please see the details of missing source or destination topics, or missing references" - f"Items still found in source topics: {items_found_in_source_topics}" - f"Number of items not found in destination topics: {len(data) - len(items_found_in_destination_topics)}" - f"Items found in destination topics: {items_found_in_destination_topics}") + print("There were issues with the creation of DDI items that implement all the topic reassignments." + " Please see the details of missing source or destination topics, or missing references") return ({"SourceTopicsNotFound": source_topic_not_found, "DestinationTopicsNotFound": destination_topic_not_found, "ItemsFoundInSourceTopics": items_found_in_source_topics, diff --git a/examples/final-usoc-topics.xlsx b/examples/final-usoc-topics.xlsx new file mode 100644 index 0000000..6c5e84d Binary files /dev/null and b/examples/final-usoc-topics.xlsx differ diff --git a/examples/lib/utility.py b/examples/lib/utility.py index 9a974b1..0ce0a4c 100644 --- a/examples/lib/utility.py +++ b/examples/lib/utility.py @@ -262,6 +262,21 @@ def get_item_from_topic_name(topic_name, SearchTerms=[str(topic_name)], SearchTargets="Name", UsePrefixSearch=False)['Results'] + containing_level_zero_group = C.search_relationship_bysubject(containing_item['AgencyId'], + containing_item['Identifier'], + item_types=C.item_code('Variable Group'), + Version=containing_item['Version'], + Descriptions=True) + if len(containing_level_zero_group)==1: + containing_level_zero_group_item=C.get_item_json(containing_level_zero_group[0]['AgencyId'], + containing_level_zero_group[0]['Identifier'], + version=containing_level_zero_group[0]['Version']) + if containing_level_zero_group_item['Concept'] == None: + datasetToZeroGroupMappings[get_urn_from_item(containing_item)]=[{ + "AgencyId": containing_level_zero_group[0]['AgencyId'], + "Identifier": containing_level_zero_group[0]['Identifier'], + "Version": containing_level_zero_group[0]['Version'], + }] if len(topic_groups)==0: if get_urn_from_item(containing_item) not in datasetToZeroGroupMappings.keys(): # If we cannot determine the level zero group for the dataset (i.e. topic_group is @@ -274,19 +289,25 @@ def get_item_from_topic_name(topic_name, level_zero_groups=[] count=0 print(f"Verifying the level zero group for {len(datasetVars)} variables in dataset {get_urn_from_item(containing_item)}...") + # First we try to find a group that is referenced by the containing item (e.g. a dataset). + #level_zero_groups.extend(containing_level_zero_group) + # If we find a group, that's the level zero group. Sometimes the reference to the level + # zero group is missing from the containing item, so we will have to determine the level + # zero group using the variables in the dataset. # We only determine the level zero group for a small sample of variables, for a faster runtime... - for var in datasetVars[0:4]: - varGroups=C.search_relationship_byobject(var['Item1']['Item3'], var['Item1']['Item1'], - Version=var['Item1']['Item2'], item_types=[topic_type]) - for varGroup in varGroups: - count=count+1 - var_group_item=C.get_item_json(varGroup['Item1']['Item3'], - varGroup['Item1']['Item1'], - version=varGroup['Item1']['Item2']) - level_zero_group=get_level_zero_group_for_topic(var_group_item, C) - if level_zero_group is not None: - level_zero_groups.append(level_zero_group) - containing_level_zero_group = [] + if len(level_zero_groups)==0: + for var in datasetVars[0:4]: + varGroups=C.search_relationship_byobject(var['Item1']['Item3'], var['Item1']['Item1'], + Version=var['Item1']['Item2'], item_types=[topic_type]) + for varGroup in varGroups: + count=count+1 + var_group_item=C.get_item_json(varGroup['Item1']['Item3'], + varGroup['Item1']['Item1'], + version=varGroup['Item1']['Item2']) + level_zero_group=get_level_zero_group_for_topic(var_group_item, C) + if level_zero_group is not None: + level_zero_groups.append(level_zero_group) + containing_level_zero_group = [] # If all the level zero groups we have found are the same group, we can assume that this is # the level zero group for the dataset specified in the containing_item argument... if len(set([x[0][2].text for x in level_zero_groups]))==1: @@ -316,12 +337,8 @@ def get_item_from_topic_name(topic_name, SearchTerms=[str(topic_name)[0:3]], UsePrefixSearch=True, # returns results if they begin with the value in SearchTerms SearchTargets="Name")['Results'] if x['ItemName']['en-GB']==str(topic_name)] + """ else: - containing_level_zero_group=C.search_relationship_bysubject(containing_item['AgencyId'], - containing_item['Identifier'], - item_types=C.item_code('Variable Group'), - Version=containing_item['Version'], - Descriptions=True) if len(containing_level_zero_group)==1: containing_level_zero_group_item=C.get_item_json(containing_level_zero_group[0]['AgencyId'], containing_level_zero_group[0]['Identifier'], @@ -332,6 +349,7 @@ def get_item_from_topic_name(topic_name, "Identifier": containing_level_zero_group[0]['Identifier'], "Version": containing_level_zero_group[0]['Version'], }] + """ for topic_group in topic_groups: if topic_group['ItemName']['en-GB']==str(topic_name) and len(item)==0: groupsInDatasets.append({ @@ -488,7 +506,10 @@ def create_input_file(input_file_name, output_file_name, C): """ data = pd.read_excel(input_file_name).drop_duplicates() new_input_df = pd.DataFrame(columns=["Container", "ItemName", "URL", "Label", "CurrentTopic", "NewTopic"]) + count=0 for topic_reassignment_details in data.iloc: + print(f"{count} of {len(data)}") + count=count+1 physical_instance_containing_variable = C.search_items( C.item_code('Data File'), SearchTerms=str(topic_reassignment_details.iloc[0]).strip(), @@ -581,7 +602,10 @@ def get_group_label(topic_name, topic_type, C, language="en-GB"): groups_with_topic=C.search_items(topic_type, SearchTerms=[topic_name], SearchTargets=["Name"]) - group_label=Counter([x['Label'][language] for x in groups_with_topic['Results']]).most_common(1)[0][0] + group_label="" + most_common_label=Counter([x['Label'][language] for x in groups_with_topic['Results']]).most_common(1) + if len(most_common_label)>0: + group_label=Counter([x['Label'][language] for x in groups_with_topic['Results']]).most_common(1)[0][0] return group_label def get_level_zero_group_for_topic(group, C, language="en-GB"): diff --git a/examples/update-usoc-topics2.xlsx b/examples/update-usoc-topics2.xlsx new file mode 100644 index 0000000..d56ae21 Binary files /dev/null and b/examples/update-usoc-topics2.xlsx differ