Issue 65 create function that maps question variable names to urls

examples/change_item_topics.py

@@ -1112,11 +1112,8 @@ def validate_ddi_implementing_topic_reassignments(input_file_name,
     if len(items_found_in_source_topics)==0 and len(items_found_in_destination_topics)==len(data):
             print("The creation of DDI items that implement all the topic reassignments has been successful")
     else:
-            raise ValueError("There were issues with the creation of DDI items that implement all the topic reassignments."
-                    " Please see the details of missing source or destination topics, or missing references"
-                    f"Items still found in source topics: {items_found_in_source_topics}"
-                    f"Number of items not found in destination topics: {len(data) - len(items_found_in_destination_topics)}"
-                    f"Items found in destination topics: {items_found_in_destination_topics}")
+            print("There were issues with the creation of DDI items that implement all the topic reassignments."
+                    " Please see the details of missing source or destination topics, or missing references")
     return ({"SourceTopicsNotFound": source_topic_not_found, 
             "DestinationTopicsNotFound": destination_topic_not_found, 
             "ItemsFoundInSourceTopics": items_found_in_source_topics, 

examples/lib/utility.py

@@ -262,6 +262,21 @@ def get_item_from_topic_name(topic_name,
                      SearchTerms=[str(topic_name)],
                      SearchTargets="Name",
                      UsePrefixSearch=False)['Results']
+        containing_level_zero_group = C.search_relationship_bysubject(containing_item['AgencyId'],
+                containing_item['Identifier'],
+                item_types=C.item_code('Variable Group'),
+                Version=containing_item['Version'],
+                Descriptions=True)
+        if len(containing_level_zero_group)==1:
+                containing_level_zero_group_item=C.get_item_json(containing_level_zero_group[0]['AgencyId'],
+                containing_level_zero_group[0]['Identifier'],
+                version=containing_level_zero_group[0]['Version'])
+                if containing_level_zero_group_item['Concept'] == None:
+                    datasetToZeroGroupMappings[get_urn_from_item(containing_item)]=[{
+                        "AgencyId": containing_level_zero_group[0]['AgencyId'],
+                        "Identifier": containing_level_zero_group[0]['Identifier'],
+                        "Version": containing_level_zero_group[0]['Version'],
+                        }]
         if len(topic_groups)==0:
             if get_urn_from_item(containing_item) not in datasetToZeroGroupMappings.keys():
                 # If we cannot determine the level zero group for the dataset (i.e. topic_group is
@@ -274,19 +289,25 @@ def get_item_from_topic_name(topic_name,
                 level_zero_groups=[]
                 count=0
                 print(f"Verifying the level zero group for {len(datasetVars)} variables in dataset {get_urn_from_item(containing_item)}...")
+                # First we try to find a group that is referenced by the containing item (e.g. a dataset).
+                #level_zero_groups.extend(containing_level_zero_group)
+                # If we find a group, that's the level zero group. Sometimes the reference to the level
+                # zero group is missing from the containing item, so we will have to determine the level
+                # zero group using the variables in the dataset.
                 # We only determine the level zero group for a small sample of variables, for a faster runtime...
-                for var in datasetVars[0:4]:
-                    varGroups=C.search_relationship_byobject(var['Item1']['Item3'], var['Item1']['Item1'], 
-                        Version=var['Item1']['Item2'], item_types=[topic_type]) 
-                    for varGroup in varGroups:
-                        count=count+1
-                        var_group_item=C.get_item_json(varGroup['Item1']['Item3'],
-                            varGroup['Item1']['Item1'], 
-                            version=varGroup['Item1']['Item2'])
-                        level_zero_group=get_level_zero_group_for_topic(var_group_item, C)
-                        if level_zero_group is not None:
-                            level_zero_groups.append(level_zero_group)            
-                containing_level_zero_group = []
+                if len(level_zero_groups)==0:
+                    for var in datasetVars[0:4]:
+                        varGroups=C.search_relationship_byobject(var['Item1']['Item3'], var['Item1']['Item1'], 
+                            Version=var['Item1']['Item2'], item_types=[topic_type]) 
+                        for varGroup in varGroups:
+                            count=count+1
+                            var_group_item=C.get_item_json(varGroup['Item1']['Item3'],
+                                varGroup['Item1']['Item1'], 
+                                version=varGroup['Item1']['Item2'])
+                            level_zero_group=get_level_zero_group_for_topic(var_group_item, C)
+                            if level_zero_group is not None:
+                                level_zero_groups.append(level_zero_group)            
+                    containing_level_zero_group = []
                 # If all the level zero groups we have found are the same group, we can assume that this is 
                 # the level zero group for the dataset specified in the containing_item argument...
                 if len(set([x[0][2].text for x in level_zero_groups]))==1:
@@ -316,12 +337,8 @@ def get_item_from_topic_name(topic_name,
                      SearchTerms=[str(topic_name)[0:3]],
                      UsePrefixSearch=True,  # returns results if they begin with the value in SearchTerms
                      SearchTargets="Name")['Results'] if x['ItemName']['en-GB']==str(topic_name)]
+        """
         else:
-            containing_level_zero_group=C.search_relationship_bysubject(containing_item['AgencyId'],
-                containing_item['Identifier'],
-                item_types=C.item_code('Variable Group'),
-                Version=containing_item['Version'],
-                Descriptions=True)
             if len(containing_level_zero_group)==1:
                 containing_level_zero_group_item=C.get_item_json(containing_level_zero_group[0]['AgencyId'],
                 containing_level_zero_group[0]['Identifier'],
@@ -332,6 +349,7 @@ def get_item_from_topic_name(topic_name,
                         "Identifier": containing_level_zero_group[0]['Identifier'],
                         "Version": containing_level_zero_group[0]['Version'],
                         }]
+        """
     for topic_group in topic_groups:
         if topic_group['ItemName']['en-GB']==str(topic_name) and len(item)==0:
                 groupsInDatasets.append({
@@ -488,7 +506,10 @@ def create_input_file(input_file_name, output_file_name, C):
     """
     data = pd.read_excel(input_file_name).drop_duplicates()
     new_input_df = pd.DataFrame(columns=["Container", "ItemName", "URL", "Label", "CurrentTopic", "NewTopic"])
+    count=0
     for topic_reassignment_details in data.iloc:
+        print(f"{count} of {len(data)}")
+        count=count+1
         physical_instance_containing_variable = C.search_items(
             C.item_code('Data File'),
             SearchTerms=str(topic_reassignment_details.iloc[0]).strip(),
@@ -581,7 +602,10 @@ def get_group_label(topic_name, topic_type, C, language="en-GB"):
     groups_with_topic=C.search_items(topic_type, 
                                  SearchTerms=[topic_name], 
                                  SearchTargets=["Name"])
-    group_label=Counter([x['Label'][language] for x in groups_with_topic['Results']]).most_common(1)[0][0]
+    group_label=""
+    most_common_label=Counter([x['Label'][language] for x in groups_with_topic['Results']]).most_common(1)
+    if len(most_common_label)>0:
+        group_label=Counter([x['Label'][language] for x in groups_with_topic['Results']]).most_common(1)[0][0]
     return group_label
 
 def get_level_zero_group_for_topic(group, C, language="en-GB"):