diff --git a/examples/change_item_topics.py b/examples/change_item_topics.py index 1489062..ad6cb25 100644 --- a/examples/change_item_topics.py +++ b/examples/change_item_topics.py @@ -2,8 +2,10 @@ A set of functions that reassign items to new topics. The item topic reassignments are defined in an Excel spreadsheet, the name of which is passed as an input argument to a function. An example of this spreadsheet ('topic_reassignments.xlsx') is provided in the -'examples' directory of this repository. The set of commands that need to be executed from -within a Python shell for changing item topics is: +'examples' directory of this repository. Before running the below commands to change item topics, +you may need to run 'pip install -r requirements.txt' in order to install all required packages. +The set of commands that need to be executed from within a Python shell for changing item +topics is: from colectica_api import ColecticaObject from examples.lib.utility import update_repository @@ -12,129 +14,931 @@ HOSTNAME = "HOSTNAME" C = ColecticaObject(HOSTNAME, USERNAME, PASSWORD, verify_ssl=False) import examples.change_item_topics -updated_groups = examples.change_item_topics.update_topics('examples/topic_reassignments.xlsx', C) -examples.lib.utility.update_repository(updated_groups, 'Repository commit message - update topics', C) +topic_reassignments=examples.change_item_topics.move_topics('examples/smallTest.xlsx', C) +update_repository(topic_reassignments['UpdatedTopics']['UpdatedTopicGroups'], 'Repository commit message - topic updates', C) + +After you have ran the above 'update_repository' command, the items in the repository should have +the topic reassignments described in 'smallTest.xlsx' applied to them. You can verify that the +topic reassignments have been successful by running the following command: + +updated_topics=examples.change_item_topics.update_topics(topic_reassignments["TopicReassignmentsDataFrame"], C, + updated_topic_groups=topic_reassignments['UpdatedTopics']['UpdatedTopicGroups']) + +In addition to logging information about the topic reassignments, you should see text similar to this if +all the topic reassignments have been successfully executed on the repository: + + 49 of 49 topic reassignments in the input file have already been performed, + 0 pair(s) of DDI Fragments implementing topic reassignments specified in the input file have been created. + The item topic reassignments in the input data file have already all been successfully executed. """ -from .lib.utility import ( +from examples.lib.utility import ( get_namespace, find_all_references, create_variable_reference, create_question_reference, - get_current_state_of_topic_group, update_list_of_topic_groups, get_urn_from_item, - get_item_from_topic_name + get_item_from_topic_name, + create_variable_group, + create_group_reference, + create_group_lookup_dict, + get_current_state_of_topic_group, + get_level_zero_group_for_topic, + get_group_label, + get_elements_of_type, + get_element_by_name, + get_element_fragment_by_name, + get_urn_from_fragment ) import defusedxml #pip install openpyxl #might need to install openpyxl, a dependency for read-excel import pandas as pd +import uuid +from collections import Counter + +def move_topics(input_file, C): + datasetToZeroGroupMappings={} + topics_to_create=find_topics_to_create(input_file, + C, + datasetToZeroGroupMappings=datasetToZeroGroupMappings) + groupsInDatasets=create_group_lookup_dict(datasetToZeroGroupMappings, C) + items_with_new_level_one_topics=create_ddi_objects_with_new_level_one_topics( + topics_to_create['levelOneGroupsToCreate'], topics_to_create['levelTwoGroupsToCreate'], C) + items_with_modified_level_one_topics=create_ddi_objects_for_modified_level_one_topics( + topics_to_create["levelOneGroupsToModify"], C) + print("""Validating the DDI objects that have been created for the new level two topics that needed + to be created, and for which level one topics also need to be created...""") + validationResultsNewLevelTwoTopics=validateLevelTwoTopics(items_with_new_level_one_topics["LevelZero"], + items_with_new_level_one_topics["LevelOne"], + items_with_new_level_one_topics["LevelTwo"], + C) + print("""Validating the DDI objects that have been created for the new level one topics that needed + to be created...""") + validationResultsNewLevelOneTopics=validateLevelOneTopics(items_with_new_level_one_topics["LevelZero"], + items_with_new_level_one_topics["LevelOne"], + C) + print("""Validating the DDI objects that have been created for the new level two topics that needed + to be created, but for which level one topics already existed...""") + validationResultsNewLevelTwoTopicsWithModifiedLevelOneTopics=validateLevelTwoTopics(items_with_modified_level_one_topics["LevelZero"], + items_with_modified_level_one_topics["LevelOne"], + items_with_modified_level_one_topics["LevelTwo"], + C) + print("""Validating the DDI objects that have been created for the existing level one topics that needed + to be modified to include references to new level two topics...""") + validationResultsModifiedLevelOneTopics=validateLevelOneTopics(items_with_modified_level_one_topics["LevelZero"], + items_with_modified_level_one_topics["LevelOne"], + C) + """ + In summary, the below tests verify the following: + 1. If we're creating a topic, it doesn't already exist. If we're modifying a topic, + it does already exist. + 2+3. We're creating/modifying all the topics we need to; we're not missing anything + """ + # 1. We need to verify that all the groups in topics_to_create["levelOneGroupsToCreate"] and + # topics_to_create["levelTwoGroupsToCreate"] do not currently exist. + for topic_to_create in topics_to_create["levelOneGroupsToCreate"]: + if len([(existing_group['DatasetName'], existing_group['VariableGroupName']) + for existing_group in groupsInDatasets + if existing_group['DatasetName']==topic_to_create['DatasetName'] + and existing_group['VariableGroupName']==topic_to_create['LevelOneGroupName']])!=0: + print("ERROR - LEVEL ONE TOPIC LISTED FOR CREATION ALREADY EXISTS") + for topic_to_create in topics_to_create["levelTwoGroupsToCreate"]: + if len([(existing_group['DatasetName'], existing_group['VariableGroupName']) + for existing_group in groupsInDatasets + if existing_group['DatasetName']==topic_to_create['DatasetName'] + and existing_group['VariableGroupName']==topic_to_create['LevelTwoGroupName']])!=0: + print(topic_to_create) + print("ERROR - LEVEL TWO TOPIC LISTED FOR CREATION ALREADY EXISTS") + # We also need to check that the level one topics in topics_to_create["levelOneGroupsToModify"] do exist, and + # that the level two topics they refer to do not exist. + for topic_to_create in topics_to_create["levelOneGroupsToModify"]: + if len([(existing_group['DatasetName'], existing_group['VariableGroupName']) + for existing_group in groupsInDatasets + if existing_group['VariableGroupName']==topic_to_create['LevelTwoGroupName'][0:3] + and existing_group['DatasetName']==topic_to_create['DatasetName']])==0: + print(topic_to_create) + print("ERROR - LEVEL ONE TOPIC LISTED FOR MODIFICATION DOES NOT EXIST") + if len([(existing_group['VariableGroupName'], existing_group['VariableGroupName']) + for existing_group in groupsInDatasets + if existing_group['VariableGroupName']==topic_to_create['LevelTwoGroupName'] + and existing_group['DatasetName']==topic_to_create['DatasetName']])!=0: + print(topic_to_create) + print("ERROR - LEVEL TWO TOPIC LISTED FOR CREATION ALREADY EXISTS") + # 2. We need to verify that the set of groups in items_with_new_level_one_topics is the same as in + # topics_to_create["LevelOneGroupsToCreate"]. + sortedLevelOneGroupsInTopicsToCreate=sorted(list(set([(topic_to_create['DatasetName'], topic_to_create['LevelOneGroupName']) + for topic_to_create in topics_to_create["levelOneGroupsToCreate"]]))) + sortedDdiItemsForNewLevelOneTopics=sorted(list(set([(data_for_creating_new_topic['DatasetName'], + get_element_by_name(data_for_creating_new_topic['Item'], 'VariableGroupName')['String']) + for data_for_creating_new_topic in items_with_new_level_one_topics["LevelOne"]]))) + if sortedLevelOneGroupsInTopicsToCreate==sortedDdiItemsForNewLevelOneTopics: + print("We have successfully created all necessary DDI items for the new level one topics.") + else: + print("Error: we have not created all necessary DDI items for the new level one topics.") + # 3. We need to verify that the total set of groups in items_with_new_level_one_topics['LevelTwo'] and + # items_with_modified_level_one_topics['LevelTwo'] is the same as in topics_to_create['LevelTwoGroupsToCreate']. + sortedLevelTwoGroupsInTopicsToCreate=sorted(list(set([(topic_to_create['DatasetName'], topic_to_create['LevelTwoGroupName']) + for topic_to_create in topics_to_create["levelTwoGroupsToCreate"]]))) + sortedDdiItemsForNewLevelTwoTopics=sorted(list(set([(data_for_creating_new_topic['DatasetName'], + get_element_by_name(data_for_creating_new_topic['Item'], 'VariableGroupName')['String']) + for data_for_creating_new_topic in items_with_new_level_one_topics["LevelTwo"]]))) + sortedDdiItemsForNewLevelTwoTopicsWithExistingLevelOneTopics=sorted(list(set([(data_for_creating_new_topic['DatasetName'], + get_element_by_name(data_for_creating_new_topic['Item'], 'VariableGroupName')['String']) + for data_for_creating_new_topic in items_with_modified_level_one_topics["LevelTwo"]]))) + if sortedLevelTwoGroupsInTopicsToCreate==sorted(sortedDdiItemsForNewLevelTwoTopics + + sortedDdiItemsForNewLevelTwoTopicsWithExistingLevelOneTopics): + print("We have successfully created all necessary DDI items for the new level two topics.") + else: + print("Error: we have not created all necessary DDI items for the new level two topics.") + # We need to verify that the set of level one groups in items_with_modified_level_one_topics is the same + # as in topics_to_create['levelOneGroupsToModify'] + sortedLevelOneGroupsInTopicsToModify=sorted(list(set([(x['DatasetName'], x['Item']['ItemName']['en-GB']) + for x in topics_to_create['levelOneGroupsToModify']]))) + sortedDdiItemsForModifiedLevelOneTopics=sorted(list(set([(x['DatasetName'], get_element_by_name(x['Item'], 'VariableGroupName')['String']) + for x in items_with_modified_level_one_topics["LevelOne"]]))) + if sortedLevelOneGroupsInTopicsToModify==sortedDdiItemsForModifiedLevelOneTopics: + print("We have successfully created all necessary DDI items for the modified level one topics.") + else: + print("Error: we have not created all necessary DDI items for the modified level one topics.") + # Create an array which contains the new/modified topic groups... + updated_topic_groups=[] + for x in items_with_new_level_one_topics["LevelZero"] + items_with_new_level_one_topics["LevelOne"] + \ + items_with_new_level_one_topics["LevelTwo"] + items_with_modified_level_one_topics["LevelZero"] + \ + items_with_modified_level_one_topics["LevelOne"] + items_with_modified_level_one_topics["LevelTwo"]: + if x['Item'] is not None: + item_agency_id = x['Item'][0][0].text.split(":")[2] + item_identifier = x['Item'][0][0].text.split(":")[3] + item_version = x['Item'][0][0].text.split(":")[4] + update_list_of_topic_groups(x['Item'], + item_agency_id, + item_identifier, + item_version, + C.item_code('Variable Group'), + updated_topic_groups, + dataset=x['DatasetName']) + # Now that we have verified that the topics to create/modify are correct, we can proceed to + # creating the urn dataframe which specifies which items should be moved to which topics + topic_reassignments_data_frame=generate_urn_dataframe_for_questions_and_variables(input_file, + updated_topic_groups, + C, + datasetToZeroGroupMappings=datasetToZeroGroupMappings, + groupsInDatasets=groupsInDatasets) + # Run the update_topics method that creates DDI objects that reassigns items to topics. Note that it uses + # the updated_topic_groups array as an input argument, this array contains the DDI items representing topics + updated_topics=update_topics(topic_reassignments_data_frame, C, updated_topic_groups=updated_topic_groups) + final_validation_results=validate_ddi_implementing_topic_reassignments(input_file, + updated_topics["UpdatedTopicGroups"], C) + return { + "UpdatedTopics": updated_topics, + "ValidationResults": final_validation_results, + "TopicReassignmentsDataFrame": topic_reassignments_data_frame + } + +def update_urns_list(urns, + item, + containing_item, + topic_type, + source_topic, + target_topic, + topic_groups, + C, + groupsInDatasets=[], + datasetToZeroGroupMappings={}, + dataset_name=""): + """Updates a dictionary containing lists of URNs used for topic reassignment. + + Arguments: urns (dict): A dictionary containing lists of URNs used for topic reassignment. + item (dict): A dictionary containing details of the item being reassigned to a new topic. + containing_item (dict): A dictionary containing details of the item containing the item being + reassigned. + topic_type (str): The item type code for the topic groups (e.g. variable group or question group). + target_topic (str): The name of the destination topic group. + topic_groups (str): A list of DDI items that are the groups representing topics that we + are updating (e.g. by adding/removing references to variables/questions, in order to reassign + these items to new topics). + C (ColecticaObject): an authenticated ColecticaObject instance. + + Keyword arguments: + dataset_name (str): The name of the dataset containing the item being reassigned. + + Note that the following are mutable arguments, we want them to be updated in place + so it can be reused later in the process (see the method move_topics for how they are used). + + groupsInDatasets: A list of dict objects that map the datasets to topic groups they contain. + datasetToZeroGroupMappings (dict): A dictionary mapping dataset names to level zero topic groups. + + Returns: + None: The function updates the urns dictionary in place. + """ + destination_topic_urn="" + containing_item_details = {"AgencyId": containing_item['AgencyId'], + "Identifier": containing_item['Identifier'], + "Version": containing_item['Version'], + } + item_urn = get_urn_from_item(item) + """source_groups = C.search_relationship_byobject(item['AgencyId'], + item['Identifier'], Version=item['Version'], + item_types=topic_type, Descriptions=True) + """ + source_groups=get_item_from_topic_name(str(source_topic), + topic_type, + containing_item_details, + C, + dataset_name=dataset_name, + groupsInDatasets=groupsInDatasets, + datasetToZeroGroupMappings=datasetToZeroGroupMappings) + for source_group in source_groups: + source_topic_urn = get_urn_from_item(source_group) + if len(source_groups)==0: + source_topic_urn = "" + destination_group=get_item_from_topic_name(str(target_topic), + topic_type, + containing_item_details, + C, + dataset_name=dataset_name, + groupsInDatasets=groupsInDatasets, + datasetToZeroGroupMappings=datasetToZeroGroupMappings) + if len(destination_group)==1: + destination_topic_urn = get_urn_from_item(destination_group[0]) + elif topic_type==C.item_code('Variable Group'): + # the topic groups all exist for questions which is why we only do the below for variable groups + destination_group_details=[x for x in topic_groups if x['DatasetName']==dataset_name + and get_elements_of_type(x['Item'], "VariableGroupName")!=[] + and get_elements_of_type(x['Item'], "VariableGroupName")[0][0].text==str(target_topic)] + if len(destination_group_details)==1: + destination_topic_urn=get_urn_from_fragment(destination_group_details[0]['Item']) + if item_urn not in urns['itemUrns'] and destination_topic_urn != "" and source_topic_urn != destination_topic_urn: + urns['itemUrns'].append(item_urn) + urns['sourceTopicGroups'].append(source_topic_urn) + urns['destinationTopicGroups'].append(destination_topic_urn) + urns['datasets'].append(dataset_name) + +def generate_urn_dataframe_for_questions_and_variables(input_file_name, + topic_groups, + C, + groupsInDatasets=[], + datasetToZeroGroupMappings={}, + ): + """Generates a dataframe containing URNs used for topic reassignment. + + Arguments: + input_file_name (str): the name of the Excel spreadsheet containing details of topic reassignments. + topic_groups (str): A list of DDI items that are the groups representing topics that we + are updating (e.g. by adding/removing references to variables/questions, in order to reassign + these items to new topics). + C (ColecticaObject): an authenticated ColecticaObject instance. + + Keyword arguments: + Note that these are mutable arguments, we want them to be updated in place + so they can be reused later in the process (see the method move_topics for how they are used). + + groupsInDatasets: A list of dict objects that map the datasets to topic groups they contain. + datasetToZeroGroupMappings (dict): a dictionary mapping dataset names to level zero topic groups. -def generate_urn_dataframe(input_file_name, C): - """Method for generating input for code that updates topics. The code iterates through - a spreadsheet containing details of new item topic assignments and generates a dataframe - of URNs that can be used as input to a method that reassigns items to new topics. + Returns: + pd.DataFrame: A dataframe containing URNs representing topic reassignment operations. """ print(f"Reading topic reassignments from {input_file_name}") data = pd.read_excel(input_file_name) - urn_data_frame={ + urns={ "itemUrns": [], "sourceTopicGroups": [], - "destinationTopicGroups": [] + "destinationTopicGroups": [], + "datasets": [] } # Iterate through the rows in the spreadsheet. Each row contains details of a topic # reassignment for an item... - for topic_reassignment_details in data.iloc: - containing_item_name = topic_reassignment_details.iloc[0] - url = topic_reassignment_details.iloc[2] - agency_id = url.split("/")[4] - identifier = url.split("/")[5] - if len(url.split("/")) == 7: - version = url.split("/")[6] - item = C.get_item_json(agency_id, identifier, version=version) + print("Creating dataframe containing URNs used for topic reassignment...") + for index, topic_reassignment_details in data.iterrows(): + print(f"Creating dataframe row {index+1} of {data.shape[0]}...") + topic_dict = create_topic_reassignment_dict(topic_reassignment_details, C) + physical_instance_containing_variable = C.search_items( + C.item_code('Data File'), + SearchTerms=str(topic_dict['dataset_name']).strip(), + SearchLatestVersion=True)['Results'] + print(len(physical_instance_containing_variable)) + if len(physical_instance_containing_variable)==1: + update_urns_list(urns, topic_dict['item'], physical_instance_containing_variable[0], + C.item_code('Variable Group'), + topic_dict['source_topic_name'], + topic_dict['destination_topic_name'], + topic_groups, C, datasetToZeroGroupMappings=datasetToZeroGroupMappings, + groupsInDatasets=groupsInDatasets, + dataset_name=topic_dict['dataset_name']) else: - item = C.get_item_json(agency_id, identifier) - version = item['Version'] - item_urn = "urn:ddi:" + agency_id + ":" + identifier + ":" + str(version) - item_type = item['ItemType'] - if item_type==C.item_code('Question'): - topic_type=C.item_code('Question Group') - containing_item_type=C.item_code('Data Collection') - elif item_type==C.item_code('Variable'): - topic_type=C.item_code('Variable Group') - containing_item_type=C.item_code('Data File') - item_urn = get_urn_from_item(item) - source_topic = get_item_from_topic_name(topic_reassignment_details.iloc[4], topic_type, containing_item_name, containing_item_type, C) - destination_topic = get_item_from_topic_name(topic_reassignment_details.iloc[5], topic_type, containing_item_name, containing_item_type, C) - urn_data_frame['itemUrns'].append(item_urn) - if len(source_topic)>0: - urn_data_frame['sourceTopicGroups'].append(get_urn_from_item(source_topic[0])) - if len(destination_topic)>0: - urn_data_frame['destinationTopicGroups'].append(get_urn_from_item(destination_topic[0])) - return pd.DataFrame(urn_data_frame) - -def update_topics(input_file_name, C): - """Method for reassigning items to new topics. The code iterates through a data frame - containing details of new item topic assignments and performs the reassignments. + raise ValueError(f"Cannot find unique physical instance with name {topic_dict['dataset_name']}, " + f"found {len(physical_instance_containing_variable)} instances") + allRelatedQuestions= C.search_relationship_bysubject(topic_dict['item']['AgencyId'], + topic_dict['item']['Identifier'], + Version=topic_dict['item']['Version'], item_types=C.item_code("Question"), Descriptions=True) + for relatedQuestion in allRelatedQuestions: + question_sets=C.query_set(relatedQuestion['AgencyId'], + relatedQuestion['Identifier'], + version=relatedQuestion['Version'], + reverseTraversal=True, + item_types=[C.item_code('Data Collection')]) + print(len(question_sets)>0 and len(set([(x['Item1']['Item3'], x['Item1']['Item1']) for x in question_sets] ))==1) + if len(question_sets)>0 and len(set([(x['Item1']['Item3'], x['Item1']['Item1']) for x in question_sets] ))==1: + latest_version_of_question_set = max([x['Item1']['Item2'] for x in question_sets]) + containing_item=C.get_item_xml(question_sets[0]['Item1']['Item3'], + question_sets[0]['Item1']['Item1'], + version=latest_version_of_question_set) + update_urns_list(urns, relatedQuestion, containing_item, + C.item_code('Question Group'), + topic_dict['source_topic_name'], + topic_dict['destination_topic_name'], + topic_groups, + C, + groupsInDatasets=groupsInDatasets, + dataset_name=topic_dict['dataset_name'], + datasetToZeroGroupMappings=datasetToZeroGroupMappings) + return (pd.DataFrame(urns)) + +def get_level_zero_group_from_dataset(physical_instance_containing_variable, all_variable_groups, C): + """Gets the level zero topic group for a dataset. If the level zero topic group cannot be found + for a dataset, an error is raised and the process crashes. + + Arguments: + physical_instance_containing_variable (dict): A dictionary containing details of a + physical instance. + all_variable_groups (list): A list of all variable groups in the repository. + C (ColecticaObject): an authenticated ColecticaObject instance. + Returns: + ElementTree.Element: An ElementTree.Element representing the level zero topic group. + """ + level_zero_group_details=C.search_relationship_byobject( + physical_instance_containing_variable['AgencyId'], + physical_instance_containing_variable['Identifier'], + Version=physical_instance_containing_variable['Version'], + item_types=[C.item_code('Variable Group')]) + level_zero_group_item=None + if len(level_zero_group_details)==0: + level_zero_group_details=[x for x in all_variable_groups if x['Label']['en-GB']== + physical_instance_containing_variable['Label']['en-GB']] + if len(level_zero_group_details)==1: + level_zero_group_item=C.get_item_xml(level_zero_group_details[0]['AgencyId'], + level_zero_group_details[0]['Identifier'], + version=level_zero_group_details[0]['Version'])['Item'] + else: + raise ValueError(f"Cannot find level zero group for dataset: " + f"Agency: {physical_instance_containing_variable['AgencyId']}, " + f"Identifier: {physical_instance_containing_variable['Identifier']}, " + f"Version: {physical_instance_containing_variable['Version']}") + else: + level_zero_group_item=C.get_item_xml(level_zero_group_details[0]['Item1']['Item3'], + level_zero_group_details[0]['Item1']['Item1'], + version=level_zero_group_details[0]['Item1']['Item2'])['Item'] + if level_zero_group_item is None: + return None + else: + return(defusedxml.ElementTree.fromstring(level_zero_group_item)) + +def create_topic_reassignment_dict(topic_reassignment_details, C): + """Creates a dictionary containing details of a topic reassignment, for convenient + reuse later in the code and to avoid code in multiple places which re-parses the same details. + + Arguments: + topic_reassignment_details (pd.Series): A pandas Series containing details of a topic reassignment. + C (ColecticaObject): an authenticated ColecticaObject instance. + + Returns: + dict: A dictionary containing details of a topic reassignment. + """ + topic_reassignment_dict={} + dataset_name = topic_reassignment_details.iloc[0] + topic_reassignment_dict['dataset_name']=dataset_name + url = topic_reassignment_details.iloc[2] + topic_reassignment_dict['source_topic_name'] = str(topic_reassignment_details.iloc[4]) + topic_reassignment_dict['destination_topic_name'] = str(topic_reassignment_details.iloc[5]) + agency_id = url.split("/")[4] + identifier = url.split("/")[5] + item = C.get_item_xml(agency_id, identifier) + item_element = defusedxml.ElementTree.fromstring(item['Item']) + topic_reassignment_dict['namespace_version'] = get_namespace(item_element.tag).split(':')[2] + topic_reassignment_dict['item'] = item + physical_instance_containing_variable = C.search_items( + C.item_code('Data File'), + SearchTerms=str(dataset_name).strip(), + SearchLatestVersion=True, + UsePrefixSearch=True )['Results'] + if len(physical_instance_containing_variable)==1: + topic_reassignment_dict['physical_instance_containing_variable'] = physical_instance_containing_variable[0] + topic_reassignment_dict['physical_instance_search_set'] = { + "AgencyId": physical_instance_containing_variable[0]['AgencyId'], + "Identifier": physical_instance_containing_variable[0]['Identifier'], + "Version": physical_instance_containing_variable[0]['Version'] + } + else: + raise ValueError(f"Cannot find physical instance with name {dataset_name}") + return topic_reassignment_dict + +def find_topics_to_create(input_file_name, C, groupsInDatasets=[], datasetToZeroGroupMappings={}): + """This code iterates through an Excel input file containing details of new variable topic + assignments, and finds variable groups representing topics that don't already exist and + will need to be created in order to perform the topic reassignments. + + Arguments: + input_file_name (str): the name of the Excel spreadsheet containing details of new + variable topic assignments. + C (ColecticaObject): an authenticated ColecticaObject instance. + + Keyword arguments: + Note that these are mutable arguments, we want them to be updated in place + so they can be reused later in the process (see the method move_topics for how they are used). + + datasetToZeroGroupMappings (dict): a dictionary mapping dataset names to level zero + topic groups. + groupsInDatasets: A list of dict objects that map the datasets to topic groups they contain. + + Returns: + dict: A dictionary containing three lists of objects: + 1. The level one groups that need to be created because a level two group is being created + that requires it (e.g. if we are creating a level two group in a dataset representing the + 10320 topic, but the 103 topic is not already present in that dataset). + 2. The level two groups that need to be created. + 3. The level one groups that need to be modified because a level two group is being created + in a dataset where the associated level one group already exists (e.g. if we are creating + a level two group in a dataset representing the 10320 topic, and the 103 topic already + exists in that dataset). + """ + print(f"Reading topic reassignments from {input_file_name}, finding topics that need to be created...") + data = pd.read_excel(input_file_name) + levelOneGroupsToCreate=[] + levelOneGroupsToModify=[] + levelTwoGroupsToCreate=[] + all_variable_groups=C.search_items(C.item_code('Variable Group'), SearchLatestVersion=True)['Results'] + for index, topic_reassignment_details in data.iterrows(): + topic_dict = create_topic_reassignment_dict(topic_reassignment_details, C) + topic_type=C.item_code('Variable Group') + level_zero_group=get_level_zero_group_from_dataset(topic_dict['physical_instance_containing_variable'], + all_variable_groups, C) + destination_topic = get_item_from_topic_name(topic_dict['destination_topic_name'], + topic_type, + topic_dict['physical_instance_search_set'], + C, + groupsInDatasets=groupsInDatasets, + dataset_name=topic_dict['dataset_name'], + datasetToZeroGroupMappings=datasetToZeroGroupMappings) + if len(destination_topic)==0: + level_one_group_name = str(topic_dict['destination_topic_name'])[0:3] + if len(topic_dict['destination_topic_name'])==5: + level_two_group_name = str(topic_dict['destination_topic_name']) + if ((topic_dict['dataset_name'], topic_dict['destination_topic_name']) + not in [(x['DatasetName'], x['LevelTwoGroupName']) for x in levelTwoGroupsToCreate]): + levelTwoGroupsToCreate.append({'DatasetName': topic_dict['dataset_name'], + 'LevelTwoGroupName': topic_dict['destination_topic_name'], + 'NamespaceVersion': topic_dict['namespace_version']}) + levelOneGroups=get_item_from_topic_name(level_one_group_name, + topic_type, + topic_dict['physical_instance_search_set'], + C, + dataset_name=topic_dict['dataset_name'], + groupsInDatasets=groupsInDatasets, + datasetToZeroGroupMappings=datasetToZeroGroupMappings) + if len(levelOneGroups)==0: + if len( [x for x in levelOneGroupsToCreate + if x['DatasetName']==topic_dict['dataset_name'] + and x['LevelOneGroupName']==level_one_group_name])==0: + levelOneGroupsToCreate.append({'DatasetName': topic_dict['dataset_name'], + 'LevelOneGroupName': level_one_group_name, + 'LevelZeroGroup': level_zero_group, + 'NamespaceVersion': topic_dict['namespace_version']}) + else: + if len( [x for x in levelOneGroupsToModify + if x['DatasetName']==topic_dict['dataset_name'] + and x['LevelTwoGroupName']==level_two_group_name])==0: + for group in levelOneGroups: + levelOneGroupsToModify.append({ + 'DatasetName': topic_dict['dataset_name'], + 'LevelTwoGroupName': level_two_group_name, + 'NamespaceVersion': topic_dict['namespace_version'], + 'Item': group + }) + return ({"levelOneGroupsToCreate": levelOneGroupsToCreate, + "levelTwoGroupsToCreate": levelTwoGroupsToCreate, + "levelOneGroupsToModify": levelOneGroupsToModify}) + +def create_ddi_objects_with_new_level_one_topics(level_one_groups_to_create, level_two_groups_to_create, C): + """Creates ddi objects that are needed for topic reassignments that require creating new level one + topics. This involves creating ddi objects for the new level one topics, and for new level two topics + if necessary, as well as modifying the appropriate level zero topics to include references to the new + level one topics. + + Arguments: + level_one_groups_to_create: the level one groups that need to be created either because an item + is being reassigned to a level one topic group that does not yet exist, or because a level + two group is being created that requires it (e.g. if we are creating a level two + group in a dataset representing the 10320 topic, but the 103 topic is not already + present in that dataset). + level_two_groups_to_create: the level two groups that need to be created. + C (ColecticaObject): an authenticated ColecticaObject instance. + + Returns: + dict: A dictionary containing three lists of objects: + 1. Objects containing information about the DDI level zero topic groups that have + been modified to include references to the new level one groups. + 2. Objects containing information about the DDI level one topic groups that have been + created. + 3. Objects containing information about the DDI level two topic groups that have been + created. + """ + allConcepts=C.search_items(C.item_code('Concept'))['Results'] + # get rid of concepts that don't properly define an itemname + concepts=[x for x in allConcepts if list(x['ItemName'].keys())==['en-GB']] + ddiObjectsLevelZero = [] + ddiObjectsLevelOne = [] + ddiObjectsLevelTwo = [] + for level_one_group_to_create in level_one_groups_to_create: + level_one_group_uuid=str(uuid.uuid4()) + level_one_group_name=level_one_group_to_create['LevelOneGroupName'] + topic_type=C.item_code('Variable Group') + namespace_version=level_one_group_to_create['NamespaceVersion'] + level_zero_group_urn=level_one_group_to_create['LevelZeroGroup'][0][0].text + zero_group_agency_id = level_zero_group_urn.split(":")[2] + zero_group_identifier = level_zero_group_urn.split(":")[3] + zero_group_version = level_zero_group_urn.split(":")[4] + level_zero_group_object = get_current_state_of_topic_group(zero_group_agency_id, + zero_group_identifier, ddiObjectsLevelZero, C, version=zero_group_version) + level_one_group_label=get_group_label(level_one_group_name, topic_type, C) + levelOneConcept=[x for x in concepts if x['ItemName']['en-GB']==level_one_group_name] + if len(levelOneConcept)==1: + level_one_group_object = create_variable_group(level_one_group_name, + level_one_group_label, + level_one_group_uuid, + namespace_version, + levelOneConcept[0]['AgencyId'], + levelOneConcept[0]['Identifier'], + levelOneConcept[0]['Version']) + level_one_group_reference=create_group_reference('uk.closer', + level_one_group_uuid, 1, namespace_version, topic_type, C) + get_element_fragment_by_name(level_zero_group_object, "VariableGroup").append(level_one_group_reference) + update_list_of_topic_groups(level_zero_group_object, + zero_group_agency_id, + zero_group_identifier, + zero_group_version, + topic_type, + ddiObjectsLevelZero, + dataset=level_one_group_to_create['DatasetName']) + level_two_groups=[x for x in level_two_groups_to_create + if x['DatasetName']==level_one_group_to_create['DatasetName'] + and x['LevelTwoGroupName'][0:3]==level_one_group_to_create['LevelOneGroupName']] + for level_two_group in level_two_groups: + level_two_group_uuid=str(uuid.uuid4()) + level_two_group_name=level_two_group['LevelTwoGroupName'] + level_two_group_label=get_group_label(level_two_group_name, + topic_type, C) + levelTwoConcept=[x for x in concepts if x['ItemName']['en-GB']==level_two_group_name] + if len(levelTwoConcept)==1: + level_two_group_object=create_variable_group(level_two_group_name, + level_two_group_label, + level_two_group_uuid, + namespace_version, + levelTwoConcept[0]['AgencyId'], + levelTwoConcept[0]['Identifier'], + levelTwoConcept[0]['Version']) + reference_to_level_two_group=create_group_reference('uk.closer', + level_two_group_uuid, + 1, + namespace_version, topic_type, + C) + level_one_group_object[0].append(reference_to_level_two_group) + ddiObjectsLevelTwo.append({'Item': level_two_group_object, + 'DatasetName': level_two_group['DatasetName']}) + ddiObjectsLevelOne.append({'Item': level_one_group_object, + 'DatasetName': level_one_group_to_create['DatasetName']}) + return({"LevelZero": ddiObjectsLevelZero, "LevelOne": ddiObjectsLevelOne, "LevelTwo": ddiObjectsLevelTwo}) + +def create_ddi_objects_for_modified_level_one_topics(level_one_objects_to_modify, C, language = "en-GB"): + """Creates ddi objects that are needed for topic reassignments where an item is being reassigned to + level two topic that doesn't yet exist, but the relevant level one topic already exists. + This involves creating ddi objects for new destination level two topics, and creating ddi objects + that represent the modified level one topics that include references to the new level two ddi objects. + + Arguments: + level_one_objects_to_modify (list): A list of items containing information about the + level one groups that need to be modified because a level two group is being + created in a dataset where the associated level one group already exists + (e.g. when we are creating a level two group in a dataset representing + the 10320 topic, and the 103 topic already exists in that dataset). + + C (ColecticaObject): an authenticated ColecticaObject instance. + + Returns: + dict: A dictionary containing three lists of objects: + 1. Objects containing information about the DDI level zero topic groups that include + references to the modified level one groups. These level zero topics aren't modified + by this method, but are included for convenience when validating the topic group + modifications/creations performed by this group. + 2. Objects containing information about the DDI level one topic groups that have been + modified. + 3. Objects containing information about the DDI level two topic groups that have been + created. + """ + allConcepts=C.search_items(C.item_code('Concept'))['Results'] + # get rid of concepts that don't properly define an itemname + concepts=[x for x in allConcepts if list(x['ItemName'].keys())==['en-GB']] + uniqueL1GroupsToModify=list([{"LevelOneTopicName": level_one_object['Item']['ItemName'][language], + "LevelTwoGroupName": level_one_object['LevelTwoGroupName'], + "Item": level_one_object['Item'], + "NamespaceVersion": level_one_object['NamespaceVersion'], + "DatasetName": level_one_object['DatasetName']} for level_one_object in level_one_objects_to_modify]) + ddiObjectsLevelZero = [] + modifiedDdiObjectsLevelOne = [] + newDdiObjectsLevelTwo = [] + for level_one_group in uniqueL1GroupsToModify: + level_one_group_agency_id=level_one_group['Item']['AgencyId'] + level_one_group_identifier=level_one_group['Item']['Identifier'] + level_one_group_version=level_one_group['Item']['Version'] + level_one_group_object = get_current_state_of_topic_group(level_one_group_agency_id, + level_one_group_identifier, + modifiedDdiObjectsLevelOne, + C, + version=level_one_group_version) + level_two_group_uuid=str(uuid.uuid4()) + level_two_group_name=level_one_group['LevelTwoGroupName'] + topic_type=level_one_group['Item']['ItemType'] + namespace_version=level_one_group['NamespaceVersion'] + level_two_group_label=get_group_label(level_two_group_name, topic_type, C) + concept=[concept for concept in concepts if concept['ItemName']['en-GB']==level_two_group_name] + if len(concept)==1: + level_two_group_object=create_variable_group(level_two_group_name, + level_two_group_label, + level_two_group_uuid, + namespace_version, + concept[0]['AgencyId'], + concept[0]['Identifier'], + concept[0]['Version']) + reference_to_level_two_group=create_group_reference('uk.closer', + level_two_group_uuid, + 1, + namespace_version, + topic_type, + C) + level_zero_group_for_level_one_topic=get_level_zero_group_for_topic(level_one_group['Item'], C) + if len([level_zero_object for level_zero_object in ddiObjectsLevelZero + if level_zero_object['Item'] is not None and + level_zero_object['Item'][0][2].text==level_zero_group_for_level_one_topic[0][2].text])==0: + ddiObjectsLevelZero.append({"Identifier": level_one_group['Item']['Identifier'], + "AgencyId": level_one_group['Item']['AgencyId'], + "Version": level_one_group['Item']['Version'], + "ItemType": level_one_group['Item']['ItemType'], + "Item": level_zero_group_for_level_one_topic, + "DatasetName": level_one_group['DatasetName']}) + get_element_fragment_by_name(level_one_group_object, "VariableGroup").append( + reference_to_level_two_group) + update_list_of_topic_groups(level_one_group_object, + level_one_group_agency_id, + level_one_group_identifier, + level_one_group_version, + topic_type, + modifiedDdiObjectsLevelOne, + dataset=level_one_group['DatasetName']) + newDdiObjectsLevelTwo.append({"Item": level_two_group_object, + "DatasetName": level_one_group['DatasetName']}) + else: + raise ValueError(f"Could not find Concept for level two group {level_two_group_name}") + return({ "LevelZero": ddiObjectsLevelZero, + "LevelOne": [{"Item": x['Item'], "DatasetName": x['DatasetName']} for x in modifiedDdiObjectsLevelOne], + "LevelTwo": newDdiObjectsLevelTwo}) + +def validateLevelTwoTopics(levelZeroTopics, levelOneTopics, levelTwoTopics, C): + """Validates level two topics by checking that: + 1. The dataset containing the level two topic exists. + 2. The level two topic is referenced from a level one topic group in the same dataset. + 3. The first three digits of the level two topic name match the level one topic name. + 4. The level one topic is referenced from a level zero topic group. + 5. The dataset label for the dataset containing the level two topic matches the label + of the level zero topic group. + + Arguments: + levelZeroTopics (list): A list of level zero topic groups. + levelOneTopics (list): A list of level one topic groups. + levelTwoTopics (list): A list of level two topic groups. + C (ColecticaObject): an authenticated ColecticaObject instance. + Returns: + dict: A dictionary object containing two lists: + 1. The validated level two topics. + 2. The invalid level two topics. + """ + validatedLevelTwoTopics = [] + invalidLevelTwoTopics = [] + print("Validating level two topics...") + for levelTwoTopic in levelTwoTopics: + levelTwoTopicName=get_elements_of_type(levelTwoTopic['Item'], + "VariableGroupName")[0][0].text + dataset=C.search_items( + C.item_code('Data File'), + SearchTerms=str(levelTwoTopic['DatasetName']).strip(), + SearchLatestVersion=True)['Results'] + found = False + if len(dataset)==1: + # Dataset containing level two topic found... + datasetLabel=dataset[0]['Label']['en-GB'] + identifier = levelTwoTopic['Item'][0][2].text + for levelOneTopic in levelOneTopics: + levelOneTopicName=get_elements_of_type(levelOneTopic['Item'], + "VariableGroupName")[0][0].text + level_one_refs=find_all_references(levelOneTopic['Item'], + 'uk.closer', + identifier) + if len(level_one_refs)==1 and levelOneTopic['DatasetName']==levelTwoTopic['DatasetName']: + # Level two reference found in level one group, and dataset names for groups match... + if levelTwoTopicName[0:3]==levelOneTopicName: + # First three digits of level two topic name matches level one topic name... + level_one_identifier=levelOneTopic['Item'][0][2].text + level_zero_refs=[] + for levelZeroTopic in levelZeroTopics: + if levelZeroTopic['Item'] is not None: + level_zero_refs=find_all_references(levelZeroTopic['Item'], + 'uk.closer', + level_one_identifier) + if len(level_zero_refs)==1: + # Level one reference found in level zero, now check labels match... + level_zero_label = get_element_by_name( + levelZeroTopic['Item'], + 'Label')['Content'] + if level_zero_label==datasetLabel: + # Labels for dataset containing level two group and level zero group match + found=True + validatedLevelTwoTopics.append(levelTwoTopic) + if not found: + invalidLevelTwoTopics.append(levelTwoTopic) + if len(validatedLevelTwoTopics) == len(levelTwoTopics): + print("""The validation has been successful. For each of the specified level two topics, + the following checks have passed: + + 1. The dataset containing the level two topic exists. + 2. The level two topic is referenced from a level one topic group in the same dataset. + 3. The first three digits of the level two topic name match the level one topic name. + 4. The level one topic is referenced from a level zero topic group. + 5. The dataset label for the dataset containing the level two topic matches the label + of the level zero topic group.""") + else: + print(f"The following level two topics are invalid: {invalidLevelTwoTopics}") + return { + "ValidatedLevelTwoTopics": validatedLevelTwoTopics, + "InvalidLevelTwoTopics": invalidLevelTwoTopics + } + +def validateLevelOneTopics(ddi_objects_level_zero, level_one_topics, C): + """Validates level one topics by checking that: + 1. The level one topic is referenced from a level zero topic group. + 2. The dataset label for the dataset containing the level one topic matches the label + of the level zero topic group. + + Arguments: + ddi_objects_level_zero (list): A list of level zero topic groups. + level_one_topics (list): A list of level one topic groups. + C (ColecticaObject): an authenticated ColecticaObject instance. + Returns: + dict: A dictionary object containing two lists: + 1. The validated level one topics. + 2. The invalid level one topics. + """ + validatedLevelOneTopics = [] + invalidLevelOneTopics = [] + found = False + print("Validating level one topics...") + for level_one_topic in level_one_topics: + level_one_identifier=level_one_topic['Item'][0][2].text + for level_zero_object in ddi_objects_level_zero: + if level_zero_object['Item'] is not None: + level_zero_refs=find_all_references(level_zero_object['Item'], + 'uk.closer', + level_one_identifier) + if len(level_zero_refs)==1: + # Level one reference found in level zero, now check dataset labels match... + physical_instance = C.search_items( + C.item_code('Data File'), + SearchTerms=str(level_one_topic['DatasetName']).strip(), + SearchLatestVersion=True)['Results'] + if len(physical_instance)==1: + levelOneDatasetLabel=physical_instance[0]['Label']['en-GB'] + levelZeroGroupLabel=(get_element_by_name( + level_zero_object['Item'], 'Label')['Content']) + if levelOneDatasetLabel==levelZeroGroupLabel: + # Dataset labels for level one and the level zero label match + validatedLevelOneTopics.append(level_one_topic) + found = True + else: + raise ValueError(f"Cannot find unique physical instance with name {level_one_topic['DatasetName']}, " + f"found {len(physical_instance)} instances") + if not found: + invalidLevelOneTopics.append(level_one_topic) + if len(validatedLevelOneTopics) == len(level_one_topics): + print("""The validation has been successful. For each of the specified level one topics, the + following checks have passed: + + 1. The level one topic is referenced from a level zero topic group. + 2. The dataset label for the dataset containing the level one topic matches the label + of the level zero topic group.""") + else: + print(f"The following level one topics are invalid: {invalidLevelOneTopics}") + return { + "ValidatedLevelOneTopics": validatedLevelOneTopics, + "InvalidLevelOneTopics": invalidLevelOneTopics + } + +def update_topics(topic_reassignments_data_frame, C, updated_topic_groups=None): + """Method used for reassigning items to new topics. The code iterates through a data frame + containing details of new item topic assignments and creates DDI objects representing + the updated topics, which will then be used to perform the reassignments. + + Arguments: + topic_reassignment_details (pd.Series): A pandas Series containing details of a topic reassignment. + C (ColecticaObject): an authenticated ColecticaObject instance. + updated_topic_groups (list): List of dict-like entries representing topic groups. + + Returns: + dict: A dictionary object containing three lists: + 1. Items that have already been moved from a source topic. + 2. Items that have already been moved to a destination topic. + 3. The list of DDI objects representing the updated topics. + """ - topic_reassignments_data_frame=generate_urn_dataframe(input_file_name, C) # Initialise lists... - item_not_present_in_source_topic = [] - item_present_in_destination_topic = [] - updated_topic_groups = [] + if updated_topic_groups is None: + updated_topic_groups = [] + items_not_present_in_source_topic = [] + items_present_in_destination_topic = [] + reference_from_source_ddi_version = None # Iterate through the rows in the data frame. Each row contains details of a topic # reassignment for a item... - for topic_reassignment_details in topic_reassignments_data_frame.iloc: - print("Performing the following topic reassignment...") - print(f"Item {topic_reassignment_details.iloc[0]} to {topic_reassignment_details.iloc[1]}") - topic_reassignment_details.iloc[0] - item_agency_id = topic_reassignment_details.iloc[0].split(":")[2] - item_identifier = topic_reassignment_details.iloc[0].split(":")[3] - item_version = topic_reassignment_details.iloc[0].split(":")[4] - source_group_item_agency_id = topic_reassignment_details.iloc[1].split(":")[2] - source_group_item_identifier = topic_reassignment_details.iloc[1].split(":")[3] - source_group_item_version = topic_reassignment_details.iloc[1].split(":")[4] - destination_group_item_agency_id = topic_reassignment_details.iloc[2].split(":")[2] - destination_group_item_identifier = topic_reassignment_details.iloc[2].split(":")[3] - destination_group_item_version = topic_reassignment_details.iloc[2].split(":")[4] - item = C.get_item_json(item_agency_id, item_identifier, version = item_version) - source_group = C.get_item_json(source_group_item_agency_id, - source_group_item_identifier, - version = source_group_item_version) - destination_group = C.get_item_json(destination_group_item_agency_id, - destination_group_item_identifier, - version = destination_group_item_version) - # We get the current state of the group containing a reference to the item. - # This group represents the topic the item is currently assigned - # to. - source_item = get_current_state_of_topic_group( + for index, topic_reassignment_details in topic_reassignments_data_frame.iterrows(): + print("Creating DDI that will perform the following topic reassignment...") + print(f"Item {topic_reassignment_details['itemUrns']} to {topic_reassignment_details['destinationTopicGroups']}") + item_agency_id = topic_reassignment_details['itemUrns'].split(":")[2] + item_identifier = topic_reassignment_details['itemUrns'].split(":")[3] + item_version = topic_reassignment_details['itemUrns'].split(":")[4] + item = C.get_item_json(item_agency_id, item_identifier, version = item_version) + topic_type="" + if item['ItemType'] == C.item_code('Variable'): + topic_type=C.item_code('Variable Group') + elif item['ItemType'] == C.item_code('Question'): + topic_type=C.item_code('Question Group') + reference_to_move=None + reference_from_source_ddi_version = None + if topic_reassignment_details['sourceTopicGroups'] !='': + source_group_item_agency_id = topic_reassignment_details['sourceTopicGroups'].split(":")[2] + source_group_item_identifier = topic_reassignment_details['sourceTopicGroups'].split(":")[3] + source_group = C.get_item_json(source_group_item_agency_id, + source_group_item_identifier) + # We get the current state of the group containing a reference to the item. + # This group represents the topic the item is currently assigned to. + source_group_item = get_current_state_of_topic_group( source_group['AgencyId'], source_group['Identifier'], updated_topic_groups, C, version=source_group['Version'] ) + # Find and remove the reference to the item in the source group/topic. + references_to_move = find_all_references( + source_group_item, + item['AgencyId'], + item['Identifier']) + if len(references_to_move) > 0: + for reference_to_move in references_to_move: + source_group_item[0].remove(reference_to_move) + # We need to take note of the namespace version for the reusable element, + # we use this later when determining if we need to update the version + # number to the version used in the destination topic group. + reference_from_source_ddi_version = ("ddi:reusable:" + f"{get_namespace(reference_to_move.tag).split(':')[2]}") + # Finally we update the array containing the most current versions of the + # group/topics with the updated source topic... + update_list_of_topic_groups(source_group_item, + source_group['AgencyId'], + source_group['Identifier'], + source_group['Version'], + source_group['ItemType'], + updated_topic_groups, + dataset=topic_reassignment_details['datasets']) + destination_group_item_agency_id = topic_reassignment_details['destinationTopicGroups'].split(":")[2] + destination_group_item_identifier = topic_reassignment_details['destinationTopicGroups'].split(":")[3] + destination_group_item_version = topic_reassignment_details['destinationTopicGroups'].split(":")[4] # We get the current state of the group that we will be adding a # reference to the item to. This group represents the topic the # item will be reassigned to. + print(destination_group_item_agency_id) + print(destination_group_item_identifier) + print(destination_group_item_version) destination_item = get_current_state_of_topic_group( - destination_group['AgencyId'], - destination_group['Identifier'], + destination_group_item_agency_id, + destination_group_item_identifier, updated_topic_groups, C, - version=destination_group['Version'] + version=destination_group_item_version, ) - # Find and remove the reference to the item in the source group/topic. - references_to_move = find_all_references( - source_item, item['AgencyId'], item['Identifier']) + print(destination_item) # We check to see if a reference to the item is already present in the - # destination group/topic. This information can be used to determine if the + # destination group/topic. This can be used to determine if the # topic reassignments described in the input file have already been # successfully performed. reference_in_destination_topic = find_all_references(destination_item, - item['AgencyId'], item['Identifier']) - if len(references_to_move) > 0 and len(reference_in_destination_topic)==0: - for reference_to_move in references_to_move: - source_item[0].remove(reference_to_move) + item['AgencyId'], + item['Identifier']) + if len(reference_in_destination_topic)==0: # We need to get the namespaces for the item reference and the # group representing the topic we are re-assigning the item to. These # namespaces begin with the text 'ddi:reusable:' and are followed by a @@ -143,8 +947,7 @@ def update_topics(input_file_name, C): # and the DDI version for the group/topic to which we want to # reassign a item to may be different. We need to ensure that when # adding a new item reference to a topic, they both have the same - # namespace, otherwise the group update will not work. - reference_from_source_ddi_version = reference_to_move.tag + # namespace, otherwise the topic group update will not work. destination_ddi_version_reusable = ("ddi:reusable:" f"{get_namespace(destination_item.tag).split(':')[2]}") destination_ddi_version_datacollection = ("ddi:datacollection:" @@ -154,67 +957,167 @@ def update_topics(input_file_name, C): # the group/topic that we want to add the reference to, we need # to create a new version of the reference which has the same namespace as # the group/topic we will be adding it to. - if reference_from_source_ddi_version != destination_ddi_version_reusable: - if destination_group['ItemType'] == '91da6c62-c2c2-4173-8958-22c518d1d40d': + if (reference_from_source_ddi_version != destination_ddi_version_reusable + or reference_to_move is None): + if topic_type == C.item_code('Variable Group'): new_reference = create_variable_reference(item_agency_id, item_identifier, item_version, - "Variable", destination_ddi_version_reusable ) else: new_reference = create_question_reference(item_agency_id, item_identifier, item_version, - "QuestionItem", destination_ddi_version_reusable, destination_ddi_version_datacollection ) else: new_reference = reference_to_move - # Finally we update the array containing the most current versions of the - # group/topics. First we update the entry for the topic/group we - # removed a reference from... - update_list_of_topic_groups(source_item, - source_group['AgencyId'], - source_group['Identifier'], - source_group['Version'], - source_group['ItemType'], - updated_topic_groups) - # ...and then if the reference isn't already in the topic/group we are adding a - # reference to, we add the reference to the group representing the topic it is being - # reassigned to... - if len(find_all_references(destination_item, reference_to_move[0].text, reference_to_move[1].text))==0: + # If the reference isn't already in the destination topic/group DDI, we add the + # reference to it... + if reference_to_move is None or len(find_all_references(destination_item, reference_to_move[0].text, reference_to_move[1].text))==0: destination_item[0].append(new_reference) - # ...and we update the entry for the destination topic in our array. + # ...and we update the entry for the destination topic in our array of topic groups. update_list_of_topic_groups(destination_item, - destination_group['AgencyId'], - destination_group['Identifier'], - destination_group['Version'], - destination_group['ItemType'], - updated_topic_groups) + destination_group_item_agency_id, + destination_group_item_identifier, + destination_group_item_version, + topic_type, + updated_topic_groups, + dataset=topic_reassignment_details['datasets'] + ) else: if len(references_to_move)==0: - print((f"Item {topic_reassignment_details.iloc[0]} " + print((f"Item {topic_reassignment_details['itemUrns']} " f" is not in topic " - f"{topic_reassignment_details.iloc[1]}")) - item_not_present_in_source_topic.append( - topic_reassignment_details.iloc[1]) + f"{topic_reassignment_details['sourceTopicGroups']}")) + items_not_present_in_source_topic.append( + topic_reassignment_details['itemUrns'] + ) if reference_in_destination_topic is not None: - print((f"Item {topic_reassignment_details.iloc[0]} " + print((f"Item {topic_reassignment_details['itemUrns']} " f" is already in topic " - f"{topic_reassignment_details.iloc[2]}")) - item_present_in_destination_topic.append( - topic_reassignment_details.iloc[1]) - number_of_topic_reassignments_already_performed = len([x for x in item_not_present_in_source_topic - if x in item_present_in_destination_topic]) + f"{topic_reassignment_details['destinationTopicGroups']}")) + items_present_in_destination_topic.append( + topic_reassignment_details['itemUrns']) + update_list_of_topic_groups(destination_item, + destination_group_item_agency_id, + destination_group_item_identifier, + destination_group_item_version, + topic_type, + updated_topic_groups, + dataset=topic_reassignment_details['datasets'] + ) + number_of_topic_reassignments_already_performed = len([x for x in items_not_present_in_source_topic + if x in items_present_in_destination_topic]) number_of_topic_reassignments_to_be_performed = len(topic_reassignments_data_frame) - number_of_topic_reassignments_already_performed print(f"{number_of_topic_reassignments_already_performed} of {len(topic_reassignments_data_frame)} topic" f" reassignments in the input file have already been performed,") print(f"{number_of_topic_reassignments_to_be_performed} pair(s) of DDI Fragments implementing topic" " reassignments specified in the input file have been created.") - if (len(item_not_present_in_source_topic) == len(topic_reassignments_data_frame) and - len(item_present_in_destination_topic) == len(topic_reassignments_data_frame)): + if (len(items_not_present_in_source_topic) == len(topic_reassignments_data_frame) and + len(items_present_in_destination_topic) == len(topic_reassignments_data_frame)): print("The item topic reassignments in the input data file have already all been " "successfully executed.") - return updated_topic_groups + return ({ + "ItemsMovedFromSourceTopic": items_not_present_in_source_topic, + "ItemsMovedToDestinationTopic": items_present_in_destination_topic, + "UpdatedTopicGroups": updated_topic_groups + }) + +def validate_ddi_implementing_topic_reassignments(input_file_name, + updated_topic_groups, + C, + language="en-GB"): + """Read topic reassignments from an Excel file and validate corresponding DDI items + and references that implement those topic reassignments. This function reads topic-reassignment + rows from the supplied Excel file, and determines if the item specified in each row is not in the + appropriate source topic group in the updated_topic_groups array, and is present in the relevant + destination topic group in the same array; i.e. if the topic reassignment for the item has been + successful. + + Arguments: + input_file_name (str): Path to the Excel file containing topic reassignment rows. + updated_topic_groups (list): List of dict-like entries representing topic groups; + we search for groups by their name and the name of the dataset that contains them (in + the case of Variable Groups) or is associated with their related variables (in the case of + Question Groups). + C (ColecticaObject): an authenticated ColecticaObject instance. + + Returns: + tuple: (source_topic_not_found, destination_topic_not_found, found_source_topics, found_destination_topics) + - source_topic_not_found (list): Rows from the input file for which the item was not + found in the source topic group. + - destination_topic_not_found (list): Rows for which the item was not found in the destination + topic group. + - found_source_topics (list): References (DDI elements) discovered in the source topic groups + for matched rows. + - found_destination_topics (list): References (DDI elements) discovered in the destination + topic groups for matched rows. + + All the lists in the above tuple should have a length of zero, except the last list + (found_destination_topics), which should have a length equal to the number of rows in the + input file, if the topic reassignments have been successfully implemented in the DDI items + representing topic groups in updated_topic_groups. + """ + print(f"Reading and validating topic reassignments from {input_file_name}...") + data = pd.read_excel(input_file_name) + source_topic_not_found=[] + destination_topic_not_found=[] + items_found_in_source_topics=[] + items_found_in_destination_topics=[] + for index, topic_reassignment_details in data.iterrows(): + url = topic_reassignment_details.iloc[2] + agency_id = url.split("/")[4] + identifier = url.split("/")[5] + version = url.split("/")[6] + item_urn = f"urn:ddi:{agency_id}:{identifier}:{str(version)}" + updated_source_topic=[topic_group for topic_group in updated_topic_groups + if topic_group['DatasetName']==topic_reassignment_details.iloc[0] + and get_elements_of_type(topic_group['Item'], "VariableGroupName")!=[] + and get_elements_of_type(topic_group['Item'], "VariableGroupName")[0][0].text + ==str(topic_reassignment_details.iloc[4])] + updated_destination_topic=[topic_group for topic_group in updated_topic_groups + if topic_group['DatasetName']==topic_reassignment_details.iloc[0] + and get_elements_of_type(topic_group['Item'], "VariableGroupName")!=[] + and get_elements_of_type(topic_group['Item'], "VariableGroupName")[0][0].text + ==str(topic_reassignment_details.iloc[5])] + if len(updated_source_topic)==1: + references_in_source_topic=find_all_references(updated_source_topic[0]['Item'], + agency_id, + identifier) + for reference_in_source_topic in references_in_source_topic: + items_found_in_source_topics.append(reference_in_source_topic) + elif str(topic_reassignment_details.iloc[4]).strip()!='no_topic': + source_topic_not_found.append(topic_reassignment_details) + if len(updated_destination_topic)==1: + print("FOUND IN DESTINATION TOPIC") + print(topic_reassignment_details) + references_in_destination_topic=find_all_references(updated_destination_topic[0]['Item'], + agency_id, + identifier) + if len(references_in_destination_topic)==0: + raise ValueError(f"No references in destination topic. Details: {topic_reassignment_details}") + for reference_in_destination_topic in references_in_destination_topic: + items_found_in_destination_topics.append(reference_in_destination_topic) + else: + destination_topic_not_found.append(topic_reassignment_details) + print("NOT FOUND") + print(len(updated_destination_topic)) + print(topic_reassignment_details) + print(f"Number of items still in DDI representing source topic: {len(items_found_in_source_topics)}") + print(f"Number of items found in DDI representing destination topic: {len(items_found_in_destination_topics)}") + print(f"Number of items not found in DDI representing destination topic: {len(data) - len(items_found_in_destination_topics)}") + if len(items_found_in_source_topics)==0 and len(items_found_in_destination_topics)==len(data): + print("The creation of DDI items that implement all the topic reassignments has been successful") + else: + raise ValueError("There were issues with the creation of DDI items that implement all the topic reassignments." + " Please see the details of missing source or destination topics, or missing references" + f"Items still found in source topics: {items_found_in_source_topics}" + f"Number of items not found in destination topics: {len(data) - len(items_found_in_destination_topics)}" + f"Items found in destination topics: {items_found_in_destination_topics}") + return ({"SourceTopicsNotFound": source_topic_not_found, + "DestinationTopicsNotFound": destination_topic_not_found, + "ItemsFoundInSourceTopics": items_found_in_source_topics, + "ItemsFoundInDestinationTopics": items_found_in_destination_topics}) diff --git a/examples/lib/utility.py b/examples/lib/utility.py index d6ce452..9a974b1 100644 --- a/examples/lib/utility.py +++ b/examples/lib/utility.py @@ -3,6 +3,8 @@ from xml.etree import ElementTree as ET from colectica_api import ColecticaObject import defusedxml +from collections import Counter +import pandas as pd def get_namespace(tag): """Get the namespace for an XML element.""" @@ -11,6 +13,7 @@ def get_namespace(tag): return m.group(1) def references_are_equal(reference1, reference2): + """Determine if two references are equal by comparing their elements.""" ref_1_elems=[] ref_2_elems=[] for elem in reference1.findall(".//"): @@ -34,7 +37,24 @@ def find_all_references(xml_tree, agency, identifier): matching_references.append(elem) return matching_references -def create_variable_reference(agency_id, item_id, version, item_type, namespace): +def create_concept_reference(agency_id, item_id, version, namespace): + """Create an XML element representing a ConceptReference""" + new_element = ET.Element(f"{{{namespace}}}ConceptReference") + agency_element = ET.Element(f"{{{namespace}}}Agency") + id_element = ET.Element(f"{{{namespace}}}ID") + version_element = ET.Element(f"{{{namespace}}}Version") + type_of_object_element = ET.Element(f"{{{namespace}}}TypeOfObject") + id_element.text = item_id + agency_element.text = agency_id + version_element.text = str(version) + type_of_object_element.text = "Concept" + new_element.append(agency_element) + new_element.append(id_element) + new_element.append(version_element) + new_element.append(type_of_object_element) + return new_element + +def create_variable_reference(agency_id, item_id, version, namespace): """Create an XML element representing a VariableReference""" new_element = ET.Element(f"{{{namespace}}}VariableReference") agency_element = ET.Element(f"{{{namespace}}}Agency") @@ -44,14 +64,15 @@ def create_variable_reference(agency_id, item_id, version, item_type, namespace) id_element.text = item_id agency_element.text = agency_id version_element.text = str(version) - type_of_object_element.text = item_type + type_of_object_element.text = "Variable" new_element.append(agency_element) new_element.append(id_element) new_element.append(version_element) new_element.append(type_of_object_element) return new_element -def create_question_reference(agency_id, item_id, version, item_type, namespace, namespace2): +def create_question_reference(agency_id, item_id, version, namespace, namespace2): + """Create an XML element representing a QuestionItemReference""" new_element = ET.Element(f"{{{namespace2}}}QuestionItemReference") agency_element=ET.Element(f"{{{namespace}}}Agency") id_element=ET.Element(f"{{{namespace}}}ID") @@ -60,13 +81,50 @@ def create_question_reference(agency_id, item_id, version, item_type, namespace, id_element.text=item_id agency_element.text=agency_id version_element.text=str(version) - type_of_object_element.text=item_type + type_of_object_element.text="QuestionItem" new_element.append(agency_element) new_element.append(id_element) new_element.append(version_element) new_element.append(type_of_object_element) return new_element +def create_group_reference(agency_id, item_id, version, namespace_version, topic_type, C): + """Create an XML element representing a VariableGroup/QuestionGroup. + + Arguments: + agency_id (str): Agency to which the item that we are creating a reference for belongs. + For example, ``"uk.cls.nextsteps"``. + item_id (str): Identifier for the item that we are creating a reference for. + For example, ``"a6f96245-5c00-4ad3-89e9-79afaefa0c28"``. + version (str): The number indicating the version of the item we are creating. + namespace_version (str): the version of the namespaces to which various elements belong in the + reference we are creating. + topic_type (uuid): the type of topic/variable we are creating a reference to. + C (ColecticaObject): an authenticated ColecticaObject instance. + Returns: + ElementTree.Element: An ElementTree.Element representing the group reference. + """ + type_of_object_element = ET.Element(f"{{ddi:reusable:{namespace_version}}}TypeOfObject") + new_element=None + if topic_type==C.item_code('Variable Group'): + new_element = ET.Element(f"{{ddi:logicalproduct:{namespace_version}}}VariableGroupReference") + type_of_object_element.text = "VariableGroup" + elif topic_type==C.item_code('Question Group'): + new_element = ET.Element(f"{{ddi:datacollection:{namespace_version}}}QuestionGroupReference") + type_of_object_element.text = "QuestionGroup" + agency_element = ET.Element(f"{{ddi:reusable:{namespace_version}}}Agency") + id_element = ET.Element(f"{{ddi:reusable:{namespace_version}}}ID") + version_element = ET.Element(f"{{ddi:reusable:{namespace_version}}}Version") + id_element.text = item_id + agency_element.text = agency_id + version_element.text = str(version) + if new_element is not None: + new_element.append(agency_element) + new_element.append(id_element) + new_element.append(version_element) + new_element.append(type_of_object_element) + return new_element + def convert_xml_element_to_json(xml_element): """Convert an XML element to a JSON representation.""" json_object = {} @@ -79,9 +137,26 @@ def get_current_state_of_topic_group(agency_id, identifier, updated_groups, C, v """We may be performing multiple updates to the topic groups, so instead of retrieving/updating/writing data using the Colectica REST API every time we need to update a group, we will retrieve the most recent version of it from the Colectica repository - using the Colectica REST API for the first update, and on subsequent updates we will modify the - in-memory version which is stored in the updated_groups array.""" - updated_referencing_item = [x for x in updated_groups if x['AgencyId'] == identifier] + using the Colectica REST API when doing the first update, and on subsequent updates we will modify + the in-memory version which is stored in the updated_groups array. + + Arguments: + agency_id (str): Agency to which the group that we are getting the current state for belongs. + For example, ``"uk.cls.nextsteps"``. + item_id (str): Identifier for the group that we are getting the current state for. + For example, ``"a6f96245-5c00-4ad3-89e9-79afaefa0c28"``. + updated_groups: list of groups within which we search for the group specified by the + agency_id and identifier and arguments, and the version keyword argument (if specified). + C (ColecticaObject): an authenticated ColecticaObject instance. + + Keyword arguments: + version (int): The number indicating the version of the group we are searching for. + + Returns: + ElementTree.Element: An ElementTree.Element representing the topic group. + """ + updated_referencing_item = [x for x in updated_groups if x['AgencyId'] == agency_id + and x['Identifier']==identifier] if len(updated_referencing_item) > 0: referencing_item = updated_referencing_item[0]['Item'] else: @@ -90,10 +165,36 @@ def get_current_state_of_topic_group(agency_id, identifier, updated_groups, C, v referencing_item = defusedxml.ElementTree.fromstring(fragment_xml) return referencing_item -def update_list_of_topic_groups(updated_group, agency, identifier, version, - item_type, updated_groups_list): +def update_list_of_topic_groups(updated_group, + agency, + identifier, + version, + item_type, + updated_groups_list, + dataset=None): """Update the in-memory list of groups representing topics. If the topic group we have - updated is not in already in the list, we append it to the list.""" + updated is not in already in the list, we append it to the list. + + Arguments: + updated_group: the value for a group which we are either inserting into updated_groups_list (if + an earlier version of the group is not there), or we are updating (if an earlier version is + already in updated_groups_list) + agency_id (str): Agency to which the group that we are updating belongs. + For example, ``"uk.cls.nextsteps"``. + identifier (str): Identifier for the group that we are updating. + For example, ``"a6f96245-5c00-4ad3-89e9-79afaefa0c28"``. + version (int): the number indicating the version of the group we are updating. + item_type(uuid): the type of the group we are updating (e.g. C.item_code('Variable Group')) + updated_groups_list: list of groups within which we search for the group specified by the + agency_id, identifier and version arguments. + + Keyword arguments: + dataset (str): the name of the dataset to which the item belongs, if applicable (i.e. if the + item_type is 'Variable Group') + + Returns: + None: The function updates updated_groups_list in place. + """ if ([x['Identifier'] for x in updated_groups_list].count(identifier) > 0): index_of_updated_ref = [x['Identifier'] for x in updated_groups_list].index(identifier) updated_groups_list[index_of_updated_ref] = { @@ -101,7 +202,8 @@ def update_list_of_topic_groups(updated_group, agency, identifier, version, "AgencyId": agency, "Version": version, "ItemType": item_type, - "Item": updated_group + "Item": updated_group, + "DatasetName": dataset } else: updated_groups_list.append({ @@ -109,47 +211,161 @@ def update_list_of_topic_groups(updated_group, agency, identifier, version, "AgencyId": agency, "Version": version, "ItemType": item_type, - "Item": updated_group + "Item": updated_group, + "DatasetName": dataset }) - -def get_item_from_topic_name(topic_name, topic_type, containing_item_name, containing_item_type, C): + +def get_item_from_topic_name(topic_name, + topic_type, + containing_item, + C, + dataset_name="", + groupsInDatasets=[], + datasetToZeroGroupMappings={}): """Method for getting a topic item given the topic's name as a string (e.g. '11609'), the topic - type (e.g. Question Group, Variable Group), and the name and type of the item within which that - topic is contained (e.g. the name and type of a Physical Instance/Data File or a Data Collection - object). + type (e.g. Question Group, Variable Group), and the item within which that topic is contained + (e.g. a Physical Instance/Data File or a Data Collection object). - Note that the item type input arguments must be provided as UUIDs (as specified at + Note that the topic_type input argument must be provided as a UUID (as specified at https://docs.colectica.com/repository/technical/item-type-identifiers/). Item types can be mapped to their identifiers using the C.item_code function, e.g. C.item_code("Question Group"), C.item_code("Data Collection"). + + Arguments: + topic_name (str): the name of the topic we are searching for (e.g. '11609'). + topic_type (str): the type of the topic we are searching for. + containing_item (dict): A dictionary containing details of the item containing the item being + reassigned. + C (ColecticaObject): an authenticated ColecticaObject instance. + + Keyword arguments: + groupsInDatasets (list): A list of dict objects that map the datasets to topic groups they contain. + datasetToZeroGroupMappings (dict): A dictionary mapping dataset names to level zero topic groups. + + Returns: + list: A list containing Variable Groups/Question Groups items that represent topics. """ - containing_item = C.search_items( - [containing_item_type], - SearchTerms=containing_item_name, - SearchLatestVersion=True)['Results'] - if len(containing_item) == 1: - # We create a JSON object representing the containing item. - search_sets = [{ - "agencyId": containing_item[0]['AgencyId'], - "identifier": containing_item[0]['Identifier'], - "version": containing_item[0]['Version'] - }] - topic_group_identifiers = C.search_items(topic_type, - SearchSets=search_sets, - SearchTerms=[str(topic_name)])['Results'] + item=[x for x in groupsInDatasets if x['DatasetName']==dataset_name + and x['VariableGroupName']==str(topic_name) and x['TopicType']==topic_type] + if len(item)==1: + if item[0]['VariableGroupUrn']=="NA": + topic_groups=[] + else: + topic_groups=[C.get_item_json( + item[0]['VariableGroupUrn'].split(":")[2], + item[0]['VariableGroupUrn'].split(":")[3], + version=item[0]['VariableGroupUrn'].split(":")[4] + )] else: - topic_group_identifiers = [] - return topic_group_identifiers + topic_groups = C.search_items(topic_type, + SearchSets=containing_item, + SearchTerms=[str(topic_name)], + SearchTargets="Name", + UsePrefixSearch=False)['Results'] + if len(topic_groups)==0: + if get_urn_from_item(containing_item) not in datasetToZeroGroupMappings.keys(): + # If we cannot determine the level zero group for the dataset (i.e. topic_group is + # empty) we must try to determine the level zero group by inspecting variables in + # the dataset... + print((f"Cannot determine level zero group for dataset {get_urn_from_item(containing_item)}, " + "inspecting variables...")) + datasetVars=C.query_set(containing_item['AgencyId'], + containing_item['Identifier'],item_types=[C.item_code('Variable')]) + level_zero_groups=[] + count=0 + print(f"Verifying the level zero group for {len(datasetVars)} variables in dataset {get_urn_from_item(containing_item)}...") + # We only determine the level zero group for a small sample of variables, for a faster runtime... + for var in datasetVars[0:4]: + varGroups=C.search_relationship_byobject(var['Item1']['Item3'], var['Item1']['Item1'], + Version=var['Item1']['Item2'], item_types=[topic_type]) + for varGroup in varGroups: + count=count+1 + var_group_item=C.get_item_json(varGroup['Item1']['Item3'], + varGroup['Item1']['Item1'], + version=varGroup['Item1']['Item2']) + level_zero_group=get_level_zero_group_for_topic(var_group_item, C) + if level_zero_group is not None: + level_zero_groups.append(level_zero_group) + containing_level_zero_group = [] + # If all the level zero groups we have found are the same group, we can assume that this is + # the level zero group for the dataset specified in the containing_item argument... + if len(set([x[0][2].text for x in level_zero_groups]))==1: + containing_level_zero_group = [{ + "AgencyId": level_zero_groups[0][0][1].text, + "Identifier": level_zero_groups[0][0][2].text, + "Version": level_zero_groups[0][0][3].text, + }] + datasetToZeroGroupMappings[get_urn_from_item(containing_item)]=containing_level_zero_group + else: + containing_level_zero_group = [] + # Do a search for the first three numbers of the topic group, and then filter + # the results in a list comprehension to find the exact match, because it's + # quicker than just searching for the exact match directly. + topic_groups = [x for x in C.search_items(topic_type, + SearchSets=containing_level_zero_group, + SearchTerms=[str(topic_name)[0:3]], + UsePrefixSearch=True, # returns results if they begin with the value in SearchTerms + SearchTargets="Name")['Results'] if x['ItemName']['en-GB']==str(topic_name)] + else: + containing_level_zero_group=datasetToZeroGroupMappings[get_urn_from_item(containing_item)] + # Do a search for the first three numbers of the topic group, and then filter + # the results in a list comprehension to find the exact match, because it's + # quicker than just searching for the exact match directly. + topic_groups = [x for x in C.search_items(topic_type, + SearchSets=containing_level_zero_group, + SearchTerms=[str(topic_name)[0:3]], + UsePrefixSearch=True, # returns results if they begin with the value in SearchTerms + SearchTargets="Name")['Results'] if x['ItemName']['en-GB']==str(topic_name)] + else: + containing_level_zero_group=C.search_relationship_bysubject(containing_item['AgencyId'], + containing_item['Identifier'], + item_types=C.item_code('Variable Group'), + Version=containing_item['Version'], + Descriptions=True) + if len(containing_level_zero_group)==1: + containing_level_zero_group_item=C.get_item_json(containing_level_zero_group[0]['AgencyId'], + containing_level_zero_group[0]['Identifier'], + version=containing_level_zero_group[0]['Version']) + if containing_level_zero_group_item['Concept'] == None: + datasetToZeroGroupMappings[get_urn_from_item(containing_item)]=[{ + "AgencyId": containing_level_zero_group[0]['AgencyId'], + "Identifier": containing_level_zero_group[0]['Identifier'], + "Version": containing_level_zero_group[0]['Version'], + }] + for topic_group in topic_groups: + if topic_group['ItemName']['en-GB']==str(topic_name) and len(item)==0: + groupsInDatasets.append({ + "DatasetName": dataset_name, + "VariableGroupName": str(topic_name), + "VariableGroupUrn": "urn:ddi:" + topic_group['AgencyId'] + ":" + topic_group['Identifier'] + ":" + str(topic_group['Version']), + "TopicType": topic_type + }) + if len(item)==0: + groupsInDatasets.append({ + "DatasetName": dataset_name, + "VariableGroupName": str(topic_name), + "VariableGroupUrn": "NA", + "TopicType": topic_type + }) + return [x for x in topic_groups if x['ItemName']['en-GB']==str(topic_name)] def get_topic_for_item(agency_id, identifier, version, item_type, C): - """This function gets the topic item(s) for an item (i.e. question/variable), given the - question/variable's agency id, identifier, version, and the UUID code representing the topic's - type (e.g. C.item_code("Variable Group")). + """This function gets the topic item(s) for an item (i.e. question/variable). + + Arguments: + agency_id(str): the agency for the question/variable we are trying to get the topic for. + identifier(str): the identifier for the question/variable we are trying to get the topic for. + version(str): the version for the question/variable we are trying to get the topic for. + item_type(uuid): the UUID code representing the topic's type (e.g. C.item_code("Variable Group")). + + Returns: + list: a list of variable/question groups representing topics. """ topics_assigned_to_item=[] related_groups = C.search_relationship_byobject(agency_id, identifier, Version=version, item_types=[item_type]) for related_group in related_groups: - related_question_group_most_recent_version=C.get_item_xml(related_group['Item1']['Item3'], related_group['Item1']['Item1']) + related_question_group_most_recent_version=C.get_item_xml(related_group['Item1']['Item3'], + related_group['Item1']['Item1']) if identifier in related_question_group_most_recent_version['Item']: topics_assigned_to_item.append(related_question_group_most_recent_version) return topics_assigned_to_item @@ -160,6 +376,13 @@ def get_url_from_item(item, hostname): def get_urn_from_item(item): return f"urn:ddi:{item['AgencyId']}:{item['Identifier']}:{str(item['Version'])}" +def get_urn_from_fragment(fragment_xml): + urnElement=get_elements_of_type(fragment_xml, "URN") + if len(urnElement)==1: + return urnElement[0].text + else: + return None + def map_between_questions_and_variables(items, C): """Method for mapping between lists of questions and variables. The 'items' input parameter contains a list of urns for questions/variables, this function returns a list containing the @@ -174,10 +397,13 @@ def map_between_questions_and_variables(items, C): version = item.split(":")[4] item_json = C.get_item_json(agency_id, identifier, version=version) item_type = C.item_code_inv(item_json['ItemType']) + all_related_items=[] if item_type == 'Variable': - all_related_items= C.search_relationship_bysubject(agency_id, identifier, Version=version, item_types=[C.item_code("Question")]) + all_related_items= C.search_relationship_bysubject(agency_id, identifier, Version=version, + item_types=[C.item_code("Question")]) elif item_type == 'Question': - all_related_items= C.search_relationship_byobject(agency_id, identifier, Version=version, item_types=[C.item_code("Variable")]) + all_related_items= C.search_relationship_byobject(agency_id, identifier, Version=version, + item_types=[C.item_code("Variable")]) for related_item in all_related_items: related_item_json=C.get_item_json(related_item['Item1']['Item3'], related_item['Item1']['Item1'], version=related_item['Item1']['Item2']) agency_id = related_item_json['AgencyId'] @@ -197,8 +423,12 @@ def update_repository(updated_items, transaction_message, C): transaction_id = transaction_response['TransactionId'] for item in updated_items: fragment_string = defusedxml.ElementTree.tostring(item['Item'], encoding='unicode') - C.add_items_to_transaction(item['AgencyId'], item['Identifier'], item['Version'], fragment_string, - item['ItemType'], transaction_id) + C.add_items_to_transaction(item['AgencyId'], + item['Identifier'], + item['Version'], + fragment_string, + item['ItemType'], + transaction_id) commit_response = C.commit_transaction(transaction_id, transaction_message, 3) return commit_response @@ -209,6 +439,15 @@ def getTriple(tripleElem): triple[elem.tag[startOfTagName:]] = elem.text return(triple) +def get_element_fragment_by_name(xmlTree, elementName): + retElem=None + for elem in xmlTree.findall(".//"): + startOfTagName = elem.tag.index("}")+1 + tagName = elem.tag[startOfTagName:] + if tagName == elementName: + retElem = elem + return retElem + def get_element_by_name(xmlTree, elementName): retElem=None for elem in xmlTree.findall(".//"): @@ -219,7 +458,6 @@ def get_element_by_name(xmlTree, elementName): return retElem def get_elements_of_type(xmlTree, elementName): - retElem=None elems=[] for elem in xmlTree.findall(".//"): startOfTagName = elem.tag.index("}")+1 @@ -235,3 +473,181 @@ def remove_elements_from_item(item, element_name, C): for y in elementRefs: item[0].remove(y) return item + +def create_input_file(input_file_name, output_file_name, C): + """This is a utility function that creates an input file for the find_topics_to_create method by + extracting information from an existing file that is not in the required format. + + Arguments: + input_file_name (str): the name for a file that we want to extract information from. + output_file_name (str): the name for the output file to create. + + Returns: + None: the output file that is in the format required by the find_topics_to_create method is created + in the current working directory. + """ + data = pd.read_excel(input_file_name).drop_duplicates() + new_input_df = pd.DataFrame(columns=["Container", "ItemName", "URL", "Label", "CurrentTopic", "NewTopic"]) + for topic_reassignment_details in data.iloc: + physical_instance_containing_variable = C.search_items( + C.item_code('Data File'), + SearchTerms=str(topic_reassignment_details.iloc[0]).strip(), + SearchLatestVersion=True)['Results'] + if len(physical_instance_containing_variable) == 1: + # We need to search within the physical instance/dataset for the variable named in the + # current row. We create a JSON object representing the physical instance/dataset. + search_sets = [{ + "agencyId": physical_instance_containing_variable[0]['AgencyId'], + "identifier": physical_instance_containing_variable[0]['Identifier'], + "version": physical_instance_containing_variable[0]['Version'] + }] + # For this search, the 'SearchTerms' keyword argument represents the name of the + # variable we are reassigning to a new topic. The 'SearchSets' keyword argument + # represents the physical instance/dataset we are searching for that variable in. + variable_metadata = C.search_items(C.item_code('Variable'), + SearchSets=search_sets, + SearchTerms=[str(topic_reassignment_details.iloc[2]).strip()])['Results'] + if len(variable_metadata) == 1: + url=get_url_from_item(variable_metadata[0], 'discovery.closer.ac.uk') + new_row={"Container": topic_reassignment_details.iloc[0], + "ItemName": topic_reassignment_details.iloc[2], + "URL": url, + "Label": topic_reassignment_details.iloc[3], + "CurrentTopic": topic_reassignment_details.iloc[4], + "NewTopic": topic_reassignment_details.iloc[6]} + if str(topic_reassignment_details.iloc[6])!='nan': + new_input_df.loc[len(new_input_df)] = new_row + new_input_df.to_excel(output_file_name, index=False) + +def create_variable_group(group_name, + group_label, + item_id, + namespace_version, + concept_agency_id, + concept_identifier, + concept_version): + """Create a variable group representing a topic. + + Arguments: + group_name: the name of the topic/variable group. + group_label: the label for the topic/variable group. + item_id: the identifier for the topic/variable group. + namespace_version: the version for the namespaces for various elements. + concept_agency_id: the agency for the concept which this topic/variable group represents. + concept_identifier: the identifier for the concept which this topic/variable group represents. + concept_version: the version for the concept which this topic/variable group represents. + + Returns: + ElementTree.Element: An ElementTree.Element representing the variable group. + """ + fragmentString = f""" + + urn:ddi:uk.closer:{item_id}:1 + uk.closer + {item_id} + 1 + + {group_name} + + + {group_label} + + + {concept_agency_id} + {concept_identifier} + {concept_version} + Concept + + + """.replace("\n", "").replace(" ", "") + return defusedxml.ElementTree.fromstring(fragmentString) + +def get_group_label(topic_name, topic_type, C, language="en-GB"): + """Retrieves the label for a topic/group by retrieving all instances of items with the topic name and + choosing the label that is the most common for all those items (in case not all the items have the + same labels). + + Arguments: + topic_name (str): the name of the topic/group we want a label for. + topic_type (uuid): the type of the topic/group (e.g. C.item_code('Variable Group')). + C (ColecticaObject): an authenticated ColecticaObject instance. + + Keyword arguments: + language (str): the language the label is in. + + Returns: + str: the label for the specified topic/group. + """ + groups_with_topic=C.search_items(topic_type, + SearchTerms=[topic_name], + SearchTargets=["Name"]) + group_label=Counter([x['Label'][language] for x in groups_with_topic['Results']]).most_common(1)[0][0] + return group_label + +def get_level_zero_group_for_topic(group, C, language="en-GB"): + """Get the level zero group for a specified group. + + Arguments: + group (dict): A dictionary representing the group for which we want to get the level zero group. + C (ColecticaObject): an authenticated ColecticaObject instance. + + Keyword arguments: + language (str): the language the topic name is in. + + Returns: + ElementTree.Element: An ElementTree.Element representing the level zero group. + """ + item_element=None + topic_name="" + if group['ItemName']!={}: + if language in group['ItemName'].keys(): + topic_name = group['ItemName'][language] + elif isinstance(group['ItemName'], str): + topic_name = group['ItemName'] + parent_group = C.search_relationship_byobject(group['AgencyId'], + group['Identifier'], Version=group['Version'], item_types=[group['ItemType']], Descriptions=True) + if len(parent_group)==1: + if len(topic_name)==3: + level_zero_group = parent_group + elif len(topic_name)==5: + level_zero_group = C.search_relationship_byobject(parent_group[0]['AgencyId'], + parent_group[0]['Identifier'], Version=parent_group[0]['Version'], + item_types=[C.item_code('Variable Group')], Descriptions=True) + if len(level_zero_group)==1: + item=C.get_item_xml(level_zero_group[0]['AgencyId'], level_zero_group[0]['Identifier'], + version=level_zero_group[0]['Version']) + item_element = defusedxml.ElementTree.fromstring(item['Item']) + return item_element + +def create_group_lookup_dict(datasetToZeroGroupMappings, C): + """Create a tuple thats used to map the datasets specified in datasetsToZeroGroupMappings to the + topic groups they contain. + + Arguments: + datasetToZeroGroupMappings (dict): A dictionary mapping dataset names to level zero topic groups. + C (ColecticaObject): an authenticated ColecticaObject instance. + + Returns: + list: A list of tuples representing groups in datasets. + """ + groupsInDatasets=[] + count=0 + for dataset in datasetToZeroGroupMappings.keys(): + count=count+1 + print(f"Processing dataset {count} of {len(datasetToZeroGroupMappings.keys())}...") + level_zero_group=datasetToZeroGroupMappings[dataset] + dataset_agency=dataset.split(":")[2] + dataset_identifier=dataset.split(":")[3] + if len(level_zero_group)==1: + varGroups=C.query_set(level_zero_group[0]['AgencyId'], level_zero_group[0]['Identifier'], + item_types=[C.item_code('Variable Group')]) + dataset_item=C.get_item_json(dataset_agency, dataset_identifier) + # We iterate through varGroups, but exclude the level zero group... + for varGroup in [group for group in varGroups if group['Item1']['Item1']!=level_zero_group[0]['Identifier']]: + var_group_item=C.get_item_json(varGroup['Item1']['Item3'], varGroup['Item1']['Item1'], version=varGroup['Item1']['Item2']) + groupsInDatasets.append({"DatasetName": dataset_item['DublinCoreMetadata']['AlternateTitle']['en-GB'], + "VariableGroupName": var_group_item['ItemName']['en-GB'], + "VariableGroupUrn": "urn:ddi:" + var_group_item['AgencyId'] + ":" + var_group_item['Identifier'] + ":" + str(var_group_item['Version']), + "TopicType": C.item_code('Variable Group') + }) + return groupsInDatasets diff --git a/examples/smallTest.xlsx b/examples/smallTest.xlsx new file mode 100644 index 0000000..a7316a9 Binary files /dev/null and b/examples/smallTest.xlsx differ diff --git a/examples/topic_reassignments.xlsx b/examples/topic_reassignments.xlsx index 1975f59..a7316a9 100644 Binary files a/examples/topic_reassignments.xlsx and b/examples/topic_reassignments.xlsx differ diff --git a/examples/update_variables.py b/examples/update_variables.py index f744e66..f32def2 100644 --- a/examples/update_variables.py +++ b/examples/update_variables.py @@ -177,7 +177,6 @@ def update_topics(input_file_name): new_reference = create_variable_reference(variable_agency_id, variable_identifier, variable_version, - 'Variable', destination_ddi_version ) else: diff --git a/requirements.txt b/requirements.txt index dbd5dab..72ac250 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ # minimum requirements pandas>=1.5.3 requests +defusedxml +openpyxl