From c0c25846e5769b31ceb536c378bcca329b8a6a22 Mon Sep 17 00:00:00 2001 From: kjod Date: Sun, 9 Apr 2017 20:20:07 +0100 Subject: [PATCH] Added manual mapping api. Added mandatory indicator_category field and unit measure field --- ZOOM/api/indicator/filters.py | 1 + ZOOM/api/manual_map/views.py | 41 +++++++++++++++++++++++++---------- ZOOM/api/validate/views.py | 4 ++-- ZOOM/lib/tools.py | 17 +++++++++++---- 4 files changed, 46 insertions(+), 17 deletions(-) diff --git a/ZOOM/api/indicator/filters.py b/ZOOM/api/indicator/filters.py index 0a17d95b..0ebed8ed 100644 --- a/ZOOM/api/indicator/filters.py +++ b/ZOOM/api/indicator/filters.py @@ -18,4 +18,5 @@ class Meta: 'source_id', 'country_id__region', 'country_id__name', + 'unit_of_measure', ) diff --git a/ZOOM/api/manual_map/views.py b/ZOOM/api/manual_map/views.py index 328fb69d..61128d7d 100644 --- a/ZOOM/api/manual_map/views.py +++ b/ZOOM/api/manual_map/views.py @@ -13,6 +13,7 @@ from rest_framework.response import Response from rest_framework import status from rest_framework.decorators import api_view +from validate.models import File import numpy as np import pandas as pd import pickle @@ -24,31 +25,40 @@ @api_view(['GET', 'POST']) def manual_mapping(request): print('Recieved request') - print(request) + #print(request.data) + #print(request.data['file_id']) + #print(request.data['dict']) + if request.method == 'POST': print('Rewuest Recieved') #check data types print(request) # add validation check here - if 'dict' in request.POST: + if 'dict' in request.data: print('here') - mappings = json.loads(request.data['dict']) - mappings.pop("null", None) - mappings.pop("unit_measure", None)#change later - mappings.pop("validate_store", None) # remove?? - df_data = pd.read_csv(request.data['file_id']) # change to use with multiple files + print(request.data['dict']) + mappings = request.data['dict']#json.loads(request.data['dict']) + mappings.pop("null", None)#not needed for api call + mappings.pop("validate_store", None) # not needed for api call + file_id = str(File.objects.get(id=request.data['file_id']).file) + df_data = pd.read_csv(file_id) # change to use with multiple files found_dtype = [] convert_to_dtype = [] error_message = [] correction_mappings = {} - dict_name = mappings.pop("empty_indicator", None)#request.session['dtypes'] + dict_name = request.data["dtypes_loc"]#request.session['dtypes'] indicator_value = mappings.pop("empty_indicator", None) country_value = mappings.pop("empty_country", None) indicator_category_value = mappings.pop("empty_indicator_cat", None) + unit_of_measure_value = mappings.pop("empty_unit_of_measure", None) relationship_dict = mappings.pop("relationship", None) left_over_dict = mappings.pop("left_over", None) + print('Here in this method') + print(file_id) + print(dict_name) + with open(dict_name, 'rb') as f: dtypes_dict = pickle.load(f) #check if exists @@ -111,9 +121,18 @@ def manual_mapping(request): df_data['indicator_category_id'] = indicator_category_value dtypes_dict[mappings['indicator_category_id'][0]] = [('str', 'str')] - if relationship_dict: - df_data = convert_df(mappings, relationship_dict, left_over_dict, df_data, dtypes_dict) + if unit_of_measure_value: + if len(unit_of_measure_value.keys()) < 2 :#chect each entry emoty unit_of measure a dict + mappings['unit_of_measure'] = ['unit_of_measure'] + df_data['unit_of_measure'] = unit_of_measure_value[unit_of_measure_value.keys()[0]] + dtypes_dict[mappings['unit_of_measure'][0]] = [('str', 'str')] + else: + mappings['unit_of_measure'] = ['unit_of_measure'] + dtypes_dict[mappings['unit_of_measure'][0]] = [('str', 'str')] + if relationship_dict: + #check if unit of measure exists + df_data = convert_df(mappings, relationship_dict, left_over_dict, df_data, dtypes_dict, unit_of_measure_value) #remove replace #Validation @@ -168,7 +187,7 @@ def manual_mapping(request): bulk_list = [] #cycle through dataset and save each line - order["file_source_id"] = request.data['file_id'];#request.session['files'][0] + order["file_source_id"] = file_id;#request.session['files'][0] instance = FileSource(file_name = order['file_source_id']) instance.save() file_id = instance.id diff --git a/ZOOM/api/validate/views.py b/ZOOM/api/validate/views.py index ecf1c721..2400212b 100644 --- a/ZOOM/api/validate/views.py +++ b/ZOOM/api/validate/views.py @@ -67,7 +67,7 @@ def validate(request): template_heading_list.append(field.name)#.get_attname_column()) template_heading_list = template_heading_list[4:len(template_heading_list)]#skip first four headings as irrelevant to user input - template_heading_list.append("unit_measure") #needed? + #template_heading_list.append("unit_measure") #needed? #count = 0# not sure if this is still needed, might need for matches dicts, _ = get_dictionaries()#get dicts for country @@ -222,4 +222,4 @@ def validate(self, file_id, request): #output need to pass allignments of mapped headings return Response({"message": "Got some data!", "data": request.data}) #return context - #return render(request, 'validate/input_report.html', context)""" \ No newline at end of file + #return render(request, 'validate/input_report.html', context)""" diff --git a/ZOOM/lib/tools.py b/ZOOM/lib/tools.py index e76dd2d5..b24fbd27 100644 --- a/ZOOM/lib/tools.py +++ b/ZOOM/lib/tools.py @@ -183,6 +183,7 @@ def check_column_data(dtypes, column_data, model_field, file_heading):#No need f #should return field to map to def check_data_type(field, dtypes): #add time + print("getting here in check data type") dtype_set = set() result = False if field == "country_id": @@ -259,8 +260,11 @@ def correct_data(df_data, correction_data):#correction_data ["country_name, iso2 #if date convert in integer #if -def convert_df(mappings,relationship_dict, left_over_dict, df_data, dtypes_dict): +def convert_df(mappings,relationship_dict, left_over_dict, df_data, dtypes_dict, empty_unit_measure_value): + if not empty_unit_measure_value: + empty_unit_measure_value = {} + columns = [] for col in df_data.columns: temp = str(col)#.replace(" ", "~")#needed? @@ -296,15 +300,20 @@ def convert_df(mappings,relationship_dict, left_over_dict, df_data, dtypes_dict) #might have to be greater than 2 if col in mappings['indicator_category_id'] and len(mappings['indicator_category_id']) > 1:#if more than one relationship ie multiple subgroupd and relationships - check = new_df[relationship_dict[col]][counter] + check = new_df[relationship_dict[col]][counter] #if supgroup already defined new_df[relationship_dict[col]][counter] = new_df[relationship_dict[col]][counter] + "|" + (col.replace("~", " "))#last part not needed - new_df[left_over_dict[col]][counter] = df_data[col.replace("~", " ")][i] + new_df[left_over_dict[col]][counter] = df_data[col.replace("~", " ")][i]#check if col_replace is there + #if empty_unit_of_measure in mappings: + #apply units of measure + #a = 5 else: #normal case - #hre.y + #hre.y new_df[relationship_dict[col]][counter] = col.replace("~", " ")#add column heading new_df[left_over_dict[col]][counter] = df_data[col.replace("~", " ")][i] + if col.replace("~", " ") in empty_unit_measure_value: + new_df['unit_of_measure'] = empty_unit_measure_value[col.replace("~", " ")] #new_df = df_data[mappings[key]] #map value