Merge pull request #16 from BecCowley/XBTnewNCFormat

Alternative NC formats enabled

Author: BecCowley

SOOP/SOOP_XBT/DELAYED/TuroXBT2IMOSnc.py

@@ -0,0 +1,49 @@
+# create a text file with the XBT_fault_and_feature_type binary values and their corresponding labels
+import pandas as pd
+from decode_flags import convert
+
+
+def create_xbtfft():
+    # Read the CSV file and convert it to a DataFrame
+    file_path = 'flag_quality_table.csv'
+    df = pd.read_csv(file_path)
+    # drop the rows with 'selected' in the rule_direction column
+    df = df[df['rule_direction'] != 'selected']
+    # keep some of the columns
+    df = df[['name', 'full_code', 'XBT_accept_code', 'XBT_reject_code', 'TEMP_quality_control','depth', 'Parameter']]
+    # replace the NaN values with 0 in depth column
+    df['depth'] = df['depth'].fillna(1)
+    # convert the depth and rule_direction columns to match categories in the xbt_config file
+    df['depth'] = df['depth'].map({0: 'ACT_CODES_FULL_PROFILE', 1: 'ACT_CODES_TO_NEXT_FLAG', 3.6: 'ACT_CODES_SINGLE_POINT'})
+    # drop the rows with NaN values in the XBT_accept_code column
+    dfa = df.dropna(subset=['XBT_accept_code'])
+    # drop the rows with NaN values in the XBT_reject_code column
+    dfr = df.dropna(subset=['XBT_reject_code'])
+
+    # convert the name and full_code
+    labels = dfa['name'].tolist()
+    full_code = dfa['full_code'].tolist()
+    tempqc = dfa['TEMP_quality_control'].tolist()
+    depth = dfa['depth'].tolist()
+    parameter = dfa['Parameter'].tolist()
+    with open('xbt_accept_code.csv', 'w') as f:
+        # write the header
+        f.write('label,code,byte_value,tempqc,group_label,parameter\n')
+        for i in range(len(dfa)):
+            # write the labels, code and the binary value with a comma in between each
+            f.write(f'{convert(2**i, labels)},{full_code[i]},{2**i},{tempqc[i]},{depth[i]},{parameter[i]}\n')
+    labels = dfr['name'].tolist()
+    full_code = dfr['full_code'].tolist()
+    tempqc = dfr['TEMP_quality_control'].tolist()
+    depth = dfr['depth'].tolist()
+    parameter = dfr['Parameter'].tolist()
+    with open('xbt_reject_code.csv', 'w') as f:
+        # write the header
+        f.write('label,code,byte_value,tempqc,group_label,parameter\n')
+        for i in range(len(dfr)):
+            # write the labels, code and the binary value with a comma in between each
+            f.write(f'{convert(2**i, labels)},{full_code[i]},{2**i},{tempqc[i]},{depth[i]},{parameter[i]}\n')
+    return dfa, dfr
+
+
+dfa, dfr = create_xbtfft()

SOOP/SOOP_XBT/DELAYED/check_parquet.ipynb

@@ -0,0 +1,22 @@
+""" Use this snippet of code to decode binary flag codes into meaningful values
+    Have to pass in flag labels or names as type list of strings.
+    And i has to be an unsigned integer
+    """
+
+
+def convert(i, labels):
+    output = ''
+    pos = 0
+    while i:
+        if i & 1:
+            output += labels[pos]
+        pos += 1
+        i >>= 1
+    return output
+
+
+# test it:
+val = 8388641
+labels = ["scientific_qc_applied","wire_stretch","leakage","electrical_interference_interpolated","high_frequency_noise_filtered","repeat_profile","temperature_inversion_confirmed","temperature_inversion_unconfirmed","temperature_offset","temperature_eddy_or_front","temperature_steps_or_structure_confirmed","temperature_steps_or_structure_unconfirmed","depth_offset","constant_temperature","time_error_corrected","latitude_error_corrected","longitude_error_corrected","probe_type_changed","spike_interpolated","fine_structure","insulation_penetration_interpolated","nub_inversion","hit_bottom","premature_launch","surface_temperature_anomaly","surface_offset","temperature_anomaly","temperature_difference_at_depth","unique_id_changed","bowing_BathySystems","cusping_BathySystems_leakage","sippicanMK_timing_delay_driver_error","BathySystem_software_fault_modulo_10_spikes_filtered","protecho_systems_leakage_fault","sippicanMK9_sticking_bit_19_point_filtered","depth_corrected_multiplied_by_10","depth_corrected_update_fall_rate_equation","bathy_data_low_resolution"]
+print(convert(val, labels))
+

SOOP/SOOP_XBT/DELAYED/create_xbtfft.py

@@ -0,0 +1,69 @@
+name,historic_extra_code,old_code,code,full_code,depth,condition,rule_direction,QC_reject_code,QC_accept_code,TIME_quality_control,LATITUDE_quality_control,LONGITUDE_quality_control,PROBE_TYPE_quality control,TEMP_quality_control,DEPTH_quality_control,Parameter
+scientific_qc_applied,0,,QC,QCA,0,,down,,1,1,1,1,1,1,1,TEMP
+wire_stretch,0,,WS,WSA,,,down,,2,,,,,2,1,TEMP
+leakage,0,,LE,LEA,,,down,,4,,,,,2,1,TEMP
+electrical_interference_interpolated,0,"SP, IP",EI,EIA,,,selected,,8,,,,,5,1,TEMP
+electrical_interference_interpolated,0,,EI,EIA,,,down,,8,,,,,2,1,TEMP
+high_frequency_noise_filtered,0,,HF,HFA,,,selected,,16,,,,,5,1,TEMP
+high_frequency_noise_filtered,0,,HF,HFA,,,down,,16,,,,,2,1,TEMP
+repeat_profile,0,,RE,REA,0,,down,,32,,,,,0,0,TEMP
+temperature_inversion_confirmed,0,,IV,IVA,,,down,,64,,,,,1,1,TEMP
+temperature_inversion_unconfirmed,0,,PI,PIA,,,down,,128,,,,,2,1,TEMP
+temperature_offset,0,"TO, TD, SA, SO",TO,TOA,,,down,,256,,,,,2,1,TEMP
+temperature_eddy_or_front,0,,EF,EFA,0,,down,,512,,,,,1,1,TEMP
+temperature_steps_or_structure_confirmed,0,FS,ST,STA,0,,down,,1024,,,,,1,1,TEMP
+temperature_steps_or_structure_unconfirmed,0,,PS,PSA,0,,down,,2048,,,,,2,1,TEMP
+depth_offset,0,"DO, PL",DO,DOA,0,,down,,4096,,,,,5,1,TEMP
+constant_temperature,0,,CT,CTA,,,down,,8192,,,,,2,1,TEMP
+time_error_corrected,0,,TE,TEA,0,,down,,16384,5,,,,2,1,TIME
+latitude_error_corrected,0,PE,LA,LAA,0,,down,,32768,,5,,,2,1,LATITUDE
+longitude_error_corrected,0,PE,LO,LOA,0,,down,,65536,,,5,,2,1,LONGITUDE
+probe_type_changed,0,,PR,PRA,0,,down,,131072,,,,5,5,5,TEMP
+spike_interpolated,1,,SP,SPA,,,selected,,262144,,,,,5,1,TEMP
+spike_interpolated,1,,SP,SPA,,,down,,262144,,,,,2,1,TEMP
+fine_structure,1,,FS,FSA,0,,down,,524288,,,,,2,1,TEMP
+insulation_penetration_interpolated,1,,IP,IPA,,,selected,,1048576,,,,,5,1,TEMP
+insulation_penetration_interpolated,1,,IP,IPA,,,down,,1048576,,,,,2,1,TEMP
+nub_inversion,1,,NU,NUA,0,,down,,2097152,,,,,1,1,TEMP
+hit_bottom,1,,HB,HBA,,,down,,4194304,,,,,2,1,TEMP
+premature_launch,1,,PL,PLA,0,,down,,8388608,,,,,2,1,TEMP
+surface_temperature_anomaly,1,,SA,SAA,0,,down,,16777216,,,,,2,1,TEMP
+surface_offset,1,,SO,SOA,0,,down,,33554432,,,,,2,1,TEMP
+temperature_anomaly,1,,TA,TAA,,,down,,67108864,,,,,2,1,TEMP
+temperature_difference_at_depth,1,,TD,TDA,,,down,,134217728,,,,,2,1,TEMP
+unique_id_changed,1,,ID,IDA,0,,down,,268435456,,,,,1,1,TEMP
+bowing_BathySystems,1,,BO,BOA,0,,down,,536870912,,,,,2,1,TEMP
+cusping_BathySystems_leakage,1,,CU,CUA,,,down,,1073741824,,,,,2,1,TEMP
+sippicanMK_timing_delay_driver_error,1,,DR,DRA,0,,down,,2147483648,,,,,2,1,TEMP
+BathySystem_software_fault_modulo_10_spikes_filtered,1,,MO,MOA,,,down,,4294967296,,,,,2,1,TEMP
+protecho_systems_leakage_fault,1,,PF,PFA,0,,down,,8589934592,,,,,2,1,TEMP
+sippicanMK9_sticking_bit_19_point_filtered,1,,SB,SBA,,,down,,17179869184,,,,,2,1,TEMP
+depth_corrected_multiplied_by_10,1,,DE,DEA,0,,down,,34359738368,,,,,1,1,TEMP
+depth_corrected_update_fall_rate_equation,1,,DP,DPA,0,,down,,68719476736,,,,,0,0,TEMP
+bathy_data_low_resolution,1,,BD,BDA,0,,down,,1.37E+11,,,,,2,1,TEMP
+surface_transient,0,,CS,CSR,3.6,,up,1,,,,,,3,1,TEMP
+wire_break,0,,WB,WBR,,,down,2,,,,,,4,1,TEMP
+wire_stretch,0,,WS,WSR,,,down,4,,,,,,3,1,TEMP
+hit_bottom,0,,HB,HBR,,,down,8,,,,,,3,3,TEMP
+leakage,0,,LE,LER,,,down,16,,,,,,3,1,TEMP
+electrical_interference,0,"SP, IP",EI,EIR,,,down,32,,,,,,4,1,TEMP
+high_frequency_noise,0,,HF,HFR,,,down,64,,,,,,4,1,TEMP
+no_good,0,,NG,NGR,,,down,128,,,,,,4,1,TEMP
+temperature_offset,0,"TO, TD, SA, SO",TO,TOR,,,down,256,,,,,,3,1,TEMP
+depth_offset,0,"DO, PL",DO,DOR,0,,down,512,,,,,,3,3,TEMP
+constant_temperature,0,,CT,CTR,,,down,1024,,,,,,3,3,TEMP
+time_error,0,,TE,TER,0,,down,2048,,3,,,,3,1,TIME
+position_error,0,,PE,PER,0,,down,4096,,,3,3,,3,1,LATITUDE LONGITUDE
+duplicate_profile,0,,DU,DUR,0,,down,8192,,,,,,4,1,TEMP
+test_probe,0,,TP,TPR,0,,down,16384,,,4,4,,4,4,TEMP
+probe_type,0,,PR,PRR,0,,down,32768,,,,,3,3,3,TEMP
+spike,1,,SP,SPR,,,down,65536,,,,,,4,1,TEMP
+insulation_penetration,1,,IP,IPR,,,down,131072,,,,,,3,1,TEMP
+surface_temperature_anomaly,1,,SA,SAR,0,,down,262144,,,,,,3,1,TEMP
+surface_offset,1,,SO,SOR,0,,down,524288,,,,,,3,1,TEMP
+no_trace,1,,NT,NTR,0,,down,1048576,,,,,,4,1,TEMP
+bowing_BathySystems,1,,BO,BOR,0,,down,2097152,,,,,,3,1,TEMP
+sippicanMK_timing_delay_driver_error,1,,DR,DRR,,,down,4194304,,,,,,3,1,TEMP
+protecho_systems_leakage_fault,1,,PF,PFR,0,,down,8388608,,,,,,3,1,TEMP
+contact_lost_with_probe,1,,CL,CLR,,,down,16777216,,,,,,3,1,TEMP
+bathy_data_low_resolution,1,,BD,BDR,0,,down,33554432,,,,,,3,1,TEMP