diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 40f632e..eba282b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.9.0 + rev: v0.8.3 hooks: # Run the linter. - id: ruff diff --git a/src/pynxtools_apm/configurations/cameca_cfg.py b/src/pynxtools_apm/configurations/cameca_cfg.py index b5fc32f..eba5871 100644 --- a/src/pynxtools_apm/configurations/cameca_cfg.py +++ b/src/pynxtools_apm/configurations/cameca_cfg.py @@ -15,92 +15,124 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""Dict mapping custom schema instances from eln_data.yaml file on concepts in NXapm.""" +"""Dict mapping custom schema instances from custom yaml file on concepts in NXapm.""" -from pynxtools_apm.utils.pint_custom_unit_registry import ureg - - -"map": [("", "fComments"), ("", "fQuality"), ("", "fPrimaryElement")], -"map_to_f8": [ -("", "fEfficiency"), -("", ureg.volt / ureg.nanometer ** 2, "fEvaporationField"), -("", ureg.millimeter, "fFlightPath"), -("", "??", "fImageCompression"), -("", "??", "fKfactor"), -("", ureg.nanometer ** 3, "fReconVolume"), -("", ureg.degrees, "fShankAngle"), -("", ureg.nanometer, "fTipRadius"), -("", ureg.nanometer, "fTipRadius0"), -("", ureg.volt, "fVoltage0"), -("", ureg.nanometer, "fXmax"), -("", ureg.nanometer, "fXmin"), -("", ureg.nanometer, "fYmax"), -("", ureg.nanometer, "fYmin"), -("", ureg.nanometer, "fZmax"), -("", ureg.nanometer, "fZmin")] - -("", "fUserID") +from typing import Any, Dict +from pynxtools_apm.utils.pint_custom_unit_registry import ureg -("", "fAcqBuildVersion"), -("", "fAcqMajorVersion"), -("", "fAcqMinorVersion"), -("", "fCernRootVersion"), -("", "fImagoRootDate"), -("", "fImagoRootVersion"), -("", "fStreamVersion"), - -("", "fSerialNumber"), - -("", "fBcaSerialRev"), -("", "fFirmwareVersion"), -("", "fFlangeSerialNumber"), -("", "fHvpsType"), -("", "fLaserModelString"), -("", "fLaserSerialNumber"), -("", "fLcbSerialRev"), -("", "fMcpSerialNumber"), -("", "fPulserType"), -("", "fTaSerialRev"), -("", "fTdcType"), - -("type", "fAcquisitionMode"), -("", "fApertureName"), -("", "fAtomProbeName"), -("", "fComments") -("", "fName"), -("", "fProjectName"), +APM_CAMECA_TO_NEXUS: Dict[str, Any] = { + "prefix_trg": "/ENTRY[entry*]", + "prefix_src": "", + "map": [ + ("reconstruction/quality", "fQuality"), + ("reconstruction/primary_element", "fPrimaryElement"), + ("measurement/instrument/local_electrode/name", "fApertureName"), + ("measurement/instrument/instrument_name", "fAtomProbeName"), + ("measurement/instrument/fabrication/model", "fLeapModel"), + ( + "measurement/instrument/pulser/SOURCE[sourceID]/fabrication/model", + "fLaserModel", + ), + ("measurement/instrument/comments", "fInstrumentComment"), + ("atom_probe/raw_data/serialized/path", "fRawPathName"), + ("measurement/status", "fResults"), + ("specimen/description", "fSpecimenCondition"), + ("specimen/alias", "fSpecimenName"), + ("start_time", "fStartISO8601"), + ], + "map_to_f8": [ + ("reconstruction/efficiency", "fEfficiency"), + ( + "reconstruction/evaporation_field", + ureg.volt / ureg.nanometer**2, + "fEvaporationField", + ), + ("reconstruction/flight_path", ureg.millimeter, "fFlightPath"), + ("reconstruction/image_compression", "??", "fImageCompression"), + ("reconstruction/kfactor", "??", "fKfactor"), + ("reconstruction/volume", ureg.nanometer**3, "fReconVolume"), + ("reconstruction/shank_angle", ureg.degrees, "fShankAngle"), + ("reconstruction/obb/xmax", ureg.nanometer, "fXmax"), + ("reconstruction/obb/xmin", ureg.nanometer, "fXmin"), + ("reconstruction/obb/ymax", ureg.nanometer, "fYmax"), + ("reconstruction/obb/ymin", ureg.nanometer, "fYmin"), + ("reconstruction/obb/zmax", ureg.nanometer, "fZmax"), + ("reconstruction/obb/zmin", ureg.nanometer, "fZmin"), + ( + "measurement/instrument/analysis_chamber/pressure", + ureg.torr, + "fAnalysisPressure", + ), + ( + "measurement/instrument/local_electrode/voltage", + ureg.volt, + "fAnodeAccelVoltage", + ), + ("elapsed_time", ureg.second, "fElapsedTime"), + ( + "measurement/instrument/pulser/pulse_frequency", + ureg.kilohertz, + "fInitialPulserFreq", + ), + ("measurement/instrument/ion_detector/mcp_efficiency", "fMcpEfficiency"), + ("measurement/instrument/ion_detector/mesh_efficiency", "fMeshEfficiency"), + ( + "measurement/instrument/analysis_chamber/flight_path", + ureg.millimeter, + "fMaximumFlightPathMm", + ), + ("measurement/stage/specimen_temperature", ureg.kelvin, "fSpecimenTemperature"), + ( + "atom_probe/voltage_and_bowl/tof_zero_estimate", + ureg.nanosecond, + "fT0Estimate", + ), + ], + "map_to_u4": [ + ("measurement/instrument/fabrication/serial_number", "fSerialNumber"), + ("run_number", "fRunNumber"), + ], +} -("", "fLeapModel"), -("", "fLaserModel"), +# second +# ("experiment_description", ["fProjectName", "fName", "fComments"]) +# ("operation_mode", "fAcquisitionMode") +# ("atom_probe/hit_finding/total_hit_quality", "fTotalEventGolden") +# ("", "fTotalEventIncomplete") +# ("", "fTotalEventMultiple") +# ("", "fTotalEventPartials") +# ("", "fTotalEventRecords") +# ("", "fAcqBuildVersion") +# ("", "fAcqMajorVersion") +# ("", "fAcqMinorVersion") +# ("", "fCernRootVersion") +# ("", "fImagoRootDate") +# ("", "fImagoRootVersion") +# ("", "fStreamVersion")] -("", ureg.torr, "fAnalysisPressure"), -("", ureg.volt, "fAnodeAccelVoltage") -("", ureg.second, "fElapsedTime"), -("", ureg.kilohertz, "fInitialPulserFreq"), -("", "fInstrumentComment"), -("", "fMaximumFlightPathMm"), -("", "fMcpEfficiency"), -("", "fMeshEfficiency"), -# fPidAlgorithmID -# fPidMaxInitialSlew -# fPidMaxTurnOnSlew -# fPidPropCoef -# fPidPulsesPerUpdate -# fPidTradHysterisis -# fPidTradStep -("", "fRawPathName"), -("", "fResults"), -("", "fRunNumber"), -("", "fSpecimenCondition"), -("", "fSpecimenName"), -("", "fSpecimenTemperature"), -("", "fStartISO8601"), -("", ureg.nanosecond, "fT0Estimate"), -# fTargetEvapRate -# fTargetPulseFraction -("", "fTotalEventGolden") -("", "fTotalEventIncomplete") -("", "fTotalEventMultiple") -("", "fTotalEventPartials") -("", "fTotalEventRecords") +# third +# ("", ureg.nanometer, "fTipRadius") +# ("", ureg.nanometer, "fTipRadius0") +# ("", ureg.volt, "fVoltage0") +# ("", "fUserID") +# ("", "fBcaSerialRev") +# ("", "fFirmwareVersion") +# ("", "fFlangeSerialNumber") +# ("", "fHvpsType") +# ("", "fLaserModelString") +# ("", "fLaserSerialNumber") +# ("", "fLcbSerialRev") +# ("", "fMcpSerialNumber") +# ("", "fPulserType") +# ("", "fTaSerialRev") +# ("", "fTdcType") +# ("", "fTargetEvapRate") +# ("", "fTargetPulseFraction") +# ("", "fPidAlgorithmID") +# ("", "fPidMaxInitialSlew") +# ("", "fPidMaxTurnOnSlew") +# ("", "fPidPropCoef") +# ("", "fPidPulsesPerUpdate") +# ("", "fPidTradHysterisis") +# ("", "fPidTradStep") diff --git a/src/pynxtools_apm/reader.py b/src/pynxtools_apm/reader.py index bc133d9..a0d14a7 100644 --- a/src/pynxtools_apm/reader.py +++ b/src/pynxtools_apm/reader.py @@ -35,6 +35,7 @@ from pynxtools_apm.utils.load_reconstruction import ( ApmReconstructionParser, ) +from pynxtools_apm.utils.oasis_apsuite_reader import NxApmNomadOasisCamecaParser from pynxtools_apm.utils.oasis_config_reader import ( NxApmNomadOasisConfigurationParser, ) @@ -81,7 +82,6 @@ def read( template.clear() entry_id = 1 - # if len(file_paths) == 1: """ # TODO::better make this an option rather than hijack and demand a # specifically named file to trigger the synthesizer @@ -134,6 +134,10 @@ def read( else: print("No input-file defined for ranging definitions!") + if len(case.apsuite) == 1: + print("Parse from a file with IVAS/APSuite-specific concepts...") + nx_apm_cameca = NxApmNomadOasisCamecaParser(case.apsuite[0], entry_id) + print("Create NeXus default plottable data...") apm_default_plot_generator(template, entry_id) diff --git a/src/pynxtools_apm/utils/generate_synthetic_data.py b/src/pynxtools_apm/utils/generate_synthetic_data.py index 1caa57b..0e12cc9 100644 --- a/src/pynxtools_apm/utils/generate_synthetic_data.py +++ b/src/pynxtools_apm/utils/generate_synthetic_data.py @@ -248,9 +248,9 @@ def place_atoms_from_periodic_table(self): for idx in self.nrm_composition: accept_reject.append(idx[3]) accept_reject = np.cumsum(accept_reject) - assert self.xyz != [], ( - "self.xyz must not be an empty dataset, create a geometry first!" - ) + assert ( + self.xyz != [] + ), "self.xyz must not be an empty dataset, create a geometry first!" # print("Accept/reject sampling m/q values for " # + str(np.shape(self.xyz)[0]) + " ions") diff --git a/src/pynxtools_apm/utils/io_case_logic.py b/src/pynxtools_apm/utils/io_case_logic.py index 55b5b14..9e85648 100644 --- a/src/pynxtools_apm/utils/io_case_logic.py +++ b/src/pynxtools_apm/utils/io_case_logic.py @@ -32,6 +32,7 @@ ".analysis", ] VALID_FILE_NAME_SUFFIX_CONFIG = [".yaml", ".yml"] +VALID_FILE_NAME_SUFFIX_CAMECA = [".cameca"] class ApmUseCaseSelector: @@ -50,6 +51,7 @@ def __init__(self, file_paths: Tuple[str] = None): self.case: Dict[str, list] = {} self.eln: List[str] = [] self.cfg: List[str] = [] + self.apsuite: List[str] = [] self.reconstruction: List[str] = [] self.ranging: List[str] = [] self.is_valid = False @@ -57,6 +59,7 @@ def __init__(self, file_paths: Tuple[str] = None): VALID_FILE_NAME_SUFFIX_RECON + VALID_FILE_NAME_SUFFIX_RANGE + VALID_FILE_NAME_SUFFIX_CONFIG + + VALID_FILE_NAME_SUFFIX_CAMECA ) print(f"self.supported_file_name_suffixes: {self.supported_file_name_suffixes}") print(f"{file_paths}") @@ -90,7 +93,8 @@ def check_validity_of_file_combinations(self): """Check if this combination of types of files is supported.""" recon_input = 0 # reconstruction relevant file e.g. POS, ePOS, APT, ATO, CSV range_input = 0 # ranging definition file, e.g. RNG, RRNG, ENV, FIG.TXT - other_input = 0 # generic ELN or Oasis-specific configurations + other_input = 0 # generic ELN, Oasis-specific configurations + apsui_input = 0 # manual yaml files composed from IVAS/AP Suite for suffix, value in self.case.items(): if suffix not in [".h5", "range_.h5"]: if suffix in VALID_FILE_NAME_SUFFIX_RECON: @@ -99,6 +103,8 @@ def check_validity_of_file_combinations(self): range_input += len(value) elif suffix in VALID_FILE_NAME_SUFFIX_CONFIG: other_input += len(value) + elif suffix in VALID_FILE_NAME_SUFFIX_CAMECA: + apsui_input += len(value) else: continue else: @@ -126,12 +132,16 @@ def check_validity_of_file_combinations(self): self.cfg += [entry] else: self.eln += [entry] + for suffix in VALID_FILE_NAME_SUFFIX_CAMECA: + self.apsuite += self.case[suffix] print( f"recon_results: {self.reconstruction}\n" f"range_results: {self.ranging}\n" f"Oasis ELN: {self.eln}\n" f"Oasis local config: {self.cfg}\n" ) + if len(self.apsuite) > 0: + print(f"IVAS/APSuite: {self.apsuite}\n") def report_workflow(self, template: dict, entry_id: int) -> dict: """Initialize the reporting of the workflow.""" diff --git a/src/pynxtools_apm/utils/oasis_apsuite_reader.py b/src/pynxtools_apm/utils/oasis_apsuite_reader.py new file mode 100644 index 0000000..e9ccc15 --- /dev/null +++ b/src/pynxtools_apm/utils/oasis_apsuite_reader.py @@ -0,0 +1,80 @@ +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Wrapping multiple parsers for vendor files with NOMAD Oasis/ELN/YAML metadata.""" + +import pathlib +from typing import Any, Dict + +import flatdict as fd +import yaml +from ase.data import chemical_symbols + +from pynxtools_apm.concepts.mapping_functors_pint import add_specific_metadata_pint +from pynxtools_apm.configurations.cameca_cfg import APM_CAMECA_TO_NEXUS + + +class NxApmNomadOasisCamecaParser: + """Parse manually collected content from an IVAS / AP Suite YAML.""" + + def __init__(self, file_path: str = "", entry_id: int = 1, verbose: bool = False): + """Construct class""" + print(f"Extracting data from IVAS/APSuite file: {file_path}") + if pathlib.Path(file_path).name.endswith(".cameca"): + self.file_path = file_path + self.entry_id = entry_id if entry_id > 0 else 1 + self.verbose = verbose + try: + with open(self.file_path, "r", encoding="utf-8") as stream: + self.yml = fd.FlatDict(yaml.safe_load(stream), delimiter="/") + if self.verbose: + for key, val in self.yml.items(): + print(f"key: {key}, value: {val}") + except (FileNotFoundError, IOError): + print(f"File {self.file_path} not found !") + self.apsuite = fd.FlatDict({}, delimiter="/") + return + + def parse_ranging_definitions(self, template: dict) -> dict: + """Interpret human-readable ELN input to generate consistent composition table.""" + src = "sample/composition" + if src in self.yml: + if isinstance(self.yml[src], list): + dct: Dict[Any, Any] = {} # IMPLEMENT ME! + prfx = f"/ENTRY[entry{self.entry_id}]/sample/chemical_composition" + ion_id = 1 + for symbol in chemical_symbols[1::]: + # ase convention is that chemical_symbols[0] == "X" + # to enable using ordinal number for indexing + if symbol in dct: + if isinstance(dct[symbol], tuple) and len(dct[symbol]) == 2: + trg = f"{prfx}/ionID[ion{ion_id}]" + template[f"{trg}/chemical_symbol"] = symbol + template[f"{trg}/composition"] = dct[symbol][0] + if dct[symbol][1] is not None: + template[f"{trg}/composition_error"] = dct[symbol][1] + ion_id += 1 + return template + + def parse(self, template: dict) -> dict: + """Copy data from self into template the appdef instance.""" + self.parse_ranging_definitions(template) + identifier = [self.entry_id] + add_specific_metadata_pint( + APM_CAMECA_TO_NEXUS, self.apsuite, identifier, template + ) + return template