Skip to content

Commit

Permalink
Implementation skeleton for YAML IVAS/APSuite parser
Browse files Browse the repository at this point in the history
  • Loading branch information
atomprobe-tc committed Jan 26, 2025
1 parent f005457 commit 54eed6e
Show file tree
Hide file tree
Showing 6 changed files with 216 additions and 90 deletions.
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
# Ruff version.
rev: v0.9.0
rev: v0.8.3
hooks:
# Run the linter.
- id: ruff
Expand Down
200 changes: 116 additions & 84 deletions src/pynxtools_apm/configurations/cameca_cfg.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,92 +15,124 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Dict mapping custom schema instances from eln_data.yaml file on concepts in NXapm."""
"""Dict mapping custom schema instances from custom yaml file on concepts in NXapm."""

from pynxtools_apm.utils.pint_custom_unit_registry import ureg


"map": [("", "fComments"), ("", "fQuality"), ("", "fPrimaryElement")],
"map_to_f8": [
("", "fEfficiency"),
("", ureg.volt / ureg.nanometer ** 2, "fEvaporationField"),
("", ureg.millimeter, "fFlightPath"),
("", "??", "fImageCompression"),
("", "??", "fKfactor"),
("", ureg.nanometer ** 3, "fReconVolume"),
("", ureg.degrees, "fShankAngle"),
("", ureg.nanometer, "fTipRadius"),
("", ureg.nanometer, "fTipRadius0"),
("", ureg.volt, "fVoltage0"),
("", ureg.nanometer, "fXmax"),
("", ureg.nanometer, "fXmin"),
("", ureg.nanometer, "fYmax"),
("", ureg.nanometer, "fYmin"),
("", ureg.nanometer, "fZmax"),
("", ureg.nanometer, "fZmin")]

("", "fUserID")
from typing import Any, Dict

from pynxtools_apm.utils.pint_custom_unit_registry import ureg

("", "fAcqBuildVersion"),
("", "fAcqMajorVersion"),
("", "fAcqMinorVersion"),
("", "fCernRootVersion"),
("", "fImagoRootDate"),
("", "fImagoRootVersion"),
("", "fStreamVersion"),

("", "fSerialNumber"),

("", "fBcaSerialRev"),
("", "fFirmwareVersion"),
("", "fFlangeSerialNumber"),
("", "fHvpsType"),
("", "fLaserModelString"),
("", "fLaserSerialNumber"),
("", "fLcbSerialRev"),
("", "fMcpSerialNumber"),
("", "fPulserType"),
("", "fTaSerialRev"),
("", "fTdcType"),

("type", "fAcquisitionMode"),
("", "fApertureName"),
("", "fAtomProbeName"),
("", "fComments")
("", "fName"),
("", "fProjectName"),
APM_CAMECA_TO_NEXUS: Dict[str, Any] = {
"prefix_trg": "/ENTRY[entry*]",
"prefix_src": "",
"map": [
("reconstruction/quality", "fQuality"),
("reconstruction/primary_element", "fPrimaryElement"),
("measurement/instrument/local_electrode/name", "fApertureName"),
("measurement/instrument/instrument_name", "fAtomProbeName"),
("measurement/instrument/fabrication/model", "fLeapModel"),
(
"measurement/instrument/pulser/SOURCE[sourceID]/fabrication/model",
"fLaserModel",
),
("measurement/instrument/comments", "fInstrumentComment"),
("atom_probe/raw_data/serialized/path", "fRawPathName"),
("measurement/status", "fResults"),
("specimen/description", "fSpecimenCondition"),
("specimen/alias", "fSpecimenName"),
("start_time", "fStartISO8601"),
],
"map_to_f8": [
("reconstruction/efficiency", "fEfficiency"),
(
"reconstruction/evaporation_field",
ureg.volt / ureg.nanometer**2,
"fEvaporationField",
),
("reconstruction/flight_path", ureg.millimeter, "fFlightPath"),
("reconstruction/image_compression", "??", "fImageCompression"),
("reconstruction/kfactor", "??", "fKfactor"),
("reconstruction/volume", ureg.nanometer**3, "fReconVolume"),
("reconstruction/shank_angle", ureg.degrees, "fShankAngle"),
("reconstruction/obb/xmax", ureg.nanometer, "fXmax"),
("reconstruction/obb/xmin", ureg.nanometer, "fXmin"),
("reconstruction/obb/ymax", ureg.nanometer, "fYmax"),
("reconstruction/obb/ymin", ureg.nanometer, "fYmin"),
("reconstruction/obb/zmax", ureg.nanometer, "fZmax"),
("reconstruction/obb/zmin", ureg.nanometer, "fZmin"),
(
"measurement/instrument/analysis_chamber/pressure",
ureg.torr,
"fAnalysisPressure",
),
(
"measurement/instrument/local_electrode/voltage",
ureg.volt,
"fAnodeAccelVoltage",
),
("elapsed_time", ureg.second, "fElapsedTime"),
(
"measurement/instrument/pulser/pulse_frequency",
ureg.kilohertz,
"fInitialPulserFreq",
),
("measurement/instrument/ion_detector/mcp_efficiency", "fMcpEfficiency"),
("measurement/instrument/ion_detector/mesh_efficiency", "fMeshEfficiency"),
(
"measurement/instrument/analysis_chamber/flight_path",
ureg.millimeter,
"fMaximumFlightPathMm",
),
("measurement/stage/specimen_temperature", ureg.kelvin, "fSpecimenTemperature"),
(
"atom_probe/voltage_and_bowl/tof_zero_estimate",
ureg.nanosecond,
"fT0Estimate",
),
],
"map_to_u4": [
("measurement/instrument/fabrication/serial_number", "fSerialNumber"),
("run_number", "fRunNumber"),
],
}

("", "fLeapModel"),
("", "fLaserModel"),
# second
# ("experiment_description", ["fProjectName", "fName", "fComments"])
# ("operation_mode", "fAcquisitionMode")
# ("atom_probe/hit_finding/total_hit_quality", "fTotalEventGolden")
# ("", "fTotalEventIncomplete")
# ("", "fTotalEventMultiple")
# ("", "fTotalEventPartials")
# ("", "fTotalEventRecords")
# ("", "fAcqBuildVersion")
# ("", "fAcqMajorVersion")
# ("", "fAcqMinorVersion")
# ("", "fCernRootVersion")
# ("", "fImagoRootDate")
# ("", "fImagoRootVersion")
# ("", "fStreamVersion")]

("", ureg.torr, "fAnalysisPressure"),
("", ureg.volt, "fAnodeAccelVoltage")
("", ureg.second, "fElapsedTime"),
("", ureg.kilohertz, "fInitialPulserFreq"),
("", "fInstrumentComment"),
("", "fMaximumFlightPathMm"),
("", "fMcpEfficiency"),
("", "fMeshEfficiency"),
# fPidAlgorithmID
# fPidMaxInitialSlew
# fPidMaxTurnOnSlew
# fPidPropCoef
# fPidPulsesPerUpdate
# fPidTradHysterisis
# fPidTradStep
("", "fRawPathName"),
("", "fResults"),
("", "fRunNumber"),
("", "fSpecimenCondition"),
("", "fSpecimenName"),
("", "fSpecimenTemperature"),
("", "fStartISO8601"),
("", ureg.nanosecond, "fT0Estimate"),
# fTargetEvapRate
# fTargetPulseFraction
("", "fTotalEventGolden")
("", "fTotalEventIncomplete")
("", "fTotalEventMultiple")
("", "fTotalEventPartials")
("", "fTotalEventRecords")
# third
# ("", ureg.nanometer, "fTipRadius")
# ("", ureg.nanometer, "fTipRadius0")
# ("", ureg.volt, "fVoltage0")
# ("", "fUserID")
# ("", "fBcaSerialRev")
# ("", "fFirmwareVersion")
# ("", "fFlangeSerialNumber")
# ("", "fHvpsType")
# ("", "fLaserModelString")
# ("", "fLaserSerialNumber")
# ("", "fLcbSerialRev")
# ("", "fMcpSerialNumber")
# ("", "fPulserType")
# ("", "fTaSerialRev")
# ("", "fTdcType")
# ("", "fTargetEvapRate")
# ("", "fTargetPulseFraction")
# ("", "fPidAlgorithmID")
# ("", "fPidMaxInitialSlew")
# ("", "fPidMaxTurnOnSlew")
# ("", "fPidPropCoef")
# ("", "fPidPulsesPerUpdate")
# ("", "fPidTradHysterisis")
# ("", "fPidTradStep")
6 changes: 5 additions & 1 deletion src/pynxtools_apm/reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
from pynxtools_apm.utils.load_reconstruction import (
ApmReconstructionParser,
)
from pynxtools_apm.utils.oasis_apsuite_reader import NxApmNomadOasisCamecaParser
from pynxtools_apm.utils.oasis_config_reader import (
NxApmNomadOasisConfigurationParser,
)
Expand Down Expand Up @@ -81,7 +82,6 @@ def read(
template.clear()

entry_id = 1
# if len(file_paths) == 1:
"""
# TODO::better make this an option rather than hijack and demand a
# specifically named file to trigger the synthesizer
Expand Down Expand Up @@ -134,6 +134,10 @@ def read(
else:
print("No input-file defined for ranging definitions!")

if len(case.apsuite) == 1:
print("Parse from a file with IVAS/APSuite-specific concepts...")
nx_apm_cameca = NxApmNomadOasisCamecaParser(case.apsuite[0], entry_id)

print("Create NeXus default plottable data...")
apm_default_plot_generator(template, entry_id)

Expand Down
6 changes: 3 additions & 3 deletions src/pynxtools_apm/utils/generate_synthetic_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -248,9 +248,9 @@ def place_atoms_from_periodic_table(self):
for idx in self.nrm_composition:
accept_reject.append(idx[3])
accept_reject = np.cumsum(accept_reject)
assert self.xyz != [], (
"self.xyz must not be an empty dataset, create a geometry first!"
)
assert (
self.xyz != []
), "self.xyz must not be an empty dataset, create a geometry first!"
# print("Accept/reject sampling m/q values for "
# + str(np.shape(self.xyz)[0]) + " ions")

Expand Down
12 changes: 11 additions & 1 deletion src/pynxtools_apm/utils/io_case_logic.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
".analysis",
]
VALID_FILE_NAME_SUFFIX_CONFIG = [".yaml", ".yml"]
VALID_FILE_NAME_SUFFIX_CAMECA = [".cameca"]


class ApmUseCaseSelector:
Expand All @@ -50,13 +51,15 @@ def __init__(self, file_paths: Tuple[str] = None):
self.case: Dict[str, list] = {}
self.eln: List[str] = []
self.cfg: List[str] = []
self.apsuite: List[str] = []
self.reconstruction: List[str] = []
self.ranging: List[str] = []
self.is_valid = False
self.supported_file_name_suffixes = (
VALID_FILE_NAME_SUFFIX_RECON
+ VALID_FILE_NAME_SUFFIX_RANGE
+ VALID_FILE_NAME_SUFFIX_CONFIG
+ VALID_FILE_NAME_SUFFIX_CAMECA
)
print(f"self.supported_file_name_suffixes: {self.supported_file_name_suffixes}")
print(f"{file_paths}")
Expand Down Expand Up @@ -90,7 +93,8 @@ def check_validity_of_file_combinations(self):
"""Check if this combination of types of files is supported."""
recon_input = 0 # reconstruction relevant file e.g. POS, ePOS, APT, ATO, CSV
range_input = 0 # ranging definition file, e.g. RNG, RRNG, ENV, FIG.TXT
other_input = 0 # generic ELN or Oasis-specific configurations
other_input = 0 # generic ELN, Oasis-specific configurations
apsui_input = 0 # manual yaml files composed from IVAS/AP Suite
for suffix, value in self.case.items():
if suffix not in [".h5", "range_.h5"]:
if suffix in VALID_FILE_NAME_SUFFIX_RECON:
Expand All @@ -99,6 +103,8 @@ def check_validity_of_file_combinations(self):
range_input += len(value)
elif suffix in VALID_FILE_NAME_SUFFIX_CONFIG:
other_input += len(value)
elif suffix in VALID_FILE_NAME_SUFFIX_CAMECA:
apsui_input += len(value)
else:
continue
else:
Expand Down Expand Up @@ -126,12 +132,16 @@ def check_validity_of_file_combinations(self):
self.cfg += [entry]
else:
self.eln += [entry]
for suffix in VALID_FILE_NAME_SUFFIX_CAMECA:
self.apsuite += self.case[suffix]
print(
f"recon_results: {self.reconstruction}\n"
f"range_results: {self.ranging}\n"
f"Oasis ELN: {self.eln}\n"
f"Oasis local config: {self.cfg}\n"
)
if len(self.apsuite) > 0:
print(f"IVAS/APSuite: {self.apsuite}\n")

def report_workflow(self, template: dict, entry_id: int) -> dict:
"""Initialize the reporting of the workflow."""
Expand Down
80 changes: 80 additions & 0 deletions src/pynxtools_apm/utils/oasis_apsuite_reader.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
#
# Copyright The NOMAD Authors.
#
# This file is part of NOMAD. See https://nomad-lab.eu for further info.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Wrapping multiple parsers for vendor files with NOMAD Oasis/ELN/YAML metadata."""

import pathlib
from typing import Any, Dict

import flatdict as fd
import yaml
from ase.data import chemical_symbols

from pynxtools_apm.concepts.mapping_functors_pint import add_specific_metadata_pint
from pynxtools_apm.configurations.cameca_cfg import APM_CAMECA_TO_NEXUS


class NxApmNomadOasisCamecaParser:
"""Parse manually collected content from an IVAS / AP Suite YAML."""

def __init__(self, file_path: str = "", entry_id: int = 1, verbose: bool = False):
"""Construct class"""
print(f"Extracting data from IVAS/APSuite file: {file_path}")
if pathlib.Path(file_path).name.endswith(".cameca"):
self.file_path = file_path
self.entry_id = entry_id if entry_id > 0 else 1
self.verbose = verbose
try:
with open(self.file_path, "r", encoding="utf-8") as stream:
self.yml = fd.FlatDict(yaml.safe_load(stream), delimiter="/")
if self.verbose:
for key, val in self.yml.items():
print(f"key: {key}, value: {val}")
except (FileNotFoundError, IOError):
print(f"File {self.file_path} not found !")
self.apsuite = fd.FlatDict({}, delimiter="/")
return

def parse_ranging_definitions(self, template: dict) -> dict:
"""Interpret human-readable ELN input to generate consistent composition table."""
src = "sample/composition"
if src in self.yml:
if isinstance(self.yml[src], list):
dct: Dict[Any, Any] = {} # IMPLEMENT ME!
prfx = f"/ENTRY[entry{self.entry_id}]/sample/chemical_composition"
ion_id = 1
for symbol in chemical_symbols[1::]:
# ase convention is that chemical_symbols[0] == "X"
# to enable using ordinal number for indexing
if symbol in dct:
if isinstance(dct[symbol], tuple) and len(dct[symbol]) == 2:
trg = f"{prfx}/ionID[ion{ion_id}]"
template[f"{trg}/chemical_symbol"] = symbol
template[f"{trg}/composition"] = dct[symbol][0]
if dct[symbol][1] is not None:
template[f"{trg}/composition_error"] = dct[symbol][1]
ion_id += 1
return template

def parse(self, template: dict) -> dict:
"""Copy data from self into template the appdef instance."""
self.parse_ranging_definitions(template)
identifier = [self.entry_id]
add_specific_metadata_pint(
APM_CAMECA_TO_NEXUS, self.apsuite, identifier, template
)
return template

0 comments on commit 54eed6e

Please sign in to comment.