Make tomo_mapper installable as a package

.github/workflows/python-app.yml

@@ -45,3 +45,5 @@ jobs:
     - name: Test with pytest
       run: |
         pytest
+      env:
+        PYTHONPATH: src

README.md

@@ -50,7 +50,7 @@ Use the `sem` subcommand for SEM mapping. The mapper expects a map file, an imag
 python -m mapping_cli sem -m <map_file> -i <image or metadata file> -o <json_output_path>
 ```
 
-For further information about the necessary map file, see [Mapping README](./src/resources/maps/mapping)
+For further information about the necessary map file, see [Mapping README](src/tomo_mapper/resources/maps/mapping)
 
 **2. Tomography Mapping**
 
@@ -59,9 +59,9 @@ Use the `tomo` subcommand for tomography mapping. The mapper expects a map file,
 python -m mapping_cli tomo -m <map_file> -i <zip_file or folder> -o <json_output_path>
 ```
 
-For further information about the necessary map file, see [Parsing README](./src/resources/maps/parsing)
+For further information about the necessary map file, see [Parsing README](src/tomo_mapper/resources/maps/parsing)
 
-For further information about mappings used internally, see [Mapping README](./src/resources/maps/mapping)
+For further information about mappings used internally, see [Mapping README](src/tomo_mapper/resources/maps/mapping)
 
 In cases with no need of tweaking the parsing map file there is a shortcut option to use the supplied vendor-specific default map.
 

mapping_cli.py

@@ -1,120 +1,8 @@
-import argparse
-import json
-import logging
-import os
-from sys import exit
-
-from src.IO.MappingAbortionError import MappingAbortionError
-from src.IO.sem.InputReader import InputReader as InputReader_SEM
-from src.IO.tomo.InputReader import InputReader as InputReader_TOMO
-from src.IO.tomo.OutputWriter import OutputWriter
-from src.resources.maps.parsing import map_from_flag
-
-# make log level configurable from ENV, defaults to info level
-logging.basicConfig(
-    level=os.environ.get('LOGLEVEL', 'INFO').upper()
-)
-
-def add_tomo_parser(subparsers):
-    parser_t = subparsers.add_parser(
-        "tomo",
-        help="Tomography mapping functionality",
-        description='Extracting of SEM FIB Tomography metadata to unified json format'
-    )
-    # Add arguments for input, output, and map
-    parser_t.add_argument('-i', '--input', help='Input zip file or folder as path', required=True)
-    parser_t.add_argument('-o', '--output', help='Path to output json file', required=True)
-
-    # Create a group for the mutually exclusive map options
-    map_group = parser_t.add_mutually_exclusive_group(required=True)
-
-    # Add the map file option to the group
-    map_group.add_argument('-m', '--map', help='Map file as path or remote URI')
-
-    # Add the default map option to the group with the allowed values
-    map_group.add_argument('-dm', '--default-map', help='Use a default map for a vendor', choices=map_from_flag.keys(), type=str.lower)
-
-    parser_t.set_defaults(func=run_tomo_mapper)
-
-
-def add_sem_parser(subparsers):
-    parser_s = subparsers.add_parser(
-        "sem",
-        help="SEM mapping functionality",
-        description='Extracting of SEM metadata to unified json format'
-    )
-    parser_s.add_argument('-i','--input', help='Input file as file path', required=True)
-    parser_s.add_argument('-m', '--map', help='Map file as path or remote URI', required=True)
-    parser_s.add_argument('-o', '--output', help='Path to output json file', required=True)
-    parser_s.set_defaults(func=run_sem_mapper)
-
-def run_cli():
-    main_parser = argparse.ArgumentParser(prog="SEM-FIB-TOMO Mapper")
-    subparsers = main_parser.add_subparsers(dest='command', help="Choose one of the subcommands to use mapper")
-    add_tomo_parser(subparsers)
-    add_sem_parser(subparsers)
-
-    args = main_parser.parse_args()
-    if args.command:
-        args.func(args)
-    else:
-        main_parser.print_help()
-
-def run_tomo_mapper(args):
-    argdict = vars(args)
-    INPUT_SOURCE = argdict.get('input')
-    MAP_SOURCE = argdict.get('map') or str(map_from_flag.get(argdict.get('default_map')))
-    OUTPUT_PATH = argdict.get('output')
-
-    reader = None
-    try:
-        reader = InputReader_TOMO(MAP_SOURCE, INPUT_SOURCE)
-        tmpdir = reader.temp_dir_path
-    except MappingAbortionError as e:
-        if reader:
-            reader.clean_up()
-        exit(e)
-
-    output = None
-    try:
-        setup_infos = reader.retrieve_setup_info()
-
-        run_infos = reader.retrieve_run_info()
-
-        imgs = reader.retrieve_image_info()
-
-        # Now all cases are taken into account
-        #si = setup_infos if len(setup_infos) >= 1 else None
-        #ri = run_infos if len(run_infos) >= 1 else None
-
-        output = OutputWriter.stitch_together(setup_infos, run_infos, imgs)
-        OutputWriter.writeOutput(output, OUTPUT_PATH)
-    except MappingAbortionError as e:
-        reader.clean_up()
-        exit(e)
-
-    logging.info("Tomography mapping completed.")
-    reader.clean_up()
-    return output
-
-def run_sem_mapper(args):
-    argdict = vars(args)
-    INPUT_SOURCE = argdict.get('input')
-    MAP_SOURCE = argdict.get('map')
-    OUTPUT_PATH = argdict.get('output')
-
-    try:
-        reader = InputReader_SEM(MAP_SOURCE, INPUT_SOURCE)
-
-        img_info = reader.retrieve_image_info(INPUT_SOURCE)
-        if not img_info:
-            logging.error('Could not retrieve image information due to unknown error. Aborting.')
-            exit(1)
-        with open(OUTPUT_PATH, 'w', encoding="utf-8") as f:
-            json.dump(img_info, f, indent=4, ensure_ascii=False)
-    except MappingAbortionError as e:
-        exit(e)
+import sys
+from pathlib import Path
 
+sys.path.insert(0, str(Path(__file__).parent / "src"))
+from tomo_mapper.mapping_main import run_cli
 
 if __name__ == '__main__':
-    run_cli()
+    run_cli()

mappingservice-plugin/build.gradle

@@ -14,9 +14,10 @@ repositories {
 }
 
 dependencies {
-    implementation 'org.springframework:spring-core:6.2.5'
-    implementation 'org.slf4j:slf4j-api:2.0.17'
-    implementation files("src/main/lib/mapping-service-plain.jar")
+    compileOnly 'org.springframework:spring-core:6.2.5'
+    compileOnly 'org.slf4j:slf4j-api:2.0.17'
+    implementation 'org.tomlj:tomlj:1.1.0'
+    compileOnly files("src/main/lib/mapping-service-plain.jar")
 }
 
 if (System.getenv('VERSION_OVERRIDE_BY_BRANCH')) {
@@ -35,17 +36,34 @@ tasks.register('printVersion') {
 //Task for creating a resource file with the version info
 tasks.register("generateVersionProps", WriteProperties) { t ->
     def generatedResourcesDir = project.layout.buildDirectory.dir(["resources", "main"].join(File.separator))
-    def outputFile = generatedResourcesDir.map { it.file("sempluginversion.properties") }
+    def outputFile = generatedResourcesDir.map { it.file("pluginversion.properties") }
 
     t.destinationFile = outputFile.get().asFile
     t.property("version", project.version)
 }
 
+tasks.register('copyTomlToResources', Copy) {
+    from '../pyproject.toml'
+    into "${layout.buildDirectory.get()}/generated-resources"
+}
+
+processResources {
+    dependsOn tasks.copyTomlToResources
+    from("${layout.buildDirectory.get()}/generated-resources") {
+        include 'pyproject.toml'
+    }
+}
+
 resolveMainClassName.dependsOn("generateVersionProps")
 
 jar {
     dependsOn(generateVersionProps)
     archiveFileName
+
+    from {
+        configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) }
+    }
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
 }
 
 bootJar {

mappingservice-plugin/src/main/java/edu/kit/datamanager/semplugin/SEMImagePlugin.java

@@ -13,22 +13,58 @@
 import java.nio.file.Path;
 import java.util.Properties;
 
+import org.tomlj.Toml;
+import org.tomlj.TomlParseResult;
+
 public class SEMImagePlugin implements IMappingPlugin{
 
     private static String version;
 
     private final Logger LOGGER = LoggerFactory.getLogger(SEMImagePlugin.class);
-    private final String REPOSITORY = "https://github.com/kit-data-manager/tomo_mapper";
+    private String REPOSITORY;
     private String TAG;
+    private String NAME;
+    private String DESCRIPTION;
+    private MimeType[] INPUT_MIME_TYPES;
+    private MimeType[] OUTPUT_MIME_TYPES;
     private Path dir;
 
-
     public SEMImagePlugin() {
+        loadVersion();
+        loadTomlConfig();
+    }
+
+    private void loadTomlConfig() {
+        ClassLoader classLoader = this.getClass().getClassLoader();
+        URL resource = classLoader.getResource("pyproject.toml");
+        LOGGER.info("Resource file: " + resource);
+
+        if (resource != null) {
+            try (InputStream input = resource.openStream()) {
+                TomlParseResult result = Toml.parse(input);
+
+                if (result.hasErrors()) {
+                    result.errors().forEach(error -> LOGGER.warn("TOML parse error: " + error.toString()));
+                } else {
+                    REPOSITORY = result.getString("project.urls.repository");
+                    if (REPOSITORY == null) throw new IllegalArgumentException("Repository URL cannot be read from config");
+                    NAME = result.getString("tool.plugin.name");
+                    if (NAME == null) throw new IllegalArgumentException("Plugin name cannot be read from config");
+                    DESCRIPTION = result.contains("tool.plugin.description") ? result.getString("tool.plugin.description") : "descrption unavailable";
+                    INPUT_MIME_TYPES = result.getArrayOrEmpty("tool.plugin.input_mimes").toList().stream().map(Object::toString).map(MimeTypeUtils::parseMimeType).toArray(MimeType[]::new);
+                    OUTPUT_MIME_TYPES = result.getArrayOrEmpty("tool.plugin.output_mimes").toList().stream().map(Object::toString).map(MimeTypeUtils::parseMimeType).toArray(MimeType[]::new);
+                }
+            } catch (Exception e) {
+                LOGGER.error("Failed to load TOML file: " + e.getMessage());
+            }
+        }
+    }
+
+    private void loadVersion() {
         try {
             // Get the context class loader
             ClassLoader classLoader = this.getClass().getClassLoader();
-            // TODO: do we need to make sure that the resource path is somehow related to the current plugin to avoid loading the wrong property file in case of identical property names?
-            URL resource = classLoader.getResource("sempluginversion.properties");
+            URL resource = classLoader.getResource("pluginversion.properties");
             LOGGER.info("Resource file: {}", resource);
             if (resource != null) {
                 // Load the properties file
@@ -50,12 +86,12 @@ public SEMImagePlugin() {
 
     @Override
     public String name() {
-        return "GenericSEMtoJSON";
+        return NAME;
     }
 
     @Override
     public String description() {
-        return "This python based tool extracts metadata from machine generated scanning microscopy images and generates a JSON file adhering to the schema.";
+        return DESCRIPTION;
     }
 
     @Override
@@ -75,7 +111,7 @@ public MimeType[] inputTypes() {
 
     @Override
     public MimeType[] outputTypes() {
-        return new MimeType[]{MimeTypeUtils.APPLICATION_JSON};
+        return OUTPUT_MIME_TYPES;
     }
 
     @Override
@@ -104,9 +140,9 @@ public void setup() {
     @Override
     public MappingPluginState mapFile(Path mappingFile, Path inputFile, Path outputFile) throws MappingPluginException {
         long startTime = System.currentTimeMillis();
-        LOGGER.trace("Run SEM-Mapping-Tool on '{}' with mapping '{}' -> '{}'", mappingFile, inputFile, outputFile);
+        LOGGER.trace("Run {} on '{}' with mapping '{}' -> '{}'", this.name(), mappingFile, inputFile, outputFile);
         //MappingPluginState result = PythonRunnerUtil.runPythonScript(dir + "/metaMapper.py", mappingFile.toString(), inputFile.toString(), outputFile.toString());
-        String[] args = {"sem", "-m", mappingFile.toString(), "-i", inputFile.toString(), "-o", outputFile.toString()};
+        String[] args = {"-m", mappingFile.toString(), "-i", inputFile.toString(), "-o", outputFile.toString()};
         MappingPluginState result = PythonRunnerUtil.runPythonScript(dir + "/plugin_wrapper.py", args);
         long endTime = System.currentTimeMillis();
         long totalTime = endTime - startTime;

plugin_wrapper.py

@@ -1,10 +1,12 @@
 # run_mapping.py
 import sys
-from mapping_cli import run_cli
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).parent / "src"))
+from tomo_mapper.mapping_main import run_cli
 
 if __name__ == "__main__":
     # Extract arguments from the command line
-    sys.argv = ["mapping_cli"] + sys.argv[1:]
+    sys.argv = ["mapping_cli", "sem"] + sys.argv[1:]
 
     # Call the run_cli function
-    run_cli()
+    run_cli()

pyproject.toml

@@ -0,0 +1,63 @@
+[project]
+name = "tomo_mapper"
+version = "1.2.0rc"
+description = "SEM-FIB-Tomography Mapper is a tool designed for mapping SEM (Scanning Electron Microscope) images and SEM-FIB Tomography metadata to a uniform, schema-compliant json format."
+keywords = ["SEM", "Tomography", "TIFF", "metadata", "extraction", "schema"]
+readme = "README.md"
+license = {  text = "Apache-2.0"}
+requires-python = ">=3.12"
+dependencies = [
+    "pillow >= 10.3",
+    "pydantic[email] >= 2.11",
+    "xmltodict >= 0.14",
+    "requests",
+    "validators >= 0.34",
+    "jsonpath-ng >= 1.7",
+    "deepmerge >= 2.0",
+    "magika >= 0.5.1",
+    "python-dateutil"
+]
+
+[[project.authors]]
+name = "Germaine Götzelmann"
+
+[[project.authors]]
+name = "Gabin Thibaut Oumbe Tekam"
+
+[[project.maintainers]]
+name = "Germaine Götzelmann"
+
+[[project.maintainers]]
+name = "Gabin Thibaut Oumbe Tekam"
+
+[build-system]
+requires = ["setuptools>=66.1.0", "setuptools-scm", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project.scripts]
+sem-tomo-mapper = "tomo_mapper.mapping_main:run_cli"
+
+[tool.setuptools]
+include-package-data = true
+#package-dir = { "" = "src" }
+
+#[tool.setuptools.packages.find]
+#where = ["src"]
+#include = ["tomo_mapper*"]
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]
+
+[tool.setuptools.dynamic]
+dependencies = {file = [ "requirements.txt" ] }
+
+[project.urls]
+Homepage = "https://github.com/kit-data-manager/tomo_mapper"
+repository = "https://github.com/kit-data-manager/tomo_mapper"
+
+[tool.plugin]
+name = "GenericSEMtoJSON"
+description = ""
+input_mimes = ["image/tiff"]
+output_mimes = ["application/json"]
+

requirements.txt

@@ -9,4 +9,4 @@ magika >= 0.5.1
 pytest >= 7.4
 pytest-cov >= 6.0
 pytest-mock >= 3.14.0
-python-dateutil
+python-dateutil