updated modules tester and compiler to support any types of extensions

.github/workflows/updater.yml

@@ -33,7 +33,7 @@ jobs:
         run: |
           export BUMP_TYPE=${BUMP_TYPE:-patch}
           python -m updater
-          git add extensions/sources/*
+          git add extensions/**/__version__.py
           git add sources.json
           git config --local user.email "[email protected]"
           git config --local user.name "GitHub Actions"

extensions/enrichers/TestEnricher/TestEnricher.py

@@ -0,0 +1,6 @@
+from FMD3.extensions.enrichers.IEnricher import IEnricher
+
+
+class TestEnricher(IEnricher):
+    ID = "TestEnricher"
+    NAME = "TestEnricher"

extensions/enrichers/TestEnricher/__version__.py

@@ -0,0 +1 @@
+__version__ = '0.0.0'

extensions/enrichers/TestEnricher/tests/__init__.py

@@ -0,0 +1,6 @@
+import unittest
+
+
+class TestEnricher(unittest.TestCase):
+    def test_enricher(self):
+        self.assertTrue(True)

extensions/sources/MangaDex/__version__.py

@@ -1 +1 @@
-__version__ = '0.0.6'
+__version__ = '0.0.4'

sources.json

@@ -1,7 +1,16 @@
 {
-  "d07c9c2425764da8ba056505f57cf40c": {
-    "name": "MangaDex",
-    "commit_hash": "f060f1b1ca623c8ddadb5432197369c938f08c61",
-    "version": "0.0.6"
+  "sources": {
+    "d07c9c2425764da8ba056505f57cf40c": {
+      "name": "MangaDex",
+      "commit_hash": "a2880c9a77ed9eac2a26aa6f49d54f6f92038add",
+      "version": "0.0.4"
+    }
+  },
+  "enrichers": {
+    "TestEnricher": {
+      "name": "TestEnricher",
+      "commit_hash": "",
+      "version": "0.0.0"
+    }
   }
 }

updater/__main__.py

@@ -8,30 +8,44 @@
 from updater.data_extractor import get_extension_id
 from updater.versioning import bump_version, parse_version
 
+
 def main():
-    output_folder = Path("output")
-    output_folder.mkdir(exist_ok=True)  # Replace with the desired output path
+
 
     sources_data = load_sources_data()
-    dirs = [d for d in os.listdir(SOURCES_PATH) if
-            d != "__pycache__" and not d.startswith("sources") and not d.endswith(".py")]
-    for module_name in dirs:
-        module_id = get_extension_id(module_name)
-        assert module_id is not None
-        if has_code_changed(module_id, module_name, SOURCES_PATH, sources_data):
-            print(f"Changes detected in extension '{module_name}'.")
-
-            if run_tests(module_name):
-                bump_version(module_name, SOURCES_PATH)
-                sources_data[module_id] = {
-                    "name": module_name,
-                    'commit_hash': get_latest_commit_hash(module_name, SOURCES_PATH),
-                    "version": parse_version(module_name, SOURCES_PATH).base_version
-                }
-
-                zip_extension(module_id, module_name, SOURCES_PATH, output_folder)
-        else:
-            print(f"No changes detected in extension '{module_name}'.")
+
+    for generic_extension_path in filter(os.path.isdir, {dir for dir in Path("extensions").iterdir() if "__pycache__" not in str(dir)}):
+        generic_extension = generic_extension_path.name
+
+        print(f"scanning {generic_extension} extensions")
+        extension_path = f"extensions/{generic_extension}"
+        output_folder = Path(f"output/{generic_extension}")
+        output_folder.mkdir(exist_ok=True,parents=True)
+
+        if sources_data.get(generic_extension,None) is None:
+            sources_data[generic_extension] = {}
+        for module_path in filter(os.path.isdir, generic_extension_path.iterdir()):
+            module_name = module_path.name
+            module = f"extensions/{generic_extension}/{module_name}"
+            if module_name == "__pycache__":
+                continue
+            # extensions/type/module_name
+            module_id = get_extension_id(extension_path, module_name)
+            assert module_id is not None
+            if has_code_changed(module_id, module_name, extension_path, sources_data):
+                print(f"Changes detected in extension '{module_name}'.")
+
+                if run_tests(f"extensions/{generic_extension}",module_name):
+                    bump_version(module_name, extension_path)
+                    sources_data[generic_extension][module_id] = {
+                        "name": module_name,
+                        'commit_hash': get_latest_commit_hash(module_name, extension_path),
+                        "version": parse_version(module_path,module_name).base_version
+                    }
+
+                    zip_extension(module_id, module_name, extension_path, output_folder)
+            else:
+                print(f"No changes detected in extension '{module_name}'.")
     save_sources_data(sources_data)
 
 

updater/data_extractor/__init__.py

@@ -1,11 +1,6 @@
-import importlib
-
-from updater.CONSTANTS import SOURCES_PATH
-
-
-def get_extension_id(module_name):
+def get_extension_id(path,module_name):
     # Construct the module name dynamically
-    file_path = f"{SOURCES_PATH}/{module_name}/{module_name}.py"
+    file_path = f"{path}/{module_name}/{module_name}.py"
 
     try:
         with open(file_path, 'r') as file:

updater/utils.py

@@ -3,14 +3,13 @@
 import os
 
 
-def run_tests(module_name):
+def run_tests(module_path,module_name):
     try:
-        tests_module = f"extensions/sources/{module_name}/tests"
-        tests = unittest.defaultTestLoader.discover(tests_module)
+        tests = unittest.defaultTestLoader.discover(f"{module_name}.tests", top_level_dir=module_path)
         result = unittest.TextTestRunner().run(tests)
         return result.wasSuccessful()
     except Exception as e:
-        print(f"Error running tests for module '{module_name}': {e}")
+        print(f"Error running tests for module '{module_path}': {e}")
 
 
 def zip_extension(module_id,module_name, source_folder, output_folder):
@@ -24,10 +23,14 @@ def zip_extension(module_id,module_name, source_folder, output_folder):
                     file_path = os.path.join(root, file)
 
                     # Skip __pycache__ directories
-                    if '__pycache__' not in file_path:
-                        arcname = os.path.relpath(file_path, source_folder)
-                        print("Adding", file_path, "as", arcname)
-                        zip_ref.write(file_path, arcname)
+                    if '__pycache__' in file_path:
+                        continue
+                    if 'tests' in file_path:
+                        continue
+
+                    arcname = os.path.relpath(file_path, source_folder)
+                    print("Adding", f"{file_path:55}", " as ", arcname)
+                    zip_ref.write(file_path, arcname)
 
         print(f"Extension '{module_name}' successfully zipped.")
     except Exception as e:

updater/versioning.py

@@ -5,7 +5,7 @@
 
 
 def bump_version(module_name, source_folder, level='patch'):
-    v = parse_version(module_name, source_folder)
+    v = parse_version(module_name=module_name, module_path= source_folder)
     level = "patch"  # Forcing it for now. Confused if it should be implemented this way or not.
     if v:
         if level == 'major':
@@ -23,8 +23,8 @@ def bump_version(module_name, source_folder, level='patch'):
         print(f"Version bumped to {v}")
 
 
-def parse_version(module_name, source_folder) -> Version:
-    version_file_path = os.path.join(source_folder, module_name, "__version__.py")
+def parse_version(module_path,module_name) -> Version:
+    version_file_path = os.path.join(module_path, "__version__.py")
 
     try:
         with open(version_file_path, 'r') as version_file: