From 2588ebed00ce21f4abd8803d9f0f031d83564816 Mon Sep 17 00:00:00 2001
From: Kevin Dinkel <1225857+dinkelk@users.noreply.github.com>
Date: Mon, 12 Feb 2024 18:24:45 -0700
Subject: [PATCH 1/7] fixing model cache bug where creation of new submodel
 does not invalidate cache for the parent model

---
 gen/models/base.py                    | 31 ++++++++++++++++++++++++++-
 redo/database/model_cache_database.py | 10 +++++++++
 2 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/gen/models/base.py b/gen/models/base.py
index cc2c133d..f45928ea 100644
--- a/gen/models/base.py
+++ b/gen/models/base.py
@@ -105,7 +105,7 @@ def is_model_cached_this_session(filename):
             return False
 
         def is_cached_model_up_to_date(filename):
-            # See if the model is stored in the database  cache stored on disk:
+            # See if the model is stored in the database cache stored on disk:
             with model_cache_database() as db:
                 # Get the time when we last cached the model from this file:
                 cache_time_stamp = db.get_model_time_stamp(filename)
@@ -129,6 +129,27 @@ def mark_model_cached_this_session(filename):
             with model_cache_database(mode=DATABASE_MODE.READ_WRITE) as db:
                 db.mark_cached_model_up_to_date_for_session(filename)
 
+        def were_new_submodels_created(filename):
+            cached_submodels = None
+            with model_cache_database() as db:
+                cached_submodels = db.get_model_submodels(filename)
+
+            if cached_submodels:
+                _, _, model_name, _, _= redo_arg.split_model_filename(filename)
+                new_submodels = model_loader._get_model_file_paths(model_name)
+                # import sys
+                # sys.stderr.write(model_name + "\n")
+                # sys.stderr.write("cached_submodels\n")
+                # sys.stderr.write(str(cached_submodels) + "\n")
+                # sys.stderr.write("new_submodels\n")
+                # sys.stderr.write(str(new_submodels) + "\n")
+                # sys.stderr.write(str(cached_submodels == new_submodels) + "\n")
+                # sys.stderr.write("\n")
+                return not (cached_submodels == new_submodels)
+
+            # If this model does not have submodels, then return False
+            return False
+
         # If the model was cached this redo session, then we know it is safe to use
         # directly from cache without any additional checking. Dependencies do not
         # need to be checked, since this would have been done earlier in this session,
@@ -136,6 +157,13 @@ def mark_model_cached_this_session(filename):
         if is_model_cached_this_session(filename):
             return do_load_from_cache(filename)
 
+        # If this model has submodels, we need to make sure a new submodel
+        # as not been created. For example, if a name.events.yaml for name.component.yaml
+        # gets created on disk, this means the cached entry for name.component.yaml is
+        # invalid. This is true for the parent model of any newly created submodel.
+        if were_new_submodels_created(filename):
+            return None
+
         # If the model was written from a previous session, then we need to check its
         # write timestamp against the file timestamp to determine if the cached entry is
         # still valid:
@@ -231,6 +259,7 @@ def __init__(self, filename, schema):
             self.schema = os.path.splitext(os.path.basename(self.full_schema))[0]
             self.time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
             self.dependencies = []
+            self.submodels = None
 
             # Save off some local functions:
             self.zip = zip
diff --git a/redo/database/model_cache_database.py b/redo/database/model_cache_database.py
index 0f52b1ae..c85cbb60 100644
--- a/redo/database/model_cache_database.py
+++ b/redo/database/model_cache_database.py
@@ -2,6 +2,7 @@
 from database.database import DATABASE_MODE
 from time import time as curr_time
 from os import environ, sep
+from util import model_loader
 
 # The purpose of the model cache is to save off YAML file model load
 # data structures after they have been read from a file, validated, and
@@ -27,6 +28,13 @@ def store_model(self, model_file, model_object):
         self.store(model_file, model_object)
         self.store(model_file + "_time@st@@", curr_time())
         self.store(model_file + "_sess@id@@", environ["ADAMANT_SESSION_ID"])
+        if model_object.submodels is not None:
+            submodel_paths = model_loader._get_model_file_paths(model_object.model_name)
+            self.store(model_file + "_submod@paths@@", submodel_paths)
+            # import sys
+            # sys.stderr.write("SAVING\n")
+            # sys.stderr.write(str(model_object.model_name) + "\n")
+            # sys.stderr.write(str(submodel_paths) + "\n")
 
     # Update the session ID for a model. We use this to indicate that a model has been
     # fully validated (ie. not outdated) for this redo session.
@@ -43,6 +51,8 @@ def get_model_session_id(self, model_file):
     def get_model_time_stamp(self, model_file):
         return self.try_fetch(model_file + "_time@st@@")
 
+    def get_model_submodels(self, model_file):
+        return self.try_fetch(model_file + "_submod@paths@@")
 
 # Create an empty model cache database file, if one does
 # not already exist:

From 4ac42e2852eced65dad629fd0043d178c56bc1d7 Mon Sep 17 00:00:00 2001
From: Kevin Dinkel <1225857+dinkelk@users.noreply.github.com>
Date: Mon, 12 Feb 2024 19:21:30 -0700
Subject: [PATCH 2/7] fixing style

---
 gen/models/base.py                    | 2 +-
 redo/database/model_cache_database.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/gen/models/base.py b/gen/models/base.py
index f45928ea..9cc188e4 100644
--- a/gen/models/base.py
+++ b/gen/models/base.py
@@ -135,7 +135,7 @@ def were_new_submodels_created(filename):
                 cached_submodels = db.get_model_submodels(filename)
 
             if cached_submodels:
-                _, _, model_name, _, _= redo_arg.split_model_filename(filename)
+                _, _, model_name, _, _ = redo_arg.split_model_filename(filename)
                 new_submodels = model_loader._get_model_file_paths(model_name)
                 # import sys
                 # sys.stderr.write(model_name + "\n")
diff --git a/redo/database/model_cache_database.py b/redo/database/model_cache_database.py
index c85cbb60..6c322294 100644
--- a/redo/database/model_cache_database.py
+++ b/redo/database/model_cache_database.py
@@ -54,6 +54,7 @@ def get_model_time_stamp(self, model_file):
     def get_model_submodels(self, model_file):
         return self.try_fetch(model_file + "_submod@paths@@")
 
+
 # Create an empty model cache database file, if one does
 # not already exist:
 def touch_model_cache_database():

From f95b7067369c7339fef21825bde0dcd6beed58f5 Mon Sep 17 00:00:00 2001
From: Kevin Dinkel <1225857+dinkelk@users.noreply.github.com>
Date: Tue, 13 Feb 2024 09:27:39 -0700
Subject: [PATCH 3/7] adding ignore_cache feature in model base class, allowing
 us to ignore the model cache for generators that need to look for component
 unit tests

---
 gen/generators/basic.py     | 10 +++++++++-
 gen/generators/component.py | 14 ++++++++++++--
 gen/models/base.py          | 30 +++++++++++++++++++++++++++---
 3 files changed, 48 insertions(+), 6 deletions(-)

diff --git a/gen/generators/basic.py b/gen/generators/basic.py
index 751aef20..fd21969d 100644
--- a/gen/generators/basic.py
+++ b/gen/generators/basic.py
@@ -14,6 +14,7 @@ def create_basic_generator(
     template_dir=None,
     has_dependencies=True,
     camel_case_filename=False,
+    ignore_cache=False
 ):
     #
     # Dynamically generate a class of the form:
@@ -31,6 +32,7 @@ def init_func(self):
             template_dir=template_dir,
             has_dependencies=has_dependencies,
             camel_case_filename=camel_case_filename,
+            ignore_cache=ignore_cache
         )
 
     cls_name = (
@@ -62,6 +64,7 @@ def add_basic_generator_to_module(
     template_dir=None,
     has_dependencies=True,
     camel_case_filename=False,
+    ignore_cache=False
 ):
     module_name = module["__name__"]
     cls = create_basic_generator(
@@ -71,6 +74,7 @@ def add_basic_generator_to_module(
         template_dir=template_dir,
         has_dependencies=has_dependencies,
         camel_case_filename=camel_case_filename,
+        ignore_cache=ignore_cache
     )
     add_class_to_module(cls, module)
 
@@ -83,6 +87,7 @@ def add_basic_generators_to_module(
     template_dir=None,
     has_dependencies=True,
     camel_case_filename=False,
+    ignore_cache=False
 ):
     for template in templates:
         add_basic_generator_to_module(
@@ -92,6 +97,7 @@ def add_basic_generators_to_module(
             template_dir=template_dir,
             has_dependencies=has_dependencies,
             camel_case_filename=camel_case_filename,
+            ignore_cache=ignore_cache
         )
 
 
@@ -138,6 +144,7 @@ def __init__(
         template_dir=None,
         has_dependencies=True,
         camel_case_filename=False,
+        ignore_cache=False
     ):
         # Set the generator model class and type name:
         self.model_cls = model_class
@@ -145,6 +152,7 @@ def __init__(
         self.model_type = self.model_cls.__name__.lower()
         self.has_dependencies = has_dependencies
         self.camel_case_filename = camel_case_filename
+        self.ignore_cache = ignore_cache
 
         # If a full template filename is not given, then form one using
         # the model_type:
@@ -162,7 +170,7 @@ def __init__(
     # Cache model object for speed:
     def model_object(self, input_filename):
         if input_filename not in self._model_obj:
-            self._model_obj[input_filename] = self.model_cls(input_filename)
+            self._model_obj[input_filename] = self.model_cls(input_filename, ignore_cache=self.ignore_cache)
         return self._model_obj[input_filename]
 
     def input_file_regex(self):
diff --git a/gen/generators/component.py b/gen/generators/component.py
index 6de437cb..7d46c03b 100644
--- a/gen/generators/component.py
+++ b/gen/generators/component.py
@@ -1,4 +1,4 @@
-from generators.basic import add_basic_generators_to_module
+from generators.basic import add_basic_generators_to_module, add_basic_generator_to_module
 from models import component
 
 # This module contains generators that produce products related
@@ -26,7 +26,6 @@
     "component/name_enums.tex",
     "component/name_interrupts.tex",
     "component/name_requirements.tex",
-    "component/name_unit_test.tex",
 ]
 
 component_templates_no_deps = ["component/name.dot", "component/name.tex"]
@@ -41,3 +40,14 @@
     module=this_module,
     has_dependencies=False,
 )
+# We cannot use the cached component model for anything to do with unit
+# tests, since unit tests are not in the path, and need to be found by
+# searching the filesystem each time. By doing this, we will always detect
+# any new unit tests models that might appear on the filesystem for a
+# component.
+add_basic_generator_to_module(
+    component.component,
+    "component/name_unit_test.tex",
+    module=this_module,
+    ignore_cache=True
+)
diff --git a/gen/models/base.py b/gen/models/base.py
index 9cc188e4..c56051b4 100644
--- a/gen/models/base.py
+++ b/gen/models/base.py
@@ -87,10 +87,25 @@ def render(self, template_file, template_path=None):
             self.__dict__, template_file, template_path, extensions=["jinja2.ext.do"]
         )
 
+# We use this meta class to intercept calls to create a new "base" object. It
+# looks for arguments that should only be passed to __new__, currently this only
+# includes an option to ignore the model cache on load, and filters those arguments
+# before passing the remaining arguments to __init__. This differs slightly from the
+# default python behavior, and allows us to not propogate arguments like "ignore_cache"
+# through the __init__ for all model objects that inherit from base.
+#
+# This meta class also extends the abc.ABCMeta class, so we get the features from that
+# as well.
+class base_meta(abc.ABCMeta, type):
+    def __call__(cls, *args, **kwargs):
+        ignore_cache = kwargs.pop("ignore_cache", False)
+        instance = cls.__new__(cls, *args, **kwargs, ignore_cache=ignore_cache)
+        cls.__init__(instance, *args, **kwargs)
+        return instance
 
 # The model base class. All python models that load yaml files should
 # inherit from this class.
-class base(renderable_object, metaclass=abc.ABCMeta):
+class base(renderable_object, metaclass=base_meta):
     #################################################
     # Model Caching:
     #################################################
@@ -204,9 +219,18 @@ def save_to_cache(self):
     def __new__(cls, filename, *args, **kwargs):
         # Try to load the model from the cache:
         if filename:
+            # See if we are requested to ignore the cache for this model
+            # load:
+            ignore_cache = kwargs.get("ignore_cache")
             full_filename = os.path.abspath(filename)
-            model = cls.load_from_cache(cls, full_filename)
-            if model:
+            model = None
+            if not ignore_cache:
+                model = cls.load_from_cache(cls, full_filename)
+
+            # If we are not ignoring the cache, and the model was found in
+            # the cache, then use it, otherwise create the model from scratch
+            # by reading in the file.
+            if not ignore_cache and model:
                 # Create from cached model:
                 self = model
                 self.from_cache = True

From 5d73f37c38f51a9e90b3e2a233fceed7cacedda3 Mon Sep 17 00:00:00 2001
From: Kevin Dinkel <1225857+dinkelk@users.noreply.github.com>
Date: Tue, 13 Feb 2024 13:46:17 -0700
Subject: [PATCH 4/7] fixing bug in tester generators where wrong model class
 was being used for template rendering

---
 gen/generators/tests.py |  6 +++++-
 gen/models/base.py      | 10 +---------
 gen/models/tests.py     |  4 ++--
 3 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/gen/generators/tests.py b/gen/generators/tests.py
index 084f8c7a..a7dc9e2b 100644
--- a/gen/generators/tests.py
+++ b/gen/generators/tests.py
@@ -1,6 +1,6 @@
 from base_classes.generator_base import generator_base
 from generators.basic import basic_generator
-from models import tests
+from models import tests, component
 from util import error
 from util import model_loader
 import os.path
@@ -134,6 +134,10 @@ def generate(self, input_filename):
             component_name, model_types="component"
         )
         if component_model_path:
+            # Override the model class. We are going to use the component model
+            # to generate this output instead of the test model, since it contains
+            # all the info needed.
+            self.model_cls = component.component
             basic_generator.generate(self, component_model_path)
 
     # Depend on the component model and any commands, data products, events models that the component uses:
diff --git a/gen/models/base.py b/gen/models/base.py
index c56051b4..7a6d5c83 100644
--- a/gen/models/base.py
+++ b/gen/models/base.py
@@ -96,7 +96,7 @@ def render(self, template_file, template_path=None):
 #
 # This meta class also extends the abc.ABCMeta class, so we get the features from that
 # as well.
-class base_meta(abc.ABCMeta, type):
+class base_meta(abc.ABCMeta):
     def __call__(cls, *args, **kwargs):
         ignore_cache = kwargs.pop("ignore_cache", False)
         instance = cls.__new__(cls, *args, **kwargs, ignore_cache=ignore_cache)
@@ -152,14 +152,6 @@ def were_new_submodels_created(filename):
             if cached_submodels:
                 _, _, model_name, _, _ = redo_arg.split_model_filename(filename)
                 new_submodels = model_loader._get_model_file_paths(model_name)
-                # import sys
-                # sys.stderr.write(model_name + "\n")
-                # sys.stderr.write("cached_submodels\n")
-                # sys.stderr.write(str(cached_submodels) + "\n")
-                # sys.stderr.write("new_submodels\n")
-                # sys.stderr.write(str(new_submodels) + "\n")
-                # sys.stderr.write(str(cached_submodels == new_submodels) + "\n")
-                # sys.stderr.write("\n")
                 return not (cached_submodels == new_submodels)
 
             # If this model does not have submodels, then return False
diff --git a/gen/models/tests.py b/gen/models/tests.py
index 5d0078dd..46cff652 100644
--- a/gen/models/tests.py
+++ b/gen/models/tests.py
@@ -47,8 +47,8 @@ def load(self):
             self.description = self.data["description"]
         if "preamble" in self.data:
             self.preamble = self.data["preamble"]
-        tests = self.data["tests"]
-        for t in tests:
+        the_tests = self.data["tests"]
+        for t in the_tests:
             name = ada.formatType(t["name"])
             description = None
             if "description" in t:

From 1542923f06dd8feb7d7d70abd6e01913d51982e8 Mon Sep 17 00:00:00 2001
From: Kevin Dinkel <1225857+dinkelk@users.noreply.github.com>
Date: Tue, 13 Feb 2024 14:02:04 -0700
Subject: [PATCH 5/7] removing unneeded comments

---
 redo/database/model_cache_database.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/redo/database/model_cache_database.py b/redo/database/model_cache_database.py
index 6c322294..8607c1ea 100644
--- a/redo/database/model_cache_database.py
+++ b/redo/database/model_cache_database.py
@@ -31,10 +31,6 @@ def store_model(self, model_file, model_object):
         if model_object.submodels is not None:
             submodel_paths = model_loader._get_model_file_paths(model_object.model_name)
             self.store(model_file + "_submod@paths@@", submodel_paths)
-            # import sys
-            # sys.stderr.write("SAVING\n")
-            # sys.stderr.write(str(model_object.model_name) + "\n")
-            # sys.stderr.write(str(submodel_paths) + "\n")
 
     # Update the session ID for a model. We use this to indicate that a model has been
     # fully validated (ie. not outdated) for this redo session.

From 0361f3c7405a337d5f35638745ef19409e19d9a4 Mon Sep 17 00:00:00 2001
From: Kevin Dinkel <1225857+dinkelk@users.noreply.github.com>
Date: Tue, 13 Feb 2024 14:56:09 -0700
Subject: [PATCH 6/7] fixing style

---
 gen/models/base.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/gen/models/base.py b/gen/models/base.py
index 7a6d5c83..85f6073f 100644
--- a/gen/models/base.py
+++ b/gen/models/base.py
@@ -87,11 +87,12 @@ def render(self, template_file, template_path=None):
             self.__dict__, template_file, template_path, extensions=["jinja2.ext.do"]
         )
 
+
 # We use this meta class to intercept calls to create a new "base" object. It
 # looks for arguments that should only be passed to __new__, currently this only
 # includes an option to ignore the model cache on load, and filters those arguments
 # before passing the remaining arguments to __init__. This differs slightly from the
-# default python behavior, and allows us to not propogate arguments like "ignore_cache"
+# default python behavior, and allows us to not propagate arguments like "ignore_cache"
 # through the __init__ for all model objects that inherit from base.
 #
 # This meta class also extends the abc.ABCMeta class, so we get the features from that
@@ -103,6 +104,7 @@ def __call__(cls, *args, **kwargs):
         cls.__init__(instance, *args, **kwargs)
         return instance
 
+
 # The model base class. All python models that load yaml files should
 # inherit from this class.
 class base(renderable_object, metaclass=base_meta):

From d9f0527829de9cfeb70da4eee3a90229abe01080 Mon Sep 17 00:00:00 2001
From: Kevin Dinkel <1225857+dinkelk@users.noreply.github.com>
Date: Tue, 13 Feb 2024 15:09:02 -0700
Subject: [PATCH 7/7] removed some comments

---
 gen/models/base.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/gen/models/base.py b/gen/models/base.py
index 85f6073f..dca0c7ac 100644
--- a/gen/models/base.py
+++ b/gen/models/base.py
@@ -231,7 +231,6 @@ def __new__(cls, filename, *args, **kwargs):
                 self.filename = os.path.basename(filename)
                 self.full_filename = full_filename
                 self.do_save_to_cache = True
-                # import sys
                 # sys.stderr.write("lcache " + self.filename + "\n")
             else:
                 # Create from scratch:
@@ -240,7 +239,6 @@ def __new__(cls, filename, *args, **kwargs):
                 self.filename = os.path.basename(filename)
                 self.full_filename = full_filename
                 self.do_save_to_cache = True
-                # import sys
                 # sys.stderr.write("lfile " + self.filename + "\n")
         else:
             # Create from scratch. This is usually only called when
@@ -372,13 +370,9 @@ def _loadYaml(yaml_text):
             import warnings
 
             warnings.simplefilter("ignore", yaml.error.UnsafeLoaderWarning)
-            # with open(self.full_filename, 'r') as stream:
             try:
                 yml = yaml.YAML(typ='rt')
                 return yml.load(yaml_text)
-                # import sys
-                # sys.stderr.write(str(self.data) + "\n")
-                # sys.stderr.write(str(type(self.data)) + "\n")
             except yaml.YAMLError as exc:
                 raise ModelException(str(exc))
 
@@ -405,11 +399,9 @@ def _loadYaml(yaml_text):
             tokens = list(jinja_lexer.tokenize(contents))
             has_jinja_directives = False
             for token in tokens:
-                # sys.stderr.write(token.type + "\n")
                 if token.type != "data":
                     has_jinja_directives = True
                     break
-            # sys.stderr.write(self.full_filename + " has jinja directives? " + str(has_jinja_directives) + "\n")
 
             # If there are jinja directives in the file, then lets render them using
             # the global project configuration YAML.