Validate observations before handling them

import_schedules.py

@@ -14,14 +14,86 @@
 
 
 class Observation:
-    def __init__(self, site, observation):
-        self.site = site
+    def __init__(self, observation):
         self.observation = observation
+        print("Validation of observation results: ", self.validate_observation_format(observation))
 
     def create_ptr_resources(self):
         self.create_calendar()
         self.create_project()
 
+    @staticmethod
+    def validate_observation_format(observation):
+        required_keys = {
+            "site": str,
+            "start": str,
+            "end": str,
+            "submitter": str,
+            "modified": str,
+            "created": str,
+            "name": str,
+            "observation_type": str,
+            "request": dict,
+        }
+
+        def validate_configuration(config):
+            required_configuration_keys = {
+                "constraints": dict,
+                "instrument_configs": list,
+                "target": dict,
+                "type": str
+            }
+            config_keys_present = all(key in config and isinstance(config[key], types)
+                    for key, types in required_configuration_keys.items())
+            targets_ok = validate_target(config.get("target", {}))
+            constraints_ok = all(key in config.get("constraints", {}) for key in ["max_airmass", "max_lunar_phase", "min_lunar_distance"])
+            instrument_configs_ok = all(validate_inst_config(inst_config) for inst_config in config.get("instrument_configs", [{}]))
+            if not config_keys_present:
+                return False, "Missing required keys in configuration"
+            if not targets_ok:
+                return False, "Targets failed validation"
+            if not constraints_ok:
+                return False, "Constraints failed validation"
+            if not instrument_configs_ok:
+                return False, "Instrument configs failed validation"
+            return True, "Validation successful"
+
+        def validate_target(target):
+            required_target_keys = {
+                "ra": (int, float),
+                "dec": (int, float)
+            }
+            return all(key in target and isinstance(target[key], types)
+                    for key, types in required_target_keys.items())
+
+        def validate_inst_config(instrument_config):
+            required_keys = {
+                "exposure_count": int,
+                "exposure_time": (int, float),
+                "mode": str,
+                "extra_params": dict,
+                "optical_elements": dict
+            }
+            keys_are_present = all(key in instrument_config and isinstance(instrument_config[key], types)
+                    for key, types in required_keys.items())
+            filter_present = "filter" in instrument_config["optical_elements"] and isinstance(instrument_config["optical_elements"]["filter"], str)
+            offsets_present = all(key in instrument_config["extra_params"] for key in ["offset_dec", "offset_ra", "rotator_angle"])
+            return keys_are_present and filter_present and offsets_present
+
+        # Validate top-level keys
+        missing_keys = [key for key in required_keys if key not in observation]
+        if missing_keys:
+            return False, f"Missing keys: {missing_keys}"
+
+        # Validate configurations
+        configurations = observation["request"].get("configurations", [])
+        for index, conf in enumerate(configurations):
+            config_passed, message = validate_configuration(conf)
+            if not isinstance(conf, dict) or not config_passed:
+                return False, f"Configuration number {index} failed validation: {message}"
+
+        return True, "Validation successful"
+
     def _translate_to_calendar(self):
         # we need the project_id, so do the project translation if it doesn't exist
         if not hasattr(self, 'project'):
@@ -272,7 +344,7 @@ def create_latest_schedule(site):
     sched = get_schedule(site)
     print(f"Number of observations to schedule: {len(sched)}")
     for obs in sched:
-        observation = Observation(site, obs) 
+        observation = Observation(obs)
         observation.create_ptr_resources()
 
 
@@ -282,5 +354,6 @@ def import_all_schedules(event={}, context={}):
     for site in sites:
         clear_old_schedule(site)
         create_latest_schedule(site)
+    # When invoked using the http endpoint, provide a valid http response
     if "httpMethod" in event:
         return create_response(200, "Import schedules routine finished")

requirements.txt

@@ -1,14 +1,33 @@
 asn1crypto==1.5.1
+attrs==24.2.0
 boto3==1.24.18
+botocore==1.27.96
 certifi==2022.6.15
 cffi==1.15.0
 chardet==5.0.0
+charset-normalizer==2.0.12
 cryptography==3.3.1
 idna==3.3
+iniconfig==2.0.0
+Jinja2==3.1.4
+jmespath==1.0.1
+MarkupSafe==3.0.2
 moto==3.1.15
+packaging==24.2
+pluggy==1.5.0
+py==1.11.0
 pycparser==2.21
 PyJWT==2.4.0
 pytest==7.1.2
-requests==2.28.0
+python-dateutil==2.9.0.post0
+pytz==2024.2
+PyYAML==6.0.2
+requests==2.32.3
+responses==0.23.1
+s3transfer==0.6.2
 six==1.16.0
+tomli==2.2.1
+types-PyYAML==6.0.12.20240917
 urllib3==1.26.9
+Werkzeug==3.1.3
+xmltodict==0.14.2

utils.py

@@ -8,7 +8,7 @@
 from botocore.exceptions import ClientError
 
 dynamodb = boto3.resource('dynamodb')
-calendar_table_name = os.environ['DYNAMODB_CALENDAR']
+calendar_table_name = os.environ.get('DYNAMODB_CALENDAR', 'dummy-name-for-testing')
 calendar_table = dynamodb.Table(calendar_table_name)
 
 
@@ -43,7 +43,7 @@ def default(self, o):
 
 def get_utc_iso_time():
     """Returns formatted UTC datetime string of current time."""
-    return datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
+    return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
 
 def create_calendar_event(event):