Update config file specification format

.gitignore

@@ -1,4 +1,132 @@
-*.pyc
-.DS_Store
+# Byte-compiled / optimized / DLL files
 __pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
 .coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Visual Studio Code
+.vscode/

onair/config/default_config.ini

@@ -1,18 +1,22 @@
-# Required Section: DEFAULT section is the basic required elements for running OnAIR
-[DEFAULT]
-# Required Key: TelemetryDataFilePath is the path for TelemetryFile
-TelemetryDataFilePath = onair/data/raw_telemetry_data/data_physics_generation/Errors
-# Required Key: TelemetryDataFile is a file used by the selected parser
-# NOTE: TelemetryDataFile is required even when selected parser does not use it
+# Required Section: FILES section contains the paths of required files for running OnAIR
+[FILES]
+# Required Key: TelemetryFilePath is the directory of TelemetryFile
+TelemetryFilePath = onair/data/raw_telemetry_data/data_physics_generation/Errors
+# Required Key: TelemetryFile is the file read by the selected parser
+# NOTE: TelemetryFile is required even when selected parser does not use it
 TelemetryFile = 700_crash_to_earth_1.csv
-# Required Key: TelemetryMetadataFilePath is the path for TelemetryMetadataFilePath
-TelemetryMetadataFilePath = onair/data/telemetry_configs/
+# Required Key: MetaFilePath is the directory of MetaFile
+MetaFilePath = onair/data/telemetry_configs/
 # Required Key: MetaFile describes frame composition of data
 MetaFile = data_physics_generation_CONFIG.json
-# Required Key: ParserFileName is the name of the parser DataSource object to use
-ParserFileName = onair/data_handling/csv_parser.py
 
-# Plugins
+# Required Section: DATA_HANDLING section specifies which data source parser to use
+[DATA_HANDLING]
+# Required Key: DataSourceFile is the name of the parser DataSource object to use
+DataSourceFile = onair/data_handling/csv_parser.py
+
+# Required Section: PLUGINS section contains the plugins wanted for OnAIR to run
+[PLUGINS]
 # NOTE: even though keys are required, they may be set to empty dicts
 # Required Key: KnowledgeRepPluginDict(s) are used by the VehicleRep
 KnowledgeRepPluginDict = {'generic':'plugins/generic/__init__.py'}
@@ -23,8 +27,8 @@ PlannersPluginDict = {'generic':'plugins/generic/__init__.py'}
 # Required Key: ComplexPluginDict(s) are used by Agent for complex reasoning
 ComplexPluginDict = {'generic':'plugins/generic/__init__.py'}
 
-# Required Section: RUN_FLAGS are settable values to change running experience
-[RUN_FLAGS]
+# Optional Section: OPTIONS are settable values to change running experience
+[OPTIONS]
 # Optional Key: IO_Flag denotes whether or not to provide console output
-# IO_Flag defaults to false when not provided
-IO_Flag = true
+# default = false
+IO_Enabled = true

onair/config/redis_example.ini

@@ -1,14 +1,15 @@
-[DEFAULT]
-TelemetryDataFilePath = onair/data/raw_telemetry_data/data_physics_generation/Errors
+[FILES]
+TelemetryFilePath = onair/data/raw_telemetry_data/data_physics_generation/Errors
 TelemetryFile = 700_crash_to_earth_1.csv
-TelemetryMetadataFilePath = onair/data/telemetry_configs/
+MetaFilePath = onair/data/telemetry_configs/
 MetaFile = redis_example_CONFIG.json
 ParserFileName = onair/data_handling/redis_adapter.py
 
+[PLUGINS]
 KnowledgeRepPluginDict = {'knowledge':'plugins/generic/__init__.py'}
 LearnersPluginDict = {'learner':'plugins/generic/__init__.py'}
 PlannersPluginDict = {'planner':'plugins/generic/__init__.py'}
 ComplexPluginDict = {'complex':'plugins/generic/__init__.py'}
 
-[RUN_FLAGS]
-IO_Flag = true
+[OPTIONS]
+IO_Enabled = true

onair/config/reporter_config.ini

@@ -1,10 +1,13 @@
-[DEFAULT]
-TelemetryDataFilePath = onair/data/raw_telemetry_data/data_physics_generation/Errors
+[FILES]
+TelemetryFilePath = onair/data/raw_telemetry_data/data_physics_generation/Errors
 TelemetryFile = 700_crash_to_earth_1.csv
-TelemetryMetadataFilePath = onair/data/telemetry_configs/
+MetaFilePath = onair/data/telemetry_configs/
 MetaFile = data_physics_generation_CONFIG.json
-ParserFileName = onair/data_handling/csv_parser.py
 
+[DATA_HANDLING]
+DataSourceFile = onair/data_handling/csv_parser.py
+
+[PLUGINS]
 KnowledgeRepPluginDict = {'Knowledge Reporter 1':'plugins/reporter',
                           'Knowledge Reporter 2':'plugins/reporter'}
 LearnersPluginDict = {'Learners Reporter 1':'plugins/reporter',
@@ -14,7 +17,5 @@ PlannersPluginDict = {'Planner Reporter 1':'plugins/reporter',
 ComplexPluginDict = {'Complex Reporter 1':'plugins/reporter',
                      'Complex Reporter 2':'plugins/reporter'}
 
-[RUN_FLAGS]
-IO_Flag = true
-Dev_Flag = false
-Viz_Flag = false
+[OPTIONS]
+IO_Enabled = true

onair/src/run_scripts/execution_engine.py

@@ -16,31 +16,32 @@
 import importlib
 import ast
 import shutil
-from distutils.dir_util import copy_tree
+from shutil import copytree
 from time import gmtime, strftime
 
 from ..run_scripts.sim import Simulator
 
+
 class ExecutionEngine:
     def __init__(self, config_file='', run_name='', save_flag=False):
 
         # Init Housekeeping
         self.run_name = run_name
         self.config_filepath = config_file
 
-        # Init Flags
-        self.IO_Flag = False
+        # Init Options
+        self.IO_Enabled = False
 
         # Init Paths
         self.dataFilePath = ''
         self.telemetryFile = ''
-        self.fullTelemetryFileName = ''
+        self.fullTelemetryFile = ''
         self.metadataFilePath = ''
         self.metaFile = ''
-        self.fullMetaDataFileName = ''
+        self.fullMetaFile = ''
 
         # Init parsing/sim info
-        self.parser_file_name = ''
+        self.data_source_file = ''
         self.simDataSource = None
         self.sim = None
 
@@ -56,7 +57,8 @@ def __init__(self, config_file='', run_name='', save_flag=False):
         if config_file != '':
             self.init_save_paths()
             self.parse_configs(config_file)
-            self.parse_data(self.parser_file_name, self.fullTelemetryFileName, self.fullMetaDataFileName)
+            self.parse_data(self.data_source_file,
+                            self.fullTelemetryFile, self.fullMetaFile)
             self.setup_sim()
 
     def parse_configs(self, config_filepath):
@@ -66,49 +68,62 @@ def parse_configs(self, config_filepath):
             raise FileNotFoundError(f"Config file at '{config_filepath}' could not be read.")
 
         try:
-            ## Parse Required Data: Telementry Data & Configuration
-            self.dataFilePath = config['DEFAULT']['TelemetryDataFilePath']
-            self.telemetryFile = config['DEFAULT']['TelemetryFile'] # Vehicle telemetry data
-            self.fullTelemetryFileName = os.path.join(self.dataFilePath, self.telemetryFile)
-            self.metadataFilePath = config['DEFAULT']['TelemetryMetadataFilePath']
-            self.metaFile = config['DEFAULT']['MetaFile'] # Config for vehicle telemetry
-            self.fullMetaDataFileName = os.path.join(self.metadataFilePath, self.metaFile)
-
-            ## Parse Required Data: Names
-            self.parser_file_name = config['DEFAULT']['ParserFileName']
-
-            ## Parse Required Data: Plugins
-            self.knowledge_rep_plugin_dict = self.parse_plugins_dict(config['DEFAULT']['KnowledgeRepPluginDict'])
-            self.learners_plugin_dict = self.parse_plugins_dict(config['DEFAULT']['LearnersPluginDict'])
-            self.planners_plugin_dict = self.parse_plugins_dict(config['DEFAULT']['PlannersPluginDict'])
-            self.complex_plugin_dict = self.parse_plugins_dict(config['DEFAULT']['ComplexPluginDict'])
-
-            ## Parse Optional Data: Flags
-            ## 'RUN_FLAGS' must exist, but individual flags return False if missing
-            self.IO_Flag = config['RUN_FLAGS'].getboolean('IO_Flag')
+            # Parse Required Data: FILES
+            self.dataFilePath = config['FILES']['TelemetryFilePath']
+            # Vehicle telemetry data
+            self.telemetryFile = config['FILES']['TelemetryFile']
+            self.fullTelemetryFile = os.path.join(
+                self.dataFilePath, self.telemetryFile)
+            self.metadataFilePath = config['FILES']['MetaFilePath']
+            # Config for vehicle telemetry
+            self.metaFile = config['FILES']['MetaFile']
+            self.fullMetaFile = os.path.join(
+                self.metadataFilePath, self.metaFile)
+
+            # Parse Required Data: DATA_HANDLING
+            self.data_source_file = config['DATA_HANDLING']['DataSourceFile']
+
+            # Parse Required Data: PLUGINS
+            self.knowledge_rep_plugin_dict = self.parse_plugins_dict(
+                config['PLUGINS']['KnowledgeRepPluginDict'])
+            self.learners_plugin_dict = self.parse_plugins_dict(
+                config['PLUGINS']['LearnersPluginDict'])
+            self.planners_plugin_dict = self.parse_plugins_dict(
+                config['PLUGINS']['PlannersPluginDict'])
+            self.complex_plugin_dict = self.parse_plugins_dict(
+                config['PLUGINS']['ComplexPluginDict'])
+
+            # Parse Optional Data: OPTIONS
+            # 'OPTIONS' must exist, but individual options return False if missing
+            if config.has_section('OPTIONS'):
+                self.IO_Enabled = config['OPTIONS'].getboolean('IO_Enabled')
+            else:
+                self.IO_Enabled = False
 
         except KeyError as e:
             new_message = f"Config file: '{config_filepath}', missing key: {e.args[0]}"
             raise KeyError(new_message) from e
 
     def parse_plugins_dict(self, config_plugin_dict):
-        ## Parse Required Data: Plugin name to path dict
+        # Parse Required Data: Plugin name to path dict
         ast_plugin_dict = self.ast_parse_eval(config_plugin_dict)
         if isinstance(ast_plugin_dict.body, ast.Dict):
             temp_plugin_dict = ast.literal_eval(config_plugin_dict)
         else:
             raise ValueError(f"Plugin dict {config_plugin_dict} from {self.config_filepath} is invalid. It must be a dict.")
 
         for plugin_file in temp_plugin_dict.values():
-            if not(os.path.exists(plugin_file)):
+            if not (os.path.exists(plugin_file)):
                 raise FileNotFoundError(f"In config file '{self.config_filepath}' Plugin path '{plugin_file}' does not exist.")
         return temp_plugin_dict
 
     def parse_data(self, parser_file_name, data_file_name, metadata_file_name, subsystems_breakdown=False):
-        data_source_spec = importlib.util.spec_from_file_location('data_source', parser_file_name)
+        data_source_spec = importlib.util.spec_from_file_location(
+            'data_source', parser_file_name)
         data_source_module = importlib.util.module_from_spec(data_source_spec)
         data_source_spec.loader.exec_module(data_source_module)
-        self.simDataSource = data_source_module.DataSource(data_file_name, metadata_file_name, subsystems_breakdown)
+        self.simDataSource = data_source_module.DataSource(
+            data_file_name, metadata_file_name, subsystems_breakdown)
 
     def setup_sim(self):
         self.sim = Simulator(self.simDataSource,
@@ -118,7 +133,7 @@ def setup_sim(self):
                              self.complex_plugin_dict)
 
     def run_sim(self):
-        self.sim.run_sim(self.IO_Flag)
+        self.sim.run_sim(self.IO_Enabled)
         if self.save_flag:
             self.save_results(self.save_name)
 
@@ -149,9 +164,10 @@ def delete_save_paths(self):
 
     def save_results(self, save_name):
         complete_time = strftime("%H-%M-%S", gmtime())
-        save_path = os.environ['ONAIR_SAVE_PATH'] + 'saved/' + save_name + '_' + complete_time
+        save_path = os.environ['ONAIR_SAVE_PATH'] + \
+            'saved/' + save_name + '_' + complete_time
         os.makedirs(save_path, exist_ok=True)
-        copy_tree(os.environ['ONAIR_TMP_SAVE_PATH'], save_path)
+        copytree(os.environ['ONAIR_TMP_SAVE_PATH'], save_path)
 
     def set_run_param(self, name, val):
         setattr(self, name, val)