From 40066e479c191295c66240b55bd78aecedb1b80d Mon Sep 17 00:00:00 2001 From: Mark Murnane Date: Mon, 15 Apr 2024 00:22:21 -0400 Subject: [PATCH 1/6] Load config from environment --- development-defaults.ini | 10 -- sideboard/config.py | 216 +++++++++----------------- sideboard/configspec.ini | 18 ++- sideboard/internal/logging.py | 51 +++--- sideboard/tests/test_configuration.py | 201 ------------------------ test-defaults.ini | 30 ---- 6 files changed, 113 insertions(+), 413 deletions(-) delete mode 100644 development-defaults.ini delete mode 100644 sideboard/tests/test_configuration.py delete mode 100644 test-defaults.ini diff --git a/development-defaults.ini b/development-defaults.ini deleted file mode 100644 index eb39f37..0000000 --- a/development-defaults.ini +++ /dev/null @@ -1,10 +0,0 @@ -debug = False -ws.auth_required = False - -[cherrypy] -server.socket_host = "0.0.0.0" -engine.autoreload.on = True -tools.cpstats.on = False - -[loggers] -root = "DEBUG" diff --git a/sideboard/config.py b/sideboard/config.py index d6a27b2..1b84896 100755 --- a/sideboard/config.py +++ b/sideboard/config.py @@ -1,9 +1,9 @@ from __future__ import unicode_literals +import pathlib +import json import os -import re from os import unlink -from collections.abc import Sized, Iterable, Mapping from copy import deepcopy from tempfile import NamedTemporaryFile @@ -11,82 +11,10 @@ from validate import Validator -def uniquify(xs): - """ - Returns an order-preserved copy of `xs` with duplicate items removed. - - >>> uniquify(['a', 'z', 'a', 'b', 'a', 'y', 'a', 'c', 'a', 'x']) - ['a', 'z', 'b', 'y', 'c', 'x'] - - """ - is_listy = isinstance(xs, Sized) \ - and isinstance(xs, Iterable) \ - and not isinstance(xs, (Mapping, type(b''), type(''))) - assert is_listy, 'uniquify requires a listy argument' - - seen = set() - return [x for x in xs if x not in seen and not seen.add(x)] - - class ConfigurationError(RuntimeError): pass -def get_config_overrides(): - """ - Returns a list of config file paths used to override the default config. - - The SIDEBOARD_CONFIG_OVERRIDES environment variable may be set to a - semicolon separated list of absolute and/or relative paths. If the - SIDEBOARD_CONFIG_OVERRIDES is set, this function returns a list of its - contents, split on semicolons:: - - # SIDEBOARD_CONFIG_OVERRIDES='/absolute/config.ini;relative/config.ini' - return ['/absolute/config.ini', 'relative/config.ini'] - - If any of the paths listed in SIDEBOARD_CONFIG_OVERRIDES ends with the - suffix "-defaults." then a similarly named path - "." will also be included:: - - # SIDEBOARD_CONFIG_OVERRIDES='test-defaults.ini' - return ['test-defaults.ini', 'test.ini'] - - If the SIDEBOARD_CONFIG_OVERRIDES environment variable is NOT set, this - function returns a list with two relative paths:: - - return ['development-defaults.ini', 'development.ini'] - """ - config_overrides = os.environ.get( - 'SIDEBOARD_CONFIG_OVERRIDES', - 'development-defaults.ini') - - defaults_re = re.compile(r'(.+)-defaults(\.\w+)$') - config_paths = [] - for config_path in uniquify([s.strip() for s in config_overrides.split(';')]): - config_paths.append(config_path) - m = defaults_re.match(config_path) - if m: - config_paths.append(m.group(1) + m.group(2)) - - return config_paths - - -def get_config_root(): - """ - Returns the config root for the system, defaults to '/etc/sideboard'. - - If the SIDEBOARD_CONFIG_ROOT environment variable is set, its contents - will be returned instead. - """ - default_root = '/etc/sideboard' - config_root = os.environ.get('SIDEBOARD_CONFIG_ROOT', default_root) - if config_root != default_root and not os.path.isdir(config_root): - raise AssertionError('cannot find {!r} directory'.format(config_root)) - elif os.path.isdir(config_root) and not os.access(config_root, os.R_OK): - raise AssertionError('{!r} directory is not readable'.format(config_root)) - return config_root - - def get_module_and_root_dirs(requesting_file_path, is_plugin): """ Returns the "module_root" and "root" directories for the given file path. @@ -114,63 +42,28 @@ def get_module_and_root_dirs(requesting_file_path, is_plugin): Sideboard itself is making the request. Returns: - tuple(str): The "module_root" and "root" directories for the + tuple(Path, Path, str): The "module_root" and "root" directories, and plugin name for the given module. """ - module_dir = os.path.dirname(os.path.abspath(requesting_file_path)) + module_dir = pathlib.Path(requesting_file_path).parents[0] if is_plugin: from sideboard.lib import config - plugin_name = os.path.basename(module_dir) - root_dir = os.path.join(config['plugins_dir'], plugin_name) - if '_' in plugin_name and not os.path.exists(root_dir): - root_dir = os.path.join(config['plugins_dir'], plugin_name.replace('_', '-')) + plugin_name = module_dir.name + root_dir = pathlib.Path(config['plugins_dir']) / plugin_name + if '_' in plugin_name and not root_dir.exists(): + root_dir = pathlib.Path(config['plugins_dir']) / plugin_name.replace('_', '-') else: - root_dir = os.path.realpath(os.path.join(module_dir, '..')) - return module_dir, root_dir + root_dir = module_dir.parents[0] + plugin_name = "sideboard" + return module_dir, root_dir, plugin_name def get_config_files(requesting_file_path, is_plugin): """ Returns a list of absolute paths to config files for the given file path. - - When the returned config files are parsed by ConfigObj each subsequent - file will override values in earlier files. - - If `is_plugin` is `True` the first of the returned files is: - - * /etc/sideboard/plugins.d/.cfg, which is the config file we - expect in production - - - If `is_plugin` is `False` the first two returned files are: - - * /etc/sideboard/sideboard-core.cfg, which is the sideboard core config - file we expect in production - - * /etc/sideboard/sideboard-server.cfg, which is the sideboard server config - file we expect in production - - - The rest of the files returned are as follows, though we wouldn't - necessarily expect these to exist on a production install (these are - controlled by SIDEBOARD_CONFIG_OVERRIDES): - - * /development-defaults.ini, which can be checked into source - control and include whatever we want to be present in a development - environment. - - * /development.ini, which shouldn't be checked into source - control, allowing a developer to include local settings not shared with - others. - - - When developing on a machine with an installed production config file, we - might want to ignore the "real" config file and limit ourselves to only the - development files. This behavior is turned on by setting the environment - variable SIDEBOARD_MODULE_TESTING to any value. (This environment variable - is also used elsewhere to turn off automatically loading all plugins in - order to facilitate testing modules which rely on Sideboard but which are - not themselves Sideboard plugins.) + + If the file is in a plugin we check the environment variable + _CONFIG_FILES and return any paths from there, seperated by ; Args: requesting_file_path (str): The Python __file__ of the module @@ -182,24 +75,53 @@ def get_config_files(requesting_file_path, is_plugin): Returns: list(str): List of absolute paths to config files for the given module. """ - config_root = get_config_root() - module_dir, root_dir = get_module_and_root_dirs(requesting_file_path, is_plugin) - module_name = os.path.basename(module_dir) - - if 'SIDEBOARD_MODULE_TESTING' in os.environ: - base_configs = [] - elif is_plugin: - base_configs = [os.path.join(config_root, 'plugins.d', '{}.cfg'.format(module_name.replace('_', '-')))] - else: - assert module_name == 'sideboard', 'Unexpected module name {!r} requesting "non-plugin" configuration files'.format(module_name) - base_configs = [ - os.path.join(config_root, 'sideboard-core.cfg'), - os.path.join(config_root, 'sideboard-server.cfg') - ] - - config_overrides = [os.path.join(root_dir, config_path) for config_path in get_config_overrides()] - return base_configs + config_overrides - + module_dir, root_dir, plugin_name = get_module_and_root_dirs(requesting_file_path, is_plugin) + config_files_str = os.environ.get(f"{plugin_name.upper()}_CONFIG_FILES", "") + absolute_config_files = [] + if config_files_str: + config_files = [pathlib.Path(x) for x in config_files_str.split(";")] + for path in config_files: + if path.is_absolute(): + if not path.exists(): + raise ValueError(f"Config file {path} specified in {plugin_name.upper()}_CONFIG_FILES does not exist!") + absolute_config_files.append(path) + else: + if not (root_dir / path).exists(): + raise ValueError(f"Config file {root_dir / path} specified in {plugin_name.upper()}_CONFIG_FILES does not exist!") + absolute_config_files.append(root_dir / path) + return absolute_config_files + +def load_section_from_environment(path, section): + """ + Looks for configuration in environment variables. + + Args: + path (str): The prefix of the current config section. For example, + sideboard.ini: + [cherrypy] + server.thread_pool: 10 + would translate to sideboard_cherrypy_server.thread_pool + section (configobj.ConfigObj): The section of the configspec to search + for the current path in. + """ + config = {} + for setting in section: + if setting == "__many__": + prefix = f"{path}_" + for envvar in os.environ: + if envvar.startswith(prefix): + config[envvar.split(prefix, 1)[1]] = os.environ[envvar] + else: + if isinstance(section[setting], configobj.Section): + child_path = f"{path}_{setting}" + child = load_section_from_environment(child_path, section[setting]) + if child: + config[setting] = child + else: + name = f"{path}_{setting}" + if name in os.environ: + config[setting] = os.environ.get(name) + return config def parse_config(requesting_file_path, is_plugin=True): """ @@ -225,18 +147,26 @@ def parse_config(requesting_file_path, is_plugin=True): Returns: ConfigObj: The resulting configuration object. """ - module_dir, root_dir = get_module_and_root_dirs(requesting_file_path, is_plugin) + module_dir, root_dir, plugin_name = get_module_and_root_dirs(requesting_file_path, is_plugin) - specfile = os.path.join(module_dir, 'configspec.ini') - spec = configobj.ConfigObj(specfile, interpolation=False, list_values=False, encoding='utf-8', _inspec=True) + specfile = module_dir / 'configspec.ini' + spec = configobj.ConfigObj(str(specfile), interpolation=False, list_values=False, encoding='utf-8', _inspec=True) # to allow more/better interpolations root_conf = ['root = "{}"\n'.format(root_dir), 'module_root = "{}"\n'.format(module_dir)] temp_config = configobj.ConfigObj(root_conf, interpolation=False, encoding='utf-8') + environment_config = load_section_from_environment(plugin_name, spec) + print(f"Environment config for {plugin_name}") + print(json.dumps(environment_config, indent=2, sort_keys=True)) + temp_config.merge(configobj.ConfigObj(environment_config, encoding='utf-8', interpolation=False)) + for config_path in get_config_files(requesting_file_path, is_plugin): # this gracefully handles nonexistent files - temp_config.merge(configobj.ConfigObj(config_path, encoding='utf-8', interpolation=False)) + file_config = configobj.ConfigObj(str(config_path), encoding='utf-8', interpolation=False) + print(f"File config for {plugin_name} from {config_path}") + print(json.dumps(file_config, indent=2, sort_keys=True)) + temp_config.merge(file_config) # combining the merge files to one file helps configspecs with interpolation with NamedTemporaryFile(delete=False) as config_outfile: diff --git a/sideboard/configspec.ini b/sideboard/configspec.ini index 87b9a6d..44f3893 100644 --- a/sideboard/configspec.ini +++ b/sideboard/configspec.ini @@ -118,15 +118,27 @@ profiling.strip_dirs = boolean(default=False) server.socket_host = string(default="127.0.0.1") server.socket_port = integer(default=8282) +server.thread_pool = integer(default=10) tools.reset_threadlocal.on = boolean(default=True) tools.sessions.on = boolean(default=True) tools.sessions.path = string(default="/") -tools.sessions.timeout = integer(default=30) +tools.sessions.timeout = integer(default=60) tools.sessions.storage_type = string(default="file") tools.sessions.storage_path = string(default="%(root)s/data/sessions") tools.sessions.secure = boolean(default=False) +tools.sessions.prefix = string(default=sideboard) + +# RedisSession specific values +tools.sessions.host = string(default="127.0.0.1") +tools.sessions.port = integer(default=6379) +tools.sessions.db = integer(default=0) +tools.sessions.password = string(default=None) +tools.sessions.tls_skip_verify = boolean(default=False) +tools.sessions.is_sentinel = boolean(default=False) +tools.sessions.ssl = boolean(default=False) +tools.sessions.user = string(default="") # Built-in CherryPy web server stats page tools.cpstats.on = boolean(default=False) @@ -145,6 +157,10 @@ cherrypy.access = option("TRACE", "DEBUG", "INFO", "WARNING", "WARN", "ERROR", " __many__ = option("TRACE", "DEBUG", "INFO", "WARN", "WARNING", "ERROR", "CRITICAL", default="INFO") [handlers] +[[stdout]] +class = string(default="logging.StreamHandler") +stream = string(default="ext://sys.stdout") +formatter = string(default="indent_multiline") [[__many__]] formatter = string(default="default") ___many___ = string() diff --git a/sideboard/internal/logging.py b/sideboard/internal/logging.py index 20e71b8..6dbe03c 100644 --- a/sideboard/internal/logging.py +++ b/sideboard/internal/logging.py @@ -1,8 +1,7 @@ from __future__ import unicode_literals, absolute_import -import os import logging.config -from sideboard.config import config, get_config_root +from sideboard.config import config class IndentMultilinesLogFormatter(logging.Formatter): @@ -18,29 +17,25 @@ def format(self, record): def _configure_logging(): - fname = os.path.join(get_config_root(), 'logging.cfg') - if os.path.exists(fname): - logging.config.fileConfig(fname, disable_existing_loggers=True) - else: - # ConfigObj doesn't support interpolation escaping, so we manually work around it here - formatters = config['formatters'].dict() - for formatter in formatters.values(): - formatter['format'] = formatter['format'].replace('$$', '%') - formatter['datefmt'] = formatter['datefmt'].replace('$$', '%') or None - formatters['indent_multiline'] = { - '()': IndentMultilinesLogFormatter, - 'format': formatters['default']['format'] - } - logging.config.dictConfig({ - 'version': 1, - 'root': { - 'level': config['loggers']['root'], - 'handlers': config['handlers'].dict().keys() - }, - 'loggers': { - name: {'level': level} - for name, level in config['loggers'].items() if name != 'root' - }, - 'handlers': config['handlers'].dict(), - 'formatters': formatters - }) + # ConfigObj doesn't support interpolation escaping, so we manually work around it here + formatters = config['formatters'].dict() + for formatter in formatters.values(): + formatter['format'] = formatter['format'].replace('$$', '%') + formatter['datefmt'] = formatter['datefmt'].replace('$$', '%') or None + formatters['indent_multiline'] = { + '()': IndentMultilinesLogFormatter, + 'format': formatters['default']['format'] + } + logging.config.dictConfig({ + 'version': 1, + 'root': { + 'level': config['loggers']['root'], + 'handlers': config['handlers'].dict().keys() + }, + 'loggers': { + name: {'level': level} + for name, level in config['loggers'].items() if name != 'root' + }, + 'handlers': config['handlers'].dict(), + 'formatters': formatters + }) diff --git a/sideboard/tests/test_configuration.py b/sideboard/tests/test_configuration.py deleted file mode 100644 index e175a42..0000000 --- a/sideboard/tests/test_configuration.py +++ /dev/null @@ -1,201 +0,0 @@ -from __future__ import unicode_literals -import os - -import pytest -from mock import Mock - -from sideboard.lib import config -from sideboard.config import get_config_files, get_config_overrides, get_config_root, get_module_and_root_dirs, parse_config, uniquify - - -def test_uniquify(): - pytest.raises(AssertionError, uniquify, None) - assert [] == uniquify([]) - assert ['a', 'b', 'c'] == uniquify(['a', 'b', 'c']) - assert ['a', 'b', 'c', 'd', 'e'] == uniquify(['a', 'b', 'a', 'c', 'a', 'd', 'a', 'e']) - assert ['a'] == uniquify(['a', 'a', 'a', 'a', 'a', 'a', 'a', 'a']) - - -@pytest.mark.skipif( - 'SIDEBOARD_CONFIG_OVERRIDES' not in os.environ, - reason='SIDEBOARD_CONFIG_OVERRIDES not set') -def test_test_defaults_ini(): - """ - Verify that the tests were launched using `test-defaults.ini`. - - All of the other sideboard tests will succeed whether `test-defaults.ini` - or `development-defaults.ini` is used. This test is actually a functional - test of sorts; it verifies the test suite itself was launched with - `SIDEBOARD_CONFIG_OVERRIDES="test-defaults.ini"`. - - The test is skipped rather than failing if the tests were launched without - setting `SIDEBOARD_CONFIG_OVERRIDES`. - """ - from sideboard.lib import config - # is_test_running is ONLY set in test-defaults.ini - assert config.get('is_test_running') - - -class SideboardConfigurationTest(object): - sideboard_root = os.path.abspath(os.path.join(__file__, '..', '..', '..')) - dev_test_plugin_path = os.path.join(sideboard_root, 'plugins', 'configuration-test', - 'configuration_test', '__init__.py') - production_test_plugin_path = os.path.join('/', 'opt', 'sideboard', 'plugins', 'configuration-test', - 'env', 'lib', 'python{py_version}', 'site-packages', - 'configuration_test-{plugin_ver}-{py_version}.egg', - 'configuration_test', '__init__.py') - - def test_parse_config_adding_root_and_module_root_for_dev(self): - test_config = parse_config(self.dev_test_plugin_path) - expected_module_root = os.path.dirname(self.dev_test_plugin_path) - expected_root = os.path.dirname(expected_module_root) - assert test_config.get('module_root') == expected_module_root - assert test_config.get('root') == expected_root - - def test_parse_config_adding_root_and_module_root_for_production(self): - for path_values in (dict(py_version='2.7', plugin_ver='1.0'),): - test_path = self.production_test_plugin_path.format(**path_values) - test_config = parse_config(test_path) - expected_module_root = os.path.dirname(test_path) - expected_root = os.path.join('/', 'opt', 'sideboard', 'plugins', 'configuration-test') - assert test_config.get('module_root') == expected_module_root - assert test_config.get('root') == expected_root - - -class TestSideboardGetConfigFiles(object): - @pytest.fixture - def config_overrides_unset(self, monkeypatch): - monkeypatch.delenv('SIDEBOARD_CONFIG_OVERRIDES', raising=False) - - @pytest.fixture - def config_overrides_set(self, monkeypatch): - monkeypatch.setenv('SIDEBOARD_CONFIG_OVERRIDES', 'test-defaults.ini') - - @pytest.fixture - def plugin_dirs(self): - module_path = '/fake/sideboard/plugins/test-plugin/test_plugin' - root_path = os.path.join(config['plugins_dir'], 'test-plugin') - return (module_path, root_path) - - @pytest.fixture - def sideboard_dirs(self): - module_path = '/fake/sideboard/sideboard' - root_path = '/fake/sideboard' - return (module_path, root_path) - - def test_get_module_and_root_dirs_plugin(self, plugin_dirs): - assert plugin_dirs == get_module_and_root_dirs( - os.path.join(plugin_dirs[0], 'config.py'), is_plugin=True) - - def test_get_module_and_root_dirs_sideboard(self, sideboard_dirs): - assert sideboard_dirs == get_module_and_root_dirs( - os.path.join(sideboard_dirs[0], 'config.py'), is_plugin=False) - - def test_get_config_files_plugin(self, plugin_dirs, config_overrides_unset): - expected = [ - '/etc/sideboard/plugins.d/test-plugin.cfg', - os.path.join(plugin_dirs[1], 'development-defaults.ini'), - os.path.join(plugin_dirs[1], 'development.ini')] - assert expected == get_config_files( - os.path.join(plugin_dirs[0], 'config.py'), is_plugin=True) - - def test_get_config_files_sideboard(self, sideboard_dirs, config_overrides_unset): - expected = [ - '/etc/sideboard/sideboard-core.cfg', - '/etc/sideboard/sideboard-server.cfg', - os.path.join(sideboard_dirs[1], 'development-defaults.ini'), - os.path.join(sideboard_dirs[1], 'development.ini')] - assert expected == get_config_files( - os.path.join(sideboard_dirs[0], 'config.py'), is_plugin=False) - - def test_get_config_files_plugin_with_overrides(self, plugin_dirs, config_overrides_set): - expected = [ - '/etc/sideboard/plugins.d/test-plugin.cfg', - os.path.join(plugin_dirs[1], 'test-defaults.ini'), - os.path.join(plugin_dirs[1], 'test.ini')] - assert expected == get_config_files( - os.path.join(plugin_dirs[0], 'config.py'), is_plugin=True) - - def test_get_config_files_sideboard_with_overrides(self, sideboard_dirs, config_overrides_set): - expected = [ - '/etc/sideboard/sideboard-core.cfg', - '/etc/sideboard/sideboard-server.cfg', - os.path.join(sideboard_dirs[1], 'test-defaults.ini'), - os.path.join(sideboard_dirs[1], 'test.ini')] - assert expected == get_config_files( - os.path.join(sideboard_dirs[0], 'config.py'), is_plugin=False) - - -class TestSideboardGetConfigOverrides(object): - @pytest.fixture(params=[ - (None, ['development-defaults.ini', 'development.ini']), - ('test-defaults.ini', ['test-defaults.ini', 'test.ini']), - ('test.ini;development.ini;test.ini', ['test.ini', 'development.ini']), - ('test-defaults.ini;test-defaults.ini', ['test-defaults.ini', 'test.ini']), - (' /absolute/path.ini ', ['/absolute/path.ini']), - ('/absolute/path.cfg', ['/absolute/path.cfg']), - (' relative/path.ini ', ['relative/path.ini']), - ('relative/path.cfg', ['relative/path.cfg']), - ('/absolute/path.cfg;relative/path.ini', ['/absolute/path.cfg', 'relative/path.ini']), - ('relative/path.cfg;/absolute/path.ini', ['relative/path.cfg', '/absolute/path.ini']), - (' /absolute/path.cfg ; relative/path.ini ', ['/absolute/path.cfg', 'relative/path.ini']), - (' /absolute/path-defaults.ini ', ['/absolute/path-defaults.ini', '/absolute/path.ini']), - ('/absolute/path-defaults.cfg', ['/absolute/path-defaults.cfg', '/absolute/path.cfg']), - (' relative/path-defaults.ini ', ['relative/path-defaults.ini', 'relative/path.ini']), - ('relative/path-defaults.cfg', ['relative/path-defaults.cfg', 'relative/path.cfg']), - ('/absolute/path-defaults.cfg;relative/path-defaults.ini', [ - '/absolute/path-defaults.cfg', - '/absolute/path.cfg', - 'relative/path-defaults.ini', - 'relative/path.ini' - ]) - ]) - def config_overrides(self, request, monkeypatch): - if request.param[0] is None: - monkeypatch.delenv('SIDEBOARD_CONFIG_OVERRIDES', raising=False) - else: - monkeypatch.setenv('SIDEBOARD_CONFIG_OVERRIDES', request.param[0]) - return request.param[1] - - def test_get_config_overrides(self, config_overrides): - assert get_config_overrides() == config_overrides - - -class TestSideboardGetConfigRoot(object): - @pytest.fixture - def dir_missing(self, monkeypatch): - monkeypatch.setattr(os.path, 'isdir', Mock(return_value=False)) - - @pytest.fixture - def dir_exists(self, monkeypatch): - monkeypatch.setattr(os.path, 'isdir', Mock(return_value=True)) - - @pytest.fixture - def dir_readable(self, monkeypatch): - monkeypatch.setattr(os, 'access', Mock(return_value=True)) - - @pytest.fixture - def dir_unreadable(self, monkeypatch): - monkeypatch.setattr(os, 'access', Mock(return_value=False)) - - @pytest.fixture - def custom_root(self, monkeypatch): - monkeypatch.setitem(os.environ, 'SIDEBOARD_CONFIG_ROOT', '/custom/location') - - def test_valid_etc_sideboard(self, dir_exists, dir_readable): - assert get_config_root() == '/etc/sideboard' - - def test_no_etc_sideboard(self, dir_missing): - assert get_config_root() == '/etc/sideboard' - - def test_etc_sideboard_unreadable(self, dir_exists, dir_unreadable): - pytest.raises(AssertionError, get_config_root) - - def test_overridden_missing(self, custom_root, dir_missing): - pytest.raises(AssertionError, get_config_root) - - def test_overridden_unreadable(self, custom_root, dir_exists, dir_unreadable): - pytest.raises(AssertionError, get_config_root) - - def test_overridden_valid(self, custom_root, dir_exists, dir_readable): - assert get_config_root() == '/custom/location' diff --git a/test-defaults.ini b/test-defaults.ini deleted file mode 100644 index 970fdb0..0000000 --- a/test-defaults.ini +++ /dev/null @@ -1,30 +0,0 @@ - -# The settings in this file (test-defaults.ini) can be overridden in test.ini. - -debug = True -ws.auth_required = False - -is_test_running = True - -[cherrypy] -engine.autoreload.on = False -profiling.on = False -server.socket_host = "127.0.0.1" - -# By default the test server runs on port 8282. If you are already using -# port 8282, you'll receive errors like: -# OSError: Port 8282 not free on '127.0.0.1' -# -# You can change this setting in test.ini to either another free port, or -# to port 0 (meaning a free port will be chosen by the OS automatically). -# Using port 0 is mostly safe, but on heavily used systems there is a -# potential race condition if another process uses the same port in the time -# between requesting an available port and actually using it. -# -# See https://eklitzke.org/binding-on-port-zero -# -server.socket_port = 8282 - - -[loggers] -root = "DEBUG" From 8e0e46c1ae74bdcd117509c02a8e01642d9e0505 Mon Sep 17 00:00:00 2001 From: Mark Murnane Date: Mon, 15 Apr 2024 00:36:43 -0400 Subject: [PATCH 2/6] Giving environment variables priority over files --- sideboard/config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sideboard/config.py b/sideboard/config.py index 1b84896..98eab09 100755 --- a/sideboard/config.py +++ b/sideboard/config.py @@ -156,11 +156,6 @@ def parse_config(requesting_file_path, is_plugin=True): root_conf = ['root = "{}"\n'.format(root_dir), 'module_root = "{}"\n'.format(module_dir)] temp_config = configobj.ConfigObj(root_conf, interpolation=False, encoding='utf-8') - environment_config = load_section_from_environment(plugin_name, spec) - print(f"Environment config for {plugin_name}") - print(json.dumps(environment_config, indent=2, sort_keys=True)) - temp_config.merge(configobj.ConfigObj(environment_config, encoding='utf-8', interpolation=False)) - for config_path in get_config_files(requesting_file_path, is_plugin): # this gracefully handles nonexistent files file_config = configobj.ConfigObj(str(config_path), encoding='utf-8', interpolation=False) @@ -168,6 +163,11 @@ def parse_config(requesting_file_path, is_plugin=True): print(json.dumps(file_config, indent=2, sort_keys=True)) temp_config.merge(file_config) + environment_config = load_section_from_environment(plugin_name, spec) + print(f"Environment config for {plugin_name}") + print(json.dumps(environment_config, indent=2, sort_keys=True)) + temp_config.merge(configobj.ConfigObj(environment_config, encoding='utf-8', interpolation=False)) + # combining the merge files to one file helps configspecs with interpolation with NamedTemporaryFile(delete=False) as config_outfile: temp_config.write(config_outfile) From 217d3fb91d5623b30a9eed7d678e220e2ab8cff1 Mon Sep 17 00:00:00 2001 From: Mark Murnane Date: Mon, 15 Apr 2024 20:43:46 -0400 Subject: [PATCH 3/6] Fixing periods in config vars --- sideboard/config.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sideboard/config.py b/sideboard/config.py index 98eab09..69c0d87 100755 --- a/sideboard/config.py +++ b/sideboard/config.py @@ -91,6 +91,9 @@ def get_config_files(requesting_file_path, is_plugin): absolute_config_files.append(root_dir / path) return absolute_config_files +def normalize_name(name): + return name.replace(".", "_") + def load_section_from_environment(path, section): """ Looks for configuration in environment variables. @@ -109,7 +112,7 @@ def load_section_from_environment(path, section): if setting == "__many__": prefix = f"{path}_" for envvar in os.environ: - if envvar.startswith(prefix): + if envvar.startswith(prefix) and not envvar.split(prefix, 1)[1] in [normalize_name(x) for x in section]: config[envvar.split(prefix, 1)[1]] = os.environ[envvar] else: if isinstance(section[setting], configobj.Section): @@ -118,9 +121,9 @@ def load_section_from_environment(path, section): if child: config[setting] = child else: - name = f"{path}_{setting}" + name = normalize_name(f"{path}_{setting}") if name in os.environ: - config[setting] = os.environ.get(name) + config[setting] = os.environ.get(normalize_name(name)) return config def parse_config(requesting_file_path, is_plugin=True): From 4a8773049bef6824dc415979afdc13889d212fe6 Mon Sep 17 00:00:00 2001 From: Mark Murnane Date: Mon, 15 Apr 2024 23:16:18 -0400 Subject: [PATCH 4/6] Only log config if we want it --- sideboard/config.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sideboard/config.py b/sideboard/config.py index 69c0d87..1cf8cb2 100755 --- a/sideboard/config.py +++ b/sideboard/config.py @@ -162,13 +162,15 @@ def parse_config(requesting_file_path, is_plugin=True): for config_path in get_config_files(requesting_file_path, is_plugin): # this gracefully handles nonexistent files file_config = configobj.ConfigObj(str(config_path), encoding='utf-8', interpolation=False) - print(f"File config for {plugin_name} from {config_path}") - print(json.dumps(file_config, indent=2, sort_keys=True)) + if os.environ.get("LOG_CONFIG", "false").lower() == "true": + print(f"File config for {plugin_name} from {config_path}") + print(json.dumps(file_config, indent=2, sort_keys=True)) temp_config.merge(file_config) environment_config = load_section_from_environment(plugin_name, spec) - print(f"Environment config for {plugin_name}") - print(json.dumps(environment_config, indent=2, sort_keys=True)) + if os.environ.get("LOG_CONFIG", "false").lower() == "true": + print(f"Environment config for {plugin_name}") + print(json.dumps(environment_config, indent=2, sort_keys=True)) temp_config.merge(configobj.ConfigObj(environment_config, encoding='utf-8', interpolation=False)) # combining the merge files to one file helps configspecs with interpolation From 756429f9b14aba6724346cf7ed9fe0ccba209019 Mon Sep 17 00:00:00 2001 From: Mark Murnane Date: Tue, 16 Apr 2024 17:17:17 -0400 Subject: [PATCH 5/6] Parse environment variables with yaml --- sideboard/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sideboard/config.py b/sideboard/config.py index 1cf8cb2..46f4951 100755 --- a/sideboard/config.py +++ b/sideboard/config.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import pathlib import json +import yaml import os from os import unlink @@ -123,7 +124,7 @@ def load_section_from_environment(path, section): else: name = normalize_name(f"{path}_{setting}") if name in os.environ: - config[setting] = os.environ.get(normalize_name(name)) + config[setting] = yaml.safe_load(os.environ.get(normalize_name(name))) return config def parse_config(requesting_file_path, is_plugin=True): From 8a8c1498799c0e30cbb387965b81f02f05c36c8a Mon Sep 17 00:00:00 2001 From: Mark Murnane Date: Tue, 16 Apr 2024 17:21:16 -0400 Subject: [PATCH 6/6] Adding pyyaml --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index b701108..db48ea6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ paver==1.3.4 pip==24.0 psutil==5.9.8 python-prctl==1.8.1; 'linux' in sys_platform +pyyaml==6.0.1 redis==5.0.3 requests==2.31.0 rpctools==0.3.1