Skip to content

Commit b6d1156

Browse files
committed
Squashed commit of the following:
commit 10539ed Author: ajnisbet <[email protected]> Date: Fri Oct 11 13:15:54 2024 -0700 Upgrade version commit d1fc1a7 Author: ajnisbet <[email protected]> Date: Fri Oct 11 13:11:43 2024 -0700 Working watchdog commit 9d51071 Author: Arne Setzer <[email protected]> Date: Fri Oct 11 20:58:43 2024 +0200 Add support for hot reloading the config.yaml * add support for hot reloading the config.yaml * watchdogs also watches on example-config.yaml * watchdog remove os.system stuff * watchdog add requirments.{txt,in} --------- Co-authored-by: Andrew Nisbet <[email protected]> commit fc407a2 Author: ajnisbet <[email protected]> Date: Fri Oct 11 11:56:00 2024 -0700 Upgrade dependencies commit 5ec7c25 Author: ajnisbet <[email protected]> Date: Fri Oct 11 11:35:31 2024 -0700 Docker tweaks commit 87c9742 Merge: cc7f997 e93ff94 Author: ajnisbet <[email protected]> Date: Fri Oct 11 11:35:00 2024 -0700 Merge branch 'master' into dev commit cc7f997 Author: ajnisbet <[email protected]> Date: Tue Feb 20 11:46:21 2024 -0800 Tidy cors
1 parent e93ff94 commit b6d1156

13 files changed

+199
-46
lines changed

Makefile

+8-2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,12 @@ build:
77
build-m1:
88
docker build --tag opentopodata:$(VERSION) --file docker/apple-silicon.Dockerfile .
99

10+
rebuild:
11+
docker build --no-cache --tag opentopodata:$(VERSION) --file docker/Dockerfile .
12+
13+
rebuild-m1:
14+
docker build --no-cache --tag opentopodata:$(VERSION) --file docker/apple-silicon.Dockerfile .
15+
1016
run:
1117
docker run --rm -it --volume "$(shell pwd)/data:/app/data:ro" -p 5000:5000 opentopodata:$(VERSION)
1218

@@ -23,10 +29,10 @@ run-local:
2329
FLASK_APP=opentopodata/api.py FLASK_DEBUG=1 flask run --port 5000
2430

2531
black:
26-
black --target-version py39 tests opentopodata
32+
black --target-version py311 tests opentopodata docker
2733

2834
black-check:
29-
docker run --rm opentopodata:$(VERSION) python -m black --check --target-version py39 tests opentopodata
35+
docker run --rm opentopodata:$(VERSION) python -m black --check --target-version py311 tests opentopodata
3036

3137
update-requirements: build
3238
# pip-compile gets confused if there's already a requirements.txt file, and

VERSION

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
1.9.0
1+
1.10.0

docker/Dockerfile

+6-4
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,22 @@
11
# Container for packages that need to be built from source but have massive dev dependencies.
2-
FROM python:3.11.8-slim-bookworm as builder
2+
FROM python:3.11.10-slim-bookworm as builder
33
RUN set -e && \
44
apt-get update && \
55
apt-get install -y --no-install-recommends \
66
gcc \
77
python3.11-dev
88

99
RUN pip config set global.disable-pip-version-check true && \
10-
pip wheel --wheel-dir=/root/wheels uwsgi==2.0.24 && \
11-
pip wheel --wheel-dir=/root/wheels regex==2023.12.25
10+
pip wheel --wheel-dir=/root/wheels uwsgi==2.0.27 && \
11+
pip wheel --wheel-dir=/root/wheels regex==2024.9.11
1212

1313
# The actual container.
14-
FROM python:3.11.8-slim-bookworm
14+
FROM python:3.11.10-slim-bookworm
1515
RUN set -e && \
1616
apt-get update && \
1717
apt-get install -y --no-install-recommends \
18+
inotify-tools \
19+
nano \
1820
nginx \
1921
memcached \
2022
supervisor && \

docker/config_watcher.py

+110
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
import logging
2+
import time
3+
from pathlib import Path
4+
import subprocess
5+
import sys
6+
7+
from watchdog.observers import Observer
8+
from watchdog.events import FileSystemEventHandler
9+
10+
11+
# Paths.
12+
CONFIG_DIR = Path("/app/")
13+
CONFIG_PATH = Path("/app/config.yaml")
14+
EXAMPLE_CONFIG_PATH = Path("/app/example-config.yaml")
15+
16+
# Debouncing: once the config has been reloaded, any queued unprocessed events should be discarded.
17+
LAST_INVOCATION_TIME = time.time()
18+
19+
20+
# Logger setup.
21+
logger = logging.getLogger("configwatcher")
22+
LOG_FORMAT = "%(asctime)s %(levelname)-8s %(message)s"
23+
formatter = logging.Formatter(LOG_FORMAT)
24+
logger.setLevel(logging.INFO)
25+
handler = logging.StreamHandler(sys.stdout)
26+
handler.setLevel(logging.INFO)
27+
handler.setFormatter(formatter)
28+
logger.addHandler(handler)
29+
30+
31+
def run_cmd(cmd, shell=False):
32+
r = subprocess.run(cmd, shell=shell, capture_output=True)
33+
is_error = r.returncode != 0
34+
stdout = r.stdout.decode("utf-8")
35+
if is_error:
36+
logger.error(f"Error running command, returncode: {r.returncode}")
37+
logger.error("cmd:")
38+
logger.error(" ".join(cmd))
39+
if r.stdout:
40+
logger.error("stdout:")
41+
logger.error(stdout)
42+
if r.stderr:
43+
logger.error("stderr:")
44+
logger.error(r.stderr.decode("utf-8"))
45+
raise ValueError
46+
return stdout
47+
48+
49+
def reload_config():
50+
global LAST_INVOCATION_TIME
51+
LAST_INVOCATION_TIME = time.time()
52+
logger.info("Restarting OTD due to config change.")
53+
run_cmd(["supervisorctl", "-c", "/app/docker/supervisord.conf", "stop", "uwsgi"])
54+
run_cmd(
55+
["supervisorctl", "-c", "/app/docker/supervisord.conf", "restart", "memcached"]
56+
)
57+
run_cmd(["supervisorctl", "-c", "/app/docker/supervisord.conf", "start", "uwsgi"])
58+
run_cmd(
59+
["supervisorctl", "-c", "/app/docker/supervisord.conf", "start", "warm_cache"]
60+
)
61+
LAST_INVOCATION_TIME = time.time()
62+
logger.info("Restarted OTD due to config change.")
63+
64+
65+
class Handler(FileSystemEventHandler):
66+
67+
def on_any_event(self, event):
68+
watch_paths_str = [
69+
EXAMPLE_CONFIG_PATH.as_posix(),
70+
CONFIG_PATH.as_posix(),
71+
]
72+
73+
# Filter unwanted events.
74+
if event.event_type not in {"modified", "created"}:
75+
logger.debug(f"Dropping event with type {event.event_type=}")
76+
return
77+
if event.is_directory:
78+
logger.debug(f"Dropping dir event")
79+
return
80+
if event.src_path not in watch_paths_str:
81+
logger.debug(f"Dropping event with path {event.src_path=}")
82+
return
83+
if not Path(event.src_path).exists():
84+
logger.debug(f"Dropping event for nonexistent path {event.src_path=}")
85+
return
86+
87+
# Debouncing.
88+
mtime = Path(event.src_path).lstat().st_mtime
89+
if mtime < LAST_INVOCATION_TIME:
90+
msg = f"Dropping event for file that hasn't been modified since the last run. {event.src_path=}"
91+
logger.debug(msg)
92+
return
93+
94+
logger.debug(f"Dispatching event on {event.src_path=}")
95+
reload_config()
96+
97+
98+
if __name__ == "__main__":
99+
100+
event_handler = Handler()
101+
observer = Observer()
102+
observer.schedule(event_handler, CONFIG_DIR, recursive=False)
103+
observer.start()
104+
105+
try:
106+
while True:
107+
time.sleep(1)
108+
finally:
109+
observer.stop()
110+
observer.join()

docker/supervisord.conf

+19
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,19 @@
22
nodaemon=true
33
user=root
44

5+
6+
# Supervisorctl config/
7+
[unix_http_server]
8+
file=/var/run/supervisor.sock
9+
chmod=0700
10+
[rpcinterface:supervisor]
11+
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
12+
[supervisorctl]
13+
serverurl=unix:///var/run/supervisor.sock
14+
15+
16+
# OTD services.
17+
518
[program:uwsgi]
619
command=/usr/local/bin/uwsgi --ini /app/docker/uwsgi.ini --processes %(ENV_N_UWSGI_THREADS)s
720
stdout_logfile=/dev/stdout
@@ -33,3 +46,9 @@ stderr_logfile=/dev/stderr
3346
stderr_logfile_maxbytes=0
3447
autorestart=false
3548

49+
[program:watch_config]
50+
command=python /app/docker/config_watcher.py
51+
stdout_logfile=/dev/stdout
52+
stdout_logfile_maxbytes=0
53+
stderr_logfile=/dev/stderr
54+
stderr_logfile_maxbytes=0

docker/warm_cache.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -34,5 +34,7 @@
3434
break
3535

3636
else:
37-
logging.error("Timeout while trying to pre-populate the cache. This probably means that Open Topo Data isn't working.")
37+
logging.error(
38+
"Timeout while trying to pre-populate the cache. This probably means that Open Topo Data isn't working."
39+
)
3840
sys.exit(1)

docs/changelog.md

+4
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22

33
This is a list of changes to Open Topo Data between each release.
44

5+
## Version 1.10.0 (11 Oct 2024)
6+
* Minior dependency upgrades
7+
* Auto-reloading of config files without restarting docker ([#100](https://github.com/ajnisbet/opentopodata/pull/100) thanks [@arnesetzer](https://github.com/arnesetzer)!)
8+
59

610
## Version 1.9.0 (19 Feb 2024)
711
* Dependency upgrades, including python to 3.11 and rasterio to 1.3.9

docs/server.md

+2
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,8 @@ opentopodata
9797

9898
which would expose `localhost:5000/v1/etopo1` and `localhost:5000/v1/srtm90m`.
9999

100+
Mofifying the config file triggers a restart of OpenTopoData (which will reload the new config).
101+
100102

101103
### Config spec
102104

example-config.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# An example of a config.yaml file showing all possible options. If no
2-
# config.yaml file exists, opentopodata will load example-config.yaml instead.
2+
# config.yaml file exists, opentopodata will load example-config.yaml instead..
33

44

55
# 400 error will be thrown above this limit.

opentopodata/api.py

+3-4
Original file line numberDiff line numberDiff line change
@@ -85,10 +85,9 @@ def apply_cors(response):
8585
of the access_control_allow_origin config option.
8686
"""
8787
try:
88-
if _load_config()["access_control_allow_origin"]:
89-
response.headers["access-control-allow-origin"] = _load_config()[
90-
"access_control_allow_origin"
91-
]
88+
cors_value = _load_config()["access_control_allow_origin"]
89+
if cors_value:
90+
response.headers["access-control-allow-origin"] = cors_value
9291
except config.ConfigError:
9392
# If the config isn't loading, allow the request to complete without
9493
# CORS so user can see error message.

opentopodata/backend.py

+7-1
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,13 @@ def _get_elevation_from_path(lats, lons, path, interpolation):
105105
oob_indices = _validate_points_lie_within_raster(
106106
xs, ys, lats, lons, f.bounds, f.res
107107
)
108-
rows, cols = tuple(f.index(xs, ys, op=_noop))
108+
print(f"{xs=}")
109+
print(f"{ys=}")
110+
tmp = f.index(xs.tolist(), ys.tolist(), op=_noop)
111+
print(f"{tmp=}")
112+
rows, cols = tuple(tmp)
113+
114+
# rows, cols = tuple(f.index(xs, ys, op=_noop))
109115

110116
# Different versions of rasterio may or may not collapse single
111117
# f.index() lookups into scalars. We want to always have an

requirements.in

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
black
2-
Flask>=2.2.2 # Some flask 2.0 deprecations got real.
2+
Flask>=2.2.2 # Flask 2.0 deprecations were enforced.
33
flask-caching
44
geographiclib
55
numpy
@@ -11,5 +11,6 @@ pytest
1111
pytest-cov
1212
pytest-timeout
1313
PyYAML
14-
rasterio>=1.3.8 # Avoid memory leak https://github.com/ajnisbet/opentopodata/issues/68
14+
rasterio>=1.3.8,<1.4.0 # 1.3.8+ avoids memory leak https://github.com/ajnisbet/opentopodata/issues/68; 1.4.0 introduces some bugs in rowcol/xy (as of 2024-10-11).
1515
requests
16+
watchdog

0 commit comments

Comments
 (0)