Merge pull request #4931 from telstra/otsdb-dump-restore

Tool for dump/restore OpenTSDB data

tools/otsdb-dump-restore/.gitignore

@@ -0,0 +1,8 @@
+__pycache__/
+*.py[cod]
+
+.cache/
+.eggs/
+*.egg-info/
+build/
+dist/

tools/otsdb-dump-restore/Dockerfile.devel

@@ -0,0 +1,19 @@
+FROM python:3.11-slim
+
+ARG devel_user_name=root
+ARG devel_uid=0
+ARG devel_gid=0
+
+WORKDIR /kilda
+COPY build-tools build-tools
+RUN build-tools/setup-devel-user.sh "${devel_user_name}" "${devel_uid}" "${devel_gid}"
+
+USER ${devel_user_name}
+
+COPY requirements*.txt ./
+RUN pip install -r requirements.txt -r requirements-dev.txt
+
+RUN pip install flake8
+
+COPY README.md setup.py build-tools ./
+COPY src src

tools/otsdb-dump-restore/Dockerfile.prod

@@ -0,0 +1,8 @@
+FROM python:3.11-slim
+
+ARG wheel
+
+WORKDIR /tmp
+COPY dist/${wheel} ./
+RUN pip install ${wheel}
+RUN rm -rf ${wheel}

tools/otsdb-dump-restore/Makefile

@@ -0,0 +1,62 @@
+#!/usr/bin/env make
+# Makefile (root)
+
+# Targets description:
+# prod - build a container image with the application, labels it with $(DOCKER_LABEL)
+# devel - build container and starts an interactive shell inside it (app dir is attached inside the container as a
+#         volume, so all changes made to the application code are available inside the container without the need to
+#         restart/rebuild it). Useful for the application debug.
+
+APP_NAME := kilda-otsdb-dump-restore
+WHEEL_NAME = $(subst -,_,$(APP_NAME))
+DOCKER_LABEL = $(APP_NAME)
+ID_DIR := build
+DIST_DIR := dist
+
+SRC_DIR := src
+SRC_DIR_PY = $(shell find $(SRC_DIR) -type d \! -name __pycache__ -print)
+SRC = requirements*.txt setup.py build-tools/* $(wildcard $(addsuffix /*.py,$(SRC_DIR_PY))) $(shell find $(SRC_DIR) -name APP_META\* -print)
+
+all: prod
+
+prod: $(ID_DIR)/prod.iid
+	docker image tag "$$(cat $<)" "$(DOCKER_LABEL)"
+
+dist/version: $(ID_DIR)/wheel.cid
+	docker cp "$$(cat $^):/kilda/dist/version" dist/version.tmp
+	docker cp "$$(cat $^):/kilda/dist/$(WHEEL_NAME)-$$(cat dist/version.tmp)-py3-none-any.whl" dist/
+	mv dist/version.tmp $@
+
+$(ID_DIR)/wheel.cid: $(ID_DIR)/wheel.iid | $(DIST_DIR)
+	if [ -f "$@" ]; then docker rm "$$(cat $@)" && rm "$@"; fi
+	docker run --cidfile=$@ "$$(cat $^)" /kilda/build-tools/make_wheel.sh
+
+$(ID_DIR)/devel.cid: $(ID_DIR)/devel.iid
+	if [ -f "$@" ]; then docker rm "$$(cat $@)" && rm "$@"; fi
+	docker create -ti \
+		--volume="/etc/resolv.conf:/etc/resolv.conf" \
+		--volume="$$PWD:/kilda" \
+		--network=host \
+		--cidfile=$@ "$$(cat $^)" /kilda/build-tools/devel-entry-point.sh
+
+$(ID_DIR)/devel.iid: Dockerfile.devel | $(ID_DIR)
+	docker build --iidfile=$@ --file=$< \
+		--build-arg=devel_user_name=$$(id -un) \
+		--build-arg=devel_uid=$$(id -u) \
+		--build-arg=devel_gid=$$(id -g) \
+		.
+
+$(ID_DIR)/wheel.iid: Dockerfile.devel $(SRC) | $(ID_DIR)
+	docker build --iidfile=$@ --file=$< .
+
+$(ID_DIR)/prod.iid: Dockerfile.prod dist/version | $(ID_DIR)
+	docker build --iidfile=$@ --file=$< \
+		--build-arg=wheel="$(WHEEL_NAME)-$$(cat dist/version)-py3-none-any.whl" \
+		.
+
+$(ID_DIR):
+	mkdir -p $@
+$(DIST_DIR):
+	mkdir -p $@
+
+.PHONY: all prod

tools/otsdb-dump-restore/README.md

@@ -0,0 +1,146 @@
+# OpenKilda TSDB dump/restore tool
+
+This packet contains python package that provide `kilda-otsdb-dump-restore` CLI tool. The tool itself
+provides following commands:
+* `kilda-otsdb-dump` download data from an OpenTSDB endpoint.
+* `kilda-otsdb-restore` restore data to an OpenTSDB capable database.
+
+## Build
+- `make prod` builds a container image of the application and label it with kilda-otsdb-dump-restore.
+- `make devel` builds a container and start an interactive shell inside it, allowing the app directory to be attached as a volume inside the container.
+This means that any changes made to the application code will be available inside the container without the need for a restart or rebuild, making it a useful tool for debugging.
+
+## Usage
+The tool is inside a Docker container. To use the it, run the following command in your terminal:
+```
+docker run kilda-otsdb-dump-restore <command_to_use>
+```
+Note: if you need to run it in localhost, you need to add the `--network="host"` docker flag.
+
+### Dump data from OpenTSDB:
+Use `kilda-otsdb-dump` command to dump data from an OpenTSDB.
+```
+Usage: kilda-otsdb-dump [OPTIONS] OPENTSDB_ENDPOINT TIME_START
+
+  This tool dumps the data from an OpenTSDB
+
+  OPENTSDB_ENDPOINT openTSDB endpoint
+
+  TIME_START time since the data is dumped
+
+  Example: kilda-otsdb-dump http://example.com:4242 2023-03-08
+
+Options:
+  --time-stop TIME_STOP       Timestamp where to stop dumping  [default: NOW]
+  --dump-dir DIRECTORY        Location where dump files will be stored
+  --query-frame-size INTEGER  OpenTSDB query time frame size (seconds)
+                              [default: 180]
+  --metrics-prefix TEXT       Only metrics that match this prefix will be
+                              dumped  [default: kilda.]
+  --remove-metadata
+  --help                      Show this message and exit.
+```
+
+### Restore data to OpenTSDB:
+Use `kilda-otsdb-restore` command to restore data previously dumped to an OpenTSDB.
+```
+Usage: kilda-otsdb-restore [OPTIONS] OPENTSDB_ENDPOINT
+
+  This tool restore the data to an OpenTSDB
+
+  OPENTSDB_ENDPOINT openTSDB endpoint
+
+  Example: kilda-otsdb-restore http://example.com:4242
+
+Options:
+  --dump-dir DIRECTORY          Location where dump files are stored
+  --request-size-limit INTEGER  Limit for "put" request payload size (bytes)
+                                [default: 4096]
+  --help                        Show this message and exit.
+```
+### Date and time formats
+```
+%Y-%m-%d, %Y-%m-%dT%H:%M:%S, %Y-%m-%d %H:%M:%S
+```
+- "2023-03-22"
+
+- "2023-03-22T23:59:59"
+
+- "2023-03-22 23:59:59"
+
+### Example of use:
+Scenario:
+* One OpenTSDB service that has stored data.
+* One empty VictoriaMetrics service with OpenTSDB capabilites.
+* We want to migrate the data from the OpenTSDB to the VictoriaMetric service.
+
+This could be a posible workflow:
+1. Create volume to store data
+```bash
+docker volume create opentsdb-data
+```
+2. Next, we need to dump the data from OpenTSDB. The following command gets the data from a specified time until now and save it to disk:
+```bash
+docker run --rm -v opentsdb-data:/tmp kilda-otsdb-dump-restore kilda-otsdb-dump http://opentsdb:4242 "2023-03-08"
+```
+3. After dumping the data, we can restore it using the following command:
+```bash
+docker run --rm -v opentsdb-data:/tmp kilda-otsdb-dump-restore kilda-otsdb-restore http://victoria:4242
+```
+4. Finally, we can remove the volume using the following command:
+```bash
+docker volume remove opentsdb-data
+```
+
+Another approach would be to use a loop that iterates over a time range, such as days. This method could be beneficial when migrating a large amount of data.
+1. Create volume to store data
+```bash
+docker volume create opentsdb-data
+```
+2. Next, we need to dump the data from OpenTSDB. Use the following command to get the data from a specified time until the --time-stop time and save it to disk:
+```bash
+docker run --rm -v opentsdb-data:/tmp kilda-otsdb-dump-restore kilda-otsdb-dump --time-stop "2023-03-22T11:00:00" http://opentsdb:4242 "2023-03-22T00:00:00"
+```
+3. After dumping the data, we can restore it using the following command:
+```bash
+docker run --rm -v opentsdb-data:/tmp kilda-otsdb-dump-restore kilda-otsdb-restore http://victoria:4242
+```
+4. Finally, we can remove the volume using the following command:
+```bash
+docker volume remove opentsdb-data
+```
+
+### How to use the `otsdb-to-vm.sh` script
+The `otsdb-to-vm.sh` script is a bash wrapper around the `kilda-otsdb-dump-restore` tool. It is used to dump data from an OpenTSDB and restore it to a VictoriaMetrics service.
+Options `--opentsdb_endpoint`, `--victoria_metrics_endpoint`, `--start_date` and `--end_date` are **mandatory**.
+
+### Usage examples:
+```bash
+# pump data using one day batch
+./otsdb-to-vm.sh -s http://opentsdb.example.com:4242 -d http://victoria-metrics.example.com:4242 --start_date 2022-01-01 --end_date 2022-01-31 --metrics_prefix 'kilda.' --interval day
+
+# pump data using one hour batch
+./otsdb-to-vm.sh -s http://opentsdb.example.com:4242 -d http://victoria-metrics.example.com:4242 --start_date 2022-01-01T00:00:00 --end_date 2022-01-01T23:59:59 --metrics_prefix 'kilda.' --interval hour
+```
+
+### How it working:
+Two processes are running simultaneously:
+* The `dump` process (foreground) is responsible for saving **batches** of data from OpenTSDB to the specified folder;
+* The `restore` process (background) is tasked with uploading dumped data to the remote Victoria metrics server;
+#
+The `dump` process runs batches in a loop, continually checking for available space/inodes to perform a dump. if it detects a lack of space, the dump cycle will wait until space become available. Once the dump successfully finishes current batch, the folder will be renamed to indicate it's ready for upload;
+
+1) When the `dump` process will complete all batches, it sends a `SIGUSR1` signal to the `restore` process and waits for the `restore` process to finish.
+2) Pressing `Ctrl+C` once increments an internal counter but doesn't take any immediate action.
+3) Pressing `Ctrl+C` twice will stop the `dump` process. The process will wait for all background `restore` tasks to finish before terminating.
+4) Pressing `Ctrl+C` three or more times will send stop signal `SIGTERM` to the background process, causing it to stop execution after completing the current upload task.
+#
+The `restore` process runs in the background, continously monitoring folders named with a 'ready' state marker and logging it's execution events to the 'background_log' file.
+
+1) During each loop cycle, the `restore` processes only one folder.
+2) If there are no new folders to process, it suspends execution and waits for a new folder to become available for upload (marker in the name).
+3) After a successfull restore operation, the folder is removed, freeing up space for use by the `dump` process.
+4) The `restore` process is prepared to capture a `Ctrl+C` or `SIGTERM` signal and terminate its loop execution after completing the current upload task.
+5) The `restore` process also listens for a `SIGUSR1` signal to gracefully finish it's loop execution.
+6) Upon receiving the `SIGUSR1` signal, the `restore` process assumes that the `dump` loop has finished. It processes and removes all ready folders and then terminates its execution.
+#

tools/otsdb-dump-restore/build-tools/devel-entry-point.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+cd -- "$(dirname -- "$0")/.." || exit 1
+
+python setup.py develop
+
+pip install pydevd-pycharm~=203.7148.57
+
+echo
+echo Use \"kilda-otsdb-dump\" or \"kilda-otsdb-restore\" to run application
+echo
+
+exec bash -l

tools/otsdb-dump-restore/build-tools/make_wheel.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -eux
+cd -- "$(dirname -- "$0")/.." || exit 1
+
+python setup.py develop
+
+flake8 --max-line-length=119 src
+
+python setup.py bdist_wheel
+python setup.py --version > dist/version

tools/otsdb-dump-restore/build-tools/setup-devel-user.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -eux
+
+echo "$@"
+
+DEVEL_NAME="${1}"
+DEVEL_UID="${2}"
+DEVEL_GID="${3}"
+
+if [ ${DEVEL_UID} -eq 0 ]; then
+  # nothing to do for root user
+  exit
+fi
+
+if [ -z "$(getent group "${DEVEL_GID}")" ]; then
+  groupadd -g "${DEVEL_GID}" "${DEVEL_NAME}"
+fi
+
+if [ -z "$(getent passwd "${DEVEL_NAME}")" ]; then
+  useradd -m -u "${DEVEL_UID}" -g "${DEVEL_GID}" -s /bin/bash "${DEVEL_NAME}"
+fi
+
+chown "${DEVEL_UID}:${DEVEL_GID}" \
+  /kilda \
+  /usr/local \
+  /usr/local/bin \
+  /usr/local/lib/python3.11/site-packages