-
Notifications
You must be signed in to change notification settings - Fork 53
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #4931 from telstra/otsdb-dump-restore
Tool for dump/restore OpenTSDB data
- Loading branch information
Showing
23 changed files
with
2,100 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
__pycache__/ | ||
*.py[cod] | ||
|
||
.cache/ | ||
.eggs/ | ||
*.egg-info/ | ||
build/ | ||
dist/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
FROM python:3.11-slim | ||
|
||
ARG devel_user_name=root | ||
ARG devel_uid=0 | ||
ARG devel_gid=0 | ||
|
||
WORKDIR /kilda | ||
COPY build-tools build-tools | ||
RUN build-tools/setup-devel-user.sh "${devel_user_name}" "${devel_uid}" "${devel_gid}" | ||
|
||
USER ${devel_user_name} | ||
|
||
COPY requirements*.txt ./ | ||
RUN pip install -r requirements.txt -r requirements-dev.txt | ||
|
||
RUN pip install flake8 | ||
|
||
COPY README.md setup.py build-tools ./ | ||
COPY src src |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
FROM python:3.11-slim | ||
|
||
ARG wheel | ||
|
||
WORKDIR /tmp | ||
COPY dist/${wheel} ./ | ||
RUN pip install ${wheel} | ||
RUN rm -rf ${wheel} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,62 @@ | ||
#!/usr/bin/env make | ||
# Makefile (root) | ||
|
||
# Targets description: | ||
# prod - build a container image with the application, labels it with $(DOCKER_LABEL) | ||
# devel - build container and starts an interactive shell inside it (app dir is attached inside the container as a | ||
# volume, so all changes made to the application code are available inside the container without the need to | ||
# restart/rebuild it). Useful for the application debug. | ||
|
||
APP_NAME := kilda-otsdb-dump-restore | ||
WHEEL_NAME = $(subst -,_,$(APP_NAME)) | ||
DOCKER_LABEL = $(APP_NAME) | ||
ID_DIR := build | ||
DIST_DIR := dist | ||
|
||
SRC_DIR := src | ||
SRC_DIR_PY = $(shell find $(SRC_DIR) -type d \! -name __pycache__ -print) | ||
SRC = requirements*.txt setup.py build-tools/* $(wildcard $(addsuffix /*.py,$(SRC_DIR_PY))) $(shell find $(SRC_DIR) -name APP_META\* -print) | ||
|
||
all: prod | ||
|
||
prod: $(ID_DIR)/prod.iid | ||
docker image tag "$$(cat $<)" "$(DOCKER_LABEL)" | ||
|
||
dist/version: $(ID_DIR)/wheel.cid | ||
docker cp "$$(cat $^):/kilda/dist/version" dist/version.tmp | ||
docker cp "$$(cat $^):/kilda/dist/$(WHEEL_NAME)-$$(cat dist/version.tmp)-py3-none-any.whl" dist/ | ||
mv dist/version.tmp $@ | ||
|
||
$(ID_DIR)/wheel.cid: $(ID_DIR)/wheel.iid | $(DIST_DIR) | ||
if [ -f "$@" ]; then docker rm "$$(cat $@)" && rm "$@"; fi | ||
docker run --cidfile=$@ "$$(cat $^)" /kilda/build-tools/make_wheel.sh | ||
|
||
$(ID_DIR)/devel.cid: $(ID_DIR)/devel.iid | ||
if [ -f "$@" ]; then docker rm "$$(cat $@)" && rm "$@"; fi | ||
docker create -ti \ | ||
--volume="/etc/resolv.conf:/etc/resolv.conf" \ | ||
--volume="$$PWD:/kilda" \ | ||
--network=host \ | ||
--cidfile=$@ "$$(cat $^)" /kilda/build-tools/devel-entry-point.sh | ||
|
||
$(ID_DIR)/devel.iid: Dockerfile.devel | $(ID_DIR) | ||
docker build --iidfile=$@ --file=$< \ | ||
--build-arg=devel_user_name=$$(id -un) \ | ||
--build-arg=devel_uid=$$(id -u) \ | ||
--build-arg=devel_gid=$$(id -g) \ | ||
. | ||
|
||
$(ID_DIR)/wheel.iid: Dockerfile.devel $(SRC) | $(ID_DIR) | ||
docker build --iidfile=$@ --file=$< . | ||
|
||
$(ID_DIR)/prod.iid: Dockerfile.prod dist/version | $(ID_DIR) | ||
docker build --iidfile=$@ --file=$< \ | ||
--build-arg=wheel="$(WHEEL_NAME)-$$(cat dist/version)-py3-none-any.whl" \ | ||
. | ||
|
||
$(ID_DIR): | ||
mkdir -p $@ | ||
$(DIST_DIR): | ||
mkdir -p $@ | ||
|
||
.PHONY: all prod |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,146 @@ | ||
# OpenKilda TSDB dump/restore tool | ||
|
||
This packet contains python package that provide `kilda-otsdb-dump-restore` CLI tool. The tool itself | ||
provides following commands: | ||
* `kilda-otsdb-dump` download data from an OpenTSDB endpoint. | ||
* `kilda-otsdb-restore` restore data to an OpenTSDB capable database. | ||
|
||
## Build | ||
- `make prod` builds a container image of the application and label it with kilda-otsdb-dump-restore. | ||
- `make devel` builds a container and start an interactive shell inside it, allowing the app directory to be attached as a volume inside the container. | ||
This means that any changes made to the application code will be available inside the container without the need for a restart or rebuild, making it a useful tool for debugging. | ||
|
||
## Usage | ||
The tool is inside a Docker container. To use the it, run the following command in your terminal: | ||
``` | ||
docker run kilda-otsdb-dump-restore <command_to_use> | ||
``` | ||
Note: if you need to run it in localhost, you need to add the `--network="host"` docker flag. | ||
|
||
### Dump data from OpenTSDB: | ||
Use `kilda-otsdb-dump` command to dump data from an OpenTSDB. | ||
``` | ||
Usage: kilda-otsdb-dump [OPTIONS] OPENTSDB_ENDPOINT TIME_START | ||
This tool dumps the data from an OpenTSDB | ||
OPENTSDB_ENDPOINT openTSDB endpoint | ||
TIME_START time since the data is dumped | ||
Example: kilda-otsdb-dump http://example.com:4242 2023-03-08 | ||
Options: | ||
--time-stop TIME_STOP Timestamp where to stop dumping [default: NOW] | ||
--dump-dir DIRECTORY Location where dump files will be stored | ||
--query-frame-size INTEGER OpenTSDB query time frame size (seconds) | ||
[default: 180] | ||
--metrics-prefix TEXT Only metrics that match this prefix will be | ||
dumped [default: kilda.] | ||
--remove-metadata | ||
--help Show this message and exit. | ||
``` | ||
|
||
### Restore data to OpenTSDB: | ||
Use `kilda-otsdb-restore` command to restore data previously dumped to an OpenTSDB. | ||
``` | ||
Usage: kilda-otsdb-restore [OPTIONS] OPENTSDB_ENDPOINT | ||
This tool restore the data to an OpenTSDB | ||
OPENTSDB_ENDPOINT openTSDB endpoint | ||
Example: kilda-otsdb-restore http://example.com:4242 | ||
Options: | ||
--dump-dir DIRECTORY Location where dump files are stored | ||
--request-size-limit INTEGER Limit for "put" request payload size (bytes) | ||
[default: 4096] | ||
--help Show this message and exit. | ||
``` | ||
### Date and time formats | ||
``` | ||
%Y-%m-%d, %Y-%m-%dT%H:%M:%S, %Y-%m-%d %H:%M:%S | ||
``` | ||
- "2023-03-22" | ||
|
||
- "2023-03-22T23:59:59" | ||
|
||
- "2023-03-22 23:59:59" | ||
|
||
### Example of use: | ||
Scenario: | ||
* One OpenTSDB service that has stored data. | ||
* One empty VictoriaMetrics service with OpenTSDB capabilites. | ||
* We want to migrate the data from the OpenTSDB to the VictoriaMetric service. | ||
|
||
This could be a posible workflow: | ||
1. Create volume to store data | ||
```bash | ||
docker volume create opentsdb-data | ||
``` | ||
2. Next, we need to dump the data from OpenTSDB. The following command gets the data from a specified time until now and save it to disk: | ||
```bash | ||
docker run --rm -v opentsdb-data:/tmp kilda-otsdb-dump-restore kilda-otsdb-dump http://opentsdb:4242 "2023-03-08" | ||
``` | ||
3. After dumping the data, we can restore it using the following command: | ||
```bash | ||
docker run --rm -v opentsdb-data:/tmp kilda-otsdb-dump-restore kilda-otsdb-restore http://victoria:4242 | ||
``` | ||
4. Finally, we can remove the volume using the following command: | ||
```bash | ||
docker volume remove opentsdb-data | ||
``` | ||
|
||
Another approach would be to use a loop that iterates over a time range, such as days. This method could be beneficial when migrating a large amount of data. | ||
1. Create volume to store data | ||
```bash | ||
docker volume create opentsdb-data | ||
``` | ||
2. Next, we need to dump the data from OpenTSDB. Use the following command to get the data from a specified time until the --time-stop time and save it to disk: | ||
```bash | ||
docker run --rm -v opentsdb-data:/tmp kilda-otsdb-dump-restore kilda-otsdb-dump --time-stop "2023-03-22T11:00:00" http://opentsdb:4242 "2023-03-22T00:00:00" | ||
``` | ||
3. After dumping the data, we can restore it using the following command: | ||
```bash | ||
docker run --rm -v opentsdb-data:/tmp kilda-otsdb-dump-restore kilda-otsdb-restore http://victoria:4242 | ||
``` | ||
4. Finally, we can remove the volume using the following command: | ||
```bash | ||
docker volume remove opentsdb-data | ||
``` | ||
|
||
### How to use the `otsdb-to-vm.sh` script | ||
The `otsdb-to-vm.sh` script is a bash wrapper around the `kilda-otsdb-dump-restore` tool. It is used to dump data from an OpenTSDB and restore it to a VictoriaMetrics service. | ||
Options `--opentsdb_endpoint`, `--victoria_metrics_endpoint`, `--start_date` and `--end_date` are **mandatory**. | ||
|
||
### Usage examples: | ||
```bash | ||
# pump data using one day batch | ||
./otsdb-to-vm.sh -s http://opentsdb.example.com:4242 -d http://victoria-metrics.example.com:4242 --start_date 2022-01-01 --end_date 2022-01-31 --metrics_prefix 'kilda.' --interval day | ||
|
||
# pump data using one hour batch | ||
./otsdb-to-vm.sh -s http://opentsdb.example.com:4242 -d http://victoria-metrics.example.com:4242 --start_date 2022-01-01T00:00:00 --end_date 2022-01-01T23:59:59 --metrics_prefix 'kilda.' --interval hour | ||
``` | ||
|
||
### How it working: | ||
Two processes are running simultaneously: | ||
* The `dump` process (foreground) is responsible for saving **batches** of data from OpenTSDB to the specified folder; | ||
* The `restore` process (background) is tasked with uploading dumped data to the remote Victoria metrics server; | ||
# | ||
The `dump` process runs batches in a loop, continually checking for available space/inodes to perform a dump. if it detects a lack of space, the dump cycle will wait until space become available. Once the dump successfully finishes current batch, the folder will be renamed to indicate it's ready for upload; | ||
|
||
1) When the `dump` process will complete all batches, it sends a `SIGUSR1` signal to the `restore` process and waits for the `restore` process to finish. | ||
2) Pressing `Ctrl+C` once increments an internal counter but doesn't take any immediate action. | ||
3) Pressing `Ctrl+C` twice will stop the `dump` process. The process will wait for all background `restore` tasks to finish before terminating. | ||
4) Pressing `Ctrl+C` three or more times will send stop signal `SIGTERM` to the background process, causing it to stop execution after completing the current upload task. | ||
# | ||
The `restore` process runs in the background, continously monitoring folders named with a 'ready' state marker and logging it's execution events to the 'background_log' file. | ||
|
||
1) During each loop cycle, the `restore` processes only one folder. | ||
2) If there are no new folders to process, it suspends execution and waits for a new folder to become available for upload (marker in the name). | ||
3) After a successfull restore operation, the folder is removed, freeing up space for use by the `dump` process. | ||
4) The `restore` process is prepared to capture a `Ctrl+C` or `SIGTERM` signal and terminate its loop execution after completing the current upload task. | ||
5) The `restore` process also listens for a `SIGUSR1` signal to gracefully finish it's loop execution. | ||
6) Upon receiving the `SIGUSR1` signal, the `restore` process assumes that the `dump` loop has finished. It processes and removes all ready folders and then terminates its execution. | ||
# |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
#!/bin/bash | ||
|
||
cd -- "$(dirname -- "$0")/.." || exit 1 | ||
|
||
python setup.py develop | ||
|
||
pip install pydevd-pycharm~=203.7148.57 | ||
|
||
echo | ||
echo Use \"kilda-otsdb-dump\" or \"kilda-otsdb-restore\" to run application | ||
echo | ||
|
||
exec bash -l |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
#!/bin/bash | ||
|
||
set -eux | ||
cd -- "$(dirname -- "$0")/.." || exit 1 | ||
|
||
python setup.py develop | ||
|
||
flake8 --max-line-length=119 src | ||
|
||
python setup.py bdist_wheel | ||
python setup.py --version > dist/version |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
#!/bin/sh | ||
|
||
set -eux | ||
|
||
echo "$@" | ||
|
||
DEVEL_NAME="${1}" | ||
DEVEL_UID="${2}" | ||
DEVEL_GID="${3}" | ||
|
||
if [ ${DEVEL_UID} -eq 0 ]; then | ||
# nothing to do for root user | ||
exit | ||
fi | ||
|
||
if [ -z "$(getent group "${DEVEL_GID}")" ]; then | ||
groupadd -g "${DEVEL_GID}" "${DEVEL_NAME}" | ||
fi | ||
|
||
if [ -z "$(getent passwd "${DEVEL_NAME}")" ]; then | ||
useradd -m -u "${DEVEL_UID}" -g "${DEVEL_GID}" -s /bin/bash "${DEVEL_NAME}" | ||
fi | ||
|
||
chown "${DEVEL_UID}:${DEVEL_GID}" \ | ||
/kilda \ | ||
/usr/local \ | ||
/usr/local/bin \ | ||
/usr/local/lib/python3.11/site-packages |
Oops, something went wrong.