add mimir support

.dockerignore

@@ -1,4 +1,5 @@
 .gitignore
-.idea/
-.vscode/
-data/
+/.idea/
+/.vscode/
+/data/
+/tmp/

.github/workflows/docker.yml

@@ -1,27 +1,45 @@
 name: CI
 on:
   push:
-    branches: [ master ]
+    branches: [master]
 
 jobs:
   build:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/setup-go@v2
-        with:
-          go-version: 1.15.1
+      - uses: actions/checkout@v4
 
-      - uses: actions/checkout@v2
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: sepa/thanos-kit
+          tags: |
+            type=raw,value={{date 'YYMMDD'}}
+            type=raw,value={{branch}}-{{sha}}
+          flavor: |
+            latest=true
 
-      - uses: actions/cache@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
         with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
+          username: sepa
+          password: ${{ secrets.DOCKER_PASSWORD }}
 
-      - run: make docker-push
-        env:
-          DOCKER_USERNAME: sepa
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v5
+        with:
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            VER=${{ env.DOCKER_METADATA_OUTPUT_VERSION }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.gitignore

@@ -1,4 +1,5 @@
-.idea/
-.vscode/
-thanos-kit
-data/
+/.idea/
+/.vscode/
+/thanos-kit
+/data/
+/tmp/

Dockerfile

@@ -1,13 +1,13 @@
-FROM golang:alpine as builder
+FROM golang:1.20 as builder
 WORKDIR /build
-RUN apk add --no-cache make git
 # try to cache deps
 COPY go.mod go.sum ./
 RUN go mod download -x
 # resets caches
 COPY . .
+ARG VER
 RUN make test && make build
 
 FROM quay.io/prometheus/busybox:latest
 COPY --from=builder /build/thanos-kit /bin/thanos-kit
-ENTRYPOINT [ "/bin/thanos-kit" ]
+ENTRYPOINT ["/bin/thanos-kit"]

Makefile

@@ -1,21 +1,11 @@
-DOCKER_IMAGE ?= 'sepa/thanos-kit'
 VER ?= `git show -s --format=%cd-%h --date=format:%y%m%d`
-OS ?= $(shell uname -s | tr '[A-Z]' '[a-z]')
-ARCH ?= $(shell uname -m)
 
 help: ## Displays help
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n\nTargets:\n"} /^[a-z0-9A-Z_-]+:.*?##/ { printf "  \033[36m%-10s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST)
 
-.PHONY: deps
-deps: ## Ensures fresh go.mod and go.sum
-	@go mod tidy
-	@go mod verify
-
-.PHONY: test
 test: ## Run unit tests
 	@go test ./...
 
-.PHONY: build
 build: ## Build binaries with version set
 	@CGO_ENABLED=0 go build -ldflags "-w -s \
 	-X github.com/prometheus/common/version.Version=${VER} \
@@ -24,20 +14,5 @@ build: ## Build binaries with version set
 	-X github.com/prometheus/common/version.BuildUser=${USER}@`hostname` \
 	-X github.com/prometheus/common/version.BuildDate=`date +%Y%m%d-%H:%M:%S`"
 
-.PHONY: docker
-ifeq ($(OS)_$(ARCH), linux_x86_64)
-docker: test build
-	@docker build -t "thanos-kit" -f ci.dockerfile .
-else
 docker: ## Builds 'thanos-kit' docker with no tag
 	@docker build -t "thanos-kit" .
-endif
-
-.PHONY: docker-push
-docker-push: docker # CI only
-	@echo "Pushing ver: ${VER}"
-ifneq (${DOCKER_PASSWORD},)
-	@echo "${DOCKER_PASSWORD}" | docker login -u="${DOCKER_USERNAME}" --password-stdin
-	@docker tag thanos-kit ${DOCKER_IMAGE}:latest && docker push ${DOCKER_IMAGE}:latest
-endif
-	@docker tag thanos-kit ${DOCKER_IMAGE}:${VER} && docker push ${DOCKER_IMAGE}:${VER}

README.md

@@ -1,80 +1,73 @@
 # thanos-kit
-Aftermarket thanos tools and utilities.  
+Tooling to work with Thanos blocks in object storage.
 
-- `inspect` - Inspect all blocks in the bucket in detailed, table-like way (same as `thanos tools bucket inspect`)
-- `analyze` - Analyze churn, label pair cardinality for specific block
-- `dump` - Dump samples from a TSDB to text format
-- `import` - Import samples to TSDB blocks. Read about [Backfill](#backfill) below
+- **ls** - List all blocks ULIDs in the bucket, also show ULID as time (same as `thanos tools bucket ls` but with mimir support)
+- **inspect** - Inspect all blocks in the bucket in detailed, table-like way (same as `thanos tools bucket inspect` but with mimir support)
+- **analyze** - Analyze churn, label pair cardinality for specific block. (same as `promtool tsdb analyze` but also show Labels suitable for block split)
+- **dump** - Dump samples from a TSDB to text format (same as `promtool tsdb dump` but to promtext format)
+- **import** - Import samples to TSDB blocks (same as `promtool tsdb create-blocks-from openmetrics` but from promtext format). Read more about [backfill](#backfill) below
 
-Cli arguments mostly are the same as for `thanos`, help is available for each sub-command:  
+Cli arguments are mostly the same as for `thanos`, help is available for each sub-command:
 ```
-$ ./thanos-kit -h
+$ docker run sepa/thanos-kit -h
 usage: thanos-kit [<flags>] <command> [<args> ...]
 
-Tooling to work with Thanos blobs in object storage
+Tooling to work with Thanos blocks in object storage
 
 Flags:
   -h, --help            Show context-sensitive help (also try --help-long and --help-man).
       --version         Show application version.
       --log.level=info  Log filtering level (info, debug)
+      --objstore.config-file=<file-path>
+                        Path to YAML file that contains object store%s configuration. See format details: https://thanos.io/tip/thanos/storage.md/
+      --objstore.config=<content>
+                        Alternative to 'objstore.config-file' flag (mutually exclusive). Content of YAML file that contains object store%s configuration. See format details: https://thanos.io/tip/thanos/storage.md/
 
 Commands:
   help [<command>...]
     Show help.
 
+  ls [<flags>]
+    List all blocks in the bucket.
+
   inspect [<flags>]
     Inspect all blocks in the bucket in detailed, table-like way
 
   analyze [<flags>] <ULID>
-    Analyze churn, label pair cardinality
+    Analyze churn, label pair cardinality and find labels to split on
 
   dump [<flags>] <ULID>...
-    Dump samples from a TSDB
+    Dump samples from a TSDB to text
 
-  import --label=<name>="<value>" [<flags>]
-    Import samples to TSDB blocks
+  import --input-file=INPUT-FILE --label=<name>="<value>" [<flags>]
+    Import samples from text to TSDB blocks
 ```
 
 ### Get it
-Master builds are available on [Docker Hub](https://hub.docker.com/repository/docker/sepa/thanos-kit/tags)
+Docker images are available on [Docker Hub](https://hub.docker.com/repository/docker/sepa/thanos-kit/tags)
 
 ### Backfill
-
 ([Original PR](https://github.com/prometheus/prometheus/pull/7586))  
-You can use it to restore another prometheus partial dump, or any metrics exported from any system. The only supported input format is Prometheus text format.
+Supported input format is Prometheus text format.
 You are free to export/convert your existing data to this format, into one **time-sorted** text file.
 
-Sample file `rrd_exported_data.txt` (`[metric]{[labels]} [number value] [timestamp ms]`):
+`metric{[labels]} value timestamp_ms`
+
+For example:
 ```ini
-collectd_df_complex{host="myserver.fqdn.com",df="var-log",dimension="free"} 5.8093906125e+10 1599771600000
-collectd_varnish_derive{host="myserver.fqdn.com",varnish="request_rate",dimension="MAIN.client_req"} 2.3021666667e+01 1599771600000
-collectd_df_complex{host="myserver.fqdn.com",df="var-log",dimension="free"} 5.8093906125e+10 1599771615000
-collectd_load{host="myserver.fqdn.com",type="midterm"} 0.0155 1599771615000
-collectd_varnish_derive{host="myserver.fqdn.com",varnish="request_rate",dimension="MAIN.client_req"} 2.3021666667e+01 1599771630000
-collectd_load{host="myserver.fqdn.com",type="midterm"} 1.5500000000e-02 1599771630000
-collectd_varnish_derive{host="myserver.fqdn.com",varnish="request_rate",dimension="MAIN.cache_hit"} 3.8054166667e+01 1599771645000
-collectd_df_complex{host="myserver.fqdn.com",df="var-log",dimension="free"} 5.8093906125e+10 1599771645000
-collectd_varnish_derive{host="myserver.fqdn.com",varnish="request_rate",dimension="MAIN.s_pipe"} 0 1599771660000
-collectd_varnish_derive{host="myserver.fqdn.com",varnish="request_rate",dimension="MAIN.cache_hit"} 3.8054166667e+01 1599771675000
-collectd_load{host="myserver.fqdn.com",type="shortterm"} 1.1000000000e-02 1599771675000
-collectd_varnish_derive{host="myserver.fqdn.com",varnish="request_rate",dimension="MAIN.client_req"} 2.3021666667e+01 1599771690000
-collectd_load{host="myserver.fqdn.com",type="shortterm"} 1.1000000000e-02 1599771690000
-collectd_load{host="myserver.fqdn.com",type="shortterm"} 1.1000000000e-02 1599771705000
-collectd_load{host="myserver.fqdn.com",type="longterm"} 2.5500000000e-02 1599771720000
-collectd_varnish_derive{host="myserver.fqdn.com",varnish="request_rate",dimension="MAIN.s_pipe"} 0 1599771735000
-collectd_load{host="myserver.fqdn.com"type="longterm"} 2.5500000000e-02 1599771735000
-collectd_varnish_derive{host="myserver.fqdn.com",varnish="request_rate",dimension="MAIN.cache_hit"} 3.8054166667e+01 1599771750000
-collectd_load{host="myserver.fqdn.com",type="midterm"} 1.5500000000e-02 1599771750000
+k8s_ns_hourly_cost{namespace="kube-system"} 5.7 1599771600000
+k8s_ns_hourly_cost{namespace="kube-system"} 5.0 1599771630000
+...
 ```
 
-Note, `[number value]` can be mixed as normal or scientific number as per your preference. You are free to put custom labels on each metric, don't forget that "relabelling rules" defined in prometheus will not be applied on them! You should produce the final labels on your import file.
+Note, `value` can be mixed as normal or scientific number as per your preference.
 
 This format is simple to produce, but not optimized or compressed, so it's normal if your data file is huge.  
-Example of a 19G OpenMetrics file, with ~20k timeseries and 200M data points (samples) on 2y period. Globally resolution is very very low in this example. 
-Import will take around 2h and uncompacted new TSDB blocks will be around 2.1G for 7600 blocks. When prometheus scan them, it starts automatically compacting them in the background. Once compaction is completed (~30min), TSDB blocks will be around 970M for 80 blocks (without loss of data points).  
+Example of a 19G OpenMetrics file, with ~20k timeseries and 200M data points (samples) on 2y period. Globally resolution is very low in this example.
+Import will take around 2h and uncompacted new TSDB blocks will be around 2.1G for 7600 blocks. When thanos-compact scan them, it starts automatically compacting them in the background. Once compaction is completed (~30min), TSDB blocks will be around 970M for 80 blocks.  
 The size, and number of blocks depends on timeseries numbers and metrics resolution, but it gives you an order of sizes.
 
-Apart from labels set for each metric in text file, you would also need to set Thanos Metadata Labels for the whole batch of blocks you are importing (consider this as prometheus `external_labels` which scraped the metrics from text file)
+Apart from labels set for each metric in text file, you would also need to set Thanos Metadata Labels for the whole batch of blocks you are importing (consider this as prometheus `external_labels` which scraped the metrics from the text file)
 
 Example of command for importing data from `data.prom` (above) to GCS bucket `bucketname`:
 ```bash
@@ -87,4 +80,14 @@ docker run -it --rm \
         --label=replica=\"prom-a\" \ 
         --label=location=\"us-east1\"
 ```
-Current implementation reads all imported data into memory, and then flushes as TSDB 2h blocks. In case you have OOM but want to import large amount of data, you can split imported file by days (or 2h windows) and do multiple separate imports. Anyway these separate blocks would be compacted to larger one on bucket side via your compactor. Please note that compactor has default `--consistency-delay=30m` which is based on file upload time (not ULID), so it could take some time before compactor would start processing these blocks.
+Please note that compactor has default `--consistency-delay=30m` which is based on file upload time (not ULID), so it could take some time before compactor would start processing these blocks.
+
+### Cache dir
+By default, `thanos-kit` will cache blocks from object storage to `./data` directory, and the dir is not cleaned up on exit. This is to speed up subsequent runs, and to avoid deleting user data when `--data-dir=/tmp` is used for example.
+
+Important note that `dump` command downloads specified blocks to cache dir, but then dump TSDB as a whole (including blocks already present there)
+
+### Alternatives
+- [thanos tools bucket](https://thanos.io/tip/components/tools.md/#bucket)
+- [promtool tsdb](https://prometheus.io/docs/prometheus/latest/command-line/promtool/#promtool-tsdb)
+- [mimirtool](https://grafana.com/docs/mimir/latest/manage/tools/mimirtool/#backfill)