Replace markblocks with rewritten mark-blocks

CHANGELOG.md

@@ -98,6 +98,7 @@
 ### Tools
 
 * [CHANGE] `copyblocks`: Remove /pprof endpoint. #10329
+* [CHANGE] `mark-blocks`: Replace `markblocks` with added features including removing markers and reading block identifiers from a file. #10597
 
 ## 2.15.0
 

Makefile

@@ -539,9 +539,9 @@ dist: ## Generates binaries for a Mimir release.
 			echo "Building mimir-continuous-test for $$os/$$arch"; \
 			GOOS=$$os GOARCH=$$arch CGO_ENABLED=0 go build $(GO_FLAGS) -o ./dist/mimir-continuous-test-$$os-$$arch$$suffix ./cmd/mimir-continuous-test; \
 			sha256sum ./dist/mimir-continuous-test-$$os-$$arch$$suffix | cut -d ' ' -f 1 > ./dist/mimir-continuous-test-$$os-$$arch$$suffix-sha-256; \
-			echo "Building markblocks for $$os/$$arch"; \
-			GOOS=$$os GOARCH=$$arch CGO_ENABLED=0 go build $(GO_FLAGS) -o ./dist/markblocks-$$os-$$arch$$suffix ./tools/markblocks; \
-			sha256sum ./dist/markblocks-$$os-$$arch$$suffix | cut -d ' ' -f 1 > ./dist/markblocks-$$os-$$arch$$suffix-sha-256; \
+			echo "Building mark-blocks for $$os/$$arch"; \
+			GOOS=$$os GOARCH=$$arch CGO_ENABLED=0 go build $(GO_FLAGS) -o ./dist/mark-blocks-$$os-$$arch$$suffix ./tools/mark-blocks; \
+			sha256sum ./dist/mark-blocks-$$os-$$arch$$suffix | cut -d ' ' -f 1 > ./dist/mark-blocks-$$os-$$arch$$suffix-sha-256; \
 			done; \
 		done; \
 		touch $@

development/tools/upload-block/upload-block.sh

@@ -31,21 +31,21 @@ function main() {
 
   echo "Block ULID is $BLOCK_ULID."
   echo "Uploading no-compact marker..."
-  markblocks \
+  mark_blocks \
     -tenant="$TENANT" \
-    -mark=no-compact \
+    -mark-type=no-compact \
     -details="block uploaded for debugging purposes" \
-    -skip-existence-check=true \
-    "$BLOCK_ULID"
+    -meta-presence-policy=none \
+    -blocks "$BLOCK_ULID"
 
   echo "Uploading block contents..."
   aws_with_creds s3 cp --recursive "$BLOCK_DIR" "s3://$S3_BUCKET_NAME/$TENANT/$BLOCK_ULID"
 
   echo "Done."
 }
 
-function markblocks() {
-  go run "$SCRIPT_DIR/../../../tools/markblocks" \
+function mark_blocks() {
+  go run "$SCRIPT_DIR/../../../tools/mark-blocks" \
     -backend="s3" \
     -s3.access-key-id="$AWS_ACCESS_KEY_ID" \
     -s3.secret-access-key="$AWS_SECRET_ACCESS_KEY" \

docs/internal/tools/mark-blocks.md

@@ -0,0 +1,80 @@
+# mark-blocks
+
+`mark-blocks` is a tool that assists in uploading or removing block markers to a specified backend.
+Marks are uploaded to both the block folder and the global marks folder for the provided tenant.
+
+Two types of marks can be created, depending on the `-mark-type` flag provided:
+
+## `deletion` mark
+
+When `-mark-type deletion` is provided, `mark-blocks` uploads a `DeletionMark` which tells the compactor that the block provided should be deleted.
+This is a **destructive operation** (although it doesn't happen immediately, the default delation delay is 12h, see `-compactor.deletion-delay` value), proceed with caution.
+
+### Example
+
+```
+$ mkdir -p tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS2R tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS7Z
+
+$ touch tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS2R/meta.json
+
+$ tree -f tenant-1
+tenant-1
+├── tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS2R
+│   └── tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS2R/meta.json
+└── tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS7Z
+
+3 directories, 1 file
+
+$ go run ./tools/mark-blocks -mark-type deletion -tenant tenant-1 -details "Corrupted blocks" -blocks "01FSCTA0A4M1YQHZQ4B2VTGS2R,01FSCTA0A4M1YQHZQ4B2VTGS7Z"
+level=info time=2025-02-10T14:23:17.594517Z msg="skipping block because its meta.json was not found: 01FSCTA0A4M1YQHZQ4B2VTGS7Z"
+level=info time=2025-02-10T14:23:17.594747Z msg="uploaded mark to tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS2R/deletion-mark.json"
+level=info time=2025-02-10T14:23:17.594945Z msg="uploaded mark to tenant-1/markers/01FSCTA0A4M1YQHZQ4B2VTGS2R-deletion-mark.json"
+
+$ tree -f tenant-1
+tenant-1
+├── tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS2R
+│   ├── tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS2R/deletion-mark.json
+│   └── tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS2R/meta.json
+├── tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS7Z
+└── tenant-1/markers
+    └── tenant-1/markers/01FSCTA0A4M1YQHZQ4B2VTGS2R-deletion-mark.json
+
+4 directories, 3 files
+```
+
+## `no-compact` mark
+
+When `-mark-type no-compact` is provided, this tool uploads a `NoCompactMark` which tells the compactor that the marked blocks should not be compacted.
+`-details` flag can be used to provide an explanation of why the block is marked as such (who marked it? issue link?).
+
+### Example
+
+```
+$ mkdir -p tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS2R tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS7Z
+
+$ touch tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS2R/meta.json
+
+$ tree -f tenant-1
+tenant-1
+├── tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS2R
+│   └── tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS2R/meta.json
+└── tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS7Z
+
+3 directories, 1 file
+
+$ go run ./tools/mark-blocks -mark-type no-compact -tenant tenant-1 -details "Blocks with out of order chunks" -blocks "01FSCTA0A4M1YQHZQ4B2VTGS2R,01FSCTA0A4M1YQHZQ4B2VTGS7Z"
+level=info time=2025-02-10T14:27:46.232169Z msg="skipping block because its meta.json was not found: 01FSCTA0A4M1YQHZQ4B2VTGS7Z"
+level=info time=2025-02-10T14:27:46.232419Z msg="uploaded mark to tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS2R/no-compact-mark.json"
+level=info time=2025-02-10T14:27:46.232621Z msg="uploaded mark to tenant-1/markers/01FSCTA0A4M1YQHZQ4B2VTGS2R-no-compact-mark.json"
+
+$ tree -f tenant-1
+tenant-1
+├── tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS2R
+│   ├── tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS2R/meta.json
+│   └── tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS2R/no-compact-mark.json
+├── tenant-1/01FSCTA0A4M1YQHZQ4B2VTGS7Z
+└── tenant-1/markers
+    └── tenant-1/markers/01FSCTA0A4M1YQHZQ4B2VTGS2R-no-compact-mark.json
+
+4 directories, 3 files
+```

docs/sources/mimir/manage/mimir-runbooks/_index.md

@@ -614,7 +614,7 @@ How to **investigate**:
         - Find source blocks for the compaction job: search for `msg="compact blocks"` and a mention of the result block ID.
         - Mark the source blocks for no compaction (in this example the object storage backend is GCS):
           ```
-          ./tools/markblocks/markblocks -backend gcs -gcs.bucket-name <bucket> -mark no-compact -tenant <tenant-id> -details "Leading to out-of-order chunks when compacting with other blocks" <block-1> <block-2>...
+          ./tools/mark-blocks/mark-blocks -backend gcs -gcs.bucket-name <bucket> -mark-type no-compact -tenant <tenant-id> -details "Leading to out-of-order chunks when compacting with other blocks" -blocks "<block-1>,<block-2>..."
           ```
   - Result block exceeds symbol table maximum size:
     - **How to detect**: Search compactor logs for `symbol table size exceeds`.
@@ -629,7 +629,7 @@ How to **investigate**:
         Where the filenames are the block IDs: `01GZS91PMTAWAWAKRYQVNV1FPP` and `01GZSC5803FN1V1ZFY6Q8PWV1E`
       - Mark the source blocks for no compaction (in this example the object storage backend is GCS):
         ```
-        ./tools/markblocks/markblocks -backend gcs -gcs.bucket-name <bucket> -mark no-compact -tenant <tenant-id> -details "Result block exceeds symbol table maximum size" <block-1> <block-2>...
+        ./tools/mark-blocks/mark-blocks -backend gcs -gcs.bucket-name <bucket> -mark-type no-compact -tenant <tenant-id> -details "Result block exceeds symbol table maximum size" -blocks "<block-1>,<block-2>..."
         ```
     - Further reading: [Compaction algorithm]({{< relref "../../references/architecture/components/compactor#compaction-algorithm" >}}).
   - Compactor network disk unresponsive:
@@ -668,7 +668,7 @@ How to **fix** it:
 - The only long-term solution is to give the compactor more disk space, as it requires more space to fit the largest single job into its disk.
 - If the number of blocks that the compactor is failing to compact is not very significant and you want to skip compacting them and focus on more recent blocks instead, consider marking the affected blocks for no compaction:
   ```
-  ./tools/markblocks/markblocks -backend gcs -gcs.bucket-name <bucket> -mark no-compact -tenant <tenant-id> -details "focus on newer blocks"
+  ./tools/mark-blocks/mark-blocks -backend gcs -gcs.bucket-name <bucket> -mark-type no-compact -tenant <tenant-id> -details "focus on newer blocks" -blocks "<block 1>,<block 2>..."
   ```
 
 ### MimirCompactorSkippedUnhealthyBlocks

tools/copyblocks/README.md

@@ -1,4 +1,4 @@
-# Copyblocks
+# `copyblocks`
 
 This program can copy Mimir blocks between two buckets.
 
@@ -18,7 +18,7 @@ The currently supported services are Amazon Simple Storage Service (S3 and S3-co
 
 ## Running
 
-Running `go build` in this directory builds the program. Then use an example below as a guide.
+Run `go build` in this directory to build the program. Then, use an example below as a guide.
 
 ### Example for Google Cloud Storage
 

tools/copyprefix/README.md

@@ -1,4 +1,4 @@
-# Copyprefix
+# `copyprefix`
 
 This program copies objects by prefix between two buckets. If your infrastructure is largely on a single object storage service their respective CLI tool may be more convenient to use.
 
@@ -15,7 +15,7 @@ The currently supported services are Amazon Simple Storage Service (S3 and S3-co
 
 ## Running
 
-Running `go build` in this directory builds the program. Then use an example below as a guide.
+Run `go build` in this directory to build the program. Then, use an example below as a guide.
 
 ### Example for Google Cloud Storage
 

tools/mark-blocks/.gitignore

@@ -0,0 +1 @@
+mark-blocks

tools/mark-blocks/README.md

@@ -0,0 +1,63 @@
+# `mark-blocks`
+
+This program creates or removes markers for blocks.
+
+## Flags
+
+- `--tenant` (required) The tenant that owns the blocks
+- `--mark-type` (required) Mark type to create or remove, valid options: `deletion`, `no-compact`
+- `--blocks` (optional) Comma separated list of blocks IDs. If non-empty, `--blocks-file` is ignored
+- `--blocks-file` (optional) File containing a block ID per-line. Defaults to standard input (`-`). Ignored if `--blocks` is non-empty
+- `--meta-presence-policy` (optional) Policy on presence of block `meta.json` files: `none`, `skip-block`, or `require`. Defaults to `skip-block`
+- `--remove` (optional) If marks should be removed rather than uploaded. Defaults to `false`
+- `--resume-index` (optional) The index of the block to resume from. This index is logged to assist in recovering from partial failures
+- `--concurrency` (optional) How many markers to upload or remove concurrently. Defaults to `16`
+- `--details` (optional) Details to include in an added mark
+- `--dry-run` (optional) Log changes that would be made instead of actually making them
+
+Each supported object storage service also has an additional set of flags (see examples in [Running](##Running)).
+
+## Running
+
+Run `go build` in this directory to build the program. Then, use an example below as a guide.
+
+### Example for Google Cloud Storage
+
+```bash
+./mark-blocks \
+  --tenant <tenant>
+  --blocks <blocks>
+  --mark-type <mark-type>
+  --backend gcs \
+  --gcs.bucket-name <bucket name> \
+  --dry-run
+```
+
+### Example for Azure Blob Storage
+
+```bash
+./mark-blocks \
+  --tenant <tenant>
+  --blocks <blocks>
+  --mark-type <mark-type>
+  --backend azure \
+  --azure.container-name <container name> \
+  --azure.account-name <account name> \
+  --azure.account-key <account key> \
+  --dry-run
+```
+
+### Example for Amazon Simple Storage Service
+
+```bash
+./mark-blocks\
+  --tenant <tenant>
+  --blocks <blocks>
+  --mark-type <mark-type>
+  --backend s3 \
+  --s3.bucket-name <bucket name> \
+  --s3.access-key-id <access key id> \
+  --s3.secret-access-key <secret access key> \
+  --s3.endpoint <endpoint> \
+  --dry-run
+```