Merge branch 'main' into chasmArchetypeSearchAttributes

Author: awln-temporal

.dockerignore

@@ -14,3 +14,4 @@ tdbg
 temporal-server
 temporal-cassandra-tool
 temporal-sql-tool
+temporal-elasticsearch-tool

.github/.golangci.yml

@@ -128,6 +128,9 @@ linters:
         - name: function-result-limit
           arguments:
             - 5
+        - name: struct-tag
+          arguments:
+            - "validate,persistence_custom_search_attributes"
         - name: unhandled-error
           arguments:
             - "fmt.*"

.github/CODEOWNERS

@@ -1,4 +1,4 @@
 # Syntax is here:
 # https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax
 
-*                @temporalio/server
+*                @temporalio/server @temporalio/cgs

.github/workflows/flaky-tests-report.yml

@@ -0,0 +1,100 @@
+name: Flaky Tests Report
+
+on:
+  schedule:
+    # Run on Wednesdays at noon Eastern time (5 PM UTC)
+    - cron: '0 17 * * 3'
+  workflow_dispatch:
+    inputs:
+      days:
+        description: 'Number of days to look back for flaky tests'
+        required: false
+        default: '7'
+        type: string
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  flaky-tests-report:
+    runs-on: ubuntu-latest    
+    steps:
+      - name: Generate token
+        id: generate_token
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ secrets.TEMPORAL_CICD_APP_ID }}
+          private-key: ${{ secrets.TEMPORAL_CICD_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: ">=0.8.0"
+
+      - name: Install Python dependencies
+        run: |
+          cd tools/flakes
+          uv sync
+
+      - name: Install tringa
+        run: |
+          uv tool install git+https://github.com/dandavison/tringa@main --force
+
+      - name: Run tringa
+        id: generate-output-file
+        env:
+          GH_TOKEN: ${{ steps.generate_token.outputs.token }}
+          DAYS_PARAM: ${{ github.event.inputs.days || '7' }}
+        run: |
+          set -euo pipefail
+          
+          # Create output directory
+          mkdir -p tools/flakes/out
+          
+          tringa --json --since-days "$DAYS_PARAM" repo sql \
+            'select classname, name, count(*) as failure_count, max(artifact) as artifact from test where passed = false and skipped = false group by classname, name order by failure_count desc' \
+            --branch main \
+            --workflow-id 80591745 \
+            https://github.com/temporalio/temporal > tools/flakes/out/out.json
+          
+          echo "✅ Tringa command completed"
+          echo "📊 Output file size: $(wc -c < tools/flakes/out/out.json) bytes"
+          echo "📄 Full output file:"
+          cat tools/flakes/out/out.json
+
+      - name: Run Python script to process flaky tests
+        id: process-flaky-tests
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+          RUN_ID: ${{ github.run_id }}
+          REF_NAME: ${{ github.ref_name }}
+          SHA: ${{ github.sha }}
+        run: |
+          set -x
+          cd tools/flakes && uv run main.py \
+            --file out/out.json \
+            --github-summary \
+            --slack-webhook "$SLACK_WEBHOOK" \
+            --run-id "$RUN_ID" \
+            --ref-name "$REF_NAME" \
+            --sha "$SHA"
+
+      - name: Upload generated reports
+        uses: actions/upload-artifact@v4
+        if: steps.process-flaky-tests.outcome == 'success'
+        with:
+          name: flaky-tests-reports-${{ github.run_number }}
+          path: tools/flakes/out/*
+          retention-days: 30

.github/workflows/run-tests.yml

@@ -252,7 +252,7 @@ jobs:
           key: go-${{ runner.os }}${{ runner.arch }}-build-${{ env.COMMIT }}
 
       - name: Run unit tests
-        timeout-minutes: 15
+        timeout-minutes: 20
         run: make unit-test-coverage
         env:
           UNIT_TEST_DIRS: ${{ inputs.unit_test_directory }}

.gitignore

@@ -41,3 +41,6 @@
 .spr.yml
 
 /proto.tmp
+
+**/.venv/
+**/.ruff_cache/

.goreleaser.yml

@@ -8,6 +8,7 @@ archives:
       - temporal-server
       - temporal-cassandra-tool
       - temporal-sql-tool
+      - temporal-elasticsearch-tool
     name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}"
     format_overrides:
       - goos: windows
@@ -52,6 +53,18 @@ builds:
     goarch:
       - amd64
       - arm64
+  - id: temporal-elasticsearch-tool
+    dir: cmd/tools/elasticsearch
+    binary: temporal-elasticsearch-tool
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - darwin
+      - windows
+    goarch:
+      - amd64
+      - arm64
   - id: tdbg
     dir: cmd/tools/tdbg
     binary: tdbg

Makefile

@@ -3,7 +3,7 @@
 install: bins
 
 # Rebuild binaries (used by Dockerfile).
-bins: temporal-server temporal-cassandra-tool temporal-sql-tool tdbg
+bins: temporal-server temporal-cassandra-tool temporal-sql-tool temporal-elasticsearch-tool tdbg
 
 # Install all tools, recompile proto files, run all possible checks and tests (long but comprehensive).
 all: clean proto bins check test
@@ -270,6 +270,9 @@ $(STAMPDIR)/protoc-gen-go-helpers-$(GO_API_VER): | $(STAMPDIR) $(LOCALBIN)
 	@touch $@
 $(PROTOC_GEN_GO_HELPERS): $(STAMPDIR)/protoc-gen-go-helpers-$(GO_API_VER)
 
+$(LOCALBIN)/protoc-gen-go-chasm: $(LOCALBIN) cmd/tools/protoc-gen-go-chasm/main.go go.mod go.sum
+	@go build -o $@ ./cmd/tools/protoc-gen-go-chasm
+
 # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
 # $1 - target path with name of binary (ideally with version)
 # $2 - package url which can be installed
@@ -300,7 +303,7 @@ $(CHASM_BINPB): $(API_BINPB) $(INTERNAL_BINPB) $(CHASM_PROTO_FILES)
 	@printf $(COLOR) "Generate CHASM proto image..."
 	@protoc --descriptor_set_in=$(API_BINPB):$(INTERNAL_BINPB) -I=. $(CHASM_PROTO_FILES) -o $@
 
-protoc: $(PROTOGEN) $(MOCKGEN) $(GOIMPORTS) $(PROTOC_GEN_GO) $(PROTOC_GEN_GO_GRPC) $(PROTOC_GEN_GO_HELPERS) $(API_BINPB)
+protoc: $(PROTOGEN) $(MOCKGEN) $(GOIMPORTS) $(PROTOC_GEN_GO) $(PROTOC_GEN_GO_GRPC) $(PROTOC_GEN_GO_HELPERS) $(API_BINPB) $(LOCALBIN)/protoc-gen-go-chasm
 	@go run ./cmd/tools/protogen \
 		-root=$(ROOT) \
 		-proto-out=$(PROTO_OUT) \
@@ -309,6 +312,7 @@ protoc: $(PROTOGEN) $(MOCKGEN) $(GOIMPORTS) $(PROTOC_GEN_GO) $(PROTOC_GEN_GO_GRP
 		-protogen-bin=$(PROTOGEN) \
 		-goimports-bin=$(GOIMPORTS) \
 		-mockgen-bin=$(MOCKGEN) \
+		-protoc-gen-go-chasm-bin=$(LOCALBIN)/protoc-gen-go-chasm \
 		-protoc-gen-go-bin=$(PROTOC_GEN_GO) \
 		-protoc-gen-go-grpc-bin=$(PROTOC_GEN_GO_GRPC) \
 		-protoc-gen-go-helpers-bin=$(PROTOC_GEN_GO_HELPERS) \
@@ -334,6 +338,7 @@ clean-bins:
 	@rm -f temporal-cassandra-tool
 	@rm -f tdbg
 	@rm -f temporal-sql-tool
+	@rm -f temporal-elasticsearch-tool
 
 temporal-server: $(ALL_SRC)
 	@printf $(COLOR) "Build temporal-server with CGO_ENABLED=$(CGO_ENABLED) for $(GOOS)/$(GOARCH)..."
@@ -351,6 +356,10 @@ temporal-sql-tool: $(ALL_SRC)
 	@printf $(COLOR) "Build temporal-sql-tool with CGO_ENABLED=$(CGO_ENABLED) for $(GOOS)/$(GOARCH)..."
 	CGO_ENABLED=$(CGO_ENABLED) go build $(BUILD_TAG_FLAG) -o temporal-sql-tool ./cmd/tools/sql
 
+temporal-elasticsearch-tool: $(ALL_SRC)
+	@printf $(COLOR) "Build temporal-elasticsearch-tool with CGO_ENABLED=$(CGO_ENABLED) for $(GOOS)/$(GOARCH)..."
+	CGO_ENABLED=$(CGO_ENABLED) go build $(BUILD_TAG_FLAG) -o temporal-elasticsearch-tool ./cmd/tools/elasticsearch
+
 temporal-server-debug: $(ALL_SRC)
 	@printf $(COLOR) "Build temporal-server-debug with CGO_ENABLED=$(CGO_ENABLED) for $(GOOS)/$(GOARCH)..."
 	CGO_ENABLED=$(CGO_ENABLED) go build $(BUILD_TAG_FLAG),TEMPORAL_DEBUG -o temporal-server-debug ./cmd/server
@@ -519,22 +528,17 @@ install-schema-postgresql12: temporal-sql-tool
 	./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) -p 5432 --pl postgres12 --db $(VISIBILITY_DB) setup-schema -v 0.0
 	./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) -p 5432 --pl postgres12 --db $(VISIBILITY_DB) update-schema -d ./schema/postgresql/v12/visibility/versioned
 
-install-schema-es:
+install-schema-es: temporal-elasticsearch-tool
 	@printf $(COLOR) "Install Elasticsearch schema..."
-	curl --fail -X PUT "http://127.0.0.1:9200/_cluster/settings" -H "Content-Type: application/json" --data-binary @./schema/elasticsearch/visibility/cluster_settings_v7.json --write-out "\n"
-	curl --fail -X PUT "http://127.0.0.1:9200/_template/temporal_visibility_v1_template" -H "Content-Type: application/json" --data-binary @./schema/elasticsearch/visibility/index_template_v7.json --write-out "\n"
-# No --fail here because create index is not idempotent operation.
-	curl -X PUT "http://127.0.0.1:9200/temporal_visibility_v1_dev" --write-out "\n"
-# curl -X PUT "http://127.0.0.1:9200/temporal_visibility_v1_secondary" --write-out "\n"
+	./temporal-elasticsearch-tool -ep http://127.0.0.1:9200 setup-schema
+	./temporal-elasticsearch-tool -ep http://127.0.0.1:9200 create-index --index temporal_visibility_v1_dev
 
-install-schema-es-secondary:
+install-schema-es-secondary: temporal-elasticsearch-tool
 	@printf $(COLOR) "Install Elasticsearch schema..."
-	curl --fail -X PUT "http://127.0.0.1:8200/_cluster/settings" -H "Content-Type: application/json" --data-binary @./schema/elasticsearch/visibility/cluster_settings_v7.json --write-out "\n"
-	curl --fail -X PUT "http://127.0.0.1:8200/_template/temporal_visibility_v1_template" -H "Content-Type: application/json" --data-binary @./schema/elasticsearch/visibility/index_template_v7.json --write-out "\n"
-# No --fail here because create index is not idempotent operation.
-	curl -X PUT "http://127.0.0.1:8200/temporal_visibility_v1_secondary" --write-out "\n"
+	./temporal-elasticsearch-tool -ep http://127.0.0.1:8200 setup-schema
+	./temporal-elasticsearch-tool -ep http://127.0.0.1:8200 create-index --index temporal_visibility_v1_secondary
 
-install-schema-xdc: temporal-cassandra-tool
+install-schema-xdc: temporal-cassandra-tool temporal-elasticsearch-tool
 	@printf $(COLOR)  "Install Cassandra schema (active)..."
 	./temporal-cassandra-tool drop -k temporal_cluster_a -f
 	./temporal-cassandra-tool create -k temporal_cluster_a --rf 1
@@ -554,15 +558,15 @@ install-schema-xdc: temporal-cassandra-tool
 	./temporal-cassandra-tool -k temporal_cluster_c update-schema -d ./schema/cassandra/temporal/versioned
 
 	@printf $(COLOR) "Install Elasticsearch schemas..."
-	curl --fail -X PUT "http://127.0.0.1:9200/_cluster/settings" -H "Content-Type: application/json" --data-binary @./schema/elasticsearch/visibility/cluster_settings_v7.json --write-out "\n"
-	curl --fail -X PUT "http://127.0.0.1:9200/_template/temporal_visibility_v1_template" -H "Content-Type: application/json" --data-binary @./schema/elasticsearch/visibility/index_template_v7.json --write-out "\n"
-# No --fail here because create index is not idempotent operation.
-	curl -X DELETE http://localhost:9200/temporal_visibility_v1_dev_cluster_a
-	curl -X DELETE http://localhost:9200/temporal_visibility_v1_dev_cluster_b
-	curl -X DELETE http://localhost:9200/temporal_visibility_v1_dev_cluster_c
-	curl -X PUT "http://127.0.0.1:9200/temporal_visibility_v1_dev_cluster_a" --write-out "\n"
-	curl -X PUT "http://127.0.0.1:9200/temporal_visibility_v1_dev_cluster_b" --write-out "\n"
-	curl -X PUT "http://127.0.0.1:9200/temporal_visibility_v1_dev_cluster_c" --write-out "\n"
+	./temporal-elasticsearch-tool -ep http://127.0.0.1:9200 setup-schema
+# Delete indices if they exist (drop-index fails silently if index doesn't exist)
+	./temporal-elasticsearch-tool -ep http://127.0.0.1:9200 drop-index --index temporal_visibility_v1_dev_cluster_a --fail
+	./temporal-elasticsearch-tool -ep http://127.0.0.1:9200 drop-index --index temporal_visibility_v1_dev_cluster_b --fail
+	./temporal-elasticsearch-tool -ep http://127.0.0.1:9200 drop-index --index temporal_visibility_v1_dev_cluster_c --fail
+# Create indices
+	./temporal-elasticsearch-tool -ep http://127.0.0.1:9200 create-index --index temporal_visibility_v1_dev_cluster_a
+	./temporal-elasticsearch-tool -ep http://127.0.0.1:9200 create-index --index temporal_visibility_v1_dev_cluster_b
+	./temporal-elasticsearch-tool -ep http://127.0.0.1:9200 create-index --index temporal_visibility_v1_dev_cluster_c
 
 ##### Run server #####
 DOCKER_COMPOSE_FILES     := -f ./develop/docker-compose/docker-compose.yml -f ./develop/docker-compose/docker-compose.$(GOOS).yml
@@ -652,7 +656,7 @@ update-dependencies-major: $(GOMAJOR)
 
 go-generate: $(MOCKGEN) $(GOIMPORTS) $(STRINGER) $(GOWRAP)
 	@printf $(COLOR) "Process go:generate directives..."
-	@go generate ./...
+	@PATH="$(ROOT)/$(LOCALBIN):$(PATH)" go generate ./...
 
 ensure-no-changes:
 	@printf $(COLOR) "Check for local changes..."