Merge branch 'trunk' into KAFKA-9366

.github/workflows/README.md

@@ -51,6 +51,66 @@ using this for very simple tasks such as applying labels or adding comments to P
 
 _We must never run the untrusted PR code in the elevated `pull_request_target` context_
 
+## Our Workflows
+
+### Trunk Build
+
+The [ci.yml](ci.yml) is run when commits are pushed to trunk. This calls into [build.yml](build.yml)
+to run our main build. In the trunk build, we do not read from the Gradle cache,
+but we do write to it. Also, the test catalog is only updated from trunk builds.
+
+### PR Build
+
+Similar to trunk, this workflow starts in [ci.yml](ci.yml) and calls into [build.yml](build.yml).
+Unlike trunk, the PR builds _will_ utilize the Gradle cache.
+
+### PR Triage
+
+In order to get the attention of committers, we have a triage workflow for Pull Requests
+opened by non-committers. This workflow consists of three files:
+
+* [pr-update.yml](pr-update.yml) When a PR is created add the `triage` label if the PR
+  was opened by a non-committer.
+* [pr-reviewed-trigger.yml](pr-reviewed-trigger.yml) Runs when any PR is reviewed. 
+  Used as a trigger for the next workflow
+* [pr-reviewed.yml](pr-reviewed.yml) Remove the `triage` label after a PR has been reviewed
+
+_The pr-update.yml workflow includes pull_request_target!_
+
+For committers to avoid having this label added, their membership in the ASF GitHub
+organization must be public. Here are the steps to take:
+
+* Navigate to the ASF organization's "People" page https://github.com/orgs/apache/people
+* Find yourself
+* Change "Organization Visibility" to Public
+
+Full documentation for this process can be found in GitHub's docs: https://docs.github.com/en/account-and-profile/setting-up-and-managing-your-personal-account-on-github/managing-your-membership-in-organizations/publicizing-or-hiding-organization-membership
+
+If you are a committer and do not want your membership in the ASF org listed as public, 
+you will need to remove the `triage` label manually.
+
+### CI Approved
+
+Due to a combination of GitHub security and ASF's policy, we required explicit
+approval of workflows on PRs submitted by non-committers (and non-contributors).
+To simply this process, we have a `ci-approved` label which automatically approves
+these workflows.
+
+There are two files related to this workflow:
+
+* [pr-labeled.yml](pr-labeled.yml) approves a pending approval for PRs that have
+been labeled with `ci-approved`
+* [ci-requested.yml](ci-requested.yml) approves future CI requests automatically
+if the PR has the `ci-approved` label
+
+_The pr-labeled.yml workflow includes pull_request_target!_
+
+### Stale PRs
+
+This one is straightforward. Using the "actions/stale" GitHub Action, we automatically
+label and eventually close PRs which have not had activity for some time. See the
+[stale.yml](stale.yml) workflow file for specifics.
+
 ## GitHub Actions Quirks
 
 ### Composite Actions

.github/workflows/pr-reviewed-trigger.yml

@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Pull Request Reviewed
+
+on:
+  pull_request_review:
+    types:
+      - submitted
+
+jobs:
+  # This job is a workaround for the fact that pull_request_review lacks necessary permissions to modify PRs.
+  # Also, there is no pull_request_target analog to pull_request_review. The approach taken here is taken from
+  # https://securitylab.github.com/resources/github-actions-preventing-pwn-requests/.
+  pr-review-trigger:
+    name: Reviewed
+    runs-on: ubuntu-latest
+    steps:
+      - name: Env
+        run: printenv
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+      - name: Capture PR Number
+        run:
+          echo ${{ github.event.pull_request.number }} >> pr-number.txt
+      - name: Archive Event
+        uses: actions/upload-artifact@v4
+        with:
+          name: pr-number.txt
+          path: pr-number.txt

.github/workflows/pr-reviewed.yml

@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Remove Triage Label
+
+on:
+  workflow_run:
+    workflows: [Pull Request Reviewed]
+    types:
+      - completed
+
+jobs:
+  # This job runs with elevated permissions and the ability to modify pull requests. The steps taken here
+  # should be limited to updating labels and adding comments to PRs. This approach is taken from
+  # https://securitylab.github.com/resources/github-actions-preventing-pwn-requests/.
+  remove-triage:
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Env
+        run: printenv
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+      - uses: actions/download-artifact@v4
+        with:
+          github-token: ${{ github.token }}
+          run-id: ${{ github.event.workflow_run.id }}
+          name: pr-number.txt
+      - name: Remove label
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            var fs = require('fs');
+            var pr_number = Number(fs.readFileSync('./pr-number.txt'));
+            await github.rest.issues.removeLabel({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: pr_number,
+              name: 'triage'
+            });

.github/workflows/pr-update.yml

@@ -25,9 +25,11 @@ on:
   # * https://securitylab.github.com/resources/github-actions-preventing-pwn-requests/
   pull_request_target:
     types: [opened, reopened, synchronize]
+    branches:
+      - trunk
 
 jobs:
-  label_PRs:
+  add-labeler-labels:
     name: Labeler
     permissions:
       contents: read
@@ -45,3 +47,24 @@ jobs:
         PR_NUM: ${{github.event.number}}
       run: |
         ./.github/scripts/label_small.sh
+
+  add-triage-label:
+    if: github.event.action == 'opened' || github.event.action == 'reopened'
+    name: Add triage label
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Env
+        run: printenv
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        # If the PR is from a non-committer, add triage label
+      - if: |  
+          github.event.pull_request.author_association != 'MEMBER' && 
+          github.event.pull_request.author_association != 'OWNER'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+          NUMBER: ${{ github.event.pull_request.number }}
+        run: gh pr edit "$NUMBER" --add-label triage

.github/workflows/stale.yml

@@ -35,6 +35,22 @@ permissions:
   pull-requests: write
 
 jobs:
+  needs-attention:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v9
+        with:
+          debug-only: ${{ inputs.dryRun || false }}
+          operations-per-run: ${{ inputs.operationsPerRun || 500 }}
+          days-before-stale: 7
+          days-before-close: -1
+          ignore-pr-updates: true
+          only-pr-labels: 'triage'
+          stale-pr-label: 'needs-attention'
+          stale-pr-message: |
+            A label of 'needs-attention' was automatically added to this PR in order to raise the
+            attention of the committers. Once this issue has been triaged, the `triage` label
+            should be removed to prevent this automation from happening again.
   stale:
     runs-on: ubuntu-latest
     steps:

clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupDescription.java

@@ -27,6 +27,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -42,9 +43,11 @@ public class ConsumerGroupDescription {
     private final GroupState groupState;
     private final Node coordinator;
     private final Set<AclOperation> authorizedOperations;
+    private final Optional<Integer> groupEpoch;
+    private final Optional<Integer> targetAssignmentEpoch;
 
     /**
-     * @deprecated Since 4.0. Use {@link #ConsumerGroupDescription(String, boolean, Collection, String, GroupState, Node)}.
+     * @deprecated Since 4.0. Use {@link #ConsumerGroupDescription(String, boolean, Collection, String, GroupType, GroupState, Node, Set, Optional, Optional)}.
      */
     @Deprecated
     public ConsumerGroupDescription(String groupId,
@@ -57,7 +60,7 @@ public ConsumerGroupDescription(String groupId,
     }
 
     /**
-     * @deprecated Since 4.0. Use {@link #ConsumerGroupDescription(String, boolean, Collection, String, GroupState, Node, Set)}.
+     * @deprecated Since 4.0. Use {@link #ConsumerGroupDescription(String, boolean, Collection, String, GroupType, GroupState, Node, Set, Optional, Optional)}.
      */
     @Deprecated
     public ConsumerGroupDescription(String groupId,
@@ -71,7 +74,7 @@ public ConsumerGroupDescription(String groupId,
     }
 
     /**
-     * @deprecated Since 4.0. Use {@link #ConsumerGroupDescription(String, boolean, Collection, String, GroupType, GroupState, Node, Set)}.
+     * @deprecated Since 4.0. Use {@link #ConsumerGroupDescription(String, boolean, Collection, String, GroupType, GroupState, Node, Set, Optional, Optional)}.
      */
     @Deprecated
     public ConsumerGroupDescription(String groupId,
@@ -90,25 +93,8 @@ public ConsumerGroupDescription(String groupId,
         this.groupState = GroupState.parse(state.name());
         this.coordinator = coordinator;
         this.authorizedOperations = authorizedOperations;
-    }
-
-    public ConsumerGroupDescription(String groupId,
-                                    boolean isSimpleConsumerGroup,
-                                    Collection<MemberDescription> members,
-                                    String partitionAssignor,
-                                    GroupState groupState,
-                                    Node coordinator) {
-        this(groupId, isSimpleConsumerGroup, members, partitionAssignor, groupState, coordinator, Collections.emptySet());
-    }
-
-    public ConsumerGroupDescription(String groupId,
-                                    boolean isSimpleConsumerGroup,
-                                    Collection<MemberDescription> members,
-                                    String partitionAssignor,
-                                    GroupState groupState,
-                                    Node coordinator,
-                                    Set<AclOperation> authorizedOperations) {
-        this(groupId, isSimpleConsumerGroup, members, partitionAssignor, GroupType.CLASSIC, groupState, coordinator, authorizedOperations);
+        this.groupEpoch = Optional.empty();
+        this.targetAssignmentEpoch = Optional.empty();
     }
 
     public ConsumerGroupDescription(String groupId,
@@ -118,7 +104,9 @@ public ConsumerGroupDescription(String groupId,
                                     GroupType type,
                                     GroupState groupState,
                                     Node coordinator,
-                                    Set<AclOperation> authorizedOperations) {
+                                    Set<AclOperation> authorizedOperations,
+                                    Optional<Integer> groupEpoch,
+                                    Optional<Integer> targetAssignmentEpoch) {
         this.groupId = groupId == null ? "" : groupId;
         this.isSimpleConsumerGroup = isSimpleConsumerGroup;
         this.members = members == null ? Collections.emptyList() : List.copyOf(members);
@@ -127,6 +115,8 @@ public ConsumerGroupDescription(String groupId,
         this.groupState = groupState;
         this.coordinator = coordinator;
         this.authorizedOperations = authorizedOperations;
+        this.groupEpoch = groupEpoch;
+        this.targetAssignmentEpoch = targetAssignmentEpoch;
     }
 
     @Override
@@ -141,12 +131,15 @@ public boolean equals(final Object o) {
             type == that.type &&
             groupState == that.groupState &&
             Objects.equals(coordinator, that.coordinator) &&
-            Objects.equals(authorizedOperations, that.authorizedOperations);
+            Objects.equals(authorizedOperations, that.authorizedOperations) &&
+            Objects.equals(groupEpoch, that.groupEpoch) &&
+            Objects.equals(targetAssignmentEpoch, that.targetAssignmentEpoch);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(groupId, isSimpleConsumerGroup, members, partitionAssignor, type, groupState, coordinator, authorizedOperations);
+        return Objects.hash(groupId, isSimpleConsumerGroup, members, partitionAssignor, type, groupState, coordinator,
+            authorizedOperations, groupEpoch, targetAssignmentEpoch);
     }
 
     /**
@@ -215,6 +208,24 @@ public  Set<AclOperation> authorizedOperations() {
         return authorizedOperations;
     }
 
+    /**
+     * The epoch of the consumer group.
+     * The optional is set to an integer if it is a {@link GroupType#CONSUMER} group, and to empty if it
+     * is a {@link GroupType#CLASSIC} group.
+     */
+    public Optional<Integer> groupEpoch() {
+        return groupEpoch;
+    }
+
+    /**
+     * The epoch of the target assignment.
+     * The optional is set to an integer if it is a {@link GroupType#CONSUMER} group, and to empty if it
+     * is a {@link GroupType#CLASSIC} group.
+     */
+    public Optional<Integer> targetAssignmentEpoch() {
+        return targetAssignmentEpoch;
+    }
+
     @Override
     public String toString() {
         return "(groupId=" + groupId +
@@ -225,6 +236,8 @@ public String toString() {
             ", groupState=" + groupState +
             ", coordinator=" + coordinator +
             ", authorizedOperations=" + authorizedOperations +
+            ", groupEpoch=" + groupEpoch.orElse(null) +
+            ", targetAssignmentEpoch=" + targetAssignmentEpoch.orElse(null) +
             ")";
     }
 }