pulsarv3-recovery always fails when executing pulsar-bookkeeper-verify-clusterid

[piemon-nyah]

question

pulsarv3-recovery always fails when executing pulsar-bookkeeper-verify-clusterid. pulsarv3-bookie, which also uses the pulsar-bookkeeper-verify-clusterid task, has no problem.

version

pulsar：3.0.7
pulsar chart: 3.3.0

pulsarv3-recovery yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: "it-milvus-pulsarv3-recovery"
  namespace: it-milvus-test
  labels:
    app: pulsarv3
    chart: pulsarv3-3.3.0
    release: it-milvus
    heritage: Helm
    cluster: it-milvus-pulsarv3
    component: recovery
spec:
  serviceName: "it-milvus-pulsarv3-recovery"
  replicas: 1
  updateStrategy:
    type: RollingUpdate
  podManagementPolicy: Parallel
  # nodeSelector:
  selector:
    matchLabels:
      app: pulsarv3
      release: it-milvus
      component: recovery
  template:
    metadata:
      labels:
        app: pulsarv3
        release: it-milvus
        cluster: it-milvus-pulsarv3
        component: recovery
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8000"
    spec:
      affinity:
      terminationGracePeriodSeconds: 30
      serviceAccountName: "it-milvus-pulsarv3-recovery"
      initContainers:
      # This initContainer will wait for bookkeeper initnewcluster to complete
      # before deploying the bookies
      - name: pulsar-bookkeeper-verify-clusterid
        image: "apachepulsar/pulsar:3.0.7"
        imagePullPolicy: IfNotPresent
        resources: 
          requests:
            cpu: 0.1
            memory: 256Mi
        command: ["sh", "-c"]
        args:
        - >
          bin/apply-config-from-env.py conf/bookkeeper.conf;until bin/bookkeeper shell whatisinstanceid; do
            sleep 3;
          done;
        envFrom:
        - configMapRef:
            name: "it-milvus-pulsarv3-recovery"
        volumeMounts:
        
      containers:
      - name: "it-milvus-pulsarv3-recovery"
        image: "apachepulsar/pulsar:3.0.7"
        imagePullPolicy: IfNotPresent
        resources:
          limits:
              cpu: 0.1
              memory: 128Mi
          requests:
            cpu: 0.1
            memory: 128Mi
        command: ["sh", "-c"]
        args:
        - >
          bin/apply-config-from-env.py conf/bookkeeper.conf;
          
          OPTS="${OPTS} -Dlog4j2.formatMsgNoLookups=true" exec bin/bookkeeper autorecovery
        ports:
        - name: http
          containerPort: 8000
        envFrom:
        - configMapRef:
            name: "it-milvus-pulsarv3-recovery"
        volumeMounts:
        
      volumes:

pulsarv3-recovery error log

[conf/bookkeeper.conf] Applying config httpServerEnabled = true
[conf/bookkeeper.conf] Applying config httpServerPort = 8000
[conf/bookkeeper.conf] Applying config statsProviderClass = org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider
[conf/bookkeeper.conf] Applying config useHostNameAsBookieID = true
[conf/bookkeeper.conf] Applying config zkLedgersRootPath = /ledgers
[conf/bookkeeper.conf] Applying config zkServers = it-milvus-pulsarv3-zookeeper:2181
[conf/bookkeeper.conf] Updating config useV2WireProtocol = true
JAVA_HOME not set, using java from PATH. (/usr/bin/java)
Killed
JAVA_HOME not set, using java from PATH. (/usr/bin/java)
Killed
JAVA_HOME not set, using java from PATH. (/usr/bin/java)
Killed
JAVA_HOME not set, using java from PATH. (/usr/bin/java)
Killed
JAVA_HOME not set, using java from PATH. (/usr/bin/java)
Killed
JAVA_HOME not set, using java from PATH. (/usr/bin/java)
Killed
JAVA_HOME not set, using java from PATH. (/usr/bin/java)
Killed
JAVA_HOME not set, using java from PATH. (/usr/bin/java)
Killed
JAVA_HOME not set, using java from PATH. (/usr/bin/java)
Killed
JAVA_HOME not set, using java from PATH. (/usr/bin/java)
Killed
JAVA_HOME not set, using java from PATH. (/usr/bin/java)
Killed
JAVA_HOME not set, using java from PATH. (/usr/bin/java)
Killed

pulsarv3-bookie yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: "it-milvus-pulsarv3-bookie"
  namespace: it-milvus-test
  labels:
    app: pulsarv3
    chart: pulsarv3-3.3.0
    release: it-milvus
    heritage: Helm
    cluster: it-milvus-pulsarv3
    component: bookie
spec:
  serviceName: "it-milvus-pulsarv3-bookie"
  replicas: 3
  selector:
    matchLabels:
      app: pulsarv3
      release: it-milvus
      component: bookie
  updateStrategy:
    type: RollingUpdate
  podManagementPolicy: Parallel
  template:
    metadata:
      labels:
        app: pulsarv3
        release: it-milvus
        cluster: it-milvus-pulsarv3
        component: bookie
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8000"
    spec:
      affinity:
      terminationGracePeriodSeconds: 30
      serviceAccountName: "it-milvus-pulsarv3-bookie"
      securityContext:
        fsGroup: 0
        fsGroupChangePolicy: OnRootMismatch
      initContainers:
      # This initContainer will wait for bookkeeper initnewcluster to complete
      # before deploying the bookies
      - name: pulsar-bookkeeper-verify-clusterid
        image: "apachepulsar/pulsar:3.0.7"
        imagePullPolicy: IfNotPresent
        resources: 
          requests:
            cpu: 0.1
            memory: 256Mi
        command: ["sh", "-c"]
        args:
        # only reformat bookie if bookkeeper is running without persistence
        - >
          
          set -e;
          bin/apply-config-from-env.py conf/bookkeeper.conf;until bin/bookkeeper shell whatisinstanceid; do
            sleep 3;
          done;
        envFrom:
        - configMapRef:
            name: "it-milvus-pulsarv3-bookie"
        volumeMounts:
        
      containers:
      - name: "it-milvus-pulsarv3-bookie"
        image: "apachepulsar/pulsar:3.0.7"
        imagePullPolicy: IfNotPresent
        livenessProbe:
          httpGet:
            path: /api/v1/bookie/state
            port: 8000
          initialDelaySeconds: 10
          periodSeconds: 30
          timeoutSeconds: 5
          failureThreshold: 60
        readinessProbe:
          httpGet:
            path: /api/v1/bookie/is_ready
            port: 8000
          initialDelaySeconds: 10
          periodSeconds: 30
          timeoutSeconds: 5
          failureThreshold: 60
        resources:
          limits:
              cpu: 0.5
              memory: 2048Mi
          requests:
            cpu: 0.5
            memory: 2048Mi
        command: ["sh", "-c"]
        args:
        - >
          bin/apply-config-from-env.py conf/bookkeeper.conf;
          
          OPTS="${OPTS} -Dlog4j2.formatMsgNoLookups=true" exec bin/pulsar bookie;
        ports:
        - name: "bookie"
          containerPort: 3181
        - name: http
          containerPort: 8000
        envFrom:
        - configMapRef:
            name: "it-milvus-pulsarv3-bookie"
        volumeMounts:
        - name: "it-milvus-pulsarv3-bookie-journal"
          mountPath: /pulsar/data/bookkeeper/journal
        - name: "it-milvus-pulsarv3-bookie-ledgers"
          mountPath: /pulsar/data/bookkeeper/ledgers
        
      volumes:
      
      
  volumeClaimTemplates:
  - metadata:
      name: "it-milvus-pulsarv3-bookie-journal"
    spec:
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 50Gi
      storageClassName: netapp-nfs
  - metadata:
      name: "it-milvus-pulsarv3-bookie-ledgers"
    spec:
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 50Gi
      storageClassName: netapp-nfs

pulsarv3-bookie normal log

[conf/bookkeeper.conf] Applying config autoRecoveryDaemonEnabled = false
[conf/bookkeeper.conf] Applying config diskCheckInterval = 1800
[conf/bookkeeper.conf] Applying config diskUsageLwmThreshold = 0.85
[conf/bookkeeper.conf] Applying config diskUsageThreshold = 0.95
[conf/bookkeeper.conf] Applying config diskUsageWarnThreshold = 0.9
[conf/bookkeeper.conf] Applying config gcWaitTime = 300000
[conf/bookkeeper.conf] Applying config httpServerEnabled = true
[conf/bookkeeper.conf] Applying config httpServerPort = 8000
[conf/bookkeeper.conf] Applying config isForceGCAllowWhenNoSpace = true
[conf/bookkeeper.conf] Applying config journalDirectories = /pulsar/data/bookkeeper/journal
[conf/bookkeeper.conf] Applying config journalMaxBackups = 0
[conf/bookkeeper.conf] Applying config ledgerDirectories = /pulsar/data/bookkeeper/ledgers
[conf/bookkeeper.conf] Applying config majorCompactionInterval = 10800
[conf/bookkeeper.conf] Applying config majorCompactionThreshold = 0.8
[conf/bookkeeper.conf] Applying config minorCompactionInterval = 360
[conf/bookkeeper.conf] Applying config minorCompactionThreshold = 0.2
[conf/bookkeeper.conf] Applying config nettyMaxFrameSizeBytes = 104867840
[conf/bookkeeper.conf] Applying config statsProviderClass = org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider
[conf/bookkeeper.conf] Applying config useHostNameAsBookieID = true
[conf/bookkeeper.conf] Applying config zkLedgersRootPath = /ledgers
[conf/bookkeeper.conf] Applying config zkServers = it-milvus-pulsarv3-zookeeper:2181
[conf/bookkeeper.conf] Updating config journalDirectories = /pulsar/data/bookkeeper/journal
JAVA_HOME not set, using java from PATH. (/usr/bin/java)
2025-05-06T06:54:59,652+0000 [main] INFO  org.apache.bookkeeper.meta.MetadataDrivers - BookKeeper metadata driver manager initialized
2025-05-06T06:54:59,750+0000 [main] INFO  org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase - Initialize zookeeper metadata driver at metadata service uri zk+null://it-milvus-pulsarv3-zookeeper:2181/ledgers : zkServers = it-milvus-pulsarv3-zookeeper:2181, ledgersRootPath = /ledgers.
2025-05-06T06:54:59,756+0000 [main] INFO  org.apache.zookeeper.ZooKeeper - Client environment:zookeeper.version=3.9.2-e454e8c7283100c7caec6dcae2bc82aaecb63023, built on 2024-02-12 20:59 UTC
2025-05-06T06:54:59,756+0000 [main] INFO  org.apache.zookeeper.ZooKeeper - Client environment:host.name=it-milvus-pulsarv3-bookie-2.it-milvus-pulsarv3-bookie.it-milvus-test.svc.cluster.local
2025-05-06T06:54:59,757+0000 [main] INFO  org.apache.zookeeper.ZooKeeper - Client environment:java.version=17.0.12
2025-05-06T06:54:59,757+0000 [main] INFO  org.apache.zookeeper.ZooKeeper - Client environment:java.vendor=Eclipse Adoptium
2025-05-06T06:54:59,757+0000 [main] INFO  org.apache.zookeeper.ZooKeeper - Client environment:java.home=/usr/lib/jvm/temurin-17-jdk-amd64
2025-05-06T06:54:59,757+0000 [main] INFO  org.apache.zookeeper.ZooKeeper - Client environment:java.class.path=/pulsar/conf:::/pulsar/lib/com.auth0-java-jwt-4.3.0.jar:/pulsar/lib/com.auth0-jwks-rsa-0.22.0.jar:/pulsar/lib/com.beust-jcommander-1.82.jar:/pulsar/lib/com.carrotsearch-hppc-0.9.1.jar:/pulsar/lib/com.fasterxml.jackson.core-jackson-annotations-2.14.2.jar:/pulsar/lib/com.fasterxml.jackson.core-jackson-core-2.14.2.jar:/pulsar/lib/com.fasterxml.jackson.core-jackson-databind-2.14.2.jar:/pulsar/lib/com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.14.2.jar:/pulsar/lib/com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.14.2.jar:/pulsar/lib/com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.14.2.jar:/pulsar/lib/com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.14.2.jar:/pulsar/lib/com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.14.2.jar:/pulsar/lib/com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.14.2.jar:/pulsar/lib/com.fasterxml.jackson.module-jackson-module-jsonSchema-2.14.2.jar:/pulsar/lib/com.fasterxml.jackson.module-jackson-module-parameter-names-2.14.2.jar:/pulsar/lib/com.github.ben-manes.caffeine-caffeine-2.9.1.jar:/pulsar/lib/com.github.oshi-oshi-core-java11-6.4.0.jar:/pulsar/lib/com.github.seancfoley-ipaddress-5.5.0.jar:/pulsar/lib/com.github.zafarkhaja-java-semver-0.9.0.jar:/pulsar/lib/com.google.api.grpc-proto-google-common-protos-2.9.0.jar:/pulsar/lib/com.google.auth-google-auth-library-credentials-1.4.0.jar:/pulsar/lib/com.google.auth-google-auth-library-oauth2-http-1.4.0.jar:/pulsar/lib/com.google.auto.value-auto-value-annotations-1.9.jar:/pulsar/lib/com.google.code.findbugs-jsr305-3.0.2.jar:/pulsar/lib/com.google.code.gson-gson-2.8.9.jar:/pulsar/lib/com.google.errorprone-error_prone_annotations-2.5.1.jar:/pulsar/lib/com.google.guava-failureaccess-1.0.1.jar:/pulsar/lib/com.google.guava-guava-32.1.1-jre.jar:/pulsar/lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/pulsar/lib/com.google.http-client-google-http-client-1.41.0.jar:/pulsar/lib/com.google.http-client-google-http-client-gson-1.41.0.jar:/pulsar/lib/com.google.j2objc-j2objc-annotations-1.3.jar:/pulsar/lib/com.google.protobuf-protobuf-java-3.25.5.jar:/pulsar/lib/com.google.protobuf-protobuf-java-util-3.25.5.jar:/pulsar/lib/com.google.re2j-re2j-1.6.jar:/pulsar/lib/com.spotify-completable-futures-0.3.6.jar:/pulsar/lib/com.squareup.okhttp3-logging-interceptor-4.9.3.jar:/pulsar/lib/com.squareup.okhttp3-okhttp-4.9.3.jar:/pulsar/lib/com.squareup.okio-okio-3.4.0.jar:/pulsar/lib/com.squareup.okio-okio-jvm-3.4.0.jar:/pulsar/lib/com.sun.activation-javax.activation-1.2.0.jar:/pulsar/lib/com.typesafe.netty-netty-reactive-streams-2.0.6.jar:/pulsar/lib/com.yahoo.datasketches-memory-0.8.3.jar:/pulsar/lib/com.yahoo.datasketches-sketches-core-0.8.3.jar:/pulsar/lib/commons-cli-commons-cli-1.5.0.jar:/pulsar/lib/commons-codec-commons-codec-1.15.jar:/pulsar/lib/commons-configuration-commons-configuration-1.10.jar:/pulsar/lib/commons-io-commons-io-2.14.0.jar:/pulsar/lib/commons-lang-commons-lang-2.6.jar:/pulsar/lib/commons-logging-commons-logging-1.1.1.jar:/pulsar/lib/dev.failsafe-failsafe-3.3.2.jar:/pulsar/lib/io.airlift-aircompressor-0.27.jar:/pulsar/lib/io.dropwizard.metrics-metrics-core-4.1.12.1.jar:/pulsar/lib/io.dropwizard.metrics-metrics-graphite-4.1.12.1.jar:/pulsar/lib/io.dropwizard.metrics-metrics-jmx-4.1.12.1.jar:/pulsar/lib/io.dropwizard.metrics-metrics-jvm-4.1.12.1.jar:/pulsar/lib/io.grpc-grpc-all-1.55.3.jar:/pulsar/lib/io.grpc-grpc-alts-1.55.3.jar:/pulsar/lib/io.grpc-grpc-api-1.55.3.jar:/pulsar/lib/io.grpc-grpc-auth-1.55.3.jar:/pulsar/lib/io.grpc-grpc-context-1.55.3.jar:/pulsar/lib/io.grpc-grpc-core-1.55.3.jar:/pulsar/lib/io.grpc-grpc-grpclb-1.55.3.jar:/pulsar/lib/io.grpc-grpc-netty-shaded-1.55.3.jar:/pulsar/lib/io.grpc-grpc-protobuf-1.55.3.jar:/pulsar/lib/io.grpc-grpc-protobuf-lite-1.55.3.jar:/pulsar/lib/io.grpc-grpc-rls-1.55.3.jar:/pulsar/lib/io.grpc-grpc-services-1.55.3.jar:/pulsar/lib/io.grpc-grpc-servlet-1.55.3.jar:/pulsar/lib/io.grpc-grpc-servlet-jakarta-1.55.3.jar:/pulsar/lib/io.grpc-grpc-stub-1.55.3.jar:/pulsar/lib/io.grpc-grpc-util-1.60.0.jar:/pulsar/lib/io.grpc-grpc-xds-1.55.3.jar:/pulsar/lib/io.gsonfire-gson-fire-1.8.5.jar:/pulsar/lib/io.jsonwebtoken-jjwt-api-0.11.1.jar:/pulsar/lib/io.jsonwebtoken-jjwt-impl-0.11.1.jar:/pulsar/lib/io.jsonwebtoken-jjwt-jackson-0.11.1.jar:/pulsar/lib/io.kubernetes-client-java-18.0.0.jar:/pulsar/lib/io.kubernetes-client-java-api-18.0.0.jar:/pulsar/lib/io.kubernetes-client-java-proto-18.0.0.jar:/pulsar/lib/io.netty-netty-buffer-4.1.113.Final.jar:/pulsar/lib/io.netty-netty-codec-4.1.113.Final.jar:/pulsar/lib/io.netty-netty-codec-dns-4.1.113.Final.jar:/pulsar/lib/io.netty-netty-codec-haproxy-4.1.113.Final.jar:/pulsar/lib/io.netty-netty-codec-http-4.1.113.Final.jar:/pulsar/lib/io.netty-netty-codec-http2-4.1.113.Final.jar:/pulsar/lib/io.netty-netty-codec-socks-4.1.113.Final.jar:/pulsar/lib/io.netty-netty-common-4.1.113.Final.jar:/pulsar/lib/io.netty-netty-handler-4.1.113.Final.jar:/pulsar/lib/io.netty-netty-handler-proxy-4.1.113.Final.jar:/pulsar/lib/io.netty-netty-resolver-4.1.113.Final.jar:/pulsar/lib/io.netty-netty-resolver-dns-4.1.113.Final.jar:/pulsar/lib/io.netty-netty-resolver-dns-classes-macos-4.1.113.Final.jar:/pulsar/lib/io.netty-netty-resolver-dns-native-macos-4.1.113.Final-osx-aarch_64.jar:/pulsar/lib/io.netty-netty-resolver-dns-native-macos-4.1.113.Final-osx-x86_64.jar:/pulsar/lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-aarch_64.jar:/pulsar/lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-x86_64.jar:/pulsar/lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-aarch_64.jar:/pulsar/lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-x86_64.jar:/pulsar/lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-windows-x86_64.jar:/pulsar/lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final.jar:/pulsar/lib/io.netty-netty-tcnative-classes-2.0.66.Final.jar:/pulsar/lib/io.netty-netty-transport-4.1.113.Final.jar:/pulsar/lib/io.netty-netty-transport-classes-epoll-4.1.113.Final.jar:/pulsar/lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-aarch_64.jar:/pulsar/lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-x86_64.jar:/pulsar/lib/io.netty-netty-transport-native-unix-common-4.1.113.Final.jar:/pulsar/lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.21.Final.jar:/pulsar/lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.21.Final-linux-aarch_64.jar:/pulsar/lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.21.Final-linux-x86_64.jar:/pulsar/lib/io.opencensus-opencensus-api-0.28.0.jar:/pulsar/lib/io.opencensus-opencensus-contrib-http-util-0.28.0.jar:/pulsar/lib/io.opencensus-opencensus-proto-0.2.0.jar:/pulsar/lib/io.perfmark-perfmark-api-0.26.0.jar:/pulsar/lib/io.prometheus-simpleclient-0.16.0.jar:/pulsar/lib/io.prometheus-simpleclient_caffeine-0.16.0.jar:/pulsar/lib/io.prometheus-simpleclient_common-0.16.0.jar:/pulsar/lib/io.prometheus-simpleclient_hotspot-0.16.0.jar:/pulsar/lib/io.prometheus-simpleclient_httpserver-0.16.0.jar:/pulsar/lib/io.prometheus-simpleclient_jetty-0.16.0.jar:/pulsar/lib/io.prometheus-simpleclient_log4j2-0.16.0.jar:/pulsar/lib/io.prometheus-simpleclient_servlet-0.16.0.jar:/pulsar/lib/io.prometheus-simpleclient_servlet_common-0.16.0.jar:/pulsar/lib/io.prometheus-simpleclient_tracer_common-0.16.0.jar:/pulsar/lib/io.prometheus-simpleclient_tracer_otel-0.16.0.jar:/pulsar/lib/io.prometheus-simpleclient_tracer_otel_agent-0.16.0.jar:/pulsar/lib/io.prometheus.jmx-collector-0.16.1.jar:/pulsar/lib/io.reactivex.rxjava3-rxjava-3.0.1.jar:/pulsar/lib/io.swagger-swagger-annotations-1.6.10.jar:/pulsar/lib/io.swagger-swagger-core-1.6.10.jar:/pulsar/lib/io.swagger-swagger-models-1.6.10.jar:/pulsar/lib/io.vertx-vertx-auth-common-4.5.10.jar:/pulsar/lib/io.vertx-vertx-bridge-common-4.5.10.jar:/pulsar/lib/io.vertx-vertx-core-4.5.10.jar:/pulsar/lib/io.vertx-vertx-web-4.5.10.jar:/pulsar/lib/io.vertx-vertx-web-common-4.5.10.jar:/pulsar/lib/jakarta.activation-jakarta.activation-api-1.2.2.jar:/pulsar/lib/jakarta.annotation-jakarta.annotation-api-1.3.5.jar:/pulsar/lib/jakarta.validation-jakarta.validation-api-2.0.2.jar:/pulsar/lib/jakarta.ws.rs-jakarta.ws.rs-api-2.1.6.jar:/pulsar/lib/jakarta.xml.bind-jakarta.xml.bind-api-2.3.3.jar:/pulsar/lib/javax.servlet-javax.servlet-api-3.1.0.jar:/pulsar/lib/javax.validation-validation-api-1.1.0.Final.jar:/pulsar/lib/javax.websocket-javax.websocket-client-api-1.0.jar:/pulsar/lib/javax.ws.rs-javax.ws.rs-api-2.1.jar:/pulsar/lib/javax.xml.bind-jaxb-api-2.3.1.jar:/pulsar/lib/jline-jline-2.14.6.jar:/pulsar/lib/net.bytebuddy-byte-buddy-1.14.12.jar:/pulsar/lib/net.java.dev.jna-jna-jpms-5.12.1.jar:/pulsar/lib/net.java.dev.jna-jna-platform-jpms-5.12.1.jar:/pulsar/lib/net.jcip-jcip-annotations-1.0.jar:/pulsar/lib/net.jodah-typetools-0.5.0.jar:/pulsar/lib/org.apache.avro-avro-1.11.4.jar:/pulsar/lib/org.apache.avro-avro-protobuf-1.11.4.jar:/pulsar/lib/org.apache.bookkeeper-bookkeeper-common-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper-bookkeeper-common-allocator-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper-bookkeeper-proto-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper-bookkeeper-server-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper-bookkeeper-slogger-api-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper-bookkeeper-tools-framework-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper-circe-checksum-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper-cpu-affinity-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper-native-io-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper-statelib-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper-stream-storage-api-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper-stream-storage-common-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper-stream-storage-java-client-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper-stream-storage-java-client-base-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper-stream-storage-proto-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper-stream-storage-server-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper-stream-storage-service-api-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper-stream-storage-service-impl-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper.http-http-server-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper.http-vertx-http-server-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.6.jar:/pulsar/lib/org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.6.jar:/pulsar/lib/org.apache.commons-commons-collections4-4.4.jar:/pulsar/lib/org.apache.commons-commons-compress-1.26.0.jar:/pulsar/lib/org.apache.commons-commons-lang3-3.11.jar:/pulsar/lib/org.apache.commons-commons-text-1.10.0.jar:/pulsar/lib/org.apache.curator-curator-client-5.1.0.jar:/pulsar/lib/org.apache.curator-curator-framework-5.1.0.jar:/pulsar/lib/org.apache.curator-curator-recipes-5.1.0.jar:/pulsar/lib/org.apache.distributedlog-distributedlog-common-4.16.6.jar:/pulsar/lib/org.apache.distributedlog-distributedlog-core-4.16.6-tests.jar:/pulsar/lib/org.apache.distributedlog-distributedlog-core-4.16.6.jar:/pulsar/lib/org.apache.distributedlog-distributedlog-protocol-4.16.6.jar:/pulsar/lib/org.apache.httpcomponents-httpclient-4.5.13.jar:/pulsar/lib/org.apache.httpcomponents-httpcore-4.4.15.jar:/pulsar/lib/org.apache.logging.log4j-log4j-api-2.18.0.jar:/pulsar/lib/org.apache.logging.log4j-log4j-core-2.18.0.jar:/pulsar/lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar:/pulsar/lib/org.apache.logging.log4j-log4j-web-2.18.0.jar:/pulsar/lib/org.apache.pulsar-bouncy-castle-bc-3.0.7-pkg.jar:/pulsar/lib/org.apache.pulsar-jetcd-core-shaded-3.0.7-shaded.jar:/pulsar/lib/org.apache.pulsar-managed-ledger-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-broker-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-broker-auth-oidc-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-broker-auth-sasl-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-broker-common-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-client-admin-api-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-client-admin-original-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-client-api-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-client-auth-sasl-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-client-messagecrypto-bc-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-client-original-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-client-tools-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-client-tools-api-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-common-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-config-validation-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-functions-api-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-functions-instance-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-functions-local-runner-original-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-functions-proto-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-functions-runtime-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-functions-secrets-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-functions-utils-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-functions-worker-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-io-core-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-metadata-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-package-bookkeeper-storage-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-package-core-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-package-filesystem-storage-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-proxy-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-testclient-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-transaction-common-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-transaction-coordinator-3.0.7.jar:/pulsar/lib/org.apache.pulsar-pulsar-websocket-3.0.7.jar:/pulsar/lib/org.apache.pulsar-testmocks-3.0.7.jar:/pulsar/lib/org.apache.thrift-libthrift-0.14.2.jar:/pulsar/lib/org.apache.yetus-audience-annotations-0.12.0.jar:/pulsar/lib/org.apache.zookeeper-zookeeper-3.9.2.jar:/pulsar/lib/org.apache.zookeeper-zookeeper-jute-3.9.2.jar:/pulsar/lib/org.apache.zookeeper-zookeeper-prometheus-metrics-3.9.2.jar:/pulsar/lib/org.asynchttpclient-async-http-client-2.12.1.jar:/pulsar/lib/org.asynchttpclient-async-http-client-netty-utils-2.12.1.jar:/pulsar/lib/org.bitbucket.b_c-jose4j-0.9.4.jar:/pulsar/lib/org.bouncycastle-bcpkix-jdk18on-1.78.1.jar:/pulsar/lib/org.bouncycastle-bcprov-jdk18on-1.78.1.jar:/pulsar/lib/org.bouncycastle-bcutil-jdk18on-1.78.1.jar:/pulsar/lib/org.checkerframework-checker-qual-3.33.0.jar:/pulsar/lib/org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar:/pulsar/lib/org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.54.v20240208.jar:/pulsar/lib/org.eclipse.jetty-jetty-alpn-server-9.4.54.v20240208.jar:/pulsar/lib/org.eclipse.jetty-jetty-client-9.4.54.v20240208.jar:/pulsar/lib/org.eclipse.jetty-jetty-continuation-9.4.54.v20240208.jar:/pulsar/lib/org.eclipse.jetty-jetty-http-9.4.54.v20240208.jar:/pulsar/lib/org.eclipse.jetty-jetty-io-9.4.54.v20240208.jar:/pulsar/lib/org.eclipse.jetty-jetty-proxy-9.4.54.v20240208.jar:/pulsar/lib/org.eclipse.jetty-jetty-security-9.4.54.v20240208.jar:/pulsar/lib/org.eclipse.jetty-jetty-server-9.4.54.v20240208.jar:/pulsar/lib/org.eclipse.jetty-jetty-servlet-9.4.54.v20240208.jar:/pulsar/lib/org.eclipse.jetty-jetty-servlets-9.4.54.v20240208.jar:/pulsar/lib/org.eclipse.jetty-jetty-util-9.4.54.v20240208.jar:/pulsar/lib/org.eclipse.jetty-jetty-util-ajax-9.4.54.v20240208.jar:/pulsar/lib/org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.54.v20240208.jar:/pulsar/lib/org.eclipse.jetty.websocket-websocket-api-9.4.54.v20240208.jar:/pulsar/lib/org.eclipse.jetty.websocket-websocket-client-9.4.54.v20240208.jar:/pulsar/lib/org.eclipse.jetty.websocket-websocket-common-9.4.54.v20240208.jar:/pulsar/lib/org.eclipse.jetty.websocket-websocket-server-9.4.54.v20240208.jar:/pulsar/lib/org.eclipse.jetty.websocket-websocket-servlet-9.4.54.v20240208.jar:/pulsar/lib/org.glassfish.hk2-hk2-api-2.6.1.jar:/pulsar/lib/org.glassfish.hk2-hk2-locator-2.6.1.jar:/pulsar/lib/org.glassfish.hk2-hk2-utils-2.6.1.jar:/pulsar/lib/org.glassfish.hk2-osgi-resource-locator-1.0.3.jar:/pulsar/lib/org.glassfish.hk2.external-aopalliance-repackaged-2.6.1.jar:/pulsar/lib/org.glassfish.hk2.external-jakarta.injec2025-05-06T06:54:59.758392356Z t-2.6.1.jar:/pulsar/lib/org.glassfish.jersey.containers-jersey-container-servlet-2.42.jar:/pulsar/lib/org.glassfish.jersey.containers-jersey-container-servlet-core-2.42.jar:/pulsar/lib/org.glassfish.jersey.core-jersey-client-2.42.jar:/pulsar/lib/org.glassfish.jersey.core-jersey-common-2.42.jar:/pulsar/lib/org.glassfish.jersey.core-jersey-server-2.42.jar:/pulsar/lib/org.glassfish.jersey.ext-jersey-entity-filtering-2.42.jar:/pulsar/lib/org.glassfish.jersey.inject-jersey-hk2-2.42.jar:/pulsar/lib/org.glassfish.jersey.media-jersey-media-json-jackson-2.42.jar:/pulsar/lib/org.glassfish.jersey.media-jersey-media-multipart-2.42.jar:/pulsar/lib/org.hdrhistogram-HdrHistogram-2.1.9.jar:/pulsar/lib/org.javassist-javassist-3.25.0-GA.jar:/pulsar/lib/org.jctools-jctools-core-2.1.2.jar:/pulsar/lib/org.jetbrains-annotations-13.0.jar:/pulsar/lib/org.jetbrains.kotlin-kotlin-stdlib-1.6.0.jar:/pulsar/lib/org.jetbrains.kotlin-kotlin-stdlib-common-1.6.0.jar:/pulsar/lib/org.jetbrains.kotlin-kotlin-stdlib-jdk7-1.6.0.jar:/pulsar/lib/org.jetbrains.kotlin-kotlin-stdlib-jdk8-1.6.0.jar:/pulsar/lib/org.jline-jline-3.21.0.jar:/pulsar/lib/org.jvnet.mimepull-mimepull-1.9.15.jar:/pulsar/lib/org.reactivestreams-reactive-streams-1.0.3.jar:/pulsar/lib/org.roaringbitmap-RoaringBitmap-1.2.0.jar:/pulsar/lib/org.rocksdb-rocksdbjni-7.9.2.jar:/pulsar/lib/org.slf4j-jcl-over-slf4j-1.7.32.jar:/pulsar/lib/org.slf4j-slf4j-api-1.7.32.jar:/pulsar/lib/org.xerial.snappy-snappy-java-1.1.10.5.jar:/pulsar/lib/org.yaml-snakeyaml-2.0.jar:/pulsar/lib/org.zeroturnaround-zt-zip-1.17.jar:
2025-05-06T06:54:59,758+0000 [main] INFO  org.apache.zookeeper.ZooKeeper - Client environment:java.library.path=/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib
2025-05-06T06:54:59,758+0000 [main] INFO  org.apache.zookeeper.ZooKeeper - Client environment:java.io.tmpdir=/tmp
2025-05-06T06:54:59,758+0000 [main] INFO  org.apache.zookeeper.ZooKeeper - Client environment:java.compiler=<NA>
2025-05-06T06:54:59,758+0000 [main] INFO  org.apache.zookeeper.ZooKeeper - Client environment:os.name=Linux
2025-05-06T06:54:59,758+0000 [main] INFO  org.apache.zookeeper.ZooKeeper - Client environment:os.arch=amd64
2025-05-06T06:54:59,758+0000 [main] INFO  org.apache.zookeeper.ZooKeeper - Client environment:os.version=5.10.134-16.2.an8.x86_64
2025-05-06T06:54:59,758+0000 [main] INFO  org.apache.zookeeper.ZooKeeper - Client environment:user.name=pulsar
2025-05-06T06:54:59,758+0000 [main] INFO  org.apache.zookeeper.ZooKeeper - Client environment:user.home=/pulsar/data
2025-05-06T06:54:59,759+0000 [main] INFO  org.apache.zookeeper.ZooKeeper - Client environment:user.dir=/pulsar
2025-05-06T06:54:59,759+0000 [main] INFO  org.apache.zookeeper.ZooKeeper - Client environment:os.memory.free=4024MB
2025-05-06T06:54:59,759+0000 [main] INFO  org.apache.zookeeper.ZooKeeper - Client environment:os.memory.max=4096MB
2025-05-06T06:54:59,759+0000 [main] INFO  org.apache.zookeeper.ZooKeeper - Client environment:os.memory.total=4096MB
2025-05-06T06:54:59,847+0000 [main] INFO  org.apache.zookeeper.ZooKeeper - Initiating client connection, connectString=it-milvus-pulsarv3-zookeeper:2181 sessionTimeout=30000 watcher=org.apache.bookkeeper.zookeeper.ZooKeeperWatcherBase@29ef6856
2025-05-06T06:54:59,943+0000 [main] INFO  org.apache.zookeeper.common.X509Util - Setting -D jdk.tls.rejectClientInitiatedRenegotiation=true to disable client-initiated TLS renegotiation
2025-05-06T06:55:02,448+0000 [main] INFO  org.apache.zookeeper.common.X509Util - Default TLS protocol is TLSv1.3, supported TLS protocols are [TLSv1.3, TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2Hello]
2025-05-06T06:55:02,644+0000 [main] INFO  org.apache.zookeeper.ClientCnxnSocket - jute.maxbuffer value is 1048575 Bytes
2025-05-06T06:55:02,748+0000 [main] INFO  org.apache.zookeeper.ClientCnxn - zookeeper.request.timeout value is 0. feature enabled=false
2025-05-06T06:55:02,945+0000 [main-SendThread(it-milvus-pulsarv3-zookeeper:2181)] INFO  org.apache.zookeeper.ClientCnxn - Opening socket connection to server it-milvus-pulsarv3-zookeeper/10.125.3.172:2181.
2025-05-06T06:55:02,945+0000 [main-SendThread(it-milvus-pulsarv3-zookeeper:2181)] INFO  org.apache.zookeeper.ClientCnxn - SASL config status: Will not attempt to authenticate using SASL (unknown error)
2025-05-06T06:55:02,952+0000 [main-SendThread(it-milvus-pulsarv3-zookeeper:2181)] INFO  org.apache.zookeeper.ClientCnxn - Socket connection established, initiating session, client: /10.125.3.174:54324, server: it-milvus-pulsarv3-zookeeper/10.125.3.172:2181
2025-05-06T06:55:03,144+0000 [main-SendThread(it-milvus-pulsarv3-zookeeper:2181)] INFO  org.apache.zookeeper.ClientCnxn - Session establishment complete on server it-milvus-pulsarv3-zookeeper/10.125.3.172:2181, session id = 0x2002f5d79e20007, negotiated timeout = 30000
2025-05-06T06:55:03,147+0000 [main-EventThread] INFO  org.apache.bookkeeper.zookeeper.ZooKeeperWatcherBase - ZooKeeper client is connected now.
2025-05-06T06:55:03,350+0000 [main] INFO  org.apache.bookkeeper.tools.cli.commands.bookies.InstanceIdCommand - Metadata Service Uri: zk+null://it-milvus-pulsarv3-zookeeper:2181/ledgers InstanceId: f063757b-1135-4499-885c-094fb6ece639
2025-05-06T06:55:03,550+0000 [main] INFO  org.apache.zookeeper.ZooKeeper - Session: 0x2002f5d79e20007 closed
2025-05-06T06:55:03,550+0000 [main-EventThread] INFO  org.apache.zookeeper.ClientCnxn - EventThread shut down for session: 0x2002f5d79e20007

[piemon-nyah]

Resolved after adjusting resources

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: "it-milvus-pulsarv3-recovery"
  namespace: it-milvus-test
  labels:
    app: pulsarv3
    chart: pulsarv3-3.3.0
    release: it-milvus
    heritage: Helm
    cluster: it-milvus-pulsarv3
    component: recovery
spec:
  serviceName: "it-milvus-pulsarv3-recovery"
  replicas: 1
  updateStrategy:
    type: RollingUpdate
  podManagementPolicy: Parallel
  # nodeSelector:
  selector:
    matchLabels:
      app: pulsarv3
      release: it-milvus
      component: recovery
  template:
    metadata:
      labels:
        app: pulsarv3
        release: it-milvus
        cluster: it-milvus-pulsarv3
        component: recovery
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8000"
    spec:
      affinity:
      terminationGracePeriodSeconds: 30
      serviceAccountName: "it-milvus-pulsarv3-recovery"
      initContainers:
      # This initContainer will wait for bookkeeper initnewcluster to complete
      # before deploying the bookies
      - name: pulsar-bookkeeper-verify-clusterid
        image: "apachepulsar/pulsar:3.0.7"
        imagePullPolicy: IfNotPresent
        resources: 
          requests:
            cpu: 0.1
            memory: 128Mi
        command: ["sh", "-c"]
        args:
        - >
          bin/apply-config-from-env.py conf/bookkeeper.conf;until bin/bookkeeper shell whatisinstanceid; do
            sleep 3;
          done;
        envFrom:
        - configMapRef:
            name: "it-milvus-pulsarv3-recovery"
        volumeMounts:
        
      containers:
      - name: "it-milvus-pulsarv3-recovery"
        image: "apachepulsar/pulsar:3.0.7"
        imagePullPolicy: IfNotPresent
        resources:
          limits:
              cpu: 0.2
              memory: 256Mi
          requests:
            cpu: 0.2
            memory: 256Mi
        command: ["sh", "-c"]
        args:
        - >
          bin/apply-config-from-env.py conf/bookkeeper.conf;
          
          OPTS="${OPTS} -Dlog4j2.formatMsgNoLookups=true" exec bin/bookkeeper autorecovery
        ports:
        - name: http
          containerPort: 8000
        envFrom:
        - configMapRef:
            name: "it-milvus-pulsarv3-recovery"
        volumeMounts:
        
      volumes: