[improve][client] PIP-393: Improve performance of Negative Acknowledgement

distribution/server/src/assemble/LICENSE.bin.txt

@@ -338,12 +338,11 @@ The Apache Software License, Version 2.0
     - io.prometheus-simpleclient_tracer_otel-0.16.0.jar
     - io.prometheus-simpleclient_tracer_otel_agent-0.16.0.jar
  * Prometheus exporter
-    - io.prometheus-prometheus-metrics-config-1.3.1.jar
-    - io.prometheus-prometheus-metrics-exporter-common-1.3.1.jar
-    - io.prometheus-prometheus-metrics-exporter-httpserver-1.3.1.jar
-    - io.prometheus-prometheus-metrics-exposition-formats-1.3.1.jar
-    - io.prometheus-prometheus-metrics-model-1.3.1.jar
-    - io.prometheus-prometheus-metrics-shaded-protobuf-1.3.1.jar
+    - io.prometheus-prometheus-metrics-config-1.3.3.jar
+    - io.prometheus-prometheus-metrics-exporter-common-1.3.3.jar
+    - io.prometheus-prometheus-metrics-exporter-httpserver-1.3.3.jar
+    - io.prometheus-prometheus-metrics-exposition-formats-1.3.3.jar
+    - io.prometheus-prometheus-metrics-model-1.3.3.jar
  * Jakarta Bean Validation API
     - jakarta.validation-jakarta.validation-api-2.0.2.jar
     - javax.validation-validation-api-1.1.0.Final.jar
@@ -481,8 +480,8 @@ The Apache Software License, Version 2.0
   * Prometheus
     - io.prometheus-simpleclient_httpserver-0.16.0.jar
   * Oxia
-    - io.streamnative.oxia-oxia-client-api-0.4.9.jar
-    - io.streamnative.oxia-oxia-client-0.4.9.jar
+    - io.streamnative.oxia-oxia-client-api-0.4.10.jar
+    - io.streamnative.oxia-oxia-client-0.4.10.jar
   * OpenHFT
     - net.openhft-zero-allocation-hashing-0.16.jar
   * Java JSON WebTokens
@@ -516,21 +515,21 @@ The Apache Software License, Version 2.0
   * RoaringBitmap
     - org.roaringbitmap-RoaringBitmap-1.2.0.jar
   * OpenTelemetry
-    - io.opentelemetry-opentelemetry-api-1.41.0.jar
-    - io.opentelemetry-opentelemetry-api-incubator-1.41.0-alpha.jar
-    - io.opentelemetry-opentelemetry-context-1.41.0.jar
-    - io.opentelemetry-opentelemetry-exporter-common-1.41.0.jar
-    - io.opentelemetry-opentelemetry-exporter-otlp-1.41.0.jar
-    - io.opentelemetry-opentelemetry-exporter-otlp-common-1.41.0.jar
-    - io.opentelemetry-opentelemetry-exporter-prometheus-1.41.0-alpha.jar
-    - io.opentelemetry-opentelemetry-exporter-sender-okhttp-1.41.0.jar
-    - io.opentelemetry-opentelemetry-sdk-1.41.0.jar
-    - io.opentelemetry-opentelemetry-sdk-common-1.41.0.jar
-    - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-1.41.0.jar
-    - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-spi-1.41.0.jar
-    - io.opentelemetry-opentelemetry-sdk-logs-1.41.0.jar
-    - io.opentelemetry-opentelemetry-sdk-metrics-1.41.0.jar
-    - io.opentelemetry-opentelemetry-sdk-trace-1.41.0.jar
+    - io.opentelemetry-opentelemetry-api-1.44.1.jar
+    - io.opentelemetry-opentelemetry-api-incubator-1.44.1-alpha.jar
+    - io.opentelemetry-opentelemetry-context-1.44.1.jar
+    - io.opentelemetry-opentelemetry-exporter-common-1.44.1.jar
+    - io.opentelemetry-opentelemetry-exporter-otlp-1.44.1.jar
+    - io.opentelemetry-opentelemetry-exporter-otlp-common-1.44.1.jar
+    - io.opentelemetry-opentelemetry-exporter-prometheus-1.44.1-alpha.jar
+    - io.opentelemetry-opentelemetry-exporter-sender-okhttp-1.44.1.jar
+    - io.opentelemetry-opentelemetry-sdk-1.44.1.jar
+    - io.opentelemetry-opentelemetry-sdk-common-1.44.1.jar
+    - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-1.44.1.jar
+    - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-spi-1.44.1.jar
+    - io.opentelemetry-opentelemetry-sdk-logs-1.44.1.jar
+    - io.opentelemetry-opentelemetry-sdk-metrics-1.44.1.jar
+    - io.opentelemetry-opentelemetry-sdk-trace-1.44.1.jar
     - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-1.33.6.jar
     - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-semconv-1.33.6-alpha.jar
     - io.opentelemetry.instrumentation-opentelemetry-resources-1.33.6-alpha.jar
@@ -572,10 +571,8 @@ Protocol Buffers License
 CDDL-1.1 -- ../licenses/LICENSE-CDDL-1.1.txt
  * Java Annotations API
     - com.sun.activation-javax.activation-1.2.0.jar
-    - javax.xml.bind-jaxb-api-2.3.1.jar
  * Java Servlet API -- javax.servlet-javax.servlet-api-3.1.0.jar
  * WebSocket Server API -- javax.websocket-javax.websocket-client-api-1.0.jar
- * Java Web Service REST API -- javax.ws.rs-javax.ws.rs-api-2.1.jar
  * HK2 - Dependency Injection Kernel
     - org.glassfish.hk2-hk2-api-2.6.1.jar
     - org.glassfish.hk2-hk2-locator-2.6.1.jar

distribution/shell/src/assemble/LICENSE.bin.txt

@@ -388,9 +388,9 @@ The Apache Software License, Version 2.0
     - log4j-slf4j2-impl-2.23.1.jar
     - log4j-web-2.23.1.jar
  * OpenTelemetry
-    - opentelemetry-api-1.41.0.jar
-    - opentelemetry-api-incubator-1.41.0-alpha.jar
-    - opentelemetry-context-1.41.0.jar
+    - opentelemetry-api-1.44.1.jar
+    - opentelemetry-api-incubator-1.44.1-alpha.jar
+    - opentelemetry-context-1.44.1.jar
 
  * BookKeeper
     - bookkeeper-common-allocator-4.17.1.jar
@@ -431,11 +431,8 @@ MIT License
 
 CDDL-1.1 -- ../licenses/LICENSE-CDDL-1.1.txt
  * Java Annotations API
-    - javax.annotation-api-1.3.2.jar
     - javax.activation-1.2.0.jar
-    - jaxb-api-2.3.1.jar
  * WebSocket Server API -- javax.websocket-client-api-1.0.jar
- * Java Web Service REST API -- javax.ws.rs-api-2.1.jar
  * HK2 - Dependency Injection Kernel
     - hk2-api-2.6.1.jar
     - hk2-locator-2.6.1.jar

docker/pulsar/Dockerfile

@@ -94,7 +94,8 @@ RUN apk add --no-cache \
             procps \
             curl \
             bind-tools \
-            openssl
+            openssl \
+            coreutils
 
 # Upgrade all packages to get latest versions with security fixes
 RUN apk upgrade --no-cache

jetcd-core-shaded/pom.xml

@@ -45,6 +45,10 @@
           <groupId>io.netty</groupId>
           <artifactId>*</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>javax.annotation</groupId>
+          <artifactId>javax.annotation-api</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>

pip/pip-373.md

@@ -0,0 +1,78 @@
+# PIP-373: Add a topic's system prop that indicates whether users have published TXN messages in before.
+
+# Background knowledge
+
+**Transaction Buffer**
+- It maintains `aborted` messages indexes and `ongoing` TXN messages indexes in memory, we call them `Two Indexes` in the following sections.
+- It helps to filter out the messages that have been aborted or are still binding with an `on-going` TXN, to avoid consuming aborted messages or TXN in-progress messages.
+
+**Transaction Buffer initializes when a topic is loading up**
+- It reads messages that contain TXN state from the original topic, to re-build `Two Indexes` into memory.
+
+**Transaction Buffer closing when a topic is unloading**
+- It takes a snapshot for `Two Indexes` to rebuild the states faster next loading up.
+
+# Motivation
+
+- Since TXN is a feature that across multiple namespaces, it can be enabled or disabled at the cluster level.
+- Transaction Buffer will try to re-build `Two Indexes` for every topic if you enable TXN.
+- If you have a huge number of topics in a namespace, the task that re-build `Two Indexes` costs huge resources(CPU, Memory).
+
+We'd better skip Transaction Buffer re-building `Two Indexes` if the topic it is related to does not contain TXN messages to save resources usage.
+
+# Goals
+
+Skip Transaction Buffer re-building `Two Indexes` if the topic it is related to does not contain TXN messages to save resources usage.
+
+## In Scope
+
+This PIP only focuses on improving the scenario in which users have never published TXN messages on a topic.
+
+## Out of Scope
+
+This PIP does not focus on the scenario that follows.
+- Published TXN messages.
+- Consumed all TXN messages that were sent before.
+- Transaction Buffer keeps re-building `Two Indexes` even if there are no TXN messages in the topic anymore, and it costs resources.
+
+# Detailed Design
+
+## Design & Implementation Details
+
+- Add a topic-level system property named `__contains_txn_messages`, the default value is `false`, and it will be set to `true` when the first TXN message is publishing.
+- Transaction Buffer skips re-building `Two Indexes` if the property is `false`.
+
+## Public-facing Changes
+
+The topic property `__contains_txn_messages` becomes to a system property, it can never be used by users anymore. 
+
+### Public API
+- You will get a `NotAllowedException` when you try to set/remove a topic property named `__contains_txn_messages` by the API `pulsar-admin topics update-properties/remove-properties`.
+- The value of the property `__contains_txn_messages` can be queried by `pulsar-admin topics get-properties`.
+
+
+### Metrics
+| Name                                       | Description                                         | Attributes | Units|
+|--------------------------------------------|-----------------------------------------------------| --- | --- |
+| `pulsar_broker_using_txn_topics_count` | Counter. The number of topics contains TXN messages. | cluster | - |
+
+# Backward & Forward Compatibility
+
+## Upgrade
+
+There are `3` scenarios that the topic's property `__contains_txn_messages` is when users try to upgrade.
+- `__contains_txn_messages` is empty: broker initializes it by confirming whether there is TXN messages or not.
+- **(Highlight)** `__contains_txn_messages` is not empty and is not typed `boolean`, rollback to the original behavior that always re-building `Two Indexes`, but you can never modify it anymore.
+- **(Highlight)** `__contains_txn_messages` is not empty and is typed `boolean`, but it is a users' property, broker assumed that it is a system property.
+
+## Downgrade / Rollback
+
+You can downgrade or roll back gracefully.
+
+## Pulsar Geo-Replication Upgrade & Downgrade/Rollback Considerations
+
+The PIP does not affect Geo-Replication.
+
+# Links
+* Mailing List discussion thread: https://lists.apache.org/thread/7mblhyvsrw5zybo0gs5512xg8f9sm67v
+* Mailing List voting thread: https://lists.apache.org/thread/s6z5gcjyw081cxf9pwz361r8dt2k8gvl

pip/pip-389.md

@@ -0,0 +1,110 @@
+# PIP-389: Add Producer config compressMinMsgBodySize to improve compression performance
+
+# Background knowledge
+Pulsar provide a way to compress messages before sending them to the broker[0]. This can be done by setting the `compressionType` in the producer configuration. 
+The compressionType can be set to one of the following values:
+- LZ4
+- ZLIB
+- ZSTD
+- SNAPPY
+
+But the compressionType is applied to all messages sent by the producer. This means that even small messages are compressed. 
+
+In our test, we found that compressing small messages can is meaningless. The compression ratio is low and spend more cpu.
+The relevant description in the official documentation:
+>The smaller the amount of data to compress, the more difficult it is to compress. This problem is common to all compression algorithms. [1]
+
+The similar configuration in RocketMQ is `compressMsgBodyOverHowmuch`[2]:
+>/**
+>* Compress message body threshold, namely, message body larger than 4k will be compressed on default.
+>*/
+>private int compressMsgBodyOverHowmuch = 1024 * 4;
+
+[0] https://pulsar.apache.org/docs/4.0.x/concepts-messaging/#compression
+[1] https://github.com/facebook/zstd?tab=readme-ov-file#the-case-for-small-data-compression
+[2] https://github.com/apache/rocketmq/blob/dd62ed0f3b16919adec5d5eece21a1050dc9c5a0/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java#L117
+
+# Motivation
+
+The motivation of this PIP is to provide a way to improve the compression performance by skipping the compression of small messages.
+We want to add a new configuration `compressMinMsgBodySize` to the producer configuration.
+This configuration will allow the user to set the minimum size of the message body that will be compressed. 
+If the message body size is less than the `compressMinMsgBodySize`, the message will not be compressed.
+
+# Goals
+
+## In Scope
+
+Add a new configuration `compressMinMsgBodySize` to the producer configuration.
+
+## Out of Scope
+
+Solve the compression problem of small data
+
+# High Level Design
+
+# Detailed Design
+
+## Design & Implementation Details
+
+Add a new configuration `compressMinMsgBodySize` to the producer configuration.
+This configuration will allow the user to set the minimum size of the message body that will be compressed.
+If the message body size is less than the `compressMinMsgBodySize`, the message will not be compressed.
+
+## Public-facing Changes
+
+Add a new configuration `compressMinMsgBodySize` to the producer configuration.
+
+
+### Public API
+NA
+### Binary protocol
+
+### Configuration
+
+### CLI
+
+### Metrics
+
+NA
+
+# Monitoring
+
+NA
+
+# Security Considerations
+
+NA
+
+# Backward & Forward Compatibility
+
+## Upgrade
+
+This is a new feature, and it does not affect the existing configuration.
+
+## Downgrade / Rollback
+
+The new configuration `compressMinMsgBodySize` will to be removed from the producer configuration.
+If you used it, you need to remove it manually.
+
+## Pulsar Geo-Replication Upgrade & Downgrade/Rollback Considerations
+
+<!--
+Describe what needs to be considered in Pulsar Geo-Replication in the upgrade and possible downgrade/rollback of this feature.
+-->
+
+# Alternatives
+
+<!--
+If there are alternatives that were already considered by the authors or, after the discussion, by the community, and were rejected, please list them here along with the reason why they were rejected.
+-->
+
+# General Notes
+
+# Links
+
+<!--
+Updated afterwards
+-->
+* Mailing List discussion thread: https://lists.apache.org/thread/vxvy7h61hg9wlgby6lcpkm9osdk9sx20
+* Mailing List voting thread: https://lists.apache.org/thread/xv7x3vmycxzsrhbdo7vmssh8lxxzyxd5