diff --git a/.github/workflows/gradle-build.yml b/.github/workflows/gradle-build.yml index 26e45812..192c76dd 100644 --- a/.github/workflows/gradle-build.yml +++ b/.github/workflows/gradle-build.yml @@ -37,7 +37,7 @@ jobs: - name: Set up gradle uses: gradle/actions/setup-gradle@v4 - name: Build - run: ./gradlew build + run: ./gradlew -Dorg.gradle.jvmargs=-Xmx1g build dependency-review: needs: build @@ -66,7 +66,7 @@ jobs: - name: Set up gradle uses: gradle/actions/setup-gradle@v4 - name: Staging artifacts - run: ./gradlew publish + run: ./gradlew -Dorg.gradle.jvmargs=-Xmx1g publish - name: Publish env: JRELEASER_GPG_SECRET_KEY: ${{ secrets.MAVEN_CENTRAL_GPG_SECRET_KEY }} @@ -77,4 +77,4 @@ jobs: JRELEASER_NEXUS2_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }} JRELEASER_NEXUS2_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }} JRELEASER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: ./gradlew :commons:jreleaserDeploy :outbox-kafka-spring:jreleaserDeploy :outbox-kafka-spring-reactive:jreleaserDeploy + run: ./gradlew -Dorg.gradle.jvmargs=-Xmx1g :commons:jreleaserDeploy :outbox-kafka-spring:jreleaserDeploy :outbox-kafka-spring-reactive:jreleaserDeploy diff --git a/README.md b/README.md index 53e19218..a9f84a3b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ [![ci](https://github.com/tomorrow-one/transactional-outbox/actions/workflows/gradle-build.yml/badge.svg)](https://github.com/tomorrow-one/transactional-outbox/actions/workflows/gradle-build.yml) [![maven: outbox-kafka-spring](https://img.shields.io/maven-central/v/one.tomorrow.transactional-outbox/outbox-kafka-spring.svg?label=maven:%20outbox-kafka-spring)](https://central.sonatype.com/search?q=one.tomorrow.transactional-outbox:outbox-kafka-spring) [![maven: outbox-kafka-spring-reactive](https://img.shields.io/maven-central/v/one.tomorrow.transactional-outbox/outbox-kafka-spring-reactive.svg?label=maven:%20outbox-kafka-spring-reactive)](https://central.sonatype.com/search?q=one.tomorrow.transactional-outbox:outbox-kafka-spring-reactive) +[![maven: outbox-kafka-quarkus](https://img.shields.io/maven-central/v/one.tomorrow.transactional-outbox/outbox-kafka-quarkus.svg?label=maven:%20outbox-kafka-quarkus)](https://central.sonatype.com/search?q=one.tomorrow.transactional-outbox:outbox-kafka-quarkus) # Transactional Outbox @@ -44,9 +45,7 @@ the message / payload), a solution would have to be found or developed. At the t experience in the team with Debezium or Kafka Connect. ### Current Limitations -* This library assumes and uses Spring (for transaction handling) -* It comes with a module for usage in classic spring and spring boot projects using sync/blocking operations (this - module uses spring-jdbc), and another module for reactive operations (uses [spring R2DBC](https://spring.io/projects/spring-data-r2dbc) for database access) +* It comes with modules for usage in Spring/Spring Boot projects (classic sync/blocking and reactive operations), and Quarkus applications * It's tested with postgresql only (verified support for other databases could be contributed) ## Installation & Configuration @@ -57,6 +56,7 @@ Depending on your application add one of the following libraries as dependency t * classic (sync/blocking): `one.tomorrow.transactional-outbox:outbox-kafka-spring:$version` * reactive: `one.tomorrow.transactional-outbox:outbox-kafka-spring-reactive:$version` +* quarkus: `one.tomorrow.transactional-outbox:outbox-kafka-quarkus:$version` #### Compatibility Matrix @@ -202,6 +202,28 @@ public class TransactionalOutboxConfig { } ``` +#### Setup the `OutboxProcessor` from `outbox-kafka-quarkus` + +For Quarkus applications, the extension automatically configures the `OutboxProcessor` - no manual setup is required! Simply add the dependency and configure via `application.properties`: + +```properties +# Kafka configuration (standard Quarkus Kafka config) +kafka.bootstrap.servers=localhost:9092 + +# Transactional Outbox configuration +one.tomorrow.transactional-outbox.enabled=true +one.tomorrow.transactional-outbox.processing-interval=PT0.2S +one.tomorrow.transactional-outbox.lock-timeout=PT5S +one.tomorrow.transactional-outbox.lock-owner-id=my-service-instance-1 +one.tomorrow.transactional-outbox.event-source=my-service + +# Optional: Automatic cleanup configuration +one.tomorrow.transactional-outbox.cleanup.interval=PT1H +one.tomorrow.transactional-outbox.cleanup.retention=P30D +``` + +For detailed Quarkus-specific configuration and usage, see the [Quarkus module README](outbox-kafka-quarkus/README.md). + ## Usage In a service that changes the database (inside a transaction), create and serialize the message/event that should @@ -254,6 +276,26 @@ public Mono doSomething(String name) { } ``` +In a **quarkus application** it would look like this: + +```java +@Inject +OutboxService outboxService; + +@Transactional +public void doSomething(String id, String name) { + + // Here s.th. else would be done within the transaction, e.g. some entity created. + + SomeEvent event = SomeEvent.newBuilder() + .setId(id) + .setName(name) + .build(); + Map headers = Map.of(KafkaHeaders.HEADERS_VALUE_TYPE_NAME, event.getDescriptorForType().getFullName()); + outboxService.saveForPublishing("some-topic", id, event.toByteArray(), headers); +} +``` + ### Tracing If you have tracing in place you're probably interested in getting the trace context propagated with Kafka messages as well. @@ -324,7 +366,7 @@ public class Cleaner { ## How-To Release To release a new version follow this step -1. In your PR with the functional change, bump the version of `commons`, `outbox-kafka-spring` or `outbox-kafka-spring-reactive` in the root `build.gradle.kts` to a non-`SNAPSHOT` version. +1. In your PR with the functional change, bump the version of `commons`, `outbox-kafka-spring`, `outbox-kafka-spring-reactive` or `outbox-kafka-quarkus` in the root `build.gradle.kts` to a non-`SNAPSHOT` version. * Try to follow semantic versioning, i.e. bump the major version for binary incompatible changes, the minor version for compatible changes with improvements/new features, and the patch version for bugfixes or non-functional changes like refactorings. 2. Merge your PR - the related pipeline will publish the new version(s) to Sonatype's staging repo (SNAPSHOTs are published to Maven Central Snapshots repository (and are kept for 90 days)). 3. To publish a release, follow https://central.sonatype.com/publishing/deployments diff --git a/build.gradle.kts b/build.gradle.kts index b013f8f9..e753e547 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -6,6 +6,8 @@ import java.util.* project(":commons").version = "3.0.1-SNAPSHOT" project(":outbox-kafka-spring").version = "4.0.1-SNAPSHOT" project(":outbox-kafka-spring-reactive").version = "4.0.1-SNAPSHOT" +project(":outbox-kafka-quarkus").version = "1.0.0-SNAPSHOT" +project(":outbox-kafka-quarkus-deployment").version = "1.0.0-SNAPSHOT" plugins { id("java-library") @@ -16,9 +18,14 @@ plugins { id("org.jreleaser") version "1.20.0" id("jacoco") id("com.github.hierynomus.license") version "0.16.1" + id("io.quarkus.extension") version "3.26.0" apply false + id("io.quarkus") version "3.26.0" apply false } +group = "one.tomorrow.transactional-outbox" + val protobufVersion by extra("3.25.5") +val quarkusVersion by extra("3.26.0") // disable JReleaser on root level jreleaser { @@ -29,18 +36,23 @@ subprojects { apply(plugin = "java-library") apply(plugin = "java-test-fixtures") apply(plugin = "io.freefair.lombok") - apply(plugin = "com.google.protobuf") + // protobuf plugin does not play nicely with quarkus, see + // https://github.com/google/protobuf-gradle-plugin/issues/659 + if (!name.contains("quarkus")) + apply(plugin = "com.google.protobuf") apply(plugin = "maven-publish") apply(plugin = "org.jreleaser") apply(plugin = "jacoco") apply(plugin = "com.github.hierynomus.license") - group = "one.tomorrow.transactional-outbox" + group = rootProject.group java { sourceCompatibility = JavaVersion.VERSION_17 - withJavadocJar() + if (name != "outbox-kafka-quarkus-deployment") { + withJavadocJar() + } withSourcesJar() registerFeature("protobufSupport") { @@ -56,14 +68,16 @@ subprojects { (options as StandardJavadocDocletOptions).addBooleanOption("Xdoclint:none", true) } - protobuf { - protoc { - artifact = "com.google.protobuf:protoc:$protobufVersion" + if (!name.contains("quarkus")) { + protobuf { + protoc { + artifact = "com.google.protobuf:protoc:$protobufVersion" + } } } license { - header = file("../LICENSE-header.txt") + header = rootDir.resolve("LICENSE-header.txt") excludes( setOf( "one/tomorrow/kafka/messages/DeserializerMessages.java", @@ -116,6 +130,11 @@ subprojects { } } + // ignore information that is not contained in the pom file and suppress the warnings: + suppressPomMetadataWarningsFor("protobufSupportApiElements") + suppressPomMetadataWarningsFor("protobufSupportRuntimeElements") + suppressPomMetadataWarningsFor("testFixturesApiElements") + suppressPomMetadataWarningsFor("testFixturesRuntimeElements") } } repositories { @@ -160,6 +179,7 @@ subprojects { allprojects { repositories { mavenCentral() + gradlePluginPortal() } tasks.withType { diff --git a/outbox-kafka-quarkus-deployment/build.gradle.kts b/outbox-kafka-quarkus-deployment/build.gradle.kts new file mode 100644 index 00000000..3b9af7d4 --- /dev/null +++ b/outbox-kafka-quarkus-deployment/build.gradle.kts @@ -0,0 +1,26 @@ +plugins { + id("java") +} + +val quarkusVersion = rootProject.extra["quarkusVersion"] + +dependencies { + implementation(platform("io.quarkus:quarkus-bom:$quarkusVersion")) + implementation("io.quarkus:quarkus-core-deployment") + implementation(project(":outbox-kafka-quarkus")) + implementation("io.quarkus:quarkus-arc-deployment") + implementation("io.quarkus:quarkus-hibernate-orm-deployment") + implementation("io.quarkus:quarkus-kafka-client-deployment") + implementation("io.quarkus:quarkus-jackson-deployment") + implementation("io.quarkus:quarkus-datasource-deployment") + + // testing + testImplementation("io.quarkus:quarkus-junit5-internal") + testImplementation("io.quarkus:quarkus-hibernate-orm") + testImplementation("io.quarkus:quarkus-jdbc-postgresql") + testImplementation("io.quarkus:quarkus-flyway") + testImplementation("io.quarkus:quarkus-kafka-client") + testImplementation("io.quarkus:quarkus-messaging-kafka") + testImplementation("io.smallrye.reactive:smallrye-reactive-messaging-in-memory") + testImplementation("org.awaitility:awaitility") +} diff --git a/outbox-kafka-quarkus-deployment/src/main/java/one/tomorrow/transactionaloutbox/quarkus/deployment/TransactionalOutboxExtensionProcessor.java b/outbox-kafka-quarkus-deployment/src/main/java/one/tomorrow/transactionaloutbox/quarkus/deployment/TransactionalOutboxExtensionProcessor.java new file mode 100644 index 00000000..9aaac7e7 --- /dev/null +++ b/outbox-kafka-quarkus-deployment/src/main/java/one/tomorrow/transactionaloutbox/quarkus/deployment/TransactionalOutboxExtensionProcessor.java @@ -0,0 +1,110 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.quarkus.deployment; + +import io.quarkus.arc.deployment.AdditionalBeanBuildItem; +import io.quarkus.deployment.Capabilities; +import io.quarkus.deployment.Capability; +import io.quarkus.deployment.annotations.BuildProducer; +import io.quarkus.deployment.annotations.BuildStep; +import io.quarkus.deployment.builditem.FeatureBuildItem; +import io.quarkus.hibernate.orm.deployment.spi.AdditionalJpaModelBuildItem; +import one.tomorrow.transactionaloutbox.config.TransactionalOutboxConfig; +import one.tomorrow.transactionaloutbox.health.OutboxProcessorHealthCheck; +import one.tomorrow.transactionaloutbox.model.OutboxLock; +import one.tomorrow.transactionaloutbox.model.OutboxRecord; +import one.tomorrow.transactionaloutbox.publisher.*; +import one.tomorrow.transactionaloutbox.repository.OutboxLockRepository; +import one.tomorrow.transactionaloutbox.repository.OutboxRepository; +import one.tomorrow.transactionaloutbox.service.OutboxLockService; +import one.tomorrow.transactionaloutbox.service.OutboxProcessor; +import one.tomorrow.transactionaloutbox.service.OutboxService; +import one.tomorrow.transactionaloutbox.tracing.NoopTracingServiceProducer; +import one.tomorrow.transactionaloutbox.tracing.OpenTelemetryTracingServiceProducer; + +import java.util.List; + +class TransactionalOutboxExtensionProcessor { + + private static final String FEATURE = "transactional-outbox"; + + @BuildStep + FeatureBuildItem feature() { + return new FeatureBuildItem(FEATURE); + } + + @BuildStep + AdditionalBeanBuildItem outboxBeans() { + return AdditionalBeanBuildItem.builder() + .addBeanClasses( + OutboxLockRepository.class, + OutboxRepository.class, + OutboxLockService.class, + TransactionalOutboxConfig.class, + TransactionalOutboxConfig.CleanupConfig.class, + OutboxService.class, + OutboxProcessor.class, + PublisherConfig.class, + KafkaProducerMessagePublisherFactory.class, + DefaultKafkaProducerFactory.class + ) + .setUnremovable() + .build(); + } + + @BuildStep + void registerTracingBeans( + Capabilities capabilities, + BuildProducer additionalBeans + ) { + // Always register the default/no-op TracingService (it is marked @DefaultBean) + additionalBeans.produce(AdditionalBeanBuildItem.builder() + .addBeanClasses(NoopTracingServiceProducer.class) + .setUnremovable() + .build()); + + // Only register the OTel-based TracingService if the OTel capability is present + if (capabilities.isPresent(Capability.OPENTELEMETRY_TRACER)) { + additionalBeans.produce(AdditionalBeanBuildItem.builder() + .addBeanClass(OpenTelemetryTracingServiceProducer.class) + .setUnremovable() + .build()); + } + } + + @BuildStep + void registerHealthCheckBean( + Capabilities capabilities, + BuildProducer additionalBeans + ) { + // Only register the OutboxProcessorHealthCheck if the smallrye healt capability is present + if (capabilities.isPresent(Capability.SMALLRYE_HEALTH)) { + additionalBeans.produce(AdditionalBeanBuildItem.builder() + .addBeanClass(OutboxProcessorHealthCheck.class) + .setUnremovable() + .build()); + } + } + + @BuildStep + List jpaModels() { + return List.of( + new AdditionalJpaModelBuildItem(OutboxLock.class.getName()), + new AdditionalJpaModelBuildItem(OutboxRecord.class.getName()) + ); + } + +} diff --git a/outbox-kafka-quarkus-deployment/src/test/java/one/tomorrow/transactionaloutbox/quarkus/deployment/TestApp.java b/outbox-kafka-quarkus-deployment/src/test/java/one/tomorrow/transactionaloutbox/quarkus/deployment/TestApp.java new file mode 100644 index 00000000..9a833217 --- /dev/null +++ b/outbox-kafka-quarkus-deployment/src/test/java/one/tomorrow/transactionaloutbox/quarkus/deployment/TestApp.java @@ -0,0 +1,27 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.quarkus.deployment; + +import jakarta.enterprise.context.ApplicationScoped; + +/** + * Test application for the Transactional Outbox extension. + * This serves as a marker class for Quarkus tests. + */ +@ApplicationScoped +public class TestApp { + // This class is intentionally empty - it just serves as a marker for the Quarkus test +} diff --git a/outbox-kafka-quarkus-deployment/src/test/java/one/tomorrow/transactionaloutbox/quarkus/deployment/TransactionalOutboxExtensionInMemoryTest.java b/outbox-kafka-quarkus-deployment/src/test/java/one/tomorrow/transactionaloutbox/quarkus/deployment/TransactionalOutboxExtensionInMemoryTest.java new file mode 100644 index 00000000..4cd6be8b --- /dev/null +++ b/outbox-kafka-quarkus-deployment/src/test/java/one/tomorrow/transactionaloutbox/quarkus/deployment/TransactionalOutboxExtensionInMemoryTest.java @@ -0,0 +1,159 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.quarkus.deployment; + +import io.quarkus.test.QuarkusUnitTest; +import io.smallrye.mutiny.tuples.Tuple2; +import io.smallrye.reactive.messaging.MutinyEmitter; +import io.smallrye.reactive.messaging.kafka.api.OutgoingKafkaRecordMetadata; +import io.smallrye.reactive.messaging.memory.InMemoryConnector; +import io.smallrye.reactive.messaging.memory.InMemorySink; +import jakarta.inject.Inject; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; +import one.tomorrow.transactionaloutbox.config.TransactionalOutboxConfig; +import one.tomorrow.transactionaloutbox.model.OutboxRecord; +import one.tomorrow.transactionaloutbox.publisher.*; +import one.tomorrow.transactionaloutbox.repository.OutboxLockRepository; +import one.tomorrow.transactionaloutbox.service.OutboxLockService; +import one.tomorrow.transactionaloutbox.service.OutboxProcessor; +import one.tomorrow.transactionaloutbox.service.OutboxService; +import one.tomorrow.transactionaloutbox.tracing.NoopTracingService; +import one.tomorrow.transactionaloutbox.tracing.NoopTracingServiceProducer; +import org.eclipse.microprofile.reactive.messaging.Channel; +import org.eclipse.microprofile.reactive.messaging.Message; +import org.eclipse.microprofile.reactive.messaging.spi.Connector; +import org.jboss.shrinkwrap.api.ShrinkWrap; +import org.jboss.shrinkwrap.api.spec.JavaArchive; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.util.List; +import java.util.Optional; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.awaitility.Awaitility.await; +import static org.hamcrest.Matchers.hasSize; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +class TransactionalOutboxExtensionInMemoryTest { + + private static final String TOPIC = "deployment-inmem-topic"; + + public static class TestEmitterResolver implements EmitterMessagePublisher.EmitterResolver { + + @Channel("testTopic") + MutinyEmitter testTopicEmitter; + + @Override + public MutinyEmitter resolveBy(String topic) { + if (TOPIC.equals(topic)) + return testTopicEmitter; + + return null; + } + } + + @RegisterExtension + static final QuarkusUnitTest config = new QuarkusUnitTest() + .withConfigurationResource("application-test.properties") + .setArchiveProducer(() -> + ShrinkWrap.create(JavaArchive.class) + .addClasses(TestApp.class) + .addClasses(OutboxLockRepository.class) + .addClasses(OutboxLockService.class) + .addClasses(TransactionalOutboxConfig.class) + .addClasses(TransactionalOutboxConfig.CleanupConfig.class) + .addClasses(OutboxProcessor.class) + .addClasses(PublisherConfig.class) + .addClasses(MessagePublisher.class) + .addClasses(MessagePublisherFactory.class) + .addClasses(TestEmitterResolver.class) + .addClasses(EmitterMessagePublisher.class) + .addClasses(EmitterMessagePublisherFactory.class) + .addClasses(KafkaProducerMessagePublisher.class) + .addClasses(KafkaProducerMessagePublisherFactory.class) + .addClasses(KafkaProducerMessagePublisherFactory.KafkaProducerFactory.class) + .addClasses(DefaultKafkaProducerFactory.class) + .addClasses(NoopTracingService.class) + .addClasses(NoopTracingServiceProducer.class) + .addAsResource("application-test.properties") + .addAsResource("db/migration/V1__add-outbox-tables.sql") + ) + .overrideConfigKey("quarkus.kafka.devservices.enabled", "false") + // original + .overrideConfigKey("mp.messaging.outgoing.testTopic.connector", "smallrye-in-memory") // in production "smallrye-kafka" + .overrideConfigKey("mp.messaging.outgoing.testTopic.topic", TOPIC); + + @Inject + EntityManager entityManager; + + @Inject + OutboxService outboxService; + + @Inject + @Connector("smallrye-in-memory") + InMemoryConnector inMemoryConnector; + + private InMemorySink testTopicSink; + + @BeforeEach + void setUp() { + testTopicSink = inMemoryConnector.sink("testTopic"); + } + + @BeforeEach + @AfterEach + @Transactional + void cleanUp() { + entityManager.createQuery("DELETE FROM OutboxRecord").executeUpdate(); + } + + @Test + void testTransactionalOutbox() { + // given / when + OutboxRecord outboxRecord = createOutboxRecord(); + assertNotNull(outboxRecord.getId()); + // then + Tuple2 keyValue = waitForNextMessage(testTopicSink); + assertEquals(outboxRecord.getKey(), keyValue.getItem1()); + assertEquals(new String(outboxRecord.getValue()), new String(keyValue.getItem2())); + } + + @Transactional + OutboxRecord createOutboxRecord() { + return outboxService.saveForPublishing(TOPIC, "k1", "value".getBytes()); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static Tuple2 waitForNextMessage(InMemorySink sink) { + List> received = + await().atMost(5, SECONDS).until(sink::received, hasSize(1)); + Message message = received.get(0); + Optional metadata = + message.getMetadata(OutgoingKafkaRecordMetadata.class); + K key = (K) metadata.orElseThrow().getKey(); + V payload = message.getPayload(); + + sink.clear(); + + return Tuple2.of(key, payload); + } + +} diff --git a/outbox-kafka-quarkus-deployment/src/test/java/one/tomorrow/transactionaloutbox/quarkus/deployment/TransactionalOutboxExtensionTest.java b/outbox-kafka-quarkus-deployment/src/test/java/one/tomorrow/transactionaloutbox/quarkus/deployment/TransactionalOutboxExtensionTest.java new file mode 100644 index 00000000..01bd3643 --- /dev/null +++ b/outbox-kafka-quarkus-deployment/src/test/java/one/tomorrow/transactionaloutbox/quarkus/deployment/TransactionalOutboxExtensionTest.java @@ -0,0 +1,131 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.quarkus.deployment; + +import io.quarkus.test.QuarkusUnitTest; +import io.smallrye.common.annotation.Identifier; +import jakarta.inject.Inject; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; +import one.tomorrow.transactionaloutbox.config.TransactionalOutboxConfig; +import one.tomorrow.transactionaloutbox.model.OutboxRecord; +import one.tomorrow.transactionaloutbox.publisher.*; +import one.tomorrow.transactionaloutbox.repository.OutboxLockRepository; +import one.tomorrow.transactionaloutbox.service.OutboxLockService; +import one.tomorrow.transactionaloutbox.service.OutboxProcessor; +import one.tomorrow.transactionaloutbox.service.OutboxService; +import one.tomorrow.transactionaloutbox.tracing.NoopTracingService; +import one.tomorrow.transactionaloutbox.tracing.NoopTracingServiceProducer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.jboss.shrinkwrap.api.ShrinkWrap; +import org.jboss.shrinkwrap.api.spec.JavaArchive; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +class TransactionalOutboxExtensionTest { + + @RegisterExtension + static final QuarkusUnitTest config = new QuarkusUnitTest() + .withConfigurationResource("application-test.properties") + .setArchiveProducer(() -> + ShrinkWrap.create(JavaArchive.class) + .addClasses(TestApp.class) + .addClasses(OutboxLockRepository.class) + .addClasses(OutboxLockService.class) + .addClasses(TransactionalOutboxConfig.class) + .addClasses(TransactionalOutboxConfig.CleanupConfig.class) + .addClasses(OutboxProcessor.class) + .addClasses(PublisherConfig.class) + .addClasses(MessagePublisher.class) + .addClasses(MessagePublisherFactory.class) + .addClasses(EmitterMessagePublisher.class) + .addClasses(EmitterMessagePublisherFactory.class) + .addClasses(KafkaProducerMessagePublisher.class) + .addClasses(KafkaProducerMessagePublisherFactory.class) + .addClasses(KafkaProducerMessagePublisherFactory.KafkaProducerFactory.class) + .addClasses(DefaultKafkaProducerFactory.class) + .addClasses(NoopTracingService.class) + .addClasses(NoopTracingServiceProducer.class) + .addAsResource("application-test.properties") + .addAsResource("db/migration/V1__add-outbox-tables.sql") + ); + + private final String topic = "deployment-topic-" + System.currentTimeMillis(); + + @Inject + EntityManager entityManager; + + @Inject + OutboxService outboxService; + + @Inject + @Identifier("default-kafka-broker") + Map kafkaConfig; + + @BeforeEach + @AfterEach + @Transactional + void cleanUp() { + entityManager.createQuery("DELETE FROM OutboxRecord").executeUpdate(); + } + + @Test + void testTransactionalOutbox() { + // given / when + OutboxRecord outboxRecord = createOutboxRecord(); + assertNotNull(outboxRecord.getId()); + try (KafkaConsumer consumer = new KafkaConsumer<>(consumerConfig())) { + consumer.subscribe(List.of(topic)); + + // then + ConsumerRecords records = consumer.poll(Duration.ofSeconds(10)); + assertEquals(1, records.count()); + ConsumerRecord consumerRecord = records.iterator().next(); + assertEquals(outboxRecord.getKey(), consumerRecord.key()); + assertEquals(new String(outboxRecord.getValue()), consumerRecord.value()); + } + } + + @Transactional + OutboxRecord createOutboxRecord() { + return outboxService.saveForPublishing(topic, "k1", "value".getBytes()); + } + + private @NotNull Map consumerConfig() { + Map consumerProps = new HashMap<>(kafkaConfig); + consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, "testDeploymentConsumer"); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + return consumerProps; + } + +} diff --git a/outbox-kafka-quarkus-deployment/src/test/resources/application-test.properties b/outbox-kafka-quarkus-deployment/src/test/resources/application-test.properties new file mode 100644 index 00000000..d3637156 --- /dev/null +++ b/outbox-kafka-quarkus-deployment/src/test/resources/application-test.properties @@ -0,0 +1,15 @@ +# Test configuration for Transactional Outbox extension +one.tomorrow.transactional-outbox.event-source=deployment +one.tomorrow.transactional-outbox.lock-owner-id=deploymentTest + +# DB configuration +quarkus.datasource.db-kind=postgresql +quarkus.datasource.devservices.enabled=true + +# Flyway configurations +quarkus.flyway.migrate-at-start=true +quarkus.flyway.locations=db/migration + +# Prevent IllegalStateException: Persistence unit [] uses Quarkus' +# main formatting facilities for JSON columns in the database. +quarkus.hibernate-orm.mapping.format.global=ignore diff --git a/outbox-kafka-quarkus-deployment/src/test/resources/db/migration/V1__add-outbox-tables.sql b/outbox-kafka-quarkus-deployment/src/test/resources/db/migration/V1__add-outbox-tables.sql new file mode 100644 index 00000000..5d332b9b --- /dev/null +++ b/outbox-kafka-quarkus-deployment/src/test/resources/db/migration/V1__add-outbox-tables.sql @@ -0,0 +1,20 @@ +CREATE SEQUENCE IF NOT EXISTS outbox_kafka_id_seq; + +CREATE TABLE IF NOT EXISTS outbox_kafka ( + id BIGINT PRIMARY KEY DEFAULT nextval('outbox_kafka_id_seq'::regclass), + created TIMESTAMP WITHOUT TIME ZONE NOT NULL, + processed TIMESTAMP WITHOUT TIME ZONE NULL, + topic CHARACTER VARYING(128) NOT NULL, + key CHARACTER VARYING(128) NULL, + value BYTEA NOT NULL, + headers JSONB NULL +); + +CREATE INDEX idx_outbox_kafka_not_processed ON outbox_kafka (id) WHERE processed IS NULL; +CREATE INDEX idx_outbox_kafka_processed ON outbox_kafka (processed); + +CREATE TABLE IF NOT EXISTS outbox_kafka_lock ( + id CHARACTER VARYING(32) PRIMARY KEY, + owner_id CHARACTER VARYING(128) NOT NULL, + valid_until TIMESTAMP WITHOUT TIME ZONE NOT NULL +); diff --git a/outbox-kafka-quarkus/README.md b/outbox-kafka-quarkus/README.md new file mode 100644 index 00000000..cc76d394 --- /dev/null +++ b/outbox-kafka-quarkus/README.md @@ -0,0 +1,403 @@ +# Transactional Outbox for Quarkus + +This module provides a Quarkus extension for the [Transactional Outbox Pattern](https://microservices.io/patterns/data/transactional-outbox.html) implementation for Kafka. + +## Features + +- **Zero-configuration setup**: The extension automatically configures the outbox processor and required beans +- **Quarkus-native**: Built as a proper Quarkus extension with compile-time optimization +- **JPA/Hibernate ORM integration**: Works seamlessly with Quarkus Hibernate ORM +- **Configuration via application.properties**: All settings configurable through standard Quarkus configuration +- **OpenTelemetry tracing support**: Automatic tracing integration when `quarkus-opentelemetry` is present +- **Health checks**: Health checks ensure that the outbox is processed properly +- **Test support**: The extension supports in-memory testing using Quarkus's SmallRye in-memory connectors + +## Installation + +Add the following dependency to your Quarkus project: + +```xml + + one.tomorrow.transactional-outbox + outbox-kafka-quarkus + ${transactional-outbox.version} + +``` + +Or with Gradle: + +```kotlin +implementation("one.tomorrow.transactional-outbox:outbox-kafka-quarkus:${transactionalOutboxVersion}") +``` + +## Required Dependencies + +The extension requires the following Quarkus capabilities to be present: + +- `quarkus-hibernate-orm` - For JPA/database operations +- `quarkus-kafka-client` - For Kafka producer functionality + +Optional dependencies: +- `quarkus-opentelemetry` - For distributed tracing support + +## Database Setup + +Create the required tables using Flyway, Liquibase, or your preferred migration tool. Use the DDL from: +[outbox tables SQL](../outbox-kafka-spring/src/test/resources/db/migration/V2020.06.19.22.29.00__add-outbox-tables.sql) + +Example with Flyway: + +```sql +-- V1__add_outbox_tables.sql +CREATE TABLE outbox_kafka +( + id BIGSERIAL PRIMARY KEY, + topic VARCHAR(249) NOT NULL, + key VARCHAR(249), + value BYTEA NOT NULL, + headers JSONB, + created TIMESTAMP NOT NULL DEFAULT NOW(), + processed TIMESTAMP +); + +CREATE TABLE outbox_kafka_lock +( + id CHARACTER VARYING(32) PRIMARY KEY, + owner_id CHARACTER VARYING(128) NOT NULL, + valid_until TIMESTAMP WITHOUT TIME ZONE NOT NULL +); + +CREATE INDEX idx_outbox_kafka_not_processed ON outbox_kafka (id) WHERE processed IS NULL; +CREATE INDEX idx_outbox_kafka_processed ON outbox_kafka (processed); +``` + +## Configuration + +Configure the outbox processor in your `application.properties`: + +```properties +# Kafka configuration (standard Quarkus Kafka config) +kafka.bootstrap.servers=localhost:9092 + +# Transactional Outbox configuration +one.tomorrow.transactional-outbox.enabled=true +one.tomorrow.transactional-outbox.processing-interval=PT0.2S +one.tomorrow.transactional-outbox.lock-timeout=PT5S +one.tomorrow.transactional-outbox.lock-owner-id=my-service-instance-1 +one.tomorrow.transactional-outbox.event-source=my-service + +# Optional: Automatic cleanup configuration +one.tomorrow.transactional-outbox.cleanup.interval=PT1H +one.tomorrow.transactional-outbox.cleanup.retention=P30D +``` + +### Configuration Properties + +| Property | Default | Description | +|----------|---------|-------------| +| `one.tomorrow.transactional-outbox.enabled` | `true` | Whether the outbox processor is enabled | +| `one.tomorrow.transactional-outbox.processing-interval` | `PT0.2S` | Interval between outbox processing cycles (should be significantly smaller than lock-timeout) | +| `one.tomorrow.transactional-outbox.lock-timeout` | `PT5S` | Time after which a lock is considered stale and can be acquired by another instance | +| `one.tomorrow.transactional-outbox.lock-owner-id` | *required* | Unique identifier for this instance (e.g., hostname, pod name) | +| `one.tomorrow.transactional-outbox.event-source` | *required* | Source identifier used in the `x-source` header of published messages | +| `one.tomorrow.transactional-outbox.cleanup.interval` | *(disabled)* | Interval for automatic cleanup of processed records | +| `one.tomorrow.transactional-outbox.cleanup.retention` | *(disabled)* | How long to keep processed records before deletion | + +**Important Notes:** +- `lock-owner-id` must be unique per instance for proper distributed locking +- `processing-interval` should be significantly smaller than `lock-timeout` +- `lock-timeout` should be higher than typical GC pauses but smaller than acceptable message delay + +## Usage + +### Basic Usage + +Inject the `OutboxService` and use it within JPA transactions: + +```java +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; +import one.tomorrow.transactionaloutbox.service.OutboxService; + +@ApplicationScoped +public class OrderService { + + @Inject + OutboxService outboxService; + + @Inject + EntityManager entityManager; + + @Transactional + public void processOrder(String orderId, String customerEmail) { + // Save your business entity + Order order = new Order(orderId, customerEmail); + entityManager.persist(order); + + // Prepare event for publishing + OrderProcessedEvent event = OrderProcessedEvent.newBuilder() + .setOrderId(orderId) + .setCustomerEmail(customerEmail) + .build(); + + // Store message in outbox (will be published after transaction commits) + outboxService.saveForPublishing( + "order-events", + orderId, + event.toByteArray() + ); + } +} +``` + +### With Custom Headers + +```java +@Transactional +public void processOrderWithHeaders(String orderId, String customerEmail) { + // ... business logic ... + + Map headers = Map.of( + "event-type", "OrderProcessed", + "event-version", "v1", + "correlation-id", UUID.randomUUID().toString() + ); + + outboxService.saveForPublishing( + "order-events", + orderId, + event.toByteArray(), + headers + ); +} +``` + +## Message Headers + +Published messages include the following headers: + +- `x-sequence`: Database sequence/ID of the outbox record (for deduplication/ordering) +- `x-source`: The configured `event-source` value, can be useful for clients and for migration scenarios +- OpenTelemetry tracing headers (when tracing is enabled) +- Any custom headers you provide + +## Tracing Integration + +When `quarkus-opentelemetry` is present, the extension automatically: + +- Creates spans for outbox processing +- Propagates trace context to Kafka messages +- Links outbox storage and message publishing in traces + +No additional configuration is required - tracing works out of the box with your existing OpenTelemetry setup. + +## Health Checks + +If `quarkus-smallrye-health` is present, the extension provides comprehensive health checks for: + +- **Outbox processor status**: Whether the processor is enabled, active, and functioning correctly +- **Lock acquisition monitoring**: Tracks when the processor last attempted to acquire the distributed lock +- **Message processing monitoring**: Checks the age of the oldest unprocessed message to detect processing stalls +- **Database connectivity**: Verifies the processor can communicate with the database +- **Stale detection**: Alerts if the processor hasn't attempted lock acquisition recently (indicating potential issues) + +The health check is available at the standard Quarkus health endpoints: +- `/q/health/ready` - Includes the outbox processor readyness check +- `/q/health` - Complete health information + +### Health Check Response + +The health check provides detailed information about the processor state: + +```json +{ + "status": "UP", + "checks": [ + { + "name": "transactional-outbox-processor", + "status": "UP", + "data": { + "enabled": true, + "lockOwnerId": "my-service-instance-1", + "active": true, + "closed": false, + "status": "active", + "lastLockAttempt": "2025-01-15T10:30:45.123Z", + "timeSinceLastLockAttempt": "PT2.5S", + "oldestUnprocessedMessageAge": "PT30S", + "oldestUnprocessedMessageCreated": "2025-01-15T10:29:15.123Z" + } + } + ] +} +``` + +**Health Check Data Fields:** +- `enabled`: Whether the outbox processor is enabled in configuration +- `lockOwnerId`: The unique ID of this processor instance +- `active`: Whether this instance currently holds the processing lock +- `closed`: Whether the processor has been shut down +- `status`: Overall status (active/inactive/disabled/closed/stale/processing-stalled/error) +- `lastLockAttempt`: Timestamp of the last lock acquisition attempt +- `timeSinceLastLockAttempt`: Duration since last lock attempt +- `oldestUnprocessedMessageAge`: Age of the oldest unprocessed message (or "none" if no messages) +- `oldestUnprocessedMessageCreated`: Creation timestamp of the oldest unprocessed message +- `reason`: Explanation for unhealthy status + +### Health Status Logic + +- **UP**: Processor is functioning normally + - Processor disabled (not an error) + - Processor inactive (normal in multi-instance setup - only one should be active) + - Processor active with fresh messages (age < 2×processing-interval) + +- **DOWN**: Processor has issues that need attention + - **processing-stalled**: Active processor has unprocessed messages older than 2×processing-interval + - **stale**: Processor hasn't attempted lock acquisition in 2×lock-timeout + - **closed**: Processor has been shut down + - **error**: Unexpected error during health check + +### Processing Stall Detection + +The health check implements intelligent stall detection: +- Only triggers when processor is **active** (holds the lock) +- Compares oldest message age against **2×processing-interval** threshold +- Example: If `processing-interval=PT0.2S`, messages older than `PT0.4S` trigger an alert +- This catches scenarios where the processor is active but not actually processing messages + +## Testing + +### In-Memory Testing (Without Kafka) + +The extension supports in-memory testing using Quarkus's SmallRye in-memory connectors, allowing you to test your application without requiring a real Kafka instance. This is particularly useful for integration tests that also work with incoming / outgoing Kafka messages. + +#### Setup for In-Memory Testing + +1. **Configure channels to use in-memory connector:** + +```properties +# Test configuration (application-test.properties or test profile) +quarkus.kafka.devservices.enabled=false + +# Configure outgoing channels to use in-memory connector +mp.messaging.outgoing.my-channel.connector=smallrye-in-memory +mp.messaging.outgoing.my-channel.topic=my-topic +``` + +2. **Create an EmitterResolver implementation:** + +```java +@ApplicationScoped +public class TestEmitterResolver implements EmitterMessagePublisher.EmitterResolver { + + @Channel("my-channel") + MutinyEmitter myChannelEmitter; + + @Override + public MutinyEmitter resolveBy(String topic) { + if ("my-topic".equals(topic)) { + return myChannelEmitter; + } + return null; + } +} +``` + +3. **Write your test:** + +```java +@QuarkusTest +public class OutboxInMemoryTest { + + @Inject + OutboxService outboxService; + + @Inject + @Connector("smallrye-in-memory") + InMemoryConnector inMemoryConnector; + + private InMemorySink sink; + + @BeforeEach + void setUp() { + sink = inMemoryConnector.sink("my-channel"); + } + + @Test + @Transactional + void testOutboxWithInMemoryKafka() { + // Store message in outbox + outboxService.saveForPublishing("my-topic", "key1", "test-message".getBytes()); + + // Wait for message to be processed and published + await().atMost(5, SECONDS) + .until(() -> sink.received(), hasSize(1)); + + // Verify the published message + Message message = sink.received().get(0); + assertEquals("test-message", new String(message.getPayload())); + + // Get Kafka metadata for key verification + Optional metadata = + message.getMetadata(OutgoingKafkaRecordMetadata.class); + assertEquals("key1", metadata.orElseThrow().getKey()); + + sink.clear(); + } +} +``` + +This testing approach is demonstrated in the `OutboxProcessorInMemoryIntegrationTest`. + +## Deployment Considerations + +### Multiple Instances + +The extension uses database-based locking to ensure only one instance processes the outbox at a time: + +- Each instance tries to acquire a lock using its `lock-owner-id` +- If an instance crashes, others can take over after `lock-timeout` +- Configure unique `lock-owner-id` per instance (e.g., using hostname or pod name) + +### Native Compilation + +The extension is optimized for GraalVM native compilation and works with Quarkus native builds out of the box. + +## Migration from Spring + +If migrating from the Spring version: + +1. Replace Spring dependency with Quarkus dependency +2. Remove manual `@Configuration` classes - the extension handles bean creation +3. Move configuration from Java code to `application.properties` +4. Replace `@Autowired` with `@Inject` +5. Use `@Transactional` (Jakarta) instead of Spring's `@Transactional` + +## Troubleshooting + +### Common Issues + +**Outbox processor not starting:** +- Check that required dependencies are present +- Verify database tables exist +- Check configuration properties + +**Lock acquisition failures:** +- Ensure `lock-owner-id` is unique per instance +- Check `lock-timeout` configuration +- Verify database connectivity + +**Messages not being published:** +- Check Kafka configuration +- Verify outbox processor health checks +- Check application logs for errors + +### Logging + +Enable debug logging for troubleshooting: + +```properties +quarkus.log.category."one.tomorrow.transactionaloutbox".level=DEBUG +``` diff --git a/outbox-kafka-quarkus/build.gradle.kts b/outbox-kafka-quarkus/build.gradle.kts new file mode 100644 index 00000000..7c8dffe0 --- /dev/null +++ b/outbox-kafka-quarkus/build.gradle.kts @@ -0,0 +1,74 @@ +import io.quarkus.deployment.Capability.* + +plugins { + id("java") + id("io.quarkus.extension") +} + +quarkusExtension { + deploymentModule = "outbox-kafka-quarkus-deployment" + capabilities { + requires(CDI) + requires(HIBERNATE_ORM) + requires(TRANSACTIONS) + requires(KAFKA) + requires(JACKSON) + } +} + +val quarkusVersion = rootProject.extra["quarkusVersion"] + +dependencies { + // Quarkus BOM + implementation(platform("io.quarkus:quarkus-bom:$quarkusVersion")) + + // Core dependencies + implementation("io.quarkus:quarkus-hibernate-orm") + implementation("io.quarkus:quarkus-kafka-client") + + // Messaging dependency - optional + compileOnly("io.quarkus:quarkus-messaging-kafka") + // Health check dependencies - optional + compileOnly("io.quarkus:quarkus-smallrye-health") + // Tracing dependencies - optional + compileOnly("io.quarkus:quarkus-opentelemetry") + compileOnly("io.opentelemetry:opentelemetry-api") + + // Test dependencies + testImplementation("io.quarkus:quarkus-junit5") + testImplementation("io.quarkus:quarkus-flyway") + testImplementation("io.quarkus:quarkus-junit5-mockito") + testImplementation("io.quarkus:quarkus-devservices-postgresql:$quarkusVersion") + testImplementation("io.quarkus:quarkus-jdbc-postgresql") + testImplementation("org.testcontainers:postgresql") + testImplementation("org.testcontainers:redpanda") + testImplementation("org.testcontainers:toxiproxy") + testImplementation("org.awaitility:awaitility") + testImplementation("io.quarkus:quarkus-smallrye-fault-tolerance") + testImplementation("io.quarkus:quarkus-messaging-kafka") + testImplementation("io.smallrye.reactive:smallrye-reactive-messaging-in-memory") + testImplementation("io.quarkus:quarkus-smallrye-health") + + // Test tracing dependencies + testImplementation("io.quarkus:quarkus-opentelemetry") + testImplementation("io.opentelemetry:opentelemetry-api") + testImplementation("io.opentelemetry:opentelemetry-sdk-testing") +} + +java { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 +} + +tasks.withType { + + // apply the quarkus plugin so that we get quarkus test support (dev services etc.) + apply(plugin = "io.quarkus") + // don't build this module as quarkus app (this leads to failures with jpa), therefore disable this task + tasks.named("quarkusAppPartsBuild") { + enabled = false + } + + systemProperty("java.util.logging.manager", "org.jboss.logmanager.LogManager") + useJUnitPlatform() +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/commons/KafkaHeaders.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/commons/KafkaHeaders.java new file mode 100644 index 00000000..d424709e --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/commons/KafkaHeaders.java @@ -0,0 +1,72 @@ +/** + * Copyright 2022 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.commons; + +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.Headers; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.math.BigInteger; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + +@SuppressWarnings("unused") +public class KafkaHeaders { + + private static final Logger logger = LoggerFactory.getLogger(KafkaHeaders.class); + + /** + * The name for the header to store service that published the event. + * Useful in a migration scenario (an event is going to be published by a different service). + */ + public static final String HEADERS_SOURCE_NAME = "x-source"; + /** + * The name for the header to store the sequence as long. Because header values are stored as byte[], + * the value is expected to be the big-endian representation of the long in an 8-element byte array.
+ * To transform a long to byte[], you can use {@link Longs#toByteArray(long)}.
+ * Note: {@link BigInteger#toByteArray()} does not return the appropriate representation! + */ + public static final String HEADERS_SEQUENCE_NAME = "x-sequence"; + /** The header to store the type of the value, so data can be deserialized to that type. */ + public static final String HEADERS_VALUE_TYPE_NAME = "x-value-type"; + public static final String HEADERS_DLT_SOURCE_NAME = "x-deadletter-source"; + public static final String HEADERS_DLT_RETRY_NAME = "x-deadletter-retry"; + + public static Map knownHeaders(Headers headers) { + Map res = new HashMap<>(); + addHeaderIfPresent(HEADERS_VALUE_TYPE_NAME, String::new, headers, res); + addHeaderIfPresent(HEADERS_SOURCE_NAME, String::new, headers, res); + addHeaderIfPresent(HEADERS_SEQUENCE_NAME, Longs::toLong, headers, res); + addHeaderIfPresent(HEADERS_DLT_SOURCE_NAME, String::new, headers, res); + addHeaderIfPresent(HEADERS_DLT_RETRY_NAME, Longs::toLong, headers, res); + return res; + } + + private static void addHeaderIfPresent(String key, Function valueTransformer, Headers headers, Map res) { + Header header = headers.lastHeader(key); + if (header != null) { + try { + res.put(key, valueTransformer.apply(header.value())); + } catch (Exception e) { + logger.warn("Failed to transform byte array {} for key {}: {}", Arrays.toString(header.value()), key, e.getMessage(), e); + } + } + } + +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/commons/Longs.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/commons/Longs.java new file mode 100644 index 00000000..3402db73 --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/commons/Longs.java @@ -0,0 +1,61 @@ +/** + * Copyright 2022 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.commons; + +public class Longs { + + /** + * Returns a big-endian representation of value in an 8-element byte array; equivalent to + * ByteBuffer.allocate(8).putLong(value).array().
+ * For example, the input value 0x1213141516171819L would yield the byte array + * {0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19}. + */ + public static byte[] toByteArray(long data) { + return new byte[] { + (byte) (data >>> 56), + (byte) (data >>> 48), + (byte) (data >>> 40), + (byte) (data >>> 32), + (byte) (data >>> 24), + (byte) (data >>> 16), + (byte) (data >>> 8), + (byte) data + }; + } + + /** + * Returns the long value whose big-endian representation is stored in the given byte array (length must be 8); + * equivalent to ByteBuffer.wrap(bytes).getLong().
+ * For example, the input byte array {0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19} would yield the + * long value 0x1213141516171819L. + * @param data array of 8 bytes, the big-endian representation of the long + * @return the long value + * @throws IllegalArgumentException if data is null or has length != 8. + */ + public static long toLong(byte[] data) { + if (data == null || data.length != 8) { + throw new IllegalArgumentException("Size of data received is not 8"); + } + + long value = 0; + for (byte b : data) { + value <<= 8; + value |= b & 0xFF; + } + return value; + } + +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/config/TransactionalOutboxConfig.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/config/TransactionalOutboxConfig.java new file mode 100644 index 00000000..2d280016 --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/config/TransactionalOutboxConfig.java @@ -0,0 +1,89 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.config; + +import io.quarkus.runtime.annotations.ConfigPhase; +import io.quarkus.runtime.annotations.ConfigRoot; +import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithDefault; + +import java.time.Duration; +import java.util.Optional; + +@ConfigMapping(prefix = "one.tomorrow.transactional-outbox", namingStrategy = ConfigMapping.NamingStrategy.KEBAB_CASE) +@ConfigRoot(phase = ConfigPhase.RUN_TIME) +public interface TransactionalOutboxConfig { + + /** + * Whether the outbox processor is enabled, default true. + */ + @WithDefault("true") + boolean enabled(); + + /** + * The interval between outbox processing cycles. + * Should be significantly smaller than the configured lock timeout. + * Recommended value: 200ms (PT0.2S) + */ + @WithDefault("PT0.2S") + Duration processingInterval(); + + /** + * The time after that a lock is considered to be lost/stale, maybe because + * the lock owner crashed or was restarted. After that time another instance + * is allowed to grab and acquire the lock in order to process the outbox. + * + * The value should be significantly higher than the configured processing-interval, + * it should be higher than gc pauses or system stalls, e.g. between 2 and 10 seconds (e.g. PT5S). + */ + @WithDefault("PT5S") + Duration lockTimeout(); + + /** + * A unique identifier for the instance attempting to acquire the lock. + * Example: the hostname of the instance. + */ + String lockOwnerId(); + + /** + * A string identifying the source of the events, used as the value for the `x-source` header. + * Example: the service name or a unique identifier for the producer. + */ + String eventSource(); + + /** + * Configuration for automatic cleanup of processed outbox records. + * If not configured, cleanup will be disabled. + */ + Optional cleanup(); + + /** + * Configuration for cleanup settings. + */ + interface CleanupConfig { + /** + * The interval at which cleanup should be performed. + * Example: PT1H (every hour) + */ + Duration interval(); + + /** + * How long to retain processed records before deleting them. + * Example: P30D (30 days) + */ + Duration retention(); + } +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/health/OutboxProcessorHealthCheck.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/health/OutboxProcessorHealthCheck.java new file mode 100644 index 00000000..07c5a53d --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/health/OutboxProcessorHealthCheck.java @@ -0,0 +1,194 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.health; + +import io.quarkus.logging.Log; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import one.tomorrow.transactionaloutbox.config.TransactionalOutboxConfig; +import one.tomorrow.transactionaloutbox.model.OutboxRecord; +import one.tomorrow.transactionaloutbox.repository.OutboxRepository; +import one.tomorrow.transactionaloutbox.service.OutboxProcessor; +import org.eclipse.microprofile.health.*; + +import java.time.Duration; +import java.time.Instant; + +/** + * Health check for the Transactional Outbox processor. + * + * This health check monitors the status of the outbox processor and provides + * detailed information about its operational state, including whether it's + * active, when it last attempted to acquire the lock, and the current + * number of pending messages in the outbox. + */ +@Readiness +@ApplicationScoped +public class OutboxProcessorHealthCheck implements HealthCheck { + + private static final String HEALTH_CHECK_NAME = "transactional-outbox-processor"; + + private final OutboxProcessor outboxProcessor; + private final OutboxRepository outboxRepository; + private final TransactionalOutboxConfig config; + private final Duration staleThreshold; + + @Inject + public OutboxProcessorHealthCheck( + OutboxProcessor outboxProcessor, + OutboxRepository outboxRepository, + TransactionalOutboxConfig config + ) { + this.outboxProcessor = outboxProcessor; + this.outboxRepository = outboxRepository; + this.config = config; + staleThreshold = config.lockTimeout().multipliedBy(2); + } + + @Override + public HealthCheckResponse call() { + HealthCheckResponseBuilder responseBuilder = HealthCheckResponse.named(HEALTH_CHECK_NAME); + + try { + // Basic processor information + addBasicInformation(responseBuilder); + + HealthCheckResponse response = null; + + // If processor is disabled, it's considered healthy + response = checkEnabledStatus(response, responseBuilder); + if (response != null) + return response; + + // If processor is closed, it's unhealthy + response = checkClosedStatus(response, responseBuilder); + if (response != null) + return response; + + // Check lock acquisition status + response = checkLockAcquisitionStatus(responseBuilder, response); + if (response != null) + return response; + + // Get oldest unprocessed message age + response = checkOldestUnprocessedMessage(responseBuilder, response); + if (response != null) + return response; + + // Overall status assessment + if (outboxProcessor.isActive()) { + responseBuilder.withData("status", "active"); + return responseBuilder.up().build(); + } else { + responseBuilder.withData("status", "inactive"); + responseBuilder.withData("reason", "Processor is not currently holding the lock"); + // Being inactive is not necessarily unhealthy in a multi-instance setup + // Only one instance should be active at a time + return responseBuilder.up().build(); + } + + } catch (Exception e) { + Log.errorv(e, "Exception in OutboxProcessorHealthCheck"); + return responseBuilder + .withData("status", "error") + .withData("error", e.getMessage()) + .down() + .build(); + } + } + + private HealthCheckResponse checkOldestUnprocessedMessage( + HealthCheckResponseBuilder responseBuilder, + HealthCheckResponse response + ) { + try { + var unprocessedRecords = outboxRepository.getUnprocessedRecords(1); + if (!unprocessedRecords.isEmpty()) { + OutboxRecord oldestRecord = unprocessedRecords.get(0); + Duration ageOfOldestMessage = Duration.between(oldestRecord.getCreated(), Instant.now()); + responseBuilder.withData("oldestUnprocessedMessageAge", ageOfOldestMessage.toString()); + responseBuilder.withData("oldestUnprocessedMessageCreated", oldestRecord.getCreated().toString()); + + // If processor is active and oldest message is older than 2x processing interval, + // this indicates the processor is not working properly + Duration processingThreshold = config.processingInterval().multipliedBy(2); + if (outboxProcessor.isActive() && ageOfOldestMessage.compareTo(processingThreshold) > 0) { + response = responseBuilder + .withData("status", "processing-stalled") + .withData("reason", String.format("Active processor has unprocessed message older than %s (actual age: %s)", + processingThreshold, ageOfOldestMessage)) + .down() + .build(); + } + } else { + responseBuilder.withData("oldestUnprocessedMessageAge", "none"); + } + } catch (Exception e) { + Log.infov("Failed to get oldestUnprocessedMessageAge: {0}", e.getMessage()); + } + return response; + } + + private HealthCheckResponse checkLockAcquisitionStatus(HealthCheckResponseBuilder responseBuilder, HealthCheckResponse response) { + Instant lastLockAttempt = outboxProcessor.getLastLockAcquisitionAttempt(); + if (lastLockAttempt != null) { + responseBuilder.withData("lastLockAttempt", lastLockAttempt.toString()); + + Duration timeSinceLastAttempt = Duration.between(lastLockAttempt, Instant.now()); + responseBuilder.withData("timeSinceLastLockAttempt", timeSinceLastAttempt.toString()); + + // Check if the processor seems stale (hasn't attempted lock acquisition recently) + if (timeSinceLastAttempt.compareTo(staleThreshold) > 0) { + response = responseBuilder + .withData("status", "stale") + .withData("reason", String.format("No lock acquisition attempt for %s", timeSinceLastAttempt)) + .down() + .build(); + } + } else { + responseBuilder.withData("lastLockAttempt", "never"); + } + return response; + } + + private HealthCheckResponse checkClosedStatus(HealthCheckResponse response, HealthCheckResponseBuilder responseBuilder) { + if (outboxProcessor.isClosed()) { + response = responseBuilder + .withData("status", "closed") + .withData("reason", "Processor has been shut down") + .down() + .build(); + } + return response; + } + + private HealthCheckResponse checkEnabledStatus(HealthCheckResponse response, HealthCheckResponseBuilder responseBuilder) { + if (!outboxProcessor.isEnabled()) { + response = responseBuilder + .withData("status", "disabled") + .up() + .build(); + } + return response; + } + + private void addBasicInformation(HealthCheckResponseBuilder responseBuilder) { + responseBuilder.withData("enabled", outboxProcessor.isEnabled()); + responseBuilder.withData("lockOwnerId", outboxProcessor.getLockOwnerId()); + responseBuilder.withData("active", outboxProcessor.isActive()); + responseBuilder.withData("closed", outboxProcessor.isClosed()); + } +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/model/OutboxLock.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/model/OutboxLock.java new file mode 100644 index 00000000..5525c822 --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/model/OutboxLock.java @@ -0,0 +1,53 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.model; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import java.time.Instant; + +@Entity +@Table(name="outbox_kafka_lock") +@NoArgsConstructor +@Getter +@Setter +public class OutboxLock { + + // the static value that is used to identify the single possible record in this table - i.e. we make + // use of the uniqueness guarantee of the database to ensure that only a single lock at the same time exists + public static final String OUTBOX_LOCK_ID = "outboxLock"; + + public OutboxLock(String ownerId, Instant validUntil) { + this.ownerId = ownerId; + this.validUntil = validUntil; + } + + @Id + @Column(name="id") + private String id = OUTBOX_LOCK_ID; + + @Column(name="owner_id") + private String ownerId; + + @Column(name="valid_until") + private Instant validUntil; +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/model/OutboxRecord.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/model/OutboxRecord.java new file mode 100644 index 00000000..cfc0bce0 --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/model/OutboxRecord.java @@ -0,0 +1,87 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.model; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.persistence.*; +import lombok.*; +import org.hibernate.annotations.CreationTimestamp; +import org.hibernate.annotations.JdbcTypeCode; + +import org.hibernate.type.SqlTypes; + +import java.time.Instant; +import java.util.Map; + +@Entity +@Table(name = "outbox_kafka") +@AllArgsConstructor +@NoArgsConstructor +@Builder +@Getter +@Setter +@ToString(exclude = "value") +public class OutboxRecord { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @CreationTimestamp + private Instant created; + + @Column(name = "processed") + private Instant processed; + + @Column(name = "topic", nullable = false) + private String topic; + + @Column(name = "key") + private String key; + + @Column(name = "value", nullable = false) + private byte[] value; + + @Column(name = "headers") + @JdbcTypeCode(SqlTypes.JSON) + @Convert(converter = MapConverter.class) + private Map headers; + + static class MapConverter implements AttributeConverter, String> { + + private static final TypeReference> TYPE_REFERENCE = new TypeReference<>() {}; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @Override + public String convertToDatabaseColumn(Map value) { + try { + return value == null ? null : OBJECT_MAPPER.writeValueAsString(value); + } catch (Exception e) { + throw new PersistenceException("Failed to serialize value to JSON: " + value, e); + } + } + + @Override + public Map convertToEntityAttribute(String dbData) { + try { + return dbData == null ? null : OBJECT_MAPPER.readValue(dbData, TYPE_REFERENCE); + } catch (Exception e) { + throw new PersistenceException("Failed to parse JSON string: " + dbData, e); + } + } + } +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/DefaultKafkaProducerFactory.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/DefaultKafkaProducerFactory.java new file mode 100644 index 00000000..3b799567 --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/DefaultKafkaProducerFactory.java @@ -0,0 +1,71 @@ +/** + * Copyright 2022 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.publisher; + +import io.quarkus.arc.DefaultBean; +import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +import static org.apache.kafka.clients.producer.ProducerConfig.*; + +@ApplicationScoped +@DefaultBean +public class DefaultKafkaProducerFactory implements KafkaProducerMessagePublisherFactory.KafkaProducerFactory { + + private static final Logger logger = LoggerFactory.getLogger(DefaultKafkaProducerFactory.class); + + private final HashMap producerProps; + + @Inject + public DefaultKafkaProducerFactory(@Identifier("default-kafka-broker") Map producerProps) { + HashMap props = new HashMap<>(producerProps); + // Settings for guaranteed ordering (via enable.idempotence) and dealing with broker failures. + // Note that with `enable.idempotence = true` ordering of messages is also checked by the broker. + if (Boolean.FALSE.equals(props.get(ENABLE_IDEMPOTENCE_CONFIG))) + logger.warn(ENABLE_IDEMPOTENCE_CONFIG + " is set to 'false' - this might lead to out-of-order messages."); + + setIfNotSet(props, ENABLE_IDEMPOTENCE_CONFIG, true); + + // serializer settings + props.put(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + props.put(VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + this.producerProps = props; + } + + private static void setIfNotSet(Map props, String prop, Object value) { + props.computeIfAbsent(prop, k -> value); + } + + @Override + public KafkaProducer createKafkaProducer() { + return new KafkaProducer<>(producerProps); + } + + @Override + public String toString() { + return "DefaultKafkaProducerFactory{producerProps=" + producerProps + '}'; + } + +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/EmitterMessagePublisher.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/EmitterMessagePublisher.java new file mode 100644 index 00000000..1e4cb2ff --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/EmitterMessagePublisher.java @@ -0,0 +1,88 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.publisher; + +import io.smallrye.reactive.messaging.MutinyEmitter; +import io.smallrye.reactive.messaging.kafka.KafkaRecord; +import io.smallrye.reactive.messaging.kafka.OutgoingKafkaRecord; +import jakarta.annotation.Nonnull; +import lombok.AllArgsConstructor; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; + +/** + * A {@link MessagePublisher} that is based on quarkus Emitter abstraction. + * This allows e.g. in tests to replace the Emitter with an InMemoryEmitter, + * if a real Kafka shall not be used in the test. + */ +@AllArgsConstructor +public class EmitterMessagePublisher implements MessagePublisher { + + /** + * Resolves a {@link MutinyEmitter} to Kafka topics. + * The implementation would statically inject emitters with named channels + * so that it can resolve an emitter to a given Kafka topic. + * + * E.g. for order events and a configuration + *

+ * mp.messaging.outgoing.orderEvents.topic=orders + *

+ * The implementation would inject + *

+ * @Channel("orderEvents") MutinyEmitter<byte[]> orderEventsEmitter; + *

+ * so that for a given topic "orders" it could resolve the orderEventsEmitter. + */ + @FunctionalInterface + public interface EmitterResolver { + MutinyEmitter resolveBy(String topic); + } + + private final EmitterResolver emitterResolver; + + @Override + public Future publish( + Long id, + String topic, + String key, + byte[] payload, + @Nonnull Map headers + ) { + MutinyEmitter emitter = emitterResolver.resolveBy(topic); + if (emitter == null) { + return CompletableFuture.failedFuture( + new IllegalArgumentException("No emitter found for topic " + topic) + ); + } + + OutgoingKafkaRecord kafkaRecord = KafkaRecord.of(key, payload); + for (Map.Entry header : headers.entrySet()) { + kafkaRecord = kafkaRecord.withHeader(header.getKey(), header.getValue()); + } + + return emitter + .sendMessage(kafkaRecord) + .subscribeAsCompletionStage(); + } + + @Override + public void close() { + // noop + } + +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/EmitterMessagePublisherFactory.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/EmitterMessagePublisherFactory.java new file mode 100644 index 00000000..3116100d --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/EmitterMessagePublisherFactory.java @@ -0,0 +1,36 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.publisher; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Alternative; +import lombok.AllArgsConstructor; +import lombok.ToString; + +@ApplicationScoped +@Alternative +@AllArgsConstructor +@ToString +public class EmitterMessagePublisherFactory implements MessagePublisherFactory { + + private final EmitterMessagePublisher.EmitterResolver emitterResolver; + + @Override + public MessagePublisher create() { + return new EmitterMessagePublisher(emitterResolver); + } + +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/KafkaProducerMessagePublisher.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/KafkaProducerMessagePublisher.java new file mode 100644 index 00000000..30249528 --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/KafkaProducerMessagePublisher.java @@ -0,0 +1,54 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.publisher; + +import jakarta.annotation.Nonnull; +import lombok.AllArgsConstructor; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.header.Headers; +import org.apache.kafka.common.header.internals.RecordHeaders; + +import java.util.Map; +import java.util.concurrent.Future; + +@AllArgsConstructor +public class KafkaProducerMessagePublisher implements MessagePublisher { + + private final KafkaProducer kafkaProducer; + + @Override + public Future publish( + Long id, + String topic, + String key, + byte[] payload, + @Nonnull Map headers + ) { + Headers kafkaHeaders = new RecordHeaders(); + headers.forEach(kafkaHeaders::add); + return kafkaProducer.send( + new ProducerRecord<>(topic, null, key, payload, kafkaHeaders) + ); + } + + @Override + public void close() { + if (kafkaProducer != null) { + kafkaProducer.close(); + } + } +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/KafkaProducerMessagePublisherFactory.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/KafkaProducerMessagePublisherFactory.java new file mode 100644 index 00000000..9989b6d7 --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/KafkaProducerMessagePublisherFactory.java @@ -0,0 +1,50 @@ +/** + * Copyright 2022 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.publisher; + +import io.quarkus.arc.DefaultBean; +import jakarta.enterprise.context.ApplicationScoped; +import lombok.AllArgsConstructor; +import org.apache.kafka.clients.producer.KafkaProducer; + +@ApplicationScoped +@DefaultBean +@AllArgsConstructor +public class KafkaProducerMessagePublisherFactory implements MessagePublisherFactory { + + /** + * Creates a new {@link KafkaProducer}. By default, {@link DefaultKafkaProducerFactory} + * is used, but you can supply your own implementation. + * Just ensure that the producer is configured with enable.idempotence=true for strict ordering. + */ + @FunctionalInterface + public interface KafkaProducerFactory { + KafkaProducer createKafkaProducer(); + } + + private final KafkaProducerFactory kafkaProducerFactory; + + @Override + public MessagePublisher create() { + return new KafkaProducerMessagePublisher(kafkaProducerFactory.createKafkaProducer()); + } + + @Override + public String toString() { + return "KafkaProducerMessageFactory{producerFactory=" + kafkaProducerFactory + '}'; + } + +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/MessagePublisher.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/MessagePublisher.java new file mode 100644 index 00000000..e58ca009 --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/MessagePublisher.java @@ -0,0 +1,39 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.publisher; + +import jakarta.annotation.Nonnull; + +import java.util.Map; +import java.util.concurrent.Future; + +/** + * Abstracts message publishing. The default implementation is {@link KafkaProducerMessagePublisher}, + * but there's also the {@link EmitterMessagePublisher} that can be useful in tests, + * to be able to use in memory channels / an in memory emitter instead of a Kafka backend. + */ +public interface MessagePublisher { + + Future publish( + Long id, + String topic, + String key, + byte[] payload, + @Nonnull Map headers); + + void close(); + +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/MessagePublisherFactory.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/MessagePublisherFactory.java new file mode 100644 index 00000000..32ddf0b6 --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/MessagePublisherFactory.java @@ -0,0 +1,27 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.publisher; + +/** + * Allows to create {@link MessagePublisher} instances. This is needed for cases, + * when the message publisher was closed (e.g. due to repeated errors) so that + * a new instance can start cleanly. + */ +public interface MessagePublisherFactory { + + MessagePublisher create(); + +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/PublisherConfig.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/PublisherConfig.java new file mode 100644 index 00000000..44df9f4c --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/publisher/PublisherConfig.java @@ -0,0 +1,65 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.publisher; + +import io.quarkus.logging.Log; +import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Instance; +import jakarta.enterprise.inject.Produces; +import one.tomorrow.transactionaloutbox.publisher.EmitterMessagePublisher.EmitterResolver; +import one.tomorrow.transactionaloutbox.publisher.KafkaProducerMessagePublisherFactory.KafkaProducerFactory; + +import java.util.Map; + +@ApplicationScoped +public class PublisherConfig { + + private final Instance> producerPropsInstance; + private final Instance emitterResolverInstance; + private final Instance kafkaProducerFactoryInstance; + + public PublisherConfig( + @Identifier("default-kafka-broker") + Instance> producerPropsInstance, + Instance emitterResolverInstance, + Instance kafkaProducerFactoryInstance) { + this.producerPropsInstance = producerPropsInstance; + this.emitterResolverInstance = emitterResolverInstance; + this.kafkaProducerFactoryInstance = kafkaProducerFactoryInstance; + } + + @Produces + @ApplicationScoped + public MessagePublisherFactory createMessagePublisherFactory() { + // Check if Kafka producer props are available and contain bootstrap.servers + if (producerPropsInstance.isResolvable() && kafkaProducerFactoryInstance.isResolvable()) { + Map producerProps = producerPropsInstance.get(); + Object bootstrapServers = producerProps.get("bootstrap.servers"); + if (bootstrapServers != null && !((String)bootstrapServers).isEmpty()) { + return new KafkaProducerMessagePublisherFactory(kafkaProducerFactoryInstance.get()); + } + } + + // Check if EmitterResolver is available + if (emitterResolverInstance.isResolvable()) { + return new EmitterMessagePublisherFactory(emitterResolverInstance.get()); + } + + throw new IllegalStateException("No suitable MessagePublisherFactory can be created. " + + "Either provide Kafka producer properties with 'bootstrap.servers' or an EmitterResolver."); + } +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/repository/OutboxLockRepository.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/repository/OutboxLockRepository.java new file mode 100644 index 00000000..e96aa02d --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/repository/OutboxLockRepository.java @@ -0,0 +1,161 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.repository; + +import jakarta.inject.Singleton; +import jakarta.persistence.EntityManager; +import jakarta.persistence.LockModeType; +import jakarta.persistence.PersistenceException; +import jakarta.persistence.TypedQuery; +import jakarta.transaction.Transactional; +import lombok.AllArgsConstructor; +import one.tomorrow.transactionaloutbox.model.OutboxLock; +import org.hibernate.exception.ConstraintViolationException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import java.util.Optional; + +import static java.time.Instant.now; + +@Singleton +@AllArgsConstructor +public class OutboxLockRepository { + + private static final Logger logger = LoggerFactory.getLogger(OutboxLockRepository.class); + + private static final Map PESSIMISTIC_LOCK_PROPS = + Map.of("jakarta.persistence.lock.timeout", 0); + + private final EntityManager entityManager; + + /** + * Acquires or refreshes a lock for the given owner ID with the specified timeout. + * + * @param ownerId the ID of the owner trying to acquire the lock + * @param timeout duration until the lock expires + * @return true if the lock was acquired or refreshed, false otherwise + */ + @Transactional(Transactional.TxType.REQUIRES_NEW) + public boolean acquireOrRefreshLock(String ownerId, Duration timeout) { + OutboxLock lock = null; + Instant now = now(); + try { + lock = entityManager.find(OutboxLock.class, OutboxLock.OUTBOX_LOCK_ID); + if (lock == null) { + logger.debug("No outbox lock found. Creating one for {}", ownerId); + lock = new OutboxLock(ownerId, now.plus(timeout)); + } else { + if (ownerId.equals(lock.getOwnerId())) { + logger.debug("Found outbox lock with requested owner {}, valid until {} - updating lock", lock.getOwnerId(), lock.getValidUntil()); + entityManager.lock(lock, LockModeType.PESSIMISTIC_WRITE, PESSIMISTIC_LOCK_PROPS); + lock.setValidUntil(now.plus(timeout)); + } else if (lock.getValidUntil().isAfter(now)) { + logger.debug("Found outbox lock with owner {}, valid until {}", lock.getOwnerId(), lock.getValidUntil()); + setRollbackOnly(); + return false; + } else { + logger.info("Found expired outbox lock with owner {}, which was valid until {} - grabbing lock for {}", lock.getOwnerId(), lock.getValidUntil(), ownerId); + entityManager.lock(lock, LockModeType.PESSIMISTIC_WRITE, PESSIMISTIC_LOCK_PROPS); + lock.setOwnerId(ownerId); + lock.setValidUntil(now.plus(timeout)); + } + } + + entityManager.persist(lock); + entityManager.flush(); + logger.info("Acquired or refreshed outbox lock for owner {}, valid until {}", ownerId, lock.getValidUntil()); + return true; + } catch (PersistenceException e) { + if (e.getCause() instanceof ConstraintViolationException) { + return handleConstraintViolation(e, ownerId); + } else { + return handleLockingException(e, ownerId, lock); + } + } catch (Throwable e) { + logger.warn("Outbox lock selection/acquisition for owner {} failed", ownerId, e); + setRollbackOnly(); + throw e; + } + } + + private boolean handleLockingException(Exception e, String ownerId, OutboxLock lock) { + String reason = e.getCause() != null ? e.getCause().toString() : e.toString(); + logger.info("Could not grab lock {} for owner {} - database row is locked: {}", lock, ownerId, reason); + setRollbackOnly(); + return false; + } + + private boolean handleConstraintViolation(Exception e, String ownerId) { + String reason = e.getCause() != null ? e.getCause().toString() : e.toString(); + logger.info("Outbox lock for owner {} could not be created, another one has been created concurrently: {}", ownerId, reason); + setRollbackOnly(); + return false; + } + + private void setRollbackOnly() { + try { + jakarta.transaction.TransactionManager transactionManager = com.arjuna.ats.jta.TransactionManager.transactionManager(); + if (transactionManager.getTransaction() != null) { + transactionManager.getTransaction().setRollbackOnly(); + } + } catch (Exception ex) { + logger.info("Caught exception while rolling back OutBox transaction", ex); + } + } + + /** + * Locks the outbox lock row for the given owner if it exists. + * Must be executed inside some outer transaction. + * + * @return true if the lock could be acquired, otherwise false. + */ + public boolean preventLockStealing(String ownerId) { + Optional lock = queryByOwnerId(ownerId) + .setLockMode(LockModeType.PESSIMISTIC_READ) + .getResultStream() + .findFirst(); + return lock.isPresent(); + } + + /** + * Releases the lock held by the specified owner + * + * @param ownerId the ID of the owner whose lock should be released + */ + @Transactional + public void releaseLock(String ownerId) { + queryByOwnerId(ownerId) + .getResultStream() + .findFirst() + .ifPresentOrElse(lock -> { + entityManager.remove(lock); + entityManager.flush(); + logger.info("Released outbox lock for owner {}", ownerId); + }, + () -> logger.debug("Outbox lock for owner {} not found", ownerId) + ); + } + + private TypedQuery queryByOwnerId(String ownerId) { + return entityManager + .createQuery("FROM OutboxLock WHERE ownerId = :ownerId", OutboxLock.class) + .setParameter("ownerId", ownerId); + } +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/repository/OutboxRepository.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/repository/OutboxRepository.java new file mode 100644 index 00000000..6f32911d --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/repository/OutboxRepository.java @@ -0,0 +1,87 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.repository; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; +import lombok.AllArgsConstructor; +import one.tomorrow.transactionaloutbox.model.OutboxRecord; + +import java.time.Instant; +import java.util.List; + +@ApplicationScoped +@AllArgsConstructor +public class OutboxRepository { + + private final EntityManager entityManager; + + /** + * Persist a new outbox record + * @param record the record to persist + */ + public void persist(OutboxRecord record) { + entityManager.persist(record); + } + + /** + * Update an existing outbox record + * @param record the record to update + */ + @Transactional + public void update(OutboxRecord record) { + entityManager.merge(record); + entityManager.flush(); + } + + /** + * Return all records that have not yet been processed (i.e. that do not have the "processed" timestamp set). + * + * @param limit the max number of records to return + * @return the requested records, sorted by id ascending + */ + @Transactional + public List getUnprocessedRecords(int limit) { + return entityManager + .createQuery("FROM OutboxRecord WHERE processed IS NULL ORDER BY id ASC", OutboxRecord.class) + .setMaxResults(limit) + .getResultList(); + } + + /** + * Count the number of unprocessed outbox records + * @return the count of records where processed is null + */ + public long countUnprocessedRecords() { + return entityManager.createQuery("SELECT COUNT(o) FROM OutboxRecord o WHERE o.processed IS NULL", Long.class) + .getSingleResult(); + } + + /** + * Delete processed records older than defined point in time + * + * @param deleteOlderThan the point in time until the processed entities shall be kept + * @return amount of deleted rows + */ + @Transactional + public int deleteOutboxRecordByProcessedNotNullAndProcessedIsBefore(Instant deleteOlderThan) { + return entityManager + .createQuery("DELETE FROM OutboxRecord or WHERE or.processed IS NOT NULL AND or.processed < :deleteOlderThan") + .setParameter("deleteOlderThan", deleteOlderThan) + .executeUpdate(); + } +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/service/OutboxLockService.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/service/OutboxLockService.java new file mode 100644 index 00000000..af3fc4c5 --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/service/OutboxLockService.java @@ -0,0 +1,64 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.service; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import jakarta.transaction.Transactional; +import one.tomorrow.transactionaloutbox.config.TransactionalOutboxConfig; +import one.tomorrow.transactionaloutbox.repository.OutboxLockRepository; + +import java.time.Duration; + +/** + * Service for acquiring and refreshing a lock for the outbox processor. + */ +@ApplicationScoped +public class OutboxLockService { + + private final OutboxLockRepository repository; + private final Duration lockTimeout; + + @Inject + public OutboxLockService( + OutboxLockRepository repository, + TransactionalOutboxConfig config) { + this.repository = repository; + this.lockTimeout = config.lockTimeout(); + } + + public boolean acquireOrRefreshLock(String ownerId) { + return repository.acquireOrRefreshLock(ownerId, lockTimeout); + } + + public void releaseLock(String ownerId) { + repository.releaseLock(ownerId); + } + + @Transactional + public boolean runWithLock(String ownerId, Runnable action) { + boolean outboxLockIsPreventedFromLockStealing = repository.preventLockStealing(ownerId); + if (outboxLockIsPreventedFromLockStealing) { + action.run(); + } + return outboxLockIsPreventedFromLockStealing; + } + + Duration getLockTimeout() { + return lockTimeout; + } + +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/service/OutboxProcessor.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/service/OutboxProcessor.java new file mode 100644 index 00000000..b8404cb3 --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/service/OutboxProcessor.java @@ -0,0 +1,326 @@ +/** + * Copyright 2022 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.service; + +import io.quarkus.runtime.Startup; +import io.smallrye.mutiny.tuples.Tuple3; +import jakarta.annotation.PreDestroy; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import lombok.Getter; +import one.tomorrow.transactionaloutbox.commons.Longs; +import one.tomorrow.transactionaloutbox.config.TransactionalOutboxConfig; +import one.tomorrow.transactionaloutbox.model.OutboxRecord; +import one.tomorrow.transactionaloutbox.publisher.KafkaProducerMessagePublisherFactory; +import one.tomorrow.transactionaloutbox.publisher.MessagePublisher; +import one.tomorrow.transactionaloutbox.publisher.MessagePublisherFactory; +import one.tomorrow.transactionaloutbox.repository.OutboxRepository; +import one.tomorrow.transactionaloutbox.tracing.NoopTracingService; +import one.tomorrow.transactionaloutbox.tracing.TracingService; +import one.tomorrow.transactionaloutbox.tracing.TracingService.TraceOutboxRecordProcessingResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.*; + +import static java.time.Instant.now; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static one.tomorrow.transactionaloutbox.commons.KafkaHeaders.HEADERS_SEQUENCE_NAME; +import static one.tomorrow.transactionaloutbox.commons.KafkaHeaders.HEADERS_SOURCE_NAME; + +@Startup +@ApplicationScoped +public class OutboxProcessor { + + private static final int BATCH_SIZE = 100; + + private static final Logger logger = LoggerFactory.getLogger(OutboxProcessor.class); + + @Getter + private final boolean enabled; + private final OutboxLockService lockService; + + /** + * Returns the lock owner ID for this processor instance. + */ + @Getter + private final String lockOwnerId; + private final OutboxRepository repository; + private final MessagePublisherFactory publisherFactory; + private final Duration processingInterval; + private final byte[] eventSource; + private final TracingService tracingService; + private MessagePublisher publisher; + + /** + * Returns whether this processor is currently active and processing the outbox. + * An active processor has acquired the lock and is processing messages. + */ + @Getter + private boolean active; + + /** + * Returns whether this processor has been closed/shut down. + */ + @Getter + private boolean closed; + + /** + * Returns the time of the last lock acquisition attempt, or null if no attempt was made yet. + */ + @Getter + private Instant lastLockAcquisitionAttempt; + + private final ScheduledExecutorService scheduledExecutor; + private final ScheduledExecutorService cleanupExecutor; + private ScheduledFuture schedule; + private ScheduledFuture cleanupSchedule; + + /** + * Constructs an {@code OutboxProcessor} to process the outbox and publish messages to Kafka. + * + * @param config The {@link TransactionalOutboxConfig} containing configuration settings. + * @param repository The {@link OutboxRepository} to retrieve and update outbox records. + * Typically, this is instantiated by the framework. + * @param publisherFactory A factory to create {@link MessagePublisher} instances for publishing messages. + * By default, the {@link KafkaProducerMessagePublisherFactory} is used. + * @param lockService The {@link OutboxLockService} to manage distributed locks for processing. + * Ensures only one instance processes the outbox at a time. + * @param tracingService The {@link TracingService} to handle distributed tracing. + */ + @Inject + public OutboxProcessor( + TransactionalOutboxConfig config, + OutboxRepository repository, + MessagePublisherFactory publisherFactory, + OutboxLockService lockService, + TracingService tracingService) { + if (config.enabled()) + logger.info("Starting outbox processor with lockOwnerId {}, source {}, processing interval {} ms" + + " and publisher factory {}", config.lockOwnerId(), config.eventSource(), config.processingInterval().toMillis(), publisherFactory); + else + logger.info("Skipping outbox processor since enabled=false"); + + this.enabled = config.enabled(); + this.repository = repository; + this.lockService = lockService; + this.processingInterval = config.processingInterval(); + this.lockOwnerId = config.lockOwnerId(); + this.eventSource = config.eventSource().getBytes(); + this.tracingService = tracingService != null ? tracingService : new NoopTracingService(); + this.publisherFactory = publisherFactory; + publisher = publisherFactory.create(); + + if (config.enabled()) { + scheduledExecutor = Executors.newSingleThreadScheduledExecutor(); + cleanupExecutor = config.cleanup() + .map(cleanup -> setupCleanupSchedule(repository, cleanup)) + .orElse(null); + tryLockAcquisition(false); + } else { + scheduledExecutor = null; + cleanupExecutor = null; + } + } + + private ScheduledExecutorService setupCleanupSchedule(OutboxRepository repository, TransactionalOutboxConfig.CleanupConfig cleanupConfig) { + final ScheduledExecutorService es = Executors.newSingleThreadScheduledExecutor(); + cleanupSchedule = es.scheduleAtFixedRate(() -> { + if (active) { + Instant processedBefore = now().minus(cleanupConfig.retention()); + logger.info("Cleaning up outbox records processed before {}", processedBefore); + repository.deleteOutboxRecordByProcessedNotNullAndProcessedIsBefore(processedBefore); + } + }, 0, cleanupConfig.interval().toMillis(), MILLISECONDS); + return es; + } + + private void scheduleProcessing() { + if (scheduledExecutor.isShutdown()) + logger.info("Not scheduling processing for lockOwnerId {} (executor is shutdown)", lockOwnerId); + else + schedule = scheduledExecutor.schedule(this::processOutboxWithLock, processingInterval.toMillis(), MILLISECONDS); + } + + private void scheduleTryLockAcquisition() { + if (scheduledExecutor.isShutdown()) + logger.info("Not scheduling acquisition of outbox lock for lockOwnerId {} (executor is shutdown)", lockOwnerId); + else + schedule = scheduledExecutor.schedule(this::tryLockAcquisitionAndProcess, lockService.getLockTimeout().toMillis(), MILLISECONDS); + } + + @PreDestroy + public void close() { + closed = true; + if (enabled) { + logger.info("Stopping OutboxProcessor."); + if (schedule != null) + schedule.cancel(false); + scheduledExecutor.shutdownNow(); + + if (cleanupSchedule != null) + cleanupSchedule.cancel(false); + if (cleanupExecutor != null) + cleanupExecutor.shutdownNow(); + + publisher.close(); + if (active) + lockService.releaseLock(lockOwnerId); + } + } + + private void tryLockAcquisitionAndProcess() { + tryLockAcquisition(true); + } + + private void tryLockAcquisition(boolean processDirectlyIfLocked) { + try { + boolean originalActive = active; + logger.debug("{} trying to acquire outbox lock", lockOwnerId); + active = lockService.acquireOrRefreshLock(lockOwnerId); + lastLockAcquisitionAttempt = now(); + if (active) { + if (originalActive) + logger.debug("{} acquired outbox lock, starting to process outbox", lockOwnerId); + else + logger.info("{} acquired outbox lock, starting to process outbox", lockOwnerId); + + if (processDirectlyIfLocked) + processOutboxWithLock(); + else + scheduleProcessing(); + } + else + scheduleTryLockAcquisition(); + } catch (Exception e) { + if (closed) { + logger.debug("After closed, failed to acquire outbox lock: {}", e.toString()); + } else { + logger.warn("Failed trying lock acquisition or processing the outbox, trying again in {}", lockService.getLockTimeout(), e); + scheduleTryLockAcquisition(); + } + } + } + + private void processOutboxWithLock() { + if (!active) + throw new IllegalStateException("processOutbox must only be run when in active state"); + + if (now().isAfter(lastLockAcquisitionAttempt.plus(lockService.getLockTimeout().dividedBy(2)))) { + tryLockAcquisitionAndProcess(); + return; + } + + boolean couldRunWithLock = tryProcessOutbox(); + if (couldRunWithLock) { + scheduleProcessing(); + } else if (!closed) { + logger.info("Lock was lost, changing to inactive, now trying to acquire lock in {} ms", lockService.getLockTimeout().toMillis()); + active = false; + scheduleTryLockAcquisition(); + } + + } + + private boolean tryProcessOutbox() { + boolean couldRunWithLock = false; + try { + couldRunWithLock = lockService.runWithLock(lockOwnerId, () -> { + try { + processOutbox(); + } catch (Throwable e) { + if (!closed) { + logger.warn("Recreating producer, due to failure while processing outbox.", e); + publisher.close(); + publisher = publisherFactory.create(); + } + } + }); + } catch (Exception e) { + if (closed) + logger.debug("After closed, caught exception when trying to run with lock: {}", e.toString()); + else + logger.warn("Caught exception when trying to run with lock", e); + } + return couldRunWithLock; + } + + void processOutbox() { + logger.debug("Processing outbox"); + repository.getUnprocessedRecords(BATCH_SIZE) + .stream() + .map(outboxRecord -> { + TraceOutboxRecordProcessingResult tracingResult = + tracingService.traceOutboxRecordProcessing(outboxRecord); + Future futureResult = publisher.publish( + outboxRecord.getId(), + outboxRecord.getTopic(), + outboxRecord.getKey(), + outboxRecord.getValue(), + getHeaders(outboxRecord, tracingResult)); + return Tuple3.of(outboxRecord, tracingResult, futureResult); + }) + // collect to List (so that map is completed for all items before awaiting futures), + // to use producer internal batching + .toList() + .forEach(tuple3 -> { + OutboxRecord outboxRecord = tuple3.getItem1(); + TraceOutboxRecordProcessingResult tracingResult = tuple3.getItem2(); + Future result = tuple3.getItem3(); + try { + await(result); + logger.info("Sent record to kafka: {}", outboxRecord); + outboxRecord.setProcessed(now()); + repository.update(outboxRecord); + tracingResult.publishCompleted(); + } catch (RuntimeException e) { + logger.warn("Failed to publish {}", outboxRecord, e); + tracingResult.publishFailed(e); + } + }); + } + + private Map getHeaders( + OutboxRecord outboxRecord, + TraceOutboxRecordProcessingResult tracingResult + ) { + Map headers = new HashMap<>(); + if (tracingResult.getHeaders() != null) { + tracingResult.getHeaders().forEach((key, value) -> + headers.put(key, value.getBytes()) + ); + } + headers.put(HEADERS_SEQUENCE_NAME, Longs.toByteArray(outboxRecord.getId())); + headers.put(HEADERS_SOURCE_NAME, eventSource); + return headers; + } + + private static void await(Future future) { + try { + future.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } catch (ExecutionException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/service/OutboxService.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/service/OutboxService.java new file mode 100644 index 00000000..f95d815e --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/service/OutboxService.java @@ -0,0 +1,92 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.service; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import jakarta.transaction.Transactional; +import one.tomorrow.transactionaloutbox.model.OutboxRecord; +import one.tomorrow.transactionaloutbox.repository.OutboxRepository; +import one.tomorrow.transactionaloutbox.tracing.TracingService; + +import java.util.HashMap; +import java.util.Map; + +import static jakarta.transaction.Transactional.TxType.MANDATORY; + +@ApplicationScoped +public class OutboxService { + + private final OutboxRepository repository; + private final TracingService tracingService; + + @Inject + public OutboxService(OutboxRepository repository, TracingService tracingService) { + this.repository = repository; + this.tracingService = tracingService; + } + + /** + * Persist a record to the outbox table, must be called inside a transaction context. + * + * @param topic the Kafka topic to send the record to + * @param key the key used for the Kafka record (can be null) + * @param value the value used for the Kafka record + * @return the ID of the persisted outbox record + */ + @Transactional(MANDATORY) + public OutboxRecord saveForPublishing(String topic, String key, byte[] value) { + return saveForPublishing(topic, key, value, null); + } + + /** + * Persist a record to the outbox table, must be called inside a transaction context. + * + * @param topic the Kafka topic to send the record to + * @param key the key used for the Kafka record (can be null) + * @param value the value used for the Kafka record + * @param headers the headers used for the Kafka record (can be null) + * @return the ID of the persisted outbox record + */ + @Transactional(MANDATORY) + public OutboxRecord saveForPublishing(String topic, String key, byte[] value, Map headers) { + Map tracingHeaders = tracingService.tracingHeadersForOutboxRecord(); + Map allHeaders = merge(headers, tracingHeaders); + OutboxRecord outboxRecord = OutboxRecord.builder() + .topic(topic) + .key(key) + .value(value) + .headers(allHeaders) + .build(); + repository.persist(outboxRecord); + return outboxRecord; + } + + public static Map merge(Map map1, Map map2) { + if (isNullOrEmpty(map1)) + return map2; + if (isNullOrEmpty(map2)) + return map1; + Map result = new HashMap<>(map1); + result.putAll(map2); + return result; + } + + public static boolean isNullOrEmpty(Map map) { + return map == null || map.isEmpty(); + } + +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/tracing/NoopTracingService.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/tracing/NoopTracingService.java new file mode 100644 index 00000000..665a4dea --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/tracing/NoopTracingService.java @@ -0,0 +1,39 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.tracing; + +import one.tomorrow.transactionaloutbox.model.OutboxRecord; + +import java.util.Collections; +import java.util.Map; + +/** + * A no-op implementation of the {@link TracingService} interface. This implementation + * will be used as a fallback when OpenTelemetry is not available on the classpath. + * The OpenTelemetryTracingService should be preferred when quarkus-opentelemetry is available. + */ +public class NoopTracingService implements TracingService { + + @Override + public Map tracingHeadersForOutboxRecord() { + return Collections.emptyMap(); + } + + @Override + public TraceOutboxRecordProcessingResult traceOutboxRecordProcessing(OutboxRecord outboxRecord) { + return new HeadersOnlyTraceOutboxRecordProcessingResult(outboxRecord.getHeaders()); + } +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/tracing/NoopTracingServiceProducer.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/tracing/NoopTracingServiceProducer.java new file mode 100644 index 00000000..73400f80 --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/tracing/NoopTracingServiceProducer.java @@ -0,0 +1,32 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.tracing; + +import io.quarkus.arc.DefaultBean; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Produces; +import jakarta.inject.Singleton; + +@ApplicationScoped +public class NoopTracingServiceProducer { + + @Produces + @Singleton + @DefaultBean + TracingService tracingService() { + return new NoopTracingService(); + } +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/tracing/OpenTelemetryTracingService.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/tracing/OpenTelemetryTracingService.java new file mode 100644 index 00000000..7ecd96f8 --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/tracing/OpenTelemetryTracingService.java @@ -0,0 +1,131 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.tracing; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.propagation.TextMapGetter; +import io.opentelemetry.context.propagation.TextMapSetter; +import one.tomorrow.transactionaloutbox.model.OutboxRecord; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +public class OpenTelemetryTracingService implements TracingService { + + static final String TO_PREFIX = "To_"; + + private final OpenTelemetry openTelemetry; + private final Tracer tracer; + + public OpenTelemetryTracingService(OpenTelemetry openTelemetry) { + this.openTelemetry = openTelemetry; + this.tracer = this.openTelemetry.getTracer("one.tomorrow.transactional-outbox"); + } + + // TextMapGetter for extracting from headers + private static final TextMapGetter> HEADER_GETTER = new TextMapGetter>() { + @Override + public Iterable keys(Map carrier) { + return carrier.keySet(); + } + + @Override + public String get(Map carrier, String key) { + return carrier.get(INTERNAL_PREFIX + key); + } + }; + + // TextMapSetter for injecting into headers + private static final TextMapSetter> HEADER_SETTER = (carrier, key, value) -> + carrier.put(INTERNAL_PREFIX + key, value); + + // TextMapSetter for injecting into Kafka headers (without internal prefix) + private static final TextMapSetter> KAFKA_HEADER_SETTER = Map::put; + + @Override + public Map tracingHeadersForOutboxRecord() { + Context current = Context.current(); + Span currentSpan = Span.fromContext(current); + + if (!currentSpan.getSpanContext().isValid()) { + return Collections.emptyMap(); + } + + Map result = new HashMap<>(); + openTelemetry.getPropagators().getTextMapPropagator() + .inject(current, result, HEADER_SETTER); + return result; + } + + @Override + public TraceOutboxRecordProcessingResult traceOutboxRecordProcessing(OutboxRecord outboxRecord) { + Set> headerEntries = outboxRecord.getHeaders().entrySet(); + boolean containsTraceInfo = headerEntries.stream().anyMatch(e -> e.getKey().startsWith(INTERNAL_PREFIX)); + + if (!containsTraceInfo) { + return new HeadersOnlyTraceOutboxRecordProcessingResult(outboxRecord.getHeaders()); + } + + // Extract context from outbox record headers + Context extractedContext = openTelemetry.getPropagators().getTextMapPropagator() + .extract(Context.current(), outboxRecord.getHeaders(), HEADER_GETTER); + + // Create outbox span with the extracted context as parent + Span outboxSpan = tracer.spanBuilder("transactional-outbox") + .setParent(extractedContext) + .setStartTimestamp(outboxRecord.getCreated().toEpochMilli(), TimeUnit.MILLISECONDS) + .startSpan(); + outboxSpan.end(); + + // Filter out internal tracing headers from the result + Map newHeaders = headerEntries.stream() + .filter(entry -> !entry.getKey().startsWith(INTERNAL_PREFIX)) + .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); + + // Create processing span for publishing to Kafka with outbox span as parent + Span processingSpan = tracer.spanBuilder(TO_PREFIX + outboxRecord.getTopic()) + .setParent(extractedContext.with(outboxSpan)) + .setSpanKind(SpanKind.PRODUCER) + .startSpan(); + + // Inject processing span context into headers for Kafka + Context processingContext = extractedContext.with(processingSpan); + openTelemetry.getPropagators().getTextMapPropagator() + .inject(processingContext, newHeaders, KAFKA_HEADER_SETTER); + + return new TraceOutboxRecordProcessingResult(newHeaders) { + @Override + public void publishCompleted() { + processingSpan.end(); + } + + @Override + public void publishFailed(Throwable t) { + processingSpan.recordException(t); + processingSpan.end(); + } + }; + } +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/tracing/OpenTelemetryTracingServiceProducer.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/tracing/OpenTelemetryTracingServiceProducer.java new file mode 100644 index 00000000..9135ba75 --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/tracing/OpenTelemetryTracingServiceProducer.java @@ -0,0 +1,30 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.tracing; + +import io.opentelemetry.api.OpenTelemetry; +import jakarta.enterprise.inject.Produces; +import jakarta.inject.Singleton; + +public class OpenTelemetryTracingServiceProducer { + + @Produces + @Singleton + TracingService tracingService(OpenTelemetry openTelemetry) { + return new OpenTelemetryTracingService(openTelemetry); + } + +} diff --git a/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/tracing/TracingService.java b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/tracing/TracingService.java new file mode 100644 index 00000000..77685e58 --- /dev/null +++ b/outbox-kafka-quarkus/src/main/java/one/tomorrow/transactionaloutbox/tracing/TracingService.java @@ -0,0 +1,72 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.tracing; + +import lombok.Data; +import one.tomorrow.transactionaloutbox.model.OutboxRecord; + +import java.util.Map; + +public interface TracingService { + + String INTERNAL_PREFIX = "_internal_:"; + + /** + * Extracts the tracing headers from the current context and returns them as a map. + * If tracing is not active, an empty map is returned. + *

+ * This is meant to be used when creating an outbox record, to store the tracing headers with the record. + *

+ */ + Map tracingHeadersForOutboxRecord(); + + /** + * Extracts the tracing headers (as created via {@link #tracingHeadersForOutboxRecord()}) from the outbox record + * to create a span for the time spent in the outbox.
+ * A new span is started for the processing and publishing to Kafka, and headers to publish to Kafka are returned. + * The span must be completed once the message is published to Kafka. + */ + TraceOutboxRecordProcessingResult traceOutboxRecordProcessing(OutboxRecord outboxRecord); + + @Data + abstract class TraceOutboxRecordProcessingResult { + + private final Map headers; + + /** Must be invoked once the outbox record was successfully sent to Kafka */ + public abstract void publishCompleted(); + /** Must be invoked if the outbox record could not be sent to Kafka */ + public abstract void publishFailed(Throwable t); + + } + + class HeadersOnlyTraceOutboxRecordProcessingResult extends TraceOutboxRecordProcessingResult { + public HeadersOnlyTraceOutboxRecordProcessingResult(Map headers) { + super(headers); + } + + @Override + public void publishCompleted() { + // no-op + } + + @Override + public void publishFailed(Throwable t) { + // no-op + } + } + +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/KafkaTestUtils.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/KafkaTestUtils.java new file mode 100644 index 00000000..f52297b5 --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/KafkaTestUtils.java @@ -0,0 +1,213 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox; + +import one.tomorrow.transactionaloutbox.model.OutboxRecord; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.consumer.*; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.serialization.*; +import org.eclipse.microprofile.config.ConfigProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.*; + +import static one.tomorrow.transactionaloutbox.commons.KafkaHeaders.HEADERS_SEQUENCE_NAME; +import static one.tomorrow.transactionaloutbox.commons.KafkaHeaders.HEADERS_SOURCE_NAME; +import static one.tomorrow.transactionaloutbox.commons.Longs.toLong; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class KafkaTestUtils { + + private static final Logger logger = LoggerFactory.getLogger(KafkaTestUtils.class); + + public static void createTopic(String bootstrapServers, String ... topics) { + Map props = producerProps(bootstrapServers); + try (AdminClient client = AdminClient.create(props)) { + List newTopics = Arrays.stream(topics) + .map(topic -> new NewTopic(topic, 1, (short) 1)) + .toList(); + try { + client.createTopics(newTopics).all().get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + public static Map producerProps() { + return producerProps(bootstrapServers()); + } + + public static String bootstrapServers() { + return ConfigProvider.getConfig().getValue("kafka.bootstrap.servers", String.class); + } + + /** + * Set up test properties for an {@code } producer. + * @param brokers the bootstrapServers property. + * @return the properties. + * @since 2.3.5 + */ + public static Map producerProps(String brokers) { + Map props = new HashMap<>(); + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokers); + props.put(ProducerConfig.BATCH_SIZE_CONFIG, "16384"); + props.put(ProducerConfig.LINGER_MS_CONFIG, 1); + props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, "33554432"); + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class); + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + return props; + } + + public static KafkaConsumer setupConsumer( + String groupId, + boolean autoCommit, + String ... topicsToSubscribe) { + return setupConsumer(groupId, + autoCommit, + StringDeserializer.class, + ByteArrayDeserializer.class, + topicsToSubscribe); + } + + public static KafkaConsumer setupConsumer( + String groupId, + boolean autoCommit, + Class> keyDeserializer, + Class> valueDeserializer, + String ... topicsToSubscribe) { + // use unique groupId, so that a new consumer does not get into conflicts with some previous one, + // which might not yet be fully shutdown + Map consumerProps = consumerProps(groupId, String.valueOf(autoCommit)); + consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, keyDeserializer); + consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, valueDeserializer); + consumerProps.put(ConsumerConfig.CLIENT_ID_CONFIG, "testConsumer-" + System.currentTimeMillis() + "-clientId"); + KafkaConsumer consumer = new KafkaConsumer<>(consumerProps); + consumer.subscribe(Arrays.asList(topicsToSubscribe)); + return consumer; + } + + public static Map consumerProps(String group, String autoCommit) { + return consumerProps(bootstrapServers(), group, autoCommit); + } + + /** + * Set up test properties for an {@code } consumer. + * @param brokers the bootstrapServers property. + * @param group the group id. + * @param autoCommit the auto commit. + * @return the properties. + */ + public static Map consumerProps(String brokers, String group, String autoCommit) { + Map props = new HashMap<>(); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, brokers); + props.put(ConsumerConfig.GROUP_ID_CONFIG, group); + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, autoCommit); + props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "10"); + props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "60000"); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, IntegerDeserializer.class); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + return props; + } + + /** + * Poll the consumer for records. + * @param consumer the consumer. + * @param timeout max time in milliseconds to wait for records; forwarded to {@link Consumer#poll(Duration)}. + * @param the key type. + * @param the value type. + * @return the records. + * @throws IllegalStateException if the poll returns null (since 2.3.4). + * @since 2.9.3 + */ + public static ConsumerRecords getRecords(Consumer consumer, Duration timeout) { + return getRecords(consumer, timeout, -1); + } + + /** + * Poll the consumer for records. + * @param consumer the consumer. + * @param timeout max time in milliseconds to wait for records; forwarded to {@link Consumer#poll(Duration)}. + * @param the key type. + * @param the value type. + * @param minRecords wait until the timeout or at least this number of records are received. + * @return the records. + * @throws IllegalStateException if the poll returns null. + * @since 2.9.3 + */ + public static ConsumerRecords getRecords(Consumer consumer, Duration timeout, int minRecords) { + logger.debug("Polling..."); + Map>> records = new HashMap<>(); + long remaining = timeout.toMillis(); + int count = 0; + do { + long t1 = System.currentTimeMillis(); + ConsumerRecords received = consumer.poll(Duration.ofMillis(remaining)); + logger.debug("Received: {}", received != null ? received.count() : null); + if (received == null) { + throw new IllegalStateException("null received from consumer.poll()"); + } + if (minRecords < 0) { + return received; + } + else { + count += received.count(); + received.partitions().forEach(tp -> { + List> recs = records.computeIfAbsent(tp, part -> new ArrayList<>()); + recs.addAll(received.records(tp)); + }); + remaining -= System.currentTimeMillis() - t1; + } + } + while (count < minRecords && remaining > 0); + return new ConsumerRecords<>(records, Map.of()); + } + + public static void assertConsumedRecord(OutboxRecord outboxRecord, + String sourceHeaderValue, + ConsumerRecord kafkaRecord) { + assertEquals( + outboxRecord.getId().longValue(), + toLong(kafkaRecord.headers().lastHeader(HEADERS_SEQUENCE_NAME).value()), + "OutboxRecord id and " + HEADERS_SEQUENCE_NAME + " headers do not match" + ); + assertEquals(sourceHeaderValue, new String(kafkaRecord.headers().lastHeader(HEADERS_SOURCE_NAME).value())); + outboxRecord.getHeaders().forEach((key, value) -> + assertEquals(new String(value.getBytes()), new String(kafkaRecord.headers().lastHeader(key).value())) + ); + assertEquals(outboxRecord.getKey(), kafkaRecord.key()); + assertArrayEquals(outboxRecord.getValue(), kafkaRecord.value()); + } + + public static void assertConsumedRecord(OutboxRecord outboxRecord, + String headerKey, + String sourceHeaderValue, + ConsumerRecord kafkaRecord) { + assertEquals(outboxRecord.getKey(), kafkaRecord.key()); + assertArrayEquals(outboxRecord.getValue(), kafkaRecord.value()); + assertArrayEquals(outboxRecord.getHeaders().get(headerKey).getBytes(), kafkaRecord.headers().lastHeader(headerKey).value()); + assertEquals(outboxRecord.getId().longValue(), toLong(kafkaRecord.headers().lastHeader(HEADERS_SEQUENCE_NAME).value())); + assertArrayEquals(sourceHeaderValue.getBytes(), kafkaRecord.headers().lastHeader(HEADERS_SOURCE_NAME).value()); + } + +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/ProxiedContainerPorts.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/ProxiedContainerPorts.java new file mode 100644 index 00000000..554bf55f --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/ProxiedContainerPorts.java @@ -0,0 +1,62 @@ +/** + * Copyright 2023 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.ServerSocket; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +public class ProxiedContainerPorts { + + private static final Logger LOGGER = LoggerFactory.getLogger(ProxiedContainerPorts.class); + + /** First and last proxied ports from {@link org.testcontainers.containers.ToxiproxyContainer} */ + private static final int FIRST_PROXIED_PORT = 8666; + private static final int LAST_PROXIED_PORT = 8666 + 31; + + private static final AtomicInteger NEXT_PORT = new AtomicInteger(FIRST_PROXIED_PORT); + private static final Map PORT_BY_SERVICE = new HashMap<>(); + + + public static int findPort(String service) { + Integer result = PORT_BY_SERVICE.get(service); + if (result != null) + return result; + + result = findNextPort(NEXT_PORT.getAndIncrement()); + LOGGER.info("Setting port for {} to {}", service, result); + PORT_BY_SERVICE.put(service, result); + return result; + } + + private static int findNextPort(int port) { + if (port > LAST_PROXIED_PORT) { + throw new RuntimeException("Could not find free port for proxied container"); + } + try (ServerSocket socket = new ServerSocket(port)) { + return socket.getLocalPort(); + } catch (IOException e) { + LOGGER.info("Port {} is already in use, trying next port", port); + return findNextPort(NEXT_PORT.getAndIncrement()); + } + } + +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/ProxiedContainerSupport.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/ProxiedContainerSupport.java new file mode 100644 index 00000000..b1c844ed --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/ProxiedContainerSupport.java @@ -0,0 +1,76 @@ +/** + * Copyright 2023 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox; + +import eu.rekawek.toxiproxy.Proxy; +import eu.rekawek.toxiproxy.ToxiproxyClient; +import eu.rekawek.toxiproxy.model.ToxicDirection; +import org.testcontainers.containers.ToxiproxyContainer; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; + +import static one.tomorrow.transactionaloutbox.ProxiedContainerPorts.findPort; + +public interface ProxiedContainerSupport { + + String CUT_CONNECTION_DOWNSTREAM = "CUT_CONNECTION_DOWNSTREAM"; + String CUT_CONNECTION_UPSTREAM = "CUT_CONNECTION_UPSTREAM"; + + AtomicBoolean isCurrentlyCut = new AtomicBoolean(false); + + Proxy getProxy(); + + /** + * Cuts the connection by setting bandwidth in both directions to zero. + * @param shouldCutConnection true if the connection should be cut, or false if it should be re-enabled + */ + default void setConnectionCut(boolean shouldCutConnection) { + synchronized (isCurrentlyCut) { + if (shouldCutConnection != isCurrentlyCut.get()) { + try { + if (shouldCutConnection) { + getProxy().toxics().bandwidth(CUT_CONNECTION_DOWNSTREAM, ToxicDirection.DOWNSTREAM, 0); + getProxy().toxics().bandwidth(CUT_CONNECTION_UPSTREAM, ToxicDirection.UPSTREAM, 0); + isCurrentlyCut.set(true); + } else { + getProxy().toxics().get(CUT_CONNECTION_DOWNSTREAM).remove(); + getProxy().toxics().get(CUT_CONNECTION_UPSTREAM).remove(); + isCurrentlyCut.set(false); + } + } catch (IOException e) { + throw new RuntimeException("Could not control proxy", e); + } + } + } + } + + static Proxy createProxy(String service, ToxiproxyContainer toxiproxy, int exposedPort) { + final ToxiproxyClient toxiproxyClient = new ToxiproxyClient(toxiproxy.getHost(), toxiproxy.getControlPort()); + Proxy proxy; + try { + proxy = toxiproxyClient.createProxy( + service, + "0.0.0.0:" + findPort(service), + service + ":" + exposedPort + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + return proxy; + } + +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/ProxiedKafkaContainer.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/ProxiedKafkaContainer.java new file mode 100644 index 00000000..47946569 --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/ProxiedKafkaContainer.java @@ -0,0 +1,81 @@ +/** + * Copyright 2022 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox; + +import eu.rekawek.toxiproxy.Proxy; +import eu.rekawek.toxiproxy.ToxiproxyClient; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.ToxiproxyContainer; +import org.testcontainers.redpanda.RedpandaContainer; +import org.testcontainers.utility.DockerImageName; + +import java.io.IOException; + +import static one.tomorrow.transactionaloutbox.ProxiedContainerPorts.findPort; +import static one.tomorrow.transactionaloutbox.ProxiedContainerSupport.createProxy; + +public class ProxiedKafkaContainer extends RedpandaContainer implements ProxiedContainerSupport { + + private static ProxiedKafkaContainer kafka; + private static ToxiproxyContainer toxiproxy; + public static Proxy kafkaProxy; + public static String bootstrapServers; + + public ProxiedKafkaContainer(DockerImageName dockerImageName) { + super(dockerImageName); + } + + public static ProxiedKafkaContainer startProxiedKafka() { + if (kafka == null) { + int exposedKafkaPort = 9092; // RedpandaContainer.REDPANDA_PORT + + Network network = Network.newNetwork(); + + kafka = (ProxiedKafkaContainer) new ProxiedKafkaContainer(DockerImageName.parse("docker.redpanda.com/redpandadata/redpanda:v23.1.2")) + .withExposedPorts(exposedKafkaPort) + .withNetwork(network) + .withNetworkAliases("kafka"); + + toxiproxy = new ToxiproxyContainer(DockerImageName.parse("ghcr.io/shopify/toxiproxy:2.5.0")) + .withNetwork(network); + toxiproxy.start(); + + kafkaProxy = createProxy("kafka", toxiproxy, exposedKafkaPort); + bootstrapServers = "PLAINTEXT://" + toxiproxy.getHost() + ":" + toxiproxy.getMappedPort(findPort("kafka")); + + kafka.start(); + } + return kafka; + } + + public static void stopProxiedKafka() { + if (toxiproxy != null) + toxiproxy.stop(); + if (kafka != null) + kafka.stop(); + } + + /** Kafka advertises its connection to connected clients, therefore we must override this */ + public String getBootstrapServers() { + return bootstrapServers; + } + + @Override + public Proxy getProxy() { + return kafkaProxy; + } + +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/ProxiedPostgreSQLContainer.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/ProxiedPostgreSQLContainer.java new file mode 100644 index 00000000..cbb99b3c --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/ProxiedPostgreSQLContainer.java @@ -0,0 +1,85 @@ +/** + * Copyright 2023 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox; + +import eu.rekawek.toxiproxy.Proxy; +import org.jetbrains.annotations.NotNull; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.containers.ToxiproxyContainer; +import org.testcontainers.utility.DockerImageName; + +import static one.tomorrow.transactionaloutbox.ProxiedContainerPorts.findPort; +import static one.tomorrow.transactionaloutbox.ProxiedContainerSupport.createProxy; + +public class ProxiedPostgreSQLContainer extends PostgreSQLContainer implements ProxiedContainerSupport { + + private static ProxiedPostgreSQLContainer postgres; + public static ToxiproxyContainer toxiproxy; + public static Proxy postgresProxy; + private static String jdbcUrl; + + public ProxiedPostgreSQLContainer(DockerImageName dockerImageName) { + super(dockerImageName); + } + + public static ProxiedPostgreSQLContainer startProxiedPostgres() { + if (postgres == null) { + int exposedPostgresPort = POSTGRESQL_PORT; + + Network network = Network.newNetwork(); + + postgres = new ProxiedPostgreSQLContainer(DockerImageName.parse("postgres:16-alpine")) + .withExposedPorts(exposedPostgresPort) + .withNetwork(network) + .withNetworkAliases("postgres"); + + toxiproxy = new ToxiproxyContainer(DockerImageName.parse("ghcr.io/shopify/toxiproxy:2.5.0")) + .withNetwork(network); + toxiproxy.start(); + + postgresProxy = createProxy("postgres", toxiproxy, exposedPostgresPort); + + jdbcUrl = "jdbc:postgresql://" + getHostAndPort() + "/" + postgres.getDatabaseName(); + + postgres.start(); + } + return postgres; + } + + @NotNull + private static String getHostAndPort() { + return toxiproxy.getHost() + ":" + toxiproxy.getMappedPort(findPort("postgres")); + } + + public static void stopProxiedPostgres() { + if (toxiproxy != null) + toxiproxy.stop(); + if (postgres != null) + postgres.stop(); + } + + @Override + public String getJdbcUrl() { + return jdbcUrl; + } + + @Override + public Proxy getProxy() { + return postgresProxy; + } + +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/TestUtils.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/TestUtils.java new file mode 100644 index 00000000..1787ba9a --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/TestUtils.java @@ -0,0 +1,58 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox; + +import one.tomorrow.transactionaloutbox.model.OutboxRecord; +import org.jetbrains.annotations.NotNull; + +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; + +public class TestUtils { + + /** + * Creates a new OutboxRecord with the given parameters + */ + public static OutboxRecord newRecord(String topic, String key, String value, Map headers) { + return newRecord(null, topic, key, value, headers); + } + + /** + * Creates a new OutboxRecord with the given parameters + */ + public static OutboxRecord newRecord(Instant processed, String topic, String key, String value, Map headers) { + return OutboxRecord.builder() + .processed(processed) + .topic(topic) + .key(key) + .value(value.getBytes(StandardCharsets.UTF_8)) + .headers(headers) + .build(); + } + + @NotNull + public static Map newHeaders(String ... keyValue) { + Map headers1 = new HashMap<>(); + if(keyValue.length % 2 != 0) + throw new IllegalArgumentException("KeyValue must be a list of pairs"); + for (int i = 0; i < keyValue.length; i += 2) { + headers1.put(keyValue[i], keyValue[i + 1]); + } + return headers1; + } +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/commons/KafkaHeadersTest.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/commons/KafkaHeadersTest.java new file mode 100644 index 00000000..07b8b931 --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/commons/KafkaHeadersTest.java @@ -0,0 +1,102 @@ +/** + * Copyright 2022 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.commons; + +import org.apache.kafka.common.header.internals.RecordHeaders; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +import static one.tomorrow.transactionaloutbox.commons.KafkaHeaders.*; +import static one.tomorrow.transactionaloutbox.commons.Longs.toByteArray; +import static org.junit.jupiter.api.Assertions.assertEquals; + +class KafkaHeadersTest { + + @Test + void knownHeadersShouldReturnGivenHeaders() { + RecordHeaders headers = recordHeaders(HEADERS_SEQUENCE_NAME, toByteArray(42)); + assertEquals(mapOf(HEADERS_SEQUENCE_NAME, 42L), knownHeaders(headers)); + + headers = recordHeaders(HEADERS_SOURCE_NAME, "foo".getBytes()); + assertEquals(mapOf(HEADERS_SOURCE_NAME, "foo"), knownHeaders(headers)); + + headers = recordHeaders(HEADERS_VALUE_TYPE_NAME, "foo".getBytes()); + assertEquals(mapOf(HEADERS_VALUE_TYPE_NAME, "foo"), knownHeaders(headers)); + + headers = recordHeaders(HEADERS_DLT_SOURCE_NAME, "foo".getBytes()); + assertEquals(mapOf(HEADERS_DLT_SOURCE_NAME, "foo"), knownHeaders(headers)); + + headers = recordHeaders(HEADERS_DLT_RETRY_NAME, toByteArray(42)); + assertEquals(mapOf(HEADERS_DLT_RETRY_NAME, 42L), knownHeaders(headers)); + + headers = new RecordHeaders(); + headers.add(HEADERS_SEQUENCE_NAME, toByteArray(42)); + headers.add(HEADERS_SOURCE_NAME, "foo".getBytes()); + headers.add(HEADERS_VALUE_TYPE_NAME, "bar".getBytes()); + headers.add(HEADERS_DLT_SOURCE_NAME, "baz".getBytes()); + headers.add(HEADERS_DLT_RETRY_NAME, toByteArray(16)); + headers.add("some-header", "baz".getBytes()); + assertEquals( + mapOf( + HEADERS_SEQUENCE_NAME, 42L, + HEADERS_SOURCE_NAME, "foo", + HEADERS_VALUE_TYPE_NAME, "bar", + HEADERS_DLT_SOURCE_NAME, "baz", + HEADERS_DLT_RETRY_NAME, 16L + ), + knownHeaders(headers) + ); + } + + @Test + void knownHeadersShouldGracefullyHandleBadSequenceRepresentation() { + RecordHeaders headers = new RecordHeaders(); + headers.add(HEADERS_SEQUENCE_NAME, new byte[0]); + headers.add(HEADERS_SOURCE_NAME, "foo".getBytes()); + assertEquals(mapOf(HEADERS_SOURCE_NAME, "foo"), knownHeaders(headers)); + } + + private RecordHeaders recordHeaders(String key, byte[] value) { + RecordHeaders headers = new RecordHeaders(); + headers.add(key, value); + return headers; + } + + private Map mapOf(String key, Object value) { + Map result = new HashMap<>(); + result.put(key, value); + return result; + } + + private Map mapOf( + String key1, Object value1, + String key2, Object value2, + String key3, Object value3, + String key4, Object value4, + String key5, Object value5 + ) { + Map result = new HashMap<>(); + result.put(key1, value1); + result.put(key2, value2); + result.put(key3, value3); + result.put(key4, value4); + result.put(key5, value5); + return result; + } + +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/commons/LongsTest.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/commons/LongsTest.java new file mode 100644 index 00000000..34265b09 --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/commons/LongsTest.java @@ -0,0 +1,55 @@ +/** + * Copyright 2022 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.commons; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import java.nio.ByteBuffer; +import java.util.Random; +import java.util.stream.LongStream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +class LongsTest { + + @ParameterizedTest(name = "{displayName} [{index}] {argumentsWithNames}") + @MethodSource("longValues") + void toLong(long value) { + byte[] bytes = ByteBuffer.allocate(8).putLong(value).array(); + long converted = Longs.toLong(bytes); + assertEquals(value, converted); + } + + @ParameterizedTest(name = "{displayName} [{index}] {argumentsWithNames}") + @MethodSource("longValues") + void toByteArray(long value) { + byte[] bytes = Longs.toByteArray(value); + byte[] expected = ByteBuffer.allocate(8).putLong(value).array(); + assertArrayEquals(expected, bytes); + } + + @SuppressWarnings("unused") + static LongStream longValues() { + Random r = new Random(); + return LongStream.concat( + LongStream.of(-1, 0, 1, 2, Long.MAX_VALUE), + LongStream.generate(() -> Math.abs(r.nextLong())) + ).limit(10); + } + +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/config/TestTransactionalOutboxConfig.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/config/TestTransactionalOutboxConfig.java new file mode 100644 index 00000000..374a8e38 --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/config/TestTransactionalOutboxConfig.java @@ -0,0 +1,93 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.config; + +import one.tomorrow.transactionaloutbox.config.TransactionalOutboxConfig.CleanupConfig; + +import java.time.Duration; +import java.util.Optional; + +/** + * Utility class for tests that provides a TransactionalOutboxConfig with default values. + */ +public class TestTransactionalOutboxConfig { + + public static TransactionalOutboxConfig createDefault() { + return createConfig(Duration.ofMillis(200), Duration.ofSeconds(5), "testLockOwnerId", "testEventSource"); + } + + public static TransactionalOutboxConfig createConfig( + Duration processingInterval, + Duration lockTimeout, + String lockOwnerId, + String eventSource) { + return createConfig(processingInterval, lockTimeout, lockOwnerId, eventSource, null); + } + + public static TransactionalOutboxConfig createConfig( + Duration processingInterval, + Duration lockTimeout, + String lockOwnerId, + String eventSource, + CleanupConfig cleanupConfig) { + + return new TransactionalOutboxConfig() { + @Override + public boolean enabled() { + return true; + } + + @Override + public Duration processingInterval() { + return processingInterval; + } + + @Override + public Duration lockTimeout() { + return lockTimeout; + } + + @Override + public String lockOwnerId() { + return lockOwnerId; + } + + @Override + public String eventSource() { + return eventSource; + } + + @Override + public Optional cleanup() { + return Optional.ofNullable(cleanupConfig); + } + }; + } + + public static CleanupConfig createCleanupConfig(Duration interval, Duration retention) { + return new CleanupConfig() { + @Override + public Duration interval() { + return interval; + } + + @Override + public Duration retention() { + return retention; + } + }; + } +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/health/OutboxProcessorHealthCheckTest.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/health/OutboxProcessorHealthCheckTest.java new file mode 100644 index 00000000..e6e01373 --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/health/OutboxProcessorHealthCheckTest.java @@ -0,0 +1,220 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.health; + +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; +import one.tomorrow.transactionaloutbox.model.OutboxRecord; +import org.eclipse.microprofile.health.HealthCheckResponse; +import org.eclipse.microprofile.health.Readiness; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +@QuarkusTest +@TestProfile(OutboxProcessorHealthCheckTest.class) +public class OutboxProcessorHealthCheckTest implements QuarkusTestProfile { + + @Inject + @Readiness + OutboxProcessorHealthCheck healthCheck; + + @Inject + EntityManager entityManager; + + @Override + public Map getConfigOverrides() { + return Map.of( + "one.tomorrow.transactional-outbox.enabled", "true", + "one.tomorrow.transactional-outbox.processing-interval", "PT0.1S", // 100ms for testing + "one.tomorrow.transactional-outbox.lock-owner-id", "health-check-test", + "one.tomorrow.transactional-outbox.lock-timeout", "PT0.5S", + "one.tomorrow.transactional-outbox.event-source", "health-test" + ); + } + + @BeforeEach + @AfterEach + @Transactional + void cleanUp() { + entityManager.createQuery("DELETE FROM OutboxRecord").executeUpdate(); + entityManager.createQuery("DELETE FROM OutboxLock").executeUpdate(); + } + + @Test + void shouldReturnHealthyWhenProcessorIsEnabled() { + // when + HealthCheckResponse response = healthCheck.call(); + + // then + assertEquals(HealthCheckResponse.Status.UP, response.getStatus()); + assertEquals("transactional-outbox-processor", response.getName()); + + // Verify basic processor information is included + Map data = response.getData().orElse(Map.of()); + assertTrue((Boolean) data.get("enabled")); + assertEquals("health-check-test", data.get("lockOwnerId")); + assertNotNull(data.get("active")); + assertFalse((Boolean) data.get("closed")); + } + + @Test + void shouldIncludeOldestMessageAgeWhenMessagesExist() { + // given - add some unprocessed messages with different ages + addUnprocessedMessage("test-topic-1", "key1", "message1", Instant.now().minus(30, ChronoUnit.SECONDS)); + addUnprocessedMessage("test-topic-2", "key2", "message2", Instant.now().minus(10, ChronoUnit.SECONDS)); + + // when + HealthCheckResponse response = healthCheck.call(); + + // then + assertEquals(HealthCheckResponse.Status.UP, response.getStatus()); + + // Should include oldest message age (30 seconds is oldest) + Map data = response.getData().orElse(Map.of()); + String oldestAge = (String) data.get("oldestUnprocessedMessageAge"); + assertNotNull(oldestAge); + + // The oldest message should be approximately 30 seconds old + // Accept some variation due to test execution time + assertTrue(oldestAge.matches("PT\\d+(\\.\\d+)?S"), + "Expected duration format like PT30S or PT30.123S, but was: " + oldestAge); + + // Should include creation timestamp + assertNotNull(data.get("oldestUnprocessedMessageCreated")); + } + + @Test + void shouldReturnDownWhenActiveProcessorHasStaleMessage() { + // given - add an old message (older than 2x processing interval = 2 * 100ms = 200ms) + Instant oldTimestamp = Instant.now().minus(1, ChronoUnit.SECONDS); // 1 second old (much older than 200ms) + addUnprocessedMessage("test-topic", "key1", "stale-message", oldTimestamp); + + // Wait a bit to ensure the processor might become active + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + // when + HealthCheckResponse response = healthCheck.call(); + + // then + Map data = response.getData().orElse(Map.of()); + + // If processor is active and message is older than 2x processing interval (200ms), should be DOWN + Boolean isActive = (Boolean) data.get("active"); + if (Boolean.TRUE.equals(isActive)) { + assertEquals(HealthCheckResponse.Status.DOWN, response.getStatus()); + assertEquals("processing-stalled", data.get("status")); + assertNotNull(data.get("reason")); + } else { + // If processor is not active, should still be UP (normal in multi-instance setup) + assertEquals(HealthCheckResponse.Status.UP, response.getStatus()); + } + } + + @Test + void shouldReturnUpWhenActiveProcessorHasFreshMessage() { + // given - add a fresh message (newer than 2x processing interval = 200ms) + Instant freshTimestamp = Instant.now().minus(50, ChronoUnit.MILLIS); // 50ms old (less than 200ms threshold) + addUnprocessedMessage("test-topic", "key1", "fresh-message", freshTimestamp); + + // when + HealthCheckResponse response = healthCheck.call(); + + // then - Even if processor is active, fresh message should not cause DOWN status + assertEquals(HealthCheckResponse.Status.UP, response.getStatus()); + + Map data = response.getData().orElse(Map.of()); + assertNotEquals("processing-stalled", data.get("status")); + } + + @Test + void shouldReturnNoneWhenNoUnprocessedMessages() { + // when (no messages added) + HealthCheckResponse response = healthCheck.call(); + + // then + assertEquals(HealthCheckResponse.Status.UP, response.getStatus()); + + Map data = response.getData().orElse(Map.of()); + assertEquals("none", data.get("oldestUnprocessedMessageAge")); + } + + @Test + void shouldIncludeLockInformation() { + // when + HealthCheckResponse response = healthCheck.call(); + + // then + assertEquals(HealthCheckResponse.Status.UP, response.getStatus()); + + // Should include lock-related information + Map data = response.getData().orElse(Map.of()); + assertNotNull(data.get("lockOwnerId")); + assertEquals("health-check-test", data.get("lockOwnerId")); + + // Should track lock acquisition attempts + assertNotNull(data.get("lastLockAttempt")); + } + + @Test + void shouldHandleRepositoryErrors() { + // The health check should gracefully handle repository errors + // In a real scenario, this might happen if the database is unavailable + + // when + HealthCheckResponse response = healthCheck.call(); + + // then + // Even if there are issues with getting oldest message, + // the health check should not fail completely + assertEquals("transactional-outbox-processor", response.getName()); + assertNotNull(response.getStatus()); + } + + @Transactional + void addUnprocessedMessage(String topic, String key, String message) { + addUnprocessedMessage(topic, key, message, Instant.now()); + } + + @Transactional + void addUnprocessedMessage(String topic, String key, String message, Instant created) { + OutboxRecord outboxRecord = new OutboxRecord(); + outboxRecord.setTopic(topic); + outboxRecord.setKey(key); + outboxRecord.setValue(message.getBytes()); + // Don't set processed timestamp - this makes it unprocessed + entityManager.persist(outboxRecord); + entityManager.flush(); // Ensure it's written to DB immediately + // update created timestamp + outboxRecord.setCreated(created); + entityManager.merge(outboxRecord); + entityManager.flush(); + } +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/publisher/EmitterMessagePublisherTest.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/publisher/EmitterMessagePublisherTest.java new file mode 100644 index 00000000..742e2c28 --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/publisher/EmitterMessagePublisherTest.java @@ -0,0 +1,72 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.publisher; + +import io.smallrye.mutiny.Uni; +import io.smallrye.reactive.messaging.MutinyEmitter; +import io.smallrye.reactive.messaging.kafka.OutgoingKafkaRecord; +import one.tomorrow.transactionaloutbox.publisher.EmitterMessagePublisher.EmitterResolver; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +@SuppressWarnings("unchecked") +class EmitterMessagePublisherTest { + + private final EmitterResolver emitterResolver = mock(); + private final MutinyEmitter emitter = mock(); + private final EmitterMessagePublisher publisher = new EmitterMessagePublisher(emitterResolver); + + @Test + void publishSuccessfullyPublishesMessage() { + when(emitterResolver.resolveBy("test-topic")).thenReturn(emitter); + when(emitter.sendMessage(any(OutgoingKafkaRecord.class))) + .thenReturn(Uni.createFrom().nullItem()); + + CompletableFuture result = (CompletableFuture) publisher.publish( + 1L, "test-topic", "key", "payload".getBytes(), Map.of("k", "v".getBytes()) + ); + + assertDoesNotThrow(result::join); + verify(emitter).sendMessage(argThat((OutgoingKafkaRecord kafkaRecord) -> { + assertEquals("key", kafkaRecord.getKey()); + assertEquals("payload", new String(kafkaRecord.getPayload())); + assertEquals(1, kafkaRecord.getHeaders().toArray().length); + assertEquals("v", new String(kafkaRecord.getHeaders().lastHeader("k").value())); + return true; + })); + } + + @Test + void publishFailsWhenEmitterNotFound() { + when(emitterResolver.resolveBy("non-existent-topic")).thenReturn(null); + + CompletableFuture result = (CompletableFuture) publisher.publish( + 1L, "non-existent-topic", "key", "payload".getBytes(), Collections.emptyMap() + ); + + assertTrue(result.isCompletedExceptionally()); + CompletionException ex = assertThrows(CompletionException.class, result::join); + assertInstanceOf(IllegalArgumentException.class, ex.getCause()); + } + +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/publisher/KafkaProducerMessagePublisherTest.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/publisher/KafkaProducerMessagePublisherTest.java new file mode 100644 index 00000000..7aa9131a --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/publisher/KafkaProducerMessagePublisherTest.java @@ -0,0 +1,74 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.publisher; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +class KafkaProducerMessagePublisherTest { + + private final KafkaProducer kafkaProducer = mock(); + private final KafkaProducerMessagePublisher publisher = new KafkaProducerMessagePublisher(kafkaProducer); + + @Test + void publishSuccessfullySendsMessage() { + CompletableFuture> future = CompletableFuture.completedFuture(null); + when(kafkaProducer.send(any())).thenAnswer(inv -> future); + + Future result = publisher.publish( + 1L, "test-topic", "key", "payload".getBytes(), Map.of("h1", "v1".getBytes()) + ); + + assertDoesNotThrow(() -> result.get()); + verify(kafkaProducer).send(argThat((ProducerRecord pr) -> { + assertEquals("test-topic", pr.topic()); + assertEquals("key", pr.key()); + assertEquals("payload", new String(pr.value())); + assertEquals(1, pr.headers().toArray().length); + assertEquals("v1", new String(pr.headers().lastHeader("h1").value())); + return true; + })); + } + + @Test + void publishFailsWhenKafkaProducerThrowsException() { + CompletableFuture> future = + CompletableFuture.failedFuture(new RuntimeException("simulated error")); + when(kafkaProducer.send(any())).thenAnswer(inv -> future); + + CompletableFuture result = (CompletableFuture) publisher.publish( + 1L, "test-topic", "key", "payload".getBytes(), Map.of("header1", "value1".getBytes()) + ); + + assertTrue(result.isCompletedExceptionally()); + assertThrows(RuntimeException.class, result::join); + } + + @Test + void closeClosesKafkaProducer() { + publisher.close(); + + verify(kafkaProducer).close(); + } +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/publisher/PublisherConfigTest.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/publisher/PublisherConfigTest.java new file mode 100644 index 00000000..9399e44c --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/publisher/PublisherConfigTest.java @@ -0,0 +1,88 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.publisher; + +import jakarta.enterprise.inject.Instance; +import one.tomorrow.transactionaloutbox.publisher.EmitterMessagePublisher.EmitterResolver; +import one.tomorrow.transactionaloutbox.publisher.KafkaProducerMessagePublisherFactory.KafkaProducerFactory; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class PublisherConfigTest { + + private final Instance> producerPropsInstance = mock(); + private final Instance emitterResolverInstance = mock(); + private final Instance kafkaProducerFactoryInstance = mock(); + private final PublisherConfig publisherConfig = new PublisherConfig( + producerPropsInstance, emitterResolverInstance, kafkaProducerFactoryInstance + ); + + @Test + void returnsKafkaProducerFactoryWhenProducerPropsAreValid() { + Map producerProps = Map.of("bootstrap.servers", "localhost:9092"); + when(producerPropsInstance.isResolvable()).thenReturn(true); + when(producerPropsInstance.get()).thenReturn(producerProps); + when(kafkaProducerFactoryInstance.isResolvable()).thenReturn(true); + + assertInstanceOf(KafkaProducerMessagePublisherFactory.class, publisherConfig.createMessagePublisherFactory()); + } + + @Test + void returnsEmitterMessagePublisherFactoryWhenEmitterResolverIsAvailable() { + when(producerPropsInstance.isResolvable()).thenReturn(false); + when(emitterResolverInstance.isResolvable()).thenReturn(true); + + assertInstanceOf(EmitterMessagePublisherFactory.class, publisherConfig.createMessagePublisherFactory()); + } + + @Test + void returnsEmitterMessagePublisherFactoryWhenProducerPropsLackBootstrapServersAndEmitterResolverIsAvailable() { + Map producerProps = Map.of("some.other.config", "value"); + when(producerPropsInstance.isResolvable()).thenReturn(true); + when(producerPropsInstance.get()).thenReturn(producerProps); + when(kafkaProducerFactoryInstance.isResolvable()).thenReturn(true); + when(emitterResolverInstance.isResolvable()).thenReturn(true); + + assertInstanceOf(EmitterMessagePublisherFactory.class, publisherConfig.createMessagePublisherFactory()); + } + + @Test + void throwsExceptionWhenNoFactoryCanBeCreated() { + when(producerPropsInstance.isResolvable()).thenReturn(false); + when(emitterResolverInstance.isResolvable()).thenReturn(false); + when(kafkaProducerFactoryInstance.isResolvable()).thenReturn(false); + + assertThrows(IllegalStateException.class, publisherConfig::createMessagePublisherFactory); + } + + @Test + void throwsExceptionWhenProducerPropsLackBootstrapServersAndNoEmitterResolverIsAvailable() { + Map producerProps = Map.of("some.other.config", "value"); + when(producerPropsInstance.isResolvable()).thenReturn(true); + when(producerPropsInstance.get()).thenReturn(producerProps); + when(kafkaProducerFactoryInstance.isResolvable()).thenReturn(true); + when(emitterResolverInstance.isResolvable()).thenReturn(false); + + assertThrows(IllegalStateException.class, publisherConfig::createMessagePublisherFactory); + } + +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/repository/OutboxLockRepositoryIntegrationTest.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/repository/OutboxLockRepositoryIntegrationTest.java new file mode 100644 index 00000000..9fd3f640 --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/repository/OutboxLockRepositoryIntegrationTest.java @@ -0,0 +1,149 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.repository; + +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; +import one.tomorrow.transactionaloutbox.model.OutboxLock; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; + +import static java.util.stream.IntStream.range; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@QuarkusTest +class OutboxLockRepositoryIntegrationTest { + + @Inject + OutboxLockRepository repository; + + @Inject + EntityManager entityManager; + + @AfterEach + @Transactional + void cleanUp() { + OutboxLock lock = entityManager.find(OutboxLock.class, OutboxLock.OUTBOX_LOCK_ID); + if (lock != null) { + entityManager.remove(lock); + } + } + + @Test + void should_AcquireSingleLockOnly() throws Exception { + // given + List ownerIds = range(0, 10).mapToObj(i -> "owner-" + i).toList(); + Duration timeout = Duration.ofSeconds(20); // this must be high enough to not produce flakiness with long gc pauses + + // executor, with warmup + ExecutorService executor = Executors.newFixedThreadPool(ownerIds.size()); + executor.invokeAll( + ownerIds.stream().map(ownerId -> (Callable) () -> null).toList() + ); + + // when + List> acquireLockCalls = ownerIds.stream().map(ownerId -> + (Callable) () -> repository.acquireOrRefreshLock(ownerId, timeout) + ).toList(); + List> resultFutures = executor.invokeAll(acquireLockCalls); + executor.shutdown(); // just simple cleanup, threads are still executed + + // then + AtomicBoolean locked = new AtomicBoolean(false); + resultFutures.forEach(resultFuture -> { + try { + Boolean lockResult = resultFuture.get(); + assertFalse(locked.get() && lockResult, "Only one lock should be acquired"); + if (!locked.get()) + locked.set(lockResult); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + assertTrue(locked.get(), "At least one lock should be acquired"); + } + + @Test + void should_RefreshLock() throws InterruptedException { + // given + String ownerId1 = "owner-1"; + String ownerId2 = "owner-2"; + Duration timeout = Duration.ofMillis(200); + + boolean locked = repository.acquireOrRefreshLock(ownerId1, timeout); + assertTrue(locked); + + // when + Thread.sleep(timeout.toMillis() / 2); + locked = repository.acquireOrRefreshLock(ownerId1, timeout); + assertTrue(locked); + + // then + Thread.sleep(timeout.toMillis() / 2); + locked = repository.acquireOrRefreshLock(ownerId2, timeout); + assertFalse(locked); + } + + @Test + void should_AcquireForeignLock_AfterTimeout() throws InterruptedException { + // given + String ownerId1 = "owner-1"; + String ownerId2 = "owner-2"; + Duration timeout = Duration.ofMillis(100); + + boolean locked = repository.acquireOrRefreshLock(ownerId1, timeout); + assertTrue(locked); + + // when + Thread.sleep(timeout.toMillis() + 10); // added a little buffer to ensure timeout has passed + + // then + locked = repository.acquireOrRefreshLock(ownerId2, timeout); + assertTrue(locked); + + locked = repository.acquireOrRefreshLock(ownerId1, timeout); + assertFalse(locked); + } + + @Test + void should_ReleaseLock() { + // given + String ownerId1 = "owner-1"; + String ownerId2 = "owner-2"; + Duration timeout = Duration.ofSeconds(10); + + boolean locked = repository.acquireOrRefreshLock(ownerId1, timeout); + assertTrue(locked); + + // when + repository.releaseLock(ownerId1); + + // then + locked = repository.acquireOrRefreshLock(ownerId2, timeout); + assertTrue(locked); + } +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/repository/OutboxRepositoryIntegrationTest.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/repository/OutboxRepositoryIntegrationTest.java new file mode 100644 index 00000000..b0daa9ae --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/repository/OutboxRepositoryIntegrationTest.java @@ -0,0 +1,110 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.repository; + +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; +import one.tomorrow.transactionaloutbox.model.OutboxRecord; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.time.Instant; +import java.util.Collections; +import java.util.List; + +import static one.tomorrow.transactionaloutbox.TestUtils.newHeaders; +import static one.tomorrow.transactionaloutbox.TestUtils.newRecord; +import static org.junit.jupiter.api.Assertions.*; + +@QuarkusTest +class OutboxRepositoryIntegrationTest { + + @Inject + OutboxRepository repository; + + @Inject + EntityManager entityManager; + + @BeforeEach + @Transactional + void cleanUp() { + entityManager + .createQuery("DELETE FROM OutboxRecord") + .executeUpdate(); + } + + @Test + @Transactional + void should_FindUnprocessedRecords() { + // given + OutboxRecord record1 = newRecord(Instant.now(), "topic1", "key1", "value1", newHeaders("h1", "v1")); + repository.persist(record1); + + OutboxRecord record2 = newRecord("topic2", "key2", "value2", newHeaders("h2", "v2")); + repository.persist(record2); + + // when + List result = repository.getUnprocessedRecords(100); + + // then + assertEquals(1, result.size()); + OutboxRecord foundRecord = result.get(0); + assertEquals(record2.getId(), foundRecord.getId()); + assertEquals(record2.getTopic(), foundRecord.getTopic()); + assertEquals(record2.getKey(), foundRecord.getKey()); + } + + @Test + @Transactional + void should_DeleteProcessedRecordsAfterRetentionTime() { + // given + OutboxRecord shouldBeKeptAsNotProcessed = newRecord(null, "topic1", "key1", "value1", Collections.emptyMap()); + repository.persist(shouldBeKeptAsNotProcessed); + + OutboxRecord shouldBeKeptAsNotInDeletionPeriod = newRecord(Instant.now().minus(Duration.ofDays(1)), "topic1", "key1", "value3", Collections.emptyMap()); + repository.persist(shouldBeKeptAsNotInDeletionPeriod); + + OutboxRecord shouldBeDeleted1 = newRecord(Instant.now().minus(Duration.ofDays(16)), "topic1", "key1", "value1", Collections.emptyMap()); + repository.persist(shouldBeDeleted1); + + OutboxRecord shouldBeDeleted2 = newRecord(Instant.now().minus(Duration.ofDays(18)), "topic1", "key1", "value2", Collections.emptyMap()); + repository.persist(shouldBeDeleted2); + + OutboxRecord shouldBeDeleted3 = newRecord(Instant.now().minus(Duration.ofDays(150)), "topic1", "key1", "value2", Collections.emptyMap()); + repository.persist(shouldBeDeleted3); + + // when + Integer result = repository.deleteOutboxRecordByProcessedNotNullAndProcessedIsBefore(Instant.now().minus(Duration.ofDays(15))); + + // then + assertEquals(3, result); + assertFalse(outboxRecordExists(shouldBeDeleted1.getId())); + assertFalse(outboxRecordExists(shouldBeDeleted2.getId())); + assertFalse(outboxRecordExists(shouldBeDeleted3.getId())); + assertTrue(outboxRecordExists(shouldBeKeptAsNotInDeletionPeriod.getId())); + assertTrue(outboxRecordExists(shouldBeKeptAsNotProcessed.getId())); + } + + private boolean outboxRecordExists(Long id) { + Long result = (Long) entityManager.createQuery("select count(*) from OutboxRecord or where or.id=:id") + .setParameter("id", id) + .getSingleResult(); + return result > 0; + } +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/ConcurrentOutboxProcessorsIntegrationTest.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/ConcurrentOutboxProcessorsIntegrationTest.java new file mode 100644 index 00000000..fe2b3379 --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/ConcurrentOutboxProcessorsIntegrationTest.java @@ -0,0 +1,165 @@ +/** + * Copyright 2022 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.service; + +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; +import one.tomorrow.transactionaloutbox.KafkaTestUtils; +import one.tomorrow.transactionaloutbox.ProxiedKafkaContainer; +import one.tomorrow.transactionaloutbox.ProxiedPostgreSQLContainer; +import one.tomorrow.transactionaloutbox.config.TestTransactionalOutboxConfig; +import one.tomorrow.transactionaloutbox.config.TransactionalOutboxConfig; +import one.tomorrow.transactionaloutbox.model.OutboxRecord; +import one.tomorrow.transactionaloutbox.publisher.DefaultKafkaProducerFactory; +import one.tomorrow.transactionaloutbox.publisher.KafkaProducerMessagePublisherFactory; +import one.tomorrow.transactionaloutbox.repository.OutboxRepository; +import one.tomorrow.transactionaloutbox.tracing.NoopTracingService; +import one.tomorrow.transactionaloutbox.tracing.TracingService; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.junit.jupiter.api.*; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static java.util.stream.IntStream.range; +import static one.tomorrow.transactionaloutbox.KafkaTestUtils.*; +import static one.tomorrow.transactionaloutbox.ProxiedKafkaContainer.bootstrapServers; +import static one.tomorrow.transactionaloutbox.ProxiedKafkaContainer.startProxiedKafka; +import static one.tomorrow.transactionaloutbox.ProxiedPostgreSQLContainer.startProxiedPostgres; +import static one.tomorrow.transactionaloutbox.TestUtils.newHeaders; +import static one.tomorrow.transactionaloutbox.TestUtils.newRecord; +import static org.junit.jupiter.api.Assertions.assertEquals; + +@QuarkusTest +@TestProfile(ConcurrentOutboxProcessorsIntegrationTest.class) +@SuppressWarnings("unused") +public class ConcurrentOutboxProcessorsIntegrationTest implements QuarkusTestProfile { + + private static final String topic = "topicConcurrentTest"; + public static final ProxiedPostgreSQLContainer postgresqlContainer = startProxiedPostgres(); + public static final ProxiedKafkaContainer kafkaContainer = startProxiedKafka(); + private static Consumer consumer; + + @Inject + EntityManager entityManager; + @Inject + OutboxRepository repository; + @Inject + TestOutboxLockRepository lockRepository; + @Inject + OutboxLockService lockService; + @Inject + TestOutboxRepository transactionalRepository; + private final TracingService tracingService = new NoopTracingService(); + + private OutboxProcessor testee1; + private OutboxProcessor testee2; + + @Override + public Map getConfigOverrides() { + return Map.of( + "one.tomorrow.transactional-outbox.lock-timeout", "PT0.2S", + // db config + "quarkus.datasource.devservices.enabled", "false", + "quarkus.datasource.jdbc.url", postgresqlContainer.getJdbcUrl(), + "quarkus.datasource.username", postgresqlContainer.getUsername(), + "quarkus.datasource.password", postgresqlContainer.getPassword(), + // kafka config + "quarkus.kafka.devservices.enabled", "false", + "kafka.bootstrap.servers", kafkaContainer.getBootstrapServers() + + ); + } + + @BeforeAll + static void beforeAll() { + createTopic(bootstrapServers, topic); + consumer = setupConsumer("testGroup", false, topic); + } + + @BeforeEach + @Transactional + void cleanUp() { + entityManager + .createQuery("DELETE FROM OutboxRecord") + .executeUpdate(); + } + + @AfterEach + void afterTest() { + testee1.close(); + testee2.close(); + } + + @AfterAll + static void afterClass() { + if (consumer != null) + consumer.close(); + } + + @Test + void should_ProcessRecordsOnceInOrder() { + // given + Duration lockTimeout = Duration.ofMillis(20); // very aggressive lock stealing + Duration processingInterval = Duration.ZERO; + KafkaProducerMessagePublisherFactory publisherFactory = new KafkaProducerMessagePublisherFactory( + new DefaultKafkaProducerFactory(producerProps())); + String eventSource = "test"; + + TransactionalOutboxConfig config1 = TestTransactionalOutboxConfig.createConfig( + processingInterval, lockTimeout, "processor1", eventSource); + + TransactionalOutboxConfig config2 = TestTransactionalOutboxConfig.createConfig( + processingInterval, lockTimeout, "processor2", eventSource); + + testee1 = new OutboxProcessor(config1, repository, publisherFactory, lockService, tracingService); + testee2 = new OutboxProcessor(config2, repository, publisherFactory, lockService, tracingService); + + // when + List outboxRecords = range(0, 1000).mapToObj( + i -> newRecord(topic, "key1", "value" + i, newHeaders("h", "v" + i)) + ).toList(); + outboxRecords.forEach(transactionalRepository::persist); + + // then + List> allRecords = new ArrayList<>(); + while (allRecords.size() < outboxRecords.size()) { + ConsumerRecords records = KafkaTestUtils.getRecords(consumer(), Duration.ofSeconds(5)); + records.iterator().forEachRemaining(allRecords::add); + } + + assertEquals(outboxRecords.size(), allRecords.size()); + Iterator> iter = allRecords.iterator(); + outboxRecords.forEach(outboxRecord -> { + ConsumerRecord kafkaRecord = iter.next(); + assertConsumedRecord(outboxRecord, eventSource, kafkaRecord); + }); + } + + private static Consumer consumer() { + return consumer; + } + +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/OutboxLockServiceIntegrationTest.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/OutboxLockServiceIntegrationTest.java new file mode 100644 index 00000000..e3ef77b2 --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/OutboxLockServiceIntegrationTest.java @@ -0,0 +1,101 @@ +/** + * Copyright 2022 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.service; + +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.concurrent.*; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +@QuarkusTest +@TestProfile(OutboxLockServiceIntegrationTest.class) +public class OutboxLockServiceIntegrationTest implements QuarkusTestProfile { + + @Inject + OutboxLockService lockService; + + private static ExecutorService executorService; + + @Override + public Map getConfigOverrides() { + return Map.of("one.tomorrow.transactional-outbox.lock-timeout", "PT0S"); + } + + @BeforeAll + static void beforeClass() { + executorService = Executors.newCachedThreadPool(); + } + + @AfterAll + static void afterClass() { + executorService.shutdown(); + } + + @Test + void should_RunWithLock_PreventLockStealing() throws ExecutionException, InterruptedException, TimeoutException { + // given + String ownerId1 = "owner-1"; + String ownerId2 = "owner-2"; + + boolean locked = lockService.acquireOrRefreshLock(ownerId1); + assumeTrue(locked); + + CyclicBarrier barrier1 = new CyclicBarrier(2); + CyclicBarrier barrier2 = new CyclicBarrier(2); + CyclicBarrier barrier3 = new CyclicBarrier(2); + + // when + Future runWithLockResult = executorService.submit(() -> { + await(barrier1); + return lockService.runWithLock(ownerId1, () -> { + await(barrier2); + await(barrier3); // exit runWithLock not before owner2 has tried to "acquireOrRefreshLock" + }); + }); + Future lockStealingAttemptResult = executorService.submit(() -> { + await(barrier1); + await(barrier2); // start acquireOrRefreshLock not before owner1 is inside "runWithLock" + boolean result = lockService.acquireOrRefreshLock(ownerId2); + await(barrier3); + return result; + }); + + // then + assertTrue(runWithLockResult.get(5, SECONDS)); + assertFalse(lockStealingAttemptResult.get(5, SECONDS)); + } + + /** Awaits the given barrier, turning checked exceptions into unchecked, for easier usage in lambdas. */ + private void await(CyclicBarrier barrier) { + try { + barrier.await(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/OutboxProcessorInMemoryIntegrationTest.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/OutboxProcessorInMemoryIntegrationTest.java new file mode 100644 index 00000000..d8c66d3b --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/OutboxProcessorInMemoryIntegrationTest.java @@ -0,0 +1,198 @@ +/** + * Copyright 2022 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.service; + +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import io.smallrye.mutiny.tuples.Tuple2; +import io.smallrye.reactive.messaging.MutinyEmitter; +import io.smallrye.reactive.messaging.kafka.api.OutgoingKafkaRecordMetadata; +import io.smallrye.reactive.messaging.memory.InMemoryConnector; +import io.smallrye.reactive.messaging.memory.InMemorySink; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; +import one.tomorrow.transactionaloutbox.ProxiedKafkaContainer; +import one.tomorrow.transactionaloutbox.ProxiedPostgreSQLContainer; +import one.tomorrow.transactionaloutbox.config.TransactionalOutboxConfig; +import one.tomorrow.transactionaloutbox.model.OutboxLock; +import one.tomorrow.transactionaloutbox.model.OutboxRecord; +import one.tomorrow.transactionaloutbox.publisher.EmitterMessagePublisher; +import one.tomorrow.transactionaloutbox.publisher.MessagePublisherFactory; +import one.tomorrow.transactionaloutbox.repository.OutboxRepository; +import one.tomorrow.transactionaloutbox.tracing.TracingService; +import org.eclipse.microprofile.reactive.messaging.Channel; +import org.eclipse.microprofile.reactive.messaging.Message; +import org.eclipse.microprofile.reactive.messaging.spi.Connector; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static one.tomorrow.transactionaloutbox.ProxiedPostgreSQLContainer.startProxiedPostgres; +import static one.tomorrow.transactionaloutbox.TestUtils.newHeaders; +import static one.tomorrow.transactionaloutbox.TestUtils.newRecord; +import static one.tomorrow.transactionaloutbox.config.TestTransactionalOutboxConfig.createConfig; +import static org.awaitility.Awaitility.await; +import static org.hamcrest.Matchers.hasSize; +import static org.junit.jupiter.api.Assertions.assertEquals; + +@QuarkusTest +@TestProfile(OutboxProcessorInMemoryIntegrationTest.class) +public class OutboxProcessorInMemoryIntegrationTest implements QuarkusTestProfile { + + private static final String TOPIC_1 = "topicOPIMIT1"; + private static final String TOPIC_2 = "topicOPIMIT2"; + private static final String LOCK_OWNER_ID = "processorInMemIT"; + + public static final ProxiedPostgreSQLContainer postgresqlContainer = startProxiedPostgres(); + static { + ProxiedKafkaContainer.stopProxiedKafka(); + } + + @ApplicationScoped + public static class TestEmitterResolver implements EmitterMessagePublisher.EmitterResolver { + + @Channel("channel1") + MutinyEmitter emitter1; + @Channel("channel2") + MutinyEmitter emitter2; + + @Override + public MutinyEmitter resolveBy(String topic) { + if (TOPIC_1.equals(topic)) + return emitter1; + if (TOPIC_2.equals(topic)) + return emitter2; + return null; + } + } + + @Inject + EntityManager entityManager; + @Inject + OutboxRepository repository; + @Inject + OutboxLockService lockService; + @Inject + TestOutboxRepository transactionalRepository; + @Inject + TracingService tracingService; + @Inject + MessagePublisherFactory publisherFactory; + + @Inject + @Connector("smallrye-in-memory") + InMemoryConnector inMemoryConnector; + + private InMemorySink channel1Sink; + private InMemorySink channel2Sink; + + private OutboxProcessor testee; + + @Override + public Map getConfigOverrides() { + return Map.of( + // db config + "quarkus.datasource.devservices.enabled", "false", + "quarkus.datasource.jdbc.url", postgresqlContainer.getJdbcUrl(), + "quarkus.datasource.username", postgresqlContainer.getUsername(), + "quarkus.datasource.password", postgresqlContainer.getPassword(), + // kafka config + "quarkus.kafka.devservices.enabled", "false", + "kafka.bootstrap.servers", "", // ensure that this is not set + // outgoing channel config + "mp.messaging.outgoing.channel1.connector", "smallrye-in-memory", + "mp.messaging.outgoing.channel1.topic", TOPIC_1, + "mp.messaging.outgoing.channel2.connector", "smallrye-in-memory", + "mp.messaging.outgoing.channel2.topic", TOPIC_2 + ); + } + + @BeforeEach + void setUp() { + channel1Sink = inMemoryConnector.sink("channel1"); + channel2Sink = inMemoryConnector.sink("channel2"); + } + + @BeforeEach + @Transactional + void cleanUp() { + OutboxLock outboxLock = entityManager.find(OutboxLock.class, OutboxLock.OUTBOX_LOCK_ID); + if (outboxLock != null && !outboxLock.getOwnerId().equals(LOCK_OWNER_ID)) { + entityManager.remove(outboxLock); + } + entityManager.createQuery("DELETE FROM OutboxRecord").executeUpdate(); + } + + @AfterEach + void afterTest() { + testee.close(); + } + + @Test + void should_ProcessNewRecords() { + // given + TransactionalOutboxConfig config = createConfig( + Duration.ofMillis(50), + Duration.ofMillis(200), + LOCK_OWNER_ID, + "eventSource" + ); + testee = new OutboxProcessor(config, repository, publisherFactory, lockService, tracingService); + + // when + OutboxRecord record1 = newRecord(TOPIC_1, "key1", "value1", newHeaders("h1", "v1")); + transactionalRepository.persist(record1); + + // then + Tuple2 kv = waitForNextMessage(channel1Sink); + assertEquals(record1.getKey(), kv.getItem1()); + assertEquals(new String(record1.getValue()), new String(kv.getItem2())); + + // and when + OutboxRecord record2 = newRecord(TOPIC_2, "key2", "value2", newHeaders("h2", "v2")); + transactionalRepository.persist(record2); + + // then + kv = waitForNextMessage(channel2Sink); + assertEquals(record2.getKey(), kv.getItem1()); + assertEquals(new String(record2.getValue()), new String(kv.getItem2())); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static Tuple2 waitForNextMessage(InMemorySink sink) { + List> received = + await().atMost(10, SECONDS).until(sink::received, hasSize(1)); + Message message = received.get(0); + Optional metadata = + message.getMetadata(OutgoingKafkaRecordMetadata.class); + K key = (K) metadata.orElseThrow().getKey(); + V payload = message.getPayload(); + + sink.clear(); + + return Tuple2.of(key, payload); + } + +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/OutboxProcessorIntegrationTest.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/OutboxProcessorIntegrationTest.java new file mode 100644 index 00000000..c824b581 --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/OutboxProcessorIntegrationTest.java @@ -0,0 +1,468 @@ +/** + * Copyright 2022 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.service; + +import eu.rekawek.toxiproxy.model.toxic.Timeout; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; +import one.tomorrow.transactionaloutbox.KafkaTestUtils; +import one.tomorrow.transactionaloutbox.ProxiedKafkaContainer; +import one.tomorrow.transactionaloutbox.ProxiedPostgreSQLContainer; +import one.tomorrow.transactionaloutbox.config.TransactionalOutboxConfig; +import one.tomorrow.transactionaloutbox.config.TransactionalOutboxConfig.CleanupConfig; +import one.tomorrow.transactionaloutbox.model.OutboxRecord; +import one.tomorrow.transactionaloutbox.publisher.DefaultKafkaProducerFactory; +import one.tomorrow.transactionaloutbox.publisher.KafkaProducerMessagePublisherFactory; +import one.tomorrow.transactionaloutbox.repository.OutboxRepository; +import one.tomorrow.transactionaloutbox.tracing.TracingAssertions; +import one.tomorrow.transactionaloutbox.tracing.TracingService; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.junit.jupiter.api.*; + +import java.io.IOException; +import java.time.Duration; +import java.util.List; +import java.util.Map; + +import static eu.rekawek.toxiproxy.model.ToxicDirection.DOWNSTREAM; +import static java.lang.Thread.sleep; +import static one.tomorrow.transactionaloutbox.KafkaTestUtils.*; +import static one.tomorrow.transactionaloutbox.ProxiedKafkaContainer.bootstrapServers; +import static one.tomorrow.transactionaloutbox.ProxiedKafkaContainer.startProxiedKafka; +import static one.tomorrow.transactionaloutbox.ProxiedPostgreSQLContainer.postgresProxy; +import static one.tomorrow.transactionaloutbox.ProxiedPostgreSQLContainer.startProxiedPostgres; +import static one.tomorrow.transactionaloutbox.TestUtils.newHeaders; +import static one.tomorrow.transactionaloutbox.TestUtils.newRecord; +import static one.tomorrow.transactionaloutbox.config.TestTransactionalOutboxConfig.createCleanupConfig; +import static one.tomorrow.transactionaloutbox.config.TestTransactionalOutboxConfig.createConfig; +import static one.tomorrow.transactionaloutbox.tracing.TracingService.INTERNAL_PREFIX; +import static org.awaitility.Awaitility.await; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + +@QuarkusTest +@TestProfile(OutboxProcessorIntegrationTest.class) +@SuppressWarnings({"unused", "resource"}) +public class OutboxProcessorIntegrationTest implements QuarkusTestProfile, TracingAssertions { + + private static final String TOPIC_1 = "topicOPIT1"; + private static final String TOPIC_2 = "topicOPIT2"; + + public static final ProxiedPostgreSQLContainer postgresqlContainer = startProxiedPostgres(); + public static final ProxiedKafkaContainer kafkaContainer = startProxiedKafka(); + private static Consumer consumer; + + @Inject + EntityManager entityManager; + @Inject + OutboxRepository repository; + @Inject + TestOutboxLockRepository lockRepository; + @Inject + OutboxLockService lockService; + @Inject + TestOutboxRepository transactionalRepository; + @Inject + TracingService tracingService; + @Inject + InMemorySpanExporter spanExporter; + @Inject + OpenTelemetry openTelemetry; + + private OutboxProcessor testee; + + @Override + public Map getConfigOverrides() { + return Map.of( + "one.tomorrow.transactional-outbox.lock-timeout", "PT0.2S", + // db config + "quarkus.datasource.devservices.enabled", "false", + "quarkus.datasource.jdbc.url", postgresqlContainer.getJdbcUrl(), + "quarkus.datasource.username", postgresqlContainer.getUsername(), + "quarkus.datasource.password", postgresqlContainer.getPassword(), + // kafka config + "quarkus.kafka.devservices.enabled", "false", + "kafka.bootstrap.servers", kafkaContainer.getBootstrapServers() + + ); + } + + @BeforeAll + static void beforeAll() { + createTopic(bootstrapServers, TOPIC_1, TOPIC_2); + consumer = setupConsumer("testConsumer-" + System.currentTimeMillis(), true, TOPIC_1, TOPIC_2); + } + + @BeforeEach + @Transactional + void cleanUp() { + entityManager + .createQuery("DELETE FROM OutboxRecord") + .executeUpdate(); + // Clear spans from previous test runs + spanExporter.reset(); + } + + @AfterEach + void afterTest() { + testee.close(); + } + + @AfterAll + static void afterAll() { + consumer.close(); + } + + @Test + void should_ProcessNewRecords() { + // given + String eventSource = "test"; + TransactionalOutboxConfig config = createConfig( + Duration.ofMillis(50), + Duration.ofMillis(200), + "processor", + eventSource + ); + testee = new OutboxProcessor(config, repository, publisherFactory(), lockService, tracingService); + + // when + OutboxRecord record1 = newRecord(TOPIC_1, "key1", "value1", newHeaders("h1", "v1")); + transactionalRepository.persist(record1); + + // then + ConsumerRecords records = getAndCommitRecords(); + assertThat(records.count(), is(1)); + ConsumerRecord kafkaRecord = records.iterator().next(); + assertConsumedRecord(record1, "h1", eventSource, kafkaRecord); + + // and when + OutboxRecord record2 = newRecord(TOPIC_2, "key2", "value2", newHeaders("h2", "v2")); + transactionalRepository.persist(record2); + + // then + records = getAndCommitRecords(); + assertThat(records.count(), is(1)); + kafkaRecord = records.iterator().next(); + assertConsumedRecord(record2, "h2", eventSource, kafkaRecord); + } + + private ConsumerRecords getAndCommitRecords() { + ConsumerRecords records = KafkaTestUtils.getRecords(consumer(), Duration.ofSeconds(10)); + consumer().commitSync(); + return records; + } + + @Test + void should_ProcessNewRecords_withTracing() { + // given + String eventSource = "test"; + TransactionalOutboxConfig config = createConfig( + Duration.ofMillis(50), + Duration.ofMillis(200), + "processor", + eventSource + ); + testee = new OutboxProcessor(config, repository, publisherFactory(), lockService, tracingService); + + // when + + String traceId1 = "0123456789abcdef0123456789abcdef"; // 32 hex + String parentSpanId1 = "0123456789abcdef"; // 16 hex + String traceparent1 = "00-" + traceId1 + "-" + parentSpanId1 + "-01"; + OutboxRecord record1 = newRecord(TOPIC_1, "key1", "value1", newHeaders( + "h1", "v1", + INTERNAL_PREFIX + "traceparent", traceparent1, + INTERNAL_PREFIX + "tracestate", "vendor=value1")); + transactionalRepository.persist(record1); + + // then + ConsumerRecords records = getAndCommitRecords(); + assertEquals(1, records.count()); + ConsumerRecord kafkaRecord = records.iterator().next(); + assertConsumedRecord(record1, "h1", eventSource, kafkaRecord); + + // verify spans: one for the transactional-outbox, one for the processing to Kafka + List spans = spanExporter.getFinishedSpanItems(); + assertEquals(2, spans.size()); + + SpanData outboxSpan = findSpanByName(spanExporter, "transactional-outbox"); + assertOutboxSpan(outboxSpan, traceId1, parentSpanId1, record1); + + SpanData processingSpan = findSpanByName(spanExporter, "To_" + TOPIC_1); + assertProcessingSpan(processingSpan, traceId1, outboxSpan.getSpanId(), TOPIC_1); + + // and when + String traceId2 = "1123456789abcdef0123456789abcdef"; // 32 hex + String parentSpanId2 = "1123456789abcdef"; // 16 hex + String traceparent2 = "00-" + traceId2 + "-" + parentSpanId2 + "-01"; + OutboxRecord record2 = newRecord(TOPIC_2, "key2", "value2", newHeaders( + "h2", "v2", + INTERNAL_PREFIX + "traceparent", traceparent2, + INTERNAL_PREFIX + "tracestate", "vendor=value2")); + transactionalRepository.persist(record2); + + // then + records = getAndCommitRecords(); + assertEquals(1, records.count()); + kafkaRecord = records.iterator().next(); + assertConsumedRecord(record2, "h2", eventSource, kafkaRecord); + + // verify spans: one for the transactional-outbox, one for the processing to Kafka + await().atMost(Duration.ofSeconds(10)).untilAsserted(() -> + assertEquals(4, spanExporter.getFinishedSpanItems().size()) + ); + spans = spanExporter.getFinishedSpanItems(); + + outboxSpan = spans.get(2); + assertOutboxSpan(outboxSpan, traceId2, parentSpanId2, record2); + + processingSpan = spans.get(3); + assertProcessingSpan(processingSpan, traceId2, outboxSpan.getSpanId(), TOPIC_2); + } + + @Test + void should_StartWhenKafkaIsNotAvailableAndProcessOutboxWhenKafkaIsAvailable() throws InterruptedException { + // given + OutboxRecord record1 = newRecord(TOPIC_1, "key1", "value1", newHeaders("h1", "v1")); + transactionalRepository.persist(record1); + + kafkaContainer.setConnectionCut(true); + + // when + Duration processingInterval = Duration.ofMillis(50); + String eventSource = "test"; + TransactionalOutboxConfig config = createConfig( + processingInterval, + Duration.ofMillis(200), + "processor", + eventSource + ); + testee = new OutboxProcessor(config, repository, publisherFactory(), lockService, tracingService); + + sleep(processingInterval.plusMillis(200).toMillis()); + kafkaContainer.setConnectionCut(false); + + // then + ConsumerRecords records = getAndCommitRecords(); + assertThat(records.count(), is(1)); + assertConsumedRecord(record1, eventSource, records.iterator().next()); + } + + @Test + void should_ContinueProcessingAfterKafkaRestart() throws InterruptedException { + // given + OutboxRecord record1 = newRecord(TOPIC_1, "key1", "value1", newHeaders("h1", "v1")); + transactionalRepository.persist(record1); + + Duration processingInterval = Duration.ofMillis(50); + String eventSource = "test"; + TransactionalOutboxConfig config = createConfig( + processingInterval, + Duration.ofMillis(200), + "processor", + eventSource + ); + testee = new OutboxProcessor(config, repository, publisherFactory(), lockService, tracingService); + + // when + ConsumerRecords records = getAndCommitRecords(); + + // then + assertThat(records.count(), is(1)); + + // and when + kafkaContainer.setConnectionCut(true); + + OutboxRecord record2 = newRecord(TOPIC_2, "key2", "value2", newHeaders("h2", "v2")); + transactionalRepository.persist(record2); + + sleep(processingInterval.plusMillis(200).toMillis()); + kafkaContainer.setConnectionCut(false); + + // then + records = getAndCommitRecords(); + assertThat(records.count(), is(1)); + assertConsumedRecord(record2, "h2", eventSource, records.iterator().next()); + } + + @Test + void should_ContinueProcessingAfterDatabaseUnavailability() throws InterruptedException, IOException { + // given + OutboxRecord record1 = newRecord(TOPIC_1, "key1", "value1", newHeaders("h1", "v1")); + transactionalRepository.persist(record1); + + Duration processingInterval = Duration.ofMillis(20); + String eventSource = "test"; + TransactionalOutboxConfig config = createConfig( + processingInterval, + Duration.ofMillis(200), + "processor", + eventSource + ); + testee = new OutboxProcessor(config, repository, publisherFactory(), lockService, tracingService); + + // when + ConsumerRecords records = getAndCommitRecords(); + + // then + assertEquals(1, records.count()); + + // and when + Timeout timeout = postgresProxy.toxics().timeout("TIMEOUT", DOWNSTREAM, 1L); + sleep(processingInterval.multipliedBy(5).toMillis()); + timeout.remove(); + + OutboxRecord record2 = newRecord(TOPIC_2, "key2", "value2", newHeaders("h2", "v2")); + transactionalRepository.persistWithRetry(record2); + + // then + records = getAndCommitRecords(); + assertEquals(1, records.count()); + assertConsumedRecord(record2, "h2", eventSource, records.iterator().next()); + } + + @Test + void should_ContinueProcessingAfterDbConnectionFailureInLockAcquisition() throws InterruptedException { + // given + Duration processingInterval = Duration.ofMillis(50); + String eventSource = "test"; + + TransactionalOutboxConfig config = createConfig( + processingInterval, + Duration.ofMillis(200), + "processor", + eventSource + ); + testee = new OutboxProcessor(config, repository, publisherFactory(), lockService, tracingService); + + // when + OutboxRecord record1 = newRecord(TOPIC_1, "key1", "value1", newHeaders("h1", "v1")); + transactionalRepository.persist(record1); + + // then + ConsumerRecords records = getAndCommitRecords(); + assertThat(records.count(), is(1)); + + // and when + lockRepository.failAcquireOrRefreshLock().set(true); + + OutboxRecord record2 = newRecord(TOPIC_2, "key2", "value2", newHeaders("h2", "v2")); + transactionalRepository.persist(record2); + + lockRepository.acquireOrRefreshLockCDL().await(); + lockRepository.failAcquireOrRefreshLock().set(false); + + // then + records = getAndCommitRecords(); + assertThat(records.count(), is(1)); + } + + @Test + void should_ContinueProcessingAfterDbConnectionFailureInPreventLockStealing() throws InterruptedException { + // given + Duration processingInterval = Duration.ofMillis(500); + String eventSource = "test"; + + TransactionalOutboxConfig config = createConfig( + processingInterval, + Duration.ofMillis(200), + "processor", + eventSource + ); + testee = new OutboxProcessor(config, repository, publisherFactory(), lockService, tracingService); + + // when + OutboxRecord record1 = newRecord(TOPIC_1, "key1", "value1", newHeaders("h1", "v1")); + transactionalRepository.persist(record1); + + // then + ConsumerRecords records = getAndCommitRecords(); + assertThat(records.count(), is(1)); + + // and when + lockRepository.failPreventLockStealing().set(true); + + OutboxRecord record2 = newRecord(TOPIC_2, "key2", "value2", newHeaders("h2", "v2")); + transactionalRepository.persist(record2); + + lockRepository.preventLockStealingCDL().await(); + lockRepository.failPreventLockStealing().set(false); + + // then + records = getAndCommitRecords(); + assertThat(records.count(), is(1)); + } + + @Test + void should_CleanupOutdatedProcessedRecords() { + // given + String eventSource = "test"; + CleanupConfig cleanupConfig = createCleanupConfig( + Duration.ofMillis(100), + Duration.ofMillis(200) + ); + TransactionalOutboxConfig config = createConfig( + Duration.ofMillis(10), + Duration.ofMillis(200), + "processor", + eventSource, + cleanupConfig + ); + testee = new OutboxProcessor(config, repository, publisherFactory(), lockService, tracingService); + + // when + OutboxRecord record1 = newRecord(TOPIC_1, "key1", "value1", newHeaders("h1", "v1")); + transactionalRepository.persist(record1); + assertEquals(1, repository.getUnprocessedRecords(1).size()); + + // then + await().atMost(Duration.ofSeconds(5)).until( + () -> repository.getUnprocessedRecords(1).isEmpty() + ); + assertEquals(1, getAndCommitRecords().count()); + + // and eventually + await().atMost(Duration.ofSeconds(5)).until( + () -> countOutboxRecords() == 0 + ); + } + + @Transactional + long countOutboxRecords() { + return entityManager + .createQuery("SELECT COUNT(r) FROM OutboxRecord r", Long.class) + .getSingleResult(); + } + + private KafkaProducerMessagePublisherFactory publisherFactory() { + return new KafkaProducerMessagePublisherFactory(new DefaultKafkaProducerFactory(producerProps())); + } + + private static Consumer consumer() { + return consumer; + } + +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/OutboxProcessorTest.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/OutboxProcessorTest.java new file mode 100644 index 00000000..f5734d20 --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/OutboxProcessorTest.java @@ -0,0 +1,131 @@ +/** + * Copyright 2023 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.service; + +import one.tomorrow.transactionaloutbox.config.TestTransactionalOutboxConfig; +import one.tomorrow.transactionaloutbox.config.TransactionalOutboxConfig; +import one.tomorrow.transactionaloutbox.model.OutboxRecord; +import one.tomorrow.transactionaloutbox.publisher.MessagePublisher; +import one.tomorrow.transactionaloutbox.publisher.MessagePublisherFactory; +import one.tomorrow.transactionaloutbox.repository.OutboxRepository; +import one.tomorrow.transactionaloutbox.tracing.NoopTracingService; +import one.tomorrow.transactionaloutbox.tracing.TracingService; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; + +import static java.lang.Thread.sleep; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.*; + +class OutboxProcessorTest { + + private final OutboxRepository repository = mock(OutboxRepository.class); + private final OutboxLockService lockService = mock(OutboxLockService.class); + private final MessagePublisherFactory publisherFactory = mock(MessagePublisherFactory.class); + private final MessagePublisher publisher = mock(MessagePublisher.class); + private final TracingService tracingService = new NoopTracingService(); + + private final String key1 = "r1"; + private final OutboxRecord record1 = mock(OutboxRecord.class, RETURNS_MOCKS); + private final String key2 = "r2"; + private final OutboxRecord record2 = mock(OutboxRecord.class, RETURNS_MOCKS); + private final List records = List.of(record1, record2); + + private final Future future1 = mock(Future.class); + private final Future future2 = mock(Future.class); + + private OutboxProcessor processor; + + @BeforeEach + void setup() { + when(publisherFactory.create()).thenReturn(publisher); + when(lockService.getLockTimeout()).thenReturn(Duration.ZERO); + + TransactionalOutboxConfig config = TestTransactionalOutboxConfig.createConfig( + Duration.ZERO, + Duration.ZERO, + "lockOwnerId", + "eventSource" + ); + + processor = new OutboxProcessor( + config, + repository, + publisherFactory, + lockService, + tracingService); + + when(record1.getKey()).thenReturn(key1); + when(record2.getKey()).thenReturn(key2); + } + + /* Verifies, that all items are submitted to producer.send before the first future.get() is invoked */ + @Test + void processOutboxShouldUseProducerInternalBatching() throws ExecutionException, InterruptedException { + when(repository.getUnprocessedRecords(anyInt())).thenReturn(records); + + AtomicInteger sendCounter = new AtomicInteger(0); + + when(publisher.publish(anyLong(), any(), eq(key1), any(), any())).thenAnswer(invocation -> { + sendCounter.incrementAndGet(); + sleep(10); + return future1; + }); + when(publisher.publish(any(), any(), eq(key2), any(), any())).thenAnswer(invocation -> { + sendCounter.incrementAndGet(); + sleep(10); + return future2; + }); + + when(future1.get()).thenAnswer(invocation -> { + assertEquals(2, sendCounter.get()); + return null; + }); + + when(future2.get()).thenAnswer(invocation -> { + assertEquals(2, sendCounter.get()); + return null; + }); + + processor.processOutbox(); + } + + @Test + void processOutboxShouldSetProcessedOnlyOnSuccess() throws Exception { + when(repository.getUnprocessedRecords(anyInt())).thenReturn(records); + + when(publisher.publish(any(), any(), eq(key1), any(), any())).thenAnswer(inv -> future1); + when(future1.get()).thenThrow(new RuntimeException("simulated exception")); + + when(publisher.publish(any(), any(), eq(key2), any(), any())).thenAnswer(inv -> future2); + when(future2.get()).thenReturn(null); + + processor.processOutbox(); + + verify(record1, never()).setProcessed(any()); + verify(repository, never()).update(record1); + + verify(record2).setProcessed(any()); + verify(repository).update(record2); + } + +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/OutboxServiceIntegrationTest.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/OutboxServiceIntegrationTest.java new file mode 100644 index 00000000..a8965a8f --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/OutboxServiceIntegrationTest.java @@ -0,0 +1,107 @@ +/** + * Copyright 2023 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.service; + +import io.quarkus.test.TestTransaction; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; +import jakarta.persistence.EntityManager; +import jakarta.transaction.TransactionRequiredException; +import jakarta.transaction.Transactional; +import jakarta.transaction.TransactionalException; +import one.tomorrow.transactionaloutbox.model.OutboxRecord; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.collection.IsMapContaining.hasEntry; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; + +@QuarkusTest +@SuppressWarnings({"unused", "ConstantConditions"}) +class OutboxServiceIntegrationTest { + + private static final Logger logger = LoggerFactory.getLogger(OutboxServiceIntegrationTest.class); + + @Inject + OutboxService testee; + + @Inject + EntityManager entityManager; + + @BeforeEach + @AfterEach + @Transactional + void cleanUp() { + entityManager + .createQuery("DELETE FROM OutboxRecord") + .executeUpdate(); + } + + @Test + void should_failOnMissingTransaction() { + byte[] value = "foo".getBytes(); + TransactionalException thrown = assertThrows( + TransactionalException.class, + () -> testee.saveForPublishing("topic", "key", value) + ); + assertInstanceOf(TransactionRequiredException.class, thrown.getCause()); + } + + @Test + @TestTransaction + void should_save_withExistingTransaction() { + // given + String message = "foo"; + + // when + OutboxRecord result = testee.saveForPublishing("topic", "key", message.getBytes()); + + // then + assertThat(result.getId(), is(notNullValue())); + + OutboxRecord foundRecord = entityManager.find(OutboxRecord.class, result.getId()); + assertThat(foundRecord, is(notNullValue())); + } + + @Test + @TestTransaction + void should_save_withAdditionalHeader() { + // given + String message = "foo"; + Map additionalHeader = Map.of("key", "value"); + + // when + OutboxRecord result = testee.saveForPublishing("topic", "key", message.getBytes(), additionalHeader); + + // then + assertThat(result.getId(), is(notNullValue())); + + OutboxRecord foundRecord = entityManager.find(OutboxRecord.class, result.getId()); + assertThat(foundRecord, is(notNullValue())); + Map.Entry entry = additionalHeader.entrySet().iterator().next(); + assertThat(foundRecord.getHeaders(), hasEntry(entry.getKey(), entry.getValue())); + } + +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/OutboxServiceTest.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/OutboxServiceTest.java new file mode 100644 index 00000000..0f558a32 --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/OutboxServiceTest.java @@ -0,0 +1,175 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.service; + +import one.tomorrow.transactionaloutbox.model.OutboxRecord; +import one.tomorrow.transactionaloutbox.repository.OutboxRepository; +import one.tomorrow.transactionaloutbox.tracing.TracingService; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +class OutboxServiceTest { + + @Mock + private OutboxRepository repository; + + @Mock + private TracingService tracingService; + + private OutboxService outboxService; + + @BeforeEach + void setUp() { + MockitoAnnotations.openMocks(this); + outboxService = new OutboxService(repository, tracingService); + } + + @Test + void should_SaveOutboxRecord_WithoutHeaders() { + // given + Map traceHeaders = Map.of(); + when(tracingService.tracingHeadersForOutboxRecord()).thenReturn(traceHeaders); + String topic = "test-topic"; + String key = "test-key"; + byte[] value = "test-value".getBytes(); + + // when + OutboxRecord result = outboxService.saveForPublishing(topic, key, value); + + // then + assertNotNull(result); + assertEquals(topic, result.getTopic()); + assertEquals(key, result.getKey()); + assertArrayEquals(value, result.getValue()); + assertEquals(traceHeaders, result.getHeaders()); + + verify(repository).persist(any(OutboxRecord.class)); + verify(tracingService).tracingHeadersForOutboxRecord(); + } + + @Test + void should_SaveOutboxRecord_WithUserHeaders() { + // given + when(tracingService.tracingHeadersForOutboxRecord()).thenReturn(Map.of()); + String topic = "test-topic"; + String key = "test-key"; + byte[] value = "test-value".getBytes(); + Map userHeaders = Map.of("user-header", "user-value"); + + // when + OutboxRecord result = outboxService.saveForPublishing(topic, key, value, userHeaders); + + // then + assertNotNull(result); + assertEquals(topic, result.getTopic()); + assertEquals(key, result.getKey()); + assertArrayEquals(value, result.getValue()); + assertEquals(userHeaders, result.getHeaders()); + + verify(repository).persist(any(OutboxRecord.class)); + verify(tracingService).tracingHeadersForOutboxRecord(); + } + + @Test + void should_SaveOutboxRecord_WithTracingHeaders() { + // given + Map tracingHeaders = Map.of( + "_internal_:traceparent", "00-trace-id-span-id-01", + "_internal_:tracestate", "vendor=value" + ); + when(tracingService.tracingHeadersForOutboxRecord()).thenReturn(tracingHeaders); + String topic = "test-topic"; + String key = "test-key"; + byte[] value = "test-value".getBytes(); + + // when + OutboxRecord result = outboxService.saveForPublishing(topic, key, value); + + // then + assertNotNull(result); + assertEquals(topic, result.getTopic()); + assertEquals(key, result.getKey()); + assertArrayEquals(value, result.getValue()); + assertEquals(tracingHeaders, result.getHeaders()); + + verify(repository).persist(any(OutboxRecord.class)); + verify(tracingService).tracingHeadersForOutboxRecord(); + } + + @Test + void should_SaveOutboxRecord_WithBothUserAndTracingHeaders() { + // given + Map tracingHeaders = Map.of( + "_internal_:traceparent", "00-trace-id-span-id-01" + ); + when(tracingService.tracingHeadersForOutboxRecord()).thenReturn(tracingHeaders); + String topic = "test-topic"; + String key = "test-key"; + byte[] value = "test-value".getBytes(); + Map userHeaders = Map.of("user-header", "user-value"); + + // when + OutboxRecord result = outboxService.saveForPublishing(topic, key, value, userHeaders); + + // then + assertNotNull(result); + assertEquals(topic, result.getTopic()); + assertEquals(key, result.getKey()); + assertArrayEquals(value, result.getValue()); + + Map expectedHeaders = new HashMap<>(userHeaders); + expectedHeaders.putAll(tracingHeaders); + assertEquals(expectedHeaders, result.getHeaders()); + + verify(repository).persist(any(OutboxRecord.class)); + verify(tracingService).tracingHeadersForOutboxRecord(); + } + + @Test + void should_OverrideUserHeaders_WithTracingHeaders_WhenSameKey() { + // given + Map tracingHeaders = Map.of( + "same-key", "tracing-value" + ); + when(tracingService.tracingHeadersForOutboxRecord()).thenReturn(tracingHeaders); + String topic = "test-topic"; + String key = "test-key"; + byte[] value = "test-value".getBytes(); + Map userHeaders = Map.of("same-key", "user-value"); + + // when + OutboxRecord result = outboxService.saveForPublishing(topic, key, value, userHeaders); + + // then + assertNotNull(result); + assertEquals(topic, result.getTopic()); + assertEquals(key, result.getKey()); + assertArrayEquals(value, result.getValue()); + assertEquals("tracing-value", result.getHeaders().get("same-key")); + + verify(repository).persist(any(OutboxRecord.class)); + verify(tracingService).tracingHeadersForOutboxRecord(); + } +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/OutboxUsageIntegrationTest.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/OutboxUsageIntegrationTest.java new file mode 100644 index 00000000..f93788e3 --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/OutboxUsageIntegrationTest.java @@ -0,0 +1,117 @@ +/** + * Copyright 2023 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.service; + +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; +import one.tomorrow.transactionaloutbox.KafkaTestUtils; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.junit.jupiter.api.*; + +import java.time.Duration; +import java.util.Map; + +import static one.tomorrow.transactionaloutbox.KafkaTestUtils.*; +import static one.tomorrow.transactionaloutbox.service.SampleService.Topics.topic1; +import static org.junit.jupiter.api.Assertions.assertEquals; + +@QuarkusTest +@TestProfile(OutboxUsageIntegrationTest.class) +public class OutboxUsageIntegrationTest implements QuarkusTestProfile { + + private static Consumer consumer; + + @Inject + SampleService sampleService; + + @Inject + EntityManager entityManager; + + @Override + public Map getConfigOverrides() { + return Map.of( + "one.tomorrow.transactional-outbox.enabled", "true", // default (only needed here because of our application.property) + "one.tomorrow.transactional-outbox.processing-interval", "PT0.05S", + "one.tomorrow.transactional-outbox.lock-owner-id", "processor", + "one.tomorrow.transactional-outbox.lock-timeout", "PT0.2S", + "one.tomorrow.transactional-outbox.event-source", "test" + ); + } + + @BeforeAll + static void beforeAll() { + createTopic(bootstrapServers(), topic1); + consumer = setupConsumer("testGroup", false, StringDeserializer.class, StringDeserializer.class, topic1); + } + + @AfterAll + static void afterClass() { + consumer.close(); + } + + @BeforeEach + @AfterEach + @Transactional + void cleanUp() { + entityManager.createQuery("DELETE FROM OutboxRecord").executeUpdate(); + } + + @Test + void should_SaveEventForPublishing() { + + // when + int id = 23; + String name = "foo bar"; + sampleService.doSomething(id, name); + + // then + ConsumerRecords records = KafkaTestUtils.getRecords(consumer, Duration.ofSeconds(5)); + assertEquals(1, records.count()); + ConsumerRecord kafkaRecord = records.iterator().next(); + assertEquals(id, Integer.parseInt(kafkaRecord.key())); + assertEquals(name, kafkaRecord.value()); + } + + @Test + void should_SaveEventForPublishing_withAdditionalHeader() { + + // when + int id = 24; + String name = "foo bar baz"; + SampleService.Header additionalHeader = new SampleService.Header("key", "value"); + sampleService.doSomethingWithAdditionalHeaders(id, name, additionalHeader); + + // then + ConsumerRecords records = KafkaTestUtils.getRecords(consumer, Duration.ofSeconds(5)); + assertEquals(1, records.count()); + ConsumerRecord kafkaRecord = records.iterator().next(); + + assertEquals(id, Integer.parseInt(kafkaRecord.key())); + assertEquals(name, kafkaRecord.value()); + + Header foundHeader = kafkaRecord.headers().lastHeader("key"); + assertEquals(additionalHeader.getValue(), new String(foundHeader.value())); + } + +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/SampleService.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/SampleService.java new file mode 100644 index 00000000..230ab2f1 --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/SampleService.java @@ -0,0 +1,70 @@ +/** + * Copyright 2023 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.service; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.transaction.Transactional; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import one.tomorrow.transactionaloutbox.model.OutboxRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.Map; +import java.util.stream.Collectors; + +import static one.tomorrow.transactionaloutbox.service.SampleService.Topics.topic1; + +@ApplicationScoped +@AllArgsConstructor +public class SampleService { + + private static final Logger logger = LoggerFactory.getLogger(SampleService.class); + + private OutboxService outboxService; + + @Transactional + public void doSomething(int id, String something) { + // Here s.th. else would be done within the transaction, e.g. some entity created. + // We record this fact with the event that shall be published to interested parties / consumers. + OutboxRecord rec = outboxService.saveForPublishing(topic1, String.valueOf(id), something.getBytes()); + logger.info("Stored event [{}] in outbox with id {} and key {}", something, rec.getId(), rec.getKey()); + } + + @Transactional + public void doSomethingWithAdditionalHeaders(int id, String something, Header...headers) { + // Here s.th. else would be done within the transaction, e.g. some entity created. + // We record this fact with the event that shall be published to interested parties / consumers. + Map headerMap = Arrays.stream(headers) + .collect(Collectors.toMap(Header::getKey, Header::getValue)); + OutboxRecord rec = outboxService.saveForPublishing(topic1, String.valueOf(id), something.getBytes(), headerMap); + logger.info("Stored event [{}] in outbox with id {}, key {} and headers {}", something, rec.getId(), rec.getKey(), rec.getHeaders()); + } + + @Getter + @RequiredArgsConstructor + public static class Header { + private final String key; + private final String value; + } + + abstract static class Topics { + public static final String topic1 = "sampleTopic"; + } + +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/TestOutboxLockRepository.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/TestOutboxLockRepository.java new file mode 100644 index 00000000..a147b306 --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/TestOutboxLockRepository.java @@ -0,0 +1,79 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.service; + +import jakarta.annotation.Priority; +import jakarta.enterprise.inject.Alternative; +import jakarta.inject.Singleton; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; +import one.tomorrow.transactionaloutbox.repository.OutboxLockRepository; + +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; + +@Alternative +@Priority(1) +@Singleton +public class TestOutboxLockRepository extends OutboxLockRepository { + + private final AtomicBoolean failAcquireOrRefreshLock = new AtomicBoolean(false); + private final CountDownLatch acquireOrRefreshLockCDL = new CountDownLatch(1); + + private final AtomicBoolean failPreventLockStealing = new AtomicBoolean(false); + private final CountDownLatch preventLockStealingCDL = new CountDownLatch(1); + + public TestOutboxLockRepository(EntityManager entityManager) { + super(entityManager); + } + + @Override + @Transactional(Transactional.TxType.REQUIRES_NEW) + public boolean acquireOrRefreshLock(String ownerId, Duration timeout) { + if (failAcquireOrRefreshLock.get()) { + acquireOrRefreshLockCDL.countDown(); + throw new RuntimeException("Simulated exception"); + } + return super.acquireOrRefreshLock(ownerId, timeout); + } + + @Override + public boolean preventLockStealing(String ownerId) { + if (failPreventLockStealing.get()) { + preventLockStealingCDL.countDown(); + throw new RuntimeException("Simulated exception"); + } + return super.preventLockStealing(ownerId); + } + + public AtomicBoolean failAcquireOrRefreshLock() { + return failAcquireOrRefreshLock; + } + + public CountDownLatch acquireOrRefreshLockCDL() { + return acquireOrRefreshLockCDL; + } + + public AtomicBoolean failPreventLockStealing() { + return failPreventLockStealing; + } + + public CountDownLatch preventLockStealingCDL() { + return preventLockStealingCDL; + } + +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/TestOutboxRepository.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/TestOutboxRepository.java new file mode 100644 index 00000000..f7723452 --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/service/TestOutboxRepository.java @@ -0,0 +1,50 @@ +/** + * Copyright 2022 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.service; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import jakarta.transaction.Transactional; +import one.tomorrow.transactionaloutbox.model.OutboxRecord; +import one.tomorrow.transactionaloutbox.repository.OutboxRepository; +import org.eclipse.microprofile.faulttolerance.Retry; + +import static java.time.temporal.ChronoUnit.MILLIS; + +/** + * Helper class, which provides the transactional boundary for {@link OutboxRepository#persist(OutboxRecord)} + */ +@ApplicationScoped +public class TestOutboxRepository { + + @Inject + OutboxRepository repository; + + public TestOutboxRepository(OutboxRepository repository) { + this.repository = repository; + } + + @Transactional + public void persist(OutboxRecord outboxRecord) { + repository.persist(outboxRecord); + } + + @Retry(maxRetries = 10, delay = 500, delayUnit = MILLIS) + public void persistWithRetry(OutboxRecord outboxRecord) { + persist(outboxRecord); + } + +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/tracing/OpenTelemetryTracingServiceTest.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/tracing/OpenTelemetryTracingServiceTest.java new file mode 100644 index 00000000..c3665358 --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/tracing/OpenTelemetryTracingServiceTest.java @@ -0,0 +1,212 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.tracing; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import one.tomorrow.transactionaloutbox.model.OutboxRecord; +import one.tomorrow.transactionaloutbox.tracing.TracingService.TraceOutboxRecordProcessingResult; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.time.temporal.ChronoUnit.MILLIS; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.junit.jupiter.api.Assertions.*; + +class OpenTelemetryTracingServiceTest implements TracingAssertions { + + private OpenTelemetry openTelemetry; + + private InMemorySpanExporter spanExporter; + private OpenTelemetryTracingService tracingService; + private Tracer tracer; + + @BeforeEach + void setUp() { + spanExporter = InMemorySpanExporter.create(); + SdkTracerProvider tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) + .build(); + + openTelemetry = OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .setPropagators(ContextPropagators.create( + W3CTraceContextPropagator.getInstance())) + .build(); + + tracingService = new OpenTelemetryTracingService(openTelemetry); + tracer = openTelemetry.getTracer("one.tomorrow.transactional-outbox"); + } + + @Test + void tracingHeadersForOutboxRecord_withActiveTraceContext_returnsHeaders() { + // given + Span span = tracer.spanBuilder("test-span").startSpan(); + + try (var ignored = span.makeCurrent()) { + // when + Map headers = tracingService.tracingHeadersForOutboxRecord(); + + // then + assertFalse(headers.isEmpty()); + String traceparentKey = TracingService.INTERNAL_PREFIX + "traceparent"; + assertTrue(headers.containsKey(traceparentKey)); + String traceparent = headers.get(traceparentKey); + String expectedTraceId = span.getSpanContext().getTraceId(); + String expectedSpanId = span.getSpanContext().getSpanId(); + // traceparent format: 00--- + String[] parts = traceparent.split("-"); + assertEquals(4, parts.length); + assertEquals("00", parts[0]); + assertEquals(expectedTraceId, parts[1]); + assertEquals(expectedSpanId, parts[2]); + } finally { + span.end(); + } + } + + @Test + void tracingHeadersForOutboxRecord_withoutActiveTraceContext_returnsEmptyMap() { + // when + Map headers = tracingService.tracingHeadersForOutboxRecord(); + + // then + assertTrue(headers.isEmpty()); + } + + @Test + void traceOutboxRecordProcessing_withValidOutboxRecord_createsAndEndsSpan() { + // given + OutboxRecord outboxRecord = new OutboxRecord(); + outboxRecord.setTopic("test-topic"); + outboxRecord.setCreated(Instant.now().minusMillis(42)); + + // Provide a synthetic W3C trace context in headers (no extra parent span gets exported) + String traceId = "0123456789abcdef0123456789abcdef"; // 32 hex + String parentSpanId = "0123456789abcdef"; // 16 hex + String traceparent = "00-" + traceId + "-" + parentSpanId + "-01"; + Map headers = new HashMap<>(); + headers.put("some", "header"); + headers.put(TracingService.INTERNAL_PREFIX + "traceparent", traceparent); + outboxRecord.setHeaders(headers); + + // when + TraceOutboxRecordProcessingResult result = tracingService.traceOutboxRecordProcessing(outboxRecord); + + // then: one finished span (outbox) and one in-flight (processing) + List finished = spanExporter.getFinishedSpanItems(); + assertEquals(1, finished.size()); + SpanData outboxSpan = finished.get(0); + assertOutboxSpan(outboxSpan, traceId, parentSpanId, outboxRecord); + + // verify returned headers: user header preserved, no internal headers, and traceparent for processing present + Map resultHeaders = result.getHeaders(); + assertEquals("header", resultHeaders.get("some")); + assertTrue(resultHeaders.keySet().stream().noneMatch(k -> k.startsWith(TracingService.INTERNAL_PREFIX))); + assertTrue(resultHeaders.containsKey("traceparent")); + String processingTraceparent = resultHeaders.get("traceparent"); + String[] processingParts = processingTraceparent.split("-"); + assertEquals(4, processingParts.length); + assertEquals(traceId, processingParts[1]); + String processingSpanIdFromHeader = processingParts[2]; + // The processing span should have its own unique ID, not equal to outbox span ID + assertNotEquals(outboxSpan.getSpanId(), processingSpanIdFromHeader); + + // now end the processing span and verify it gets exported correctly + Instant before = Instant.now(); + result.publishCompleted(); + Instant after = Instant.now(); + + finished = spanExporter.getFinishedSpanItems(); + assertEquals(2, finished.size()); + SpanData processingSpan = finished.get(finished.size() - 1); + assertProcessingSpan(processingSpan, traceId, outboxSpan.getSpanId(), outboxRecord.getTopic()); + + // verify that the processing span is ended correctly + assertThat(before.toEpochMilli(), lessThanOrEqualTo(processingSpan.getEndEpochNanos() / 1_000_000)); + assertThat(after.toEpochMilli(), greaterThanOrEqualTo(processingSpan.getEndEpochNanos() / 1_000_000)); + } + + @Test + void traceOutboxRecordProcessing_withoutTraceHeaders_ignoresTracing() { + // given + OutboxRecord outboxRecord = new OutboxRecord(); + outboxRecord.setTopic("test-topic"); + outboxRecord.setCreated(Instant.now()); + + Map headers = new HashMap<>(); + headers.put("some", "header"); + outboxRecord.setHeaders(headers); + + // when + TraceOutboxRecordProcessingResult result = tracingService.traceOutboxRecordProcessing(outboxRecord); + + // then + assertTrue(spanExporter.getFinishedSpanItems().isEmpty()); + Map resultHeaders = result.getHeaders(); + assertEquals(1, resultHeaders.size()); + assertEquals("header", resultHeaders.get("some")); + + // Complete the processing (should be no-op) + result.publishCompleted(); + + // Still no spans + assertTrue(spanExporter.getFinishedSpanItems().isEmpty()); + } + + @Test + void traceOutboxRecordProcessing_whenPublishFails_recordsException() { + // given + OutboxRecord outboxRecord = new OutboxRecord(); + outboxRecord.setTopic("test-topic"); + outboxRecord.setCreated(Instant.now()); + + String traceId = "fedcba9876543210fedcba9876543210"; + String parentSpanId = "abcdef0123456789"; + String traceparent = "00-" + traceId + "-" + parentSpanId + "-01"; + Map headers = new HashMap<>(); + headers.put(TracingService.INTERNAL_PREFIX + "traceparent", traceparent); + outboxRecord.setHeaders(headers); + + // when + TraceOutboxRecordProcessingResult result = tracingService.traceOutboxRecordProcessing(outboxRecord); + RuntimeException exception = new RuntimeException("Test exception"); + result.publishFailed(exception); + + // then - both spans exported and processing span contains an exception event + List finished = spanExporter.getFinishedSpanItems(); + assertEquals(2, finished.size()); + SpanData processingSpan = finished.get(finished.size() - 1); + assertEquals(OpenTelemetryTracingService.TO_PREFIX + outboxRecord.getTopic(), processingSpan.getName()); + assertTrue(processingSpan.getEvents().stream().anyMatch(e -> "exception".equals(e.getName()))); + } +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/tracing/OpenTelemetryTracingTestConfig.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/tracing/OpenTelemetryTracingTestConfig.java new file mode 100644 index 00000000..617c933f --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/tracing/OpenTelemetryTracingTestConfig.java @@ -0,0 +1,51 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.tracing; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Produces; +import jakarta.inject.Singleton; + +@ApplicationScoped +public class OpenTelemetryTracingTestConfig { + + @Produces + @Singleton + public InMemorySpanExporter inMemorySpanExporter() { + return InMemorySpanExporter.create(); + } + + @Produces + @Singleton + public OpenTelemetry openTelemetry(InMemorySpanExporter spanExporter) { + SdkTracerProvider tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) + .build(); + + return OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .setPropagators(ContextPropagators.create( + W3CTraceContextPropagator.getInstance())) + .build(); + } +} diff --git a/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/tracing/TracingAssertions.java b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/tracing/TracingAssertions.java new file mode 100644 index 00000000..de2017b6 --- /dev/null +++ b/outbox-kafka-quarkus/src/test/java/one/tomorrow/transactionaloutbox/tracing/TracingAssertions.java @@ -0,0 +1,73 @@ +/** + * Copyright 2025 Tomorrow GmbH @ https://tomorrow.one + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package one.tomorrow.transactionaloutbox.tracing; + +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.data.SpanData; +import one.tomorrow.transactionaloutbox.model.OutboxRecord; + +import java.time.Instant; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +public interface TracingAssertions { + + default void assertOutboxSpan(SpanData outboxSpan, String expectedTraceId, String expectedParentSpanId, OutboxRecord outboxRecord) { + SpanContext spanContext = outboxSpan.getSpanContext(); + assertEquals(expectedTraceId, spanContext.getTraceId()); + assertEquals(expectedParentSpanId, outboxSpan.getParentSpanId()); + assertEquals("transactional-outbox", outboxSpan.getName()); + + // Verify timing - the span should start at the outbox record creation time + Instant expectedStartTime = outboxRecord.getCreated(); + Instant actualStartTime = Instant.ofEpochMilli(outboxSpan.getStartEpochNanos() / 1_000_000); + // Allow some tolerance for timing differences + assertTrue( + Math.abs(expectedStartTime.toEpochMilli() - actualStartTime.toEpochMilli()) < 1000, + "Start time should be close to outbox record creation time" + ); + + assertTrue(outboxSpan.getEndEpochNanos() > outboxSpan.getStartEpochNanos()); + assertNotNull(spanContext.getSpanId()); + } + + default void assertProcessingSpan(SpanData processingSpan, String expectedTraceId, String expectedParentSpanId, String expectedTopic) { + SpanContext spanContext = processingSpan.getSpanContext(); + assertEquals(expectedTraceId, spanContext.getTraceId()); + assertEquals(expectedParentSpanId, processingSpan.getParentSpanId()); + assertEquals("To_" + expectedTopic, processingSpan.getName()); + assertEquals(io.opentelemetry.api.trace.SpanKind.PRODUCER, processingSpan.getKind()); + assertNotNull(spanContext.getSpanId()); + } + + default SpanData findSpanByName(InMemorySpanExporter spanExporter, String spanName) { + List spans = spanExporter.getFinishedSpanItems(); + return spans.stream() + .filter(span -> spanName.equals(span.getName())) + + .findFirst() + .orElseThrow(() -> new AssertionError("No span found with name: " + spanName)); + } + + default List findSpansByNamePrefix(InMemorySpanExporter spanExporter, String namePrefix) { + List spans = spanExporter.getFinishedSpanItems(); + return spans.stream() + .filter(span -> span.getName().startsWith(namePrefix)) + .toList(); + } +} diff --git a/outbox-kafka-quarkus/src/test/resources/application.properties b/outbox-kafka-quarkus/src/test/resources/application.properties new file mode 100644 index 00000000..096704c5 --- /dev/null +++ b/outbox-kafka-quarkus/src/test/resources/application.properties @@ -0,0 +1,33 @@ +# transactional-outbox configuration +one.tomorrow.transactional-outbox.event-source=test +one.tomorrow.transactional-outbox.lock-owner-id=${quarkus.uuid} +one.tomorrow.transactional-outbox.lock-timeout=PT0.2S +one.tomorrow.transactional-outbox.enabled=false + +# Use TestContainers for PostgreSQL in tests +quarkus.datasource.db-kind=postgresql +%test.quarkus.datasource.devservices.enabled=true +%test.quarkus.datasource.devservices.db-name=outbox_test + +# ensure that we're not using XA +quarkus.datasource.jdbc.transactions=enabled +quarkus.datasource.jdbc.enable-recovery=true + +# Hibernate ORM configurations +quarkus.hibernate-orm.log.sql=false +quarkus.hibernate-orm.mapping.format.global=ignore + +# Flyway configurations +%test.quarkus.flyway.migrate-at-start=true +%test.quarkus.flyway.baseline-on-migrate=true + +# Kafka DevServices configuration for tests +%test.quarkus.devservices.enabled=true +%test.quarkus.kafka.devservices.enabled=true +%test.quarkus.kafka.devservices.port=9092 + +%test.quarkus.log.category."one.tomorrow.transactionaloutbox".level=DEBUG + +# Optional: specific configuration for the integration test profile +%integration-test.quarkus.datasource.devservices.enabled=true +%integration-test.quarkus.kafka.devservices.enabled=true diff --git a/outbox-kafka-quarkus/src/test/resources/db/migration/V1__add-outbox-tables.sql b/outbox-kafka-quarkus/src/test/resources/db/migration/V1__add-outbox-tables.sql new file mode 100644 index 00000000..5d332b9b --- /dev/null +++ b/outbox-kafka-quarkus/src/test/resources/db/migration/V1__add-outbox-tables.sql @@ -0,0 +1,20 @@ +CREATE SEQUENCE IF NOT EXISTS outbox_kafka_id_seq; + +CREATE TABLE IF NOT EXISTS outbox_kafka ( + id BIGINT PRIMARY KEY DEFAULT nextval('outbox_kafka_id_seq'::regclass), + created TIMESTAMP WITHOUT TIME ZONE NOT NULL, + processed TIMESTAMP WITHOUT TIME ZONE NULL, + topic CHARACTER VARYING(128) NOT NULL, + key CHARACTER VARYING(128) NULL, + value BYTEA NOT NULL, + headers JSONB NULL +); + +CREATE INDEX idx_outbox_kafka_not_processed ON outbox_kafka (id) WHERE processed IS NULL; +CREATE INDEX idx_outbox_kafka_processed ON outbox_kafka (processed); + +CREATE TABLE IF NOT EXISTS outbox_kafka_lock ( + id CHARACTER VARYING(32) PRIMARY KEY, + owner_id CHARACTER VARYING(128) NOT NULL, + valid_until TIMESTAMP WITHOUT TIME ZONE NOT NULL +); diff --git a/settings.gradle.kts b/settings.gradle.kts index ad6e38f3..55ef8912 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -2,3 +2,5 @@ rootProject.name = "transactional-outbox" include("commons") include("outbox-kafka-spring") include("outbox-kafka-spring-reactive") +include("outbox-kafka-quarkus") +include("outbox-kafka-quarkus-deployment")