diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 2dd38d1c6..5571f9c75 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -8,5 +8,6 @@ jobs: uses: ThreeDotsLabs/watermill/.github/workflows/tests.yml@master with: stress-tests: true + codecov: true secrets: codecov_token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index df550719f..f9fb11042 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -4,3 +4,7 @@ on: jobs: ci: uses: ThreeDotsLabs/watermill/.github/workflows/tests.yml@master + with: + codecov: true + secrets: + codecov_token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1425b34c8..d217874b0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,6 +7,10 @@ on: required: false type: boolean default: false + codecov: + required: false + type: boolean + default: false secrets: codecov_token: required: false @@ -57,6 +61,7 @@ jobs: - run: make wait - run: make test_codecov - uses: codecov/codecov-action@v4 + if: ${{ inputs.codecov }} with: fail_ci_if_error: true files: ./coverage.out diff --git a/docs/config/_default/params.toml b/docs/config/_default/params.toml index df13d7a0c..0543a6aac 100644 --- a/docs/config/_default/params.toml +++ b/docs/config/_default/params.toml @@ -120,7 +120,7 @@ mainSections = ["docs"] [seo] [seo.title] separator = " | " - suffix = "" + suffix = "Watermill | Event-Driven in Go" [seo.favicons] sizes = [] icon = "favicon.png" # favicon.png (default) diff --git a/docs/config/production/hugo.toml b/docs/config/production/hugo.toml index 06ae0cafe..6ce2a03cb 100644 --- a/docs/config/production/hugo.toml +++ b/docs/config/production/hugo.toml @@ -1,2 +1 @@ # Overrides for production environment -baseurl = "https://watermill.io/" diff --git a/docs/content/pubsubs/amqp.md b/docs/content/pubsubs/amqp.md index 819a30aeb..5b51443a0 100644 --- a/docs/content/pubsubs/amqp.md +++ b/docs/content/pubsubs/amqp.md @@ -12,13 +12,15 @@ We are providing Pub/Sub implementation based on [github.com/rabbitmq/amqp091-go {{% load-snippet-partial file="src-link/watermill-amqp/pkg/amqp/doc.go" first_line_contains="// AMQP" last_line_contains="package amqp" padding_after="0" %}} -### Installation +You can find a fully functional example with RabbitMQ in the [Watermill examples](https://github.com/ThreeDotsLabs/watermill/tree/master/_examples/pubsubs/amqp). + +## Installation ```bash go get github.com/ThreeDotsLabs/watermill-amqp/v3 ``` -#### Characteristics +### Characteristics | Feature | Implements | Note | | ------- | ---------- | ---- | @@ -27,7 +29,7 @@ go get github.com/ThreeDotsLabs/watermill-amqp/v3 | GuaranteedOrder | yes | yes, please check https://www.rabbitmq.com/semantics.html#ordering | | Persistent | yes* | when using `NewDurablePubSubConfig` or `NewDurableQueueConfig` | -#### Configuration +### Configuration Our AMQP is shipped with some pre-created configurations: @@ -35,25 +37,25 @@ Our AMQP is shipped with some pre-created configurations: For detailed configuration description, please check [watermill-amqp/pkg/amqp/config.go](https://github.com/ThreeDotsLabs/watermill-amqp/tree/master/pkg/amqp/config.go) -##### TLS Config +#### TLS Config TLS config can be passed to `Config.TLSConfig`. -##### Connecting +#### Connecting {{% load-snippet-partial file="src-link/_examples/pubsubs/amqp/main.go" first_line_contains="publisher, err :=" last_line_contains="panic(err)" padding_after="1" %}} {{% load-snippet-partial file="src-link/_examples/pubsubs/amqp/main.go" first_line_contains="subscriber, err :=" last_line_contains="panic(err)" padding_after="1" %}} -#### Publishing +### Publishing {{% load-snippet-partial file="src-link/watermill-amqp/pkg/amqp/publisher.go" first_line_contains="// Publish" last_line_contains="func (p *Publisher) Publish" %}} -#### Subscribing +### Subscribing {{% load-snippet-partial file="src-link/watermill-amqp/pkg/amqp/subscriber.go" first_line_contains="// Subscribe" last_line_contains="func (s *Subscriber) Subscribe" %}} -#### Marshaler +### Marshaler Marshaler is responsible for mapping AMQP's messages to Watermill's messages. @@ -62,7 +64,7 @@ If you need to customize thing in `amqp.Delivery`, you can do it `PostprocessPub {{% load-snippet-partial file="src-link/watermill-amqp/pkg/amqp/marshaler.go" first_line_contains="// Marshaler" last_line_contains="func (d DefaultMarshaler)" padding_after="0" %}} -#### AMQP "Consumer Groups" +### AMQP "Consumer Groups" AMQP doesn't provide mechanism like Kafka's "consumer groups". You can still achieve similar behaviour with `GenerateQueueNameTopicNameWithSuffix` and `NewDurablePubSubConfig`. @@ -70,7 +72,6 @@ AMQP doesn't provide mechanism like Kafka's "consumer groups". You can still ach In this example both `pubSub1` and `pubSub2` will receive some messages independently. -#### AMQP `TopologyBuilder` +### AMQP `TopologyBuilder` {{% load-snippet-partial file="src-link/watermill-amqp/pkg/amqp/topology_builder.go" first_line_contains="// TopologyBuilder" last_line_contains="}" padding_after="0" %}} - diff --git a/docs/content/pubsubs/bolt.md b/docs/content/pubsubs/bolt.md index c09e14fcf..d31571435 100644 --- a/docs/content/pubsubs/bolt.md +++ b/docs/content/pubsubs/bolt.md @@ -16,13 +16,13 @@ want to publish messages in transaction when saving other data. Bolt documentation: https://github.com/etcd-io/bbolt -### Installation +## Installation ```bash go get github.com/ThreeDotsLabs/watermill-bolt ``` -#### Characteristics +### Characteristics | Feature | Implements | Note | | ------------------- | ---------- | ---- | @@ -31,7 +31,7 @@ go get github.com/ThreeDotsLabs/watermill-bolt | GuaranteedOrder | no | | | Persistent | yes | | -#### Configuration +### Configuration {{% load-snippet-partial file="src-link/watermill-bolt/pkg/bolt/bolt.go" first_line_contains="type CommonConfig struct " last_line_equals="}" %}} @@ -39,7 +39,7 @@ go get github.com/ThreeDotsLabs/watermill-bolt {{% load-snippet-partial file="src-link/watermill-bolt/pkg/bolt/bolt.go" first_line_contains="type SubscriberConfig struct " last_line_equals="}" %}} -##### Subscription name +#### Subscription name To receive messages published to a topic, you must create a subscription to that topic. Only messages published to the topic after the subscription is @@ -53,7 +53,7 @@ In Watermill, the subscription is created automatically during calling `SubscriberConfig.GenerateSubscriptionName`. By default, it is the topic name with the string `_sub` appended to it. -##### Marshaler +#### Marshaler Watermill's messages cannot be directly saved in Bolt which operates on byte slices. Marshaller converts the messages to and from byte slices. The default @@ -63,5 +63,3 @@ unless a very large messages are used within your system. If that is the case you may want to consider implementing a more efficient marshaler. {{% load-snippet-partial file="src-link/watermill-bolt/pkg/bolt/marshaler.go" first_line_contains="// Marshaler" last_line_equals="}" %}} - - diff --git a/docs/content/pubsubs/firestore.md b/docs/content/pubsubs/firestore.md index 7cef72644..20f043b14 100644 --- a/docs/content/pubsubs/firestore.md +++ b/docs/content/pubsubs/firestore.md @@ -25,13 +25,13 @@ Godoc: Firestore documentation: -### Installation +## Installation ```bash go get github.com/ThreeDotsLabs/watermill-firestore ``` -#### Characteristics +### Characteristics | Feature | Implements | Note | | ------- | ---------- | ---- | @@ -40,17 +40,17 @@ go get github.com/ThreeDotsLabs/watermill-firestore | GuaranteedOrder | no | | | Persistent | yes | | -#### Configuration +### Configuration -##### Publisher configuration +#### Publisher configuration {{% load-snippet-partial file="src-link/watermill-firestore/pkg/firestore/publisher.go" first_line_contains="type PublisherConfig struct {" last_line_equals="}" %}} -##### Subscriber configuration +#### Subscriber configuration {{% load-snippet-partial file="src-link/watermill-firestore/pkg/firestore/subscriber.go" first_line_contains="type SubscriberConfig struct {" last_line_equals="}" %}} -##### Subscription name +#### Subscription name To receive messages published to a topic, you must create a subscription to that topic. Only messages published to the topic after the subscription is @@ -68,7 +68,7 @@ If you want to consume messages from a topic with multiple subscribers processing the incoming messages in a different way, you should use a custom function to generate unique subscription names for each subscriber. -#### Marshaler +### Marshaler Watermill's messages cannot be stored directly in Firestore. The marshaler is responsible for converting them to a type which can be stored by Firestore. @@ -76,5 +76,3 @@ The default implementation should be enough for most applications so it is unlikely that you need to implement your own marshaler. {{% load-snippet-partial file="src-link/watermill-firestore/pkg/firestore/marshaler.go" first_line_contains="// Marshaler" last_line_equals="}" padding_after="0" %}} - - diff --git a/docs/content/pubsubs/gochannel.md b/docs/content/pubsubs/gochannel.md index dead08cf4..8858a02a9 100644 --- a/docs/content/pubsubs/gochannel.md +++ b/docs/content/pubsubs/gochannel.md @@ -8,7 +8,9 @@ weight = 40 {{% load-snippet-partial file="src-link/pubsub/gochannel/pubsub.go" first_line_contains="// GoChannel" last_line_contains="type GoChannel struct {" %}} -#### Characteristics +You can find a fully functional example with Go Channels in the [Watermill examples](https://github.com/ThreeDotsLabs/watermill/tree/master/_examples/pubsubs/go-channel). + +### Characteristics | Feature | Implements | Note | | ------- | ---------- | ---- | @@ -17,21 +19,20 @@ weight = 40 | GuaranteedOrder | yes | | | Persistent | no| | -#### Configuration +### Configuration You can inject configuration via the constructor. {{% load-snippet-partial file="src-link/pubsub/gochannel/pubsub.go" first_line_contains="func NewGoChannel" last_line_contains="logger:" %}} -#### Publishing +### Publishing {{% load-snippet-partial file="src-link/pubsub/gochannel/pubsub.go" first_line_contains="// Publish" last_line_contains="func (g *GoChannel) Publish" %}} -#### Subscribing +### Subscribing {{% load-snippet-partial file="src-link/pubsub/gochannel/pubsub.go" first_line_contains="// Subscribe" last_line_contains="func (g *GoChannel) Subscribe" %}} -#### Marshaler +### Marshaler No marshaling is needed when sending messages within the process. - diff --git a/docs/content/pubsubs/googlecloud.md b/docs/content/pubsubs/googlecloud.md index 919ae6e45..fe2beb439 100644 --- a/docs/content/pubsubs/googlecloud.md +++ b/docs/content/pubsubs/googlecloud.md @@ -17,15 +17,17 @@ it allows for secure and highly available communication among independently writ Cloud Pub/Sub delivers low-latency, durable messaging that helps developers quickly integrate systems hosted on the Google Cloud Platform and externally. -Documentation: [https://cloud.google.com/pubsub/docs/](https://cloud.google.com/pubsub/docs/overview) +Official Documentation: [https://cloud.google.com/pubsub/docs/](https://cloud.google.com/pubsub/docs/overview) -### Installation +You can find a fully functional example with Google Cloud Pub/Sub in the [Watermill examples](https://github.com/ThreeDotsLabs/watermill/tree/master/_examples/pubsubs/googlecloud). + +## Installation ```bash go get github.com/ThreeDotsLabs/watermill-googlecloud ``` -#### Characteristics +### Characteristics | Feature | Implements | Note | | ------- | ---------- | ---- | @@ -34,13 +36,13 @@ go get github.com/ThreeDotsLabs/watermill-googlecloud | GuaranteedOrder | no | | | Persistent | yes* | maximum retention time is 7 days | -#### Configuration +### Configuration {{% load-snippet-partial file="src-link/watermill-googlecloud/pkg/googlecloud/publisher.go" first_line_contains="type PublisherConfig struct " last_line_contains="func NewPublisher" %}} {{% load-snippet-partial file="src-link/watermill-googlecloud/pkg/googlecloud/subscriber.go" first_line_contains="type SubscriberConfig struct {" last_line_contains="func NewSubscriber(" %}} -##### Subscription name +#### Subscription name To receive messages published to a topic, you must create a subscription to that topic. Only messages published to the topic after the subscription is created are available to subscriber @@ -58,7 +60,7 @@ By default, it is just the topic name (`TopicSubscriptionName`). When you want to consume messages from a topic with multiple subscribers, you should use `TopicSubscriptionNameWithSuffix` or your custom function to generate the subscription name. -#### Connecting +### Connecting Watermill will connect to the instance of Google Cloud Pub/Sub indicated by the environment variables. For production setup, set the `GOOGLE_APPLICATION_CREDENTIALS` env, as described in [the official Google Cloud Pub/Sub docs](https://cloud.google.com/pubsub/docs/quickstart-client-libraries#pubsub-client-libraries-go). Note that you won't need to install the Cloud SDK, as Watermill will take care of the administrative tasks (creating topics/subscriptions) with the default settings and proper permissions. @@ -68,18 +70,16 @@ For development, you can use a Docker image with the emulator and the `PUBSUB_EM {{% load-snippet-partial file="src-link/_examples/pubsubs/googlecloud/main.go" first_line_contains="subscriber, err :=" last_line_contains="panic(err)" padding_after="1" %}} -#### Publishing +### Publishing {{% load-snippet-partial file="src-link/watermill-googlecloud/pkg/googlecloud/publisher.go" first_line_contains="// Publish" last_line_contains="func (p *Publisher) Publish" %}} -#### Subscribing +### Subscribing {{% load-snippet-partial file="src-link/watermill-googlecloud/pkg/googlecloud/subscriber.go" first_line_contains="// Subscribe " last_line_contains="func (s *Subscriber) Subscribe" %}} -#### Marshaler +### Marshaler Watermill's messages cannot be directly sent to Google Cloud Pub/Sub - they need to be marshaled. You can implement your marshaler or use the default implementation. {{% load-snippet-partial file="src-link/watermill-googlecloud/pkg/googlecloud/marshaler.go" first_line_contains="// Marshaler" last_line_contains="type DefaultMarshalerUnmarshaler " padding_after="0" %}} - - diff --git a/docs/content/pubsubs/http.md b/docs/content/pubsubs/http.md index f3bf2a60a..b1f72207a 100644 --- a/docs/content/pubsubs/http.md +++ b/docs/content/pubsubs/http.md @@ -11,13 +11,13 @@ You can then post them to any Publisher. Here is an example with [sending HTTP m The HTTP publisher sends HTTP requests as specified in its configuration. Here is an example with [transforming Kafka messages into HTTP webhook requests](https://github.com/ThreeDotsLabs/watermill/tree/master/_examples/real-world-examples/sending-webhooks). -### Installation +## Installation ```bash go get github.com/ThreeDotsLabs/watermill-http/v2 ``` -#### Characteristics +### Characteristics | Feature | Implements | Note | | ------- | ---------- | ---- | @@ -26,7 +26,7 @@ go get github.com/ThreeDotsLabs/watermill-http/v2 | GuaranteedOrder | yes | | | Persistent | no| | -#### Subscriber configuration +### Subscriber configuration Subscriber configuration is done via the config struct passed to the constructor: @@ -35,20 +35,20 @@ Subscriber configuration is done via the config struct passed to the constructor You can use the `Router` config option to `SubscriberConfig` to pass your own `chi.Router` (see [chi](https://github.com/go-chi/chi)). This may be helpful if you'd like to add your own HTTP handlers (e.g. a health check endpoint). -#### Publisher configuration +### Publisher configuration Publisher configuration is done via the config struct passed to the constructor: {{% load-snippet-partial file="src-link/watermill-http/pkg/http/publisher.go" first_line_contains="type PublisherConfig struct" last_line_contains="}" %}} -How the message topic and body translate into the URL, method, headers, and payload of the HTTP request is highly configurable through the use of `MarshalMessageFunc`. +How the message topic and body translate into the URL, method, headers, and payload of the HTTP request is highly configurable through the use of `MarshalMessageFunc`. Use the provided `DefaultMarshalMessageFunc` to send POST requests to a specific url: {{% load-snippet-partial file="src-link/watermill-http/pkg/http/publisher.go" first_line_contains="// MarshalMessageFunc" last_line_contains="return req, nil" padding_after="2" %}} -You can pass your own `http.Client` to execute the requests or use Golang's default client. +You can pass your own `http.Client` to execute the requests or use Golang's default client. -#### Running +### Running To run HTTP subscriber you need to run `StartHTTPServer()`. It needs to be run after `Subscribe()`. @@ -59,11 +59,11 @@ When using with the router, you should wait for the router to start. httpSubscriber.StartHTTPServer() ``` -#### Subscribing +### Subscribing {{% load-snippet-partial file="src-link/watermill-http/pkg/http/subscriber.go" first_line_contains="// Subscribe adds" last_line_contains="func (s *Subscriber) Subscribe" %}} -##### Custom HTTP status codes +#### Custom HTTP status codes To specify a custom HTTP status code, which will returned as response, you can use following call during message handling: diff --git a/docs/content/pubsubs/io.md b/docs/content/pubsubs/io.md index 70b0fdd13..52f65c11c 100644 --- a/docs/content/pubsubs/io.md +++ b/docs/content/pubsubs/io.md @@ -14,13 +14,13 @@ Note that these aren't full-fledged Pub/Subs like Kafka, RabbitMQ, or the likes, * Subscribing for data on a file or stdin and packaging it as messages * Interfacing with third-party libraries that implement `io.Writer` or `io.Reader`, like [github.com/colinmarc/hdfs](https://github.com/colinmarc/hdfs) or [github.com/mholt/archiver](https://github.com/mholt/archiver). -### Installation +## Installation ```bash go get github.com/ThreeDotsLabs/watermill-io ``` -#### Characteristics +### Characteristics This is a very bare-bones implementation for now, so no extra features are supported. However, it is still sufficient for applications like a [CLI producer/consumer](https://github.com/ThreeDotsLabs/watermill/tree/master/tools/mill). @@ -31,7 +31,7 @@ This is a very bare-bones implementation for now, so no extra features are suppo | GuaranteedOrder | no | | | Persistent | no | | -#### Configuration +### Configuration The publisher configuration is relatively simple. @@ -47,7 +47,7 @@ The continuous reading may be used, for example, to emulate the behaviour of a ` {{% load-snippet-partial file="docs/snippets/tail-log-file/main.go" first_line_contains="// this will" last_line_contains="return false" padding_after="1" %}} -#### Marshaling/Unmarshaling +### Marshaling/Unmarshaling The MarshalFunc is an important part of `io.Publisher`, because it fully controls the format in the underlying `io.Writer` will obtain the messages. @@ -59,7 +59,7 @@ Correspondingly, the UnmarshalFunc regulates how the bytes read by the `io.Reade The package comes with some predefined marshal and unmarshal functions, but you might want to write your own marshaler/unmarshaler to work with the specific implementation of `io.Writer/io.Reader` that you are working with. -#### Topic +### Topic For the Publisher/Subscriber implementation itself, the topic has no meaning. It is difficult to interpret the meaning of topic in the general context of `io.Writer` and `io.Reader` interfaces. diff --git a/docs/content/pubsubs/kafka.md b/docs/content/pubsubs/kafka.md index cfcae8ece..6e9bb03ad 100644 --- a/docs/content/pubsubs/kafka.md +++ b/docs/content/pubsubs/kafka.md @@ -6,15 +6,17 @@ bref = "A distributed streaming platform from Apache" weight = 80 +++ -Apache Kafka is one of the most popular Pub/Subs. We are providing Pub/Sub implementation based on [Shopify's Sarama](https://github.com/Shopify/sarama). +Apache Kafka is one of the most popular Pub/Subs. We are providing Pub/Sub implementation based on [IBM Sarama](https://github.com/IBM/sarama). -### Installation +You can find a fully functional example with Kafka in the [Watermill examples](https://github.com/ThreeDotsLabs/watermill/tree/master/_examples/pubsubs/kafka). + +## Installation ```bash go get github.com/ThreeDotsLabs/watermill-kafka/v3 ``` -#### Characteristics +### Characteristics | Feature | Implements | Note | | ------- | ---------- | ---- | @@ -23,47 +25,47 @@ go get github.com/ThreeDotsLabs/watermill-kafka/v3 | GuaranteedOrder | yes | require [partition key usage](#partitioning) | | Persistent | yes| | -#### Configuration +### Configuration {{% load-snippet-partial file="src-link/watermill-kafka/pkg/kafka/subscriber.go" first_line_contains="type SubscriberConfig struct" last_line_contains="// Subscribe" %}} -##### Passing custom `Sarama` config +#### Passing custom `Sarama` config You can pass [custom config](https://github.com/Shopify/sarama/blob/master/config.go#L20) parameters via `overwriteSaramaConfig *sarama.Config` in `NewSubscriber` and `NewPublisher`. When `nil` is passed, default config is used (`DefaultSaramaSubscriberConfig`). {{% load-snippet-partial file="src-link/watermill-kafka/pkg/kafka/subscriber.go" first_line_contains="// DefaultSaramaSubscriberConfig" last_line_contains="return config" padding_after="1" %}} -#### Connecting +### Connecting -##### Publisher +#### Publisher {{% load-snippet-partial file="src-link/watermill-kafka/pkg/kafka/publisher.go" first_line_contains="// NewPublisher" last_line_contains="(*Publisher, error)" padding_after="0" %}} Example: {{% load-snippet-partial file="src-link/_examples/pubsubs/kafka/main.go" first_line_contains="saramaSubscriberConfig :=" last_line_contains="panic(err)" padding_after="1" %}} -##### Subscriber +#### Subscriber {{% load-snippet-partial file="src-link/watermill-kafka/pkg/kafka/subscriber.go" first_line_contains="// NewSubscriber" last_line_contains="(*Subscriber, error)" padding_after="0" %}} Example: {{% load-snippet-partial file="src-link/_examples/pubsubs/kafka/main.go" first_line_contains="publisher, err := kafka.NewPublisher" last_line_contains="panic(err)" padding_after="1" %}} -#### Publishing +### Publishing {{% load-snippet-partial file="src-link/watermill-kafka/pkg/kafka/publisher.go" first_line_contains="// Publish" last_line_contains="func (p *Publisher) Publish" %}} -#### Subscribing +### Subscribing {{% load-snippet-partial file="src-link/watermill-kafka/pkg/kafka/subscriber.go" first_line_contains="// Subscribe" last_line_contains="func (s *Subscriber) Subscribe" %}} -#### Marshaler +### Marshaler Watermill's messages cannot be directly sent to Kafka - they need to be marshaled. You can implement your marshaler or use default implementation. {{% load-snippet-partial file="src-link/watermill-kafka/pkg/kafka/marshaler.go" first_line_contains="// Marshaler" last_line_contains="func (DefaultMarshaler)" padding_after="0" %}} -#### Partitioning +### Partitioning Our Publisher has support for the partitioning mechanism. diff --git a/docs/content/pubsubs/nats.md b/docs/content/pubsubs/nats.md index 58fa61b4d..ffce9c466 100644 --- a/docs/content/pubsubs/nats.md +++ b/docs/content/pubsubs/nats.md @@ -8,18 +8,20 @@ weight = 90 NATS Jetstream is a data streaming system powered by NATS, and written in the Go programming language. -As of v2.0.2 this middleware will contain a beta implementation in `pkg/jetstream` based on the -[nats.go Jetstream package](https://github.com/nats-io/nats.go/tree/main/jetstream). This implementation is -considered experimental tracking with the upstream client though we target a stable watermill API by v2.1. +As of v2.0.2 this middleware will contain a beta implementation in `pkg/jetstream` based on the +[nats.go Jetstream package](https://github.com/nats-io/nats.go/tree/main/jetstream). This implementation is +considered experimental tracking with the upstream client though we target a stable watermill API by v2.1. For production use it is recommended to use the pubsub implementations in `pkg/nats` with Jetstream enabled. -### Installation +You can find a fully functional example with NATS JetStream in the [Watermill examples](https://github.com/ThreeDotsLabs/watermill/tree/master/_examples/pubsubs/nats-jetstream). + +## Installation ```bash go get github.com/ThreeDotsLabs/watermill-nats/v2 ``` -#### Characteristics +### Characteristics | Feature | Implements | Note | |---------------------|------------|-----------------------------------------------------------------------------------------------------------------------| @@ -28,7 +30,7 @@ go get github.com/ThreeDotsLabs/watermill-nats/v2 | GuaranteedOrder | no | [with the redelivery feature, order can't be guaranteed](https://github.com/nats-io/nats-streaming-server/issues/187) | | Persistent | yes | | -#### Configuration +### Configuration Configuration is done through PublisherConfig and SubscriberConfig types. These share a common JetStreamConfig. To use the experimental nats-core support, set Disabled=true. @@ -42,7 +44,7 @@ Subscriber Config: {{% load-snippet-partial file="src-link/watermill-nats/pkg/nats/subscriber.go" first_line_contains="type SubscriberConfig struct" last_line_contains="type Subscriber struct" %}} -#### Connecting +### Connecting By default NATS client will try to connect to `localhost:4222`. If you are using different hostname or port you should specify using the URL property of `SubscriberConfig` and `PublisherConfig`. @@ -58,15 +60,15 @@ Example: You can also use `NewSubscriberWithNatsConn` and `NewPublisherWithNatsConn` to use a custom `*nats.Conn`. -#### Publishing +### Publishing {{% load-snippet-partial file="src-link/watermill-nats/pkg/nats/publisher.go" first_line_contains="// Publish publishes" last_line_contains="func (p *Publisher) Publish" %}} -#### Subscribing +### Subscribing {{% load-snippet-partial file="src-link/watermill-nats/pkg/nats/subscriber.go" first_line_contains="// Subscribe " last_line_contains="func (s *Subscriber) Subscribe" %}} -#### Marshaler +### Marshaler NATS provides a header passing mechanism that allows conveying the watermill message details as metadata. This is done by default with only the binary payload sent in the message body. The header `_watermill_message_uuid` is reserved. @@ -79,6 +81,6 @@ When you have your own format of the messages, you can implement your own Marsha When needed, you can bypass both [UUID]({{< ref "message#message" >}}) and [Metadata]({{< ref "message#message" >}}) and send just a `message.Payload`, but some standard [middlewares]({{< ref "messages-router#middleware" >}}) may be not working. -### Core-Nats +## Core-Nats This package also includes limited support for connecting to [core-nats](https://docs.nats.io/nats-concepts/core-nats). While core-nats does not support many of the streaming features needed for a perfect fit with watermill and most acks end up implemented as no-ops, in environments with a mix of jetstream and core-nats messaging in play it can be nice to use watermill consistently on the application side. diff --git a/docs/content/pubsubs/redisstream.md b/docs/content/pubsubs/redisstream.md index ce3f50268..d943a27e0 100644 --- a/docs/content/pubsubs/redisstream.md +++ b/docs/content/pubsubs/redisstream.md @@ -8,13 +8,15 @@ weight = 110 Redis is the open source, in-memory data store used by millions of developers. Redis stream is a data structure that acts like an append-only log in Redis. We are providing Pub/Sub implementation based on [redis/go-redis](https://github.com/redis/go-redis). -### Installation +You can find a fully functional example with Redis Stream in the [Watermill examples](https://github.com/ThreeDotsLabs/watermill/tree/master/_examples/pubsubs/redisstream). + +## Installation ```bash go get github.com/ThreeDotsLabs/watermill-redisstream ``` -#### Characteristics +### Characteristics | Feature | Implements | Note | | ------- | ---------- | ---- | @@ -24,38 +26,38 @@ go get github.com/ThreeDotsLabs/watermill-redisstream | Persistent | yes | | | FanOut | yes | use XREAD to fan out messages when there is no consumer group | -#### Configuration +### Configuration {{% load-snippet-partial file="src-link/watermill-redisstream/pkg/redisstream/publisher.go" first_line_contains="type PublisherConfig struct" last_line_contains="// Publish publishes message to redis stream" %}} {{% load-snippet-partial file="src-link/watermill-redisstream/pkg/redisstream/subscriber.go" first_line_contains="type SubscriberConfig struct" last_line_contains="func (s *Subscriber) Subscribe" %}} -##### Passing `redis.UniversalClient` +#### Passing `redis.UniversalClient` You need to configure and pass your own go-redis client via `Client redis.UniversalClient` in `NewSubscriber` and `NewPublisher`. The client can be either `redis.Client` or `redis.ClusterClient`. -##### Publisher +#### Publisher {{% load-snippet-partial file="src-link/watermill-redisstream/pkg/redisstream/publisher.go" first_line_contains="// NewPublisher" last_line_contains="(*Publisher, error)" padding_after="0" %}} Example: {{% load-snippet-partial file="src-link/_examples/pubsubs/redisstream/main.go" first_line_contains="pubClient := redis.NewClient" last_line_contains="panic(err)" padding_after="1" %}} -##### Subscriber +#### Subscriber {{% load-snippet-partial file="src-link/watermill-redisstream/pkg/redisstream/subscriber.go" first_line_contains="// NewSubscriber" last_line_contains="(*Subscriber, error)" padding_after="0" %}} Example: {{% load-snippet-partial file="src-link/_examples/pubsubs/redisstream/main.go" first_line_contains="subClient := redis.NewClient" last_line_contains="panic(err)" padding_after="1" %}} -#### Publishing +### Publishing {{% load-snippet-partial file="src-link/watermill-redisstream/pkg/redisstream/publisher.go" first_line_contains="// Publish" last_line_contains="func (p *Publisher) Publish" %}} -#### Subscribing +### Subscribing {{% load-snippet-partial file="src-link/watermill-redisstream/pkg/redisstream/subscriber.go" first_line_contains="func (s *Subscriber) Subscribe" last_line_contains="func (s *Subscriber) Subscribe" %}} -#### Marshaler +### Marshaler Watermill's messages cannot be directly sent to Redis - they need to be marshaled. You can implement your marshaler or use default implementation. The default implementation uses [MessagePack](https://msgpack.org/index.html) for efficient serialization. diff --git a/docs/content/pubsubs/sql.md b/docs/content/pubsubs/sql.md index f71f6301d..5413c9436 100644 --- a/docs/content/pubsubs/sql.md +++ b/docs/content/pubsubs/sql.md @@ -11,7 +11,7 @@ SQL Pub/Sub executes queries on any SQL database, using it like a messaging syst While the performance of this approach isn't the best, it fits many use cases, where eventual consistency is acceptable. It can also be useful for projects that are not using any specialized message queue at the moment, but have access to a SQL database. -The SQL subscriber runs a `SELECT` query within short periods, remembering the position of the last record. If it finds +The SQL subscriber runs a `SELECT` query within short periods, remembering the position of the last record. If it finds any new records, they are returned. One handy use case is consuming events from a database table, that can be later published on some kind of message queue. @@ -22,13 +22,13 @@ SQL Pub/Sub is also a good choice for implementing Outbox pattern with [Forwarde See also the [SQL example](https://github.com/ThreeDotsLabs/watermill/tree/master/_examples/pubsubs/sql). -### Installation +## Installation ```bash go get github.com/ThreeDotsLabs/watermill-sql/v3 ``` -#### Characteristics +### Characteristics | Feature | Implements | Note | |---------------------|------------|-------------------------------------------| @@ -37,7 +37,7 @@ go get github.com/ThreeDotsLabs/watermill-sql/v3 | GuaranteedOrder | yes | | | Persistent | yes | | -#### Schema +### Schema SQL Pub/Sub uses user-defined schema to handle select and insert queries. You need to implement `SchemaAdapter` and pass it to `SubscriberConfig` or `PublisherConfig`. @@ -47,7 +47,7 @@ it to `SubscriberConfig` or `PublisherConfig`. There is a default schema provided for each supported engine (`DefaultMySQLSchema` and `DefaultPostgreSQLSchema`). It supports the most common use case (storing events in a table). You can base your schema on one of these, extending only chosen methods. -##### Extending schema +#### Extending schema Consider an example project, where you're fine with using the default schema, but would like to use `BINARY(16)` for storing the `uuid` column, instead of `VARCHAR(36)`. In that case, you have to define two methods: @@ -60,13 +60,13 @@ Note that you don't have to use the initialization queries provided by Watermill {{% load-snippet-partial file="src-link/watermill-sql/pkg/sql/schema_adapter_mysql.go" first_line_contains="// DefaultMySQLSchema" last_line_contains="type DefaultMySQLSchema" %}} -#### Configuration +### Configuration {{% load-snippet-partial file="src-link/watermill-sql/pkg/sql/publisher.go" first_line_contains="type PublisherConfig struct" last_line_contains="}" %}} {{% load-snippet-partial file="src-link/watermill-sql/pkg/sql/subscriber.go" first_line_contains="type SubscriberConfig struct" last_line_contains="}" %}} -### Publishing +## Publishing {{% load-snippet-partial file="src-link/watermill-sql/pkg/sql/publisher.go" first_line_contains="func NewPublisher" last_line_contains="func NewPublisher" %}} @@ -75,15 +75,15 @@ Example: {{% load-snippet-partial file="src-link/watermill-sql/pkg/sql/publisher.go" first_line_contains="// Publish " last_line_contains="func (p *Publisher) Publish" %}} -#### Transactions +### Transactions If you need to publish messages within a database transaction, you have to pass a `*sql.Tx` in the `NewPublisher` -constructor. You have to create one publisher for each transaction. +constructor. You have to create one publisher for each transaction. Example: {{% load-snippet-partial file="src-link/_examples/real-world-examples/transactional-events/main.go" first_line_contains="func simulateEvents" last_line_contains="return pub.Publish(" padding_after="3" %}} -#### Subscribing +### Subscribing To create a subscriber, you need to pass not only proper schema adapter, but also an offsets adapter. @@ -97,7 +97,7 @@ Example: {{% load-snippet-partial file="src-link/watermill-sql/pkg/sql/subscriber.go" first_line_contains="func (s *Subscriber) Subscribe" last_line_contains="func (s *Subscriber) Subscribe" %}} -#### Offsets Adapter +### Offsets Adapter The logic for storing offsets of messages is provided by the `OffsetsAdapter`. If your schema uses auto-incremented integer as the row ID, it should work out of the box with default offset adapters. diff --git a/docs/layouts/_default/_markup/render-link.html b/docs/layouts/_default/_markup/render-link.html new file mode 100644 index 000000000..eb8714dbc --- /dev/null +++ b/docs/layouts/_default/_markup/render-link.html @@ -0,0 +1 @@ +{{ .Text | safeHTML }} diff --git a/docs/package.json b/docs/package.json index 678cfae41..8a7c8106a 100644 --- a/docs/package.json +++ b/docs/package.json @@ -8,7 +8,8 @@ "create": "hugo new", "dev": "hugo server --disableFastRender --noHTTPCache", "format": "prettier **/** -w -c", - "build": "hugo --minify --gc", + "build": "hugo --minify --gc -b ${URL}", + "build:branch": "hugo --minify --gc -b ${DEPLOY_URL}", "preview": "vite preview --outDir public" }, "dependencies": { diff --git a/netlify.toml b/netlify.toml index 1584e7b7f..e213b7c54 100644 --- a/netlify.toml +++ b/netlify.toml @@ -8,6 +8,12 @@ NPM_VERSION = "10.2.4" HUGO_VERSION = "0.127.0" +[context.deploy-preview] + command = "./build.sh --copy && npm run build:branch" + +[context.branch-deploy] + command = "./build.sh --copy && npm run build:branch" + [[redirects]] from = "/api/event" to = "https://academy-api.threedots.tech/api/event"