Release v1.0.0 - Initial Commit

.github/actions-rs/grcov.yml

@@ -0,0 +1,2 @@
+output-type: lcov
+output-file: ./lcov.info

.github/workflows/rust.yml

@@ -0,0 +1,55 @@
+name: Rust
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --release --verbose
+
+  test:
+    name: Test with Coverage
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: cachix/install-nix-action@v22
+        with:
+          nix_path: nixpkgs=channel:nixos-20.03
+      - uses: Swatinem/rust-cache@v2
+      - name: Install cargo-llvm-cov
+        uses: taiki-e/install-action@cargo-llvm-cov
+      - uses: actions-rs/cargo@v1
+        with:
+          command: llvm-cov
+          args: --workspace --lcov --output-path lcov.info
+#      - name: Upload coverage to Codecov
+#        uses: codecov/codecov-action@v3
+#        with:
+#          token: ${{ secrets.CODECOV_TOKEN }}
+#          files: lcov.info
+
+  clippy:
+    name: Clippy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: --all

.gitignore

@@ -0,0 +1,4 @@
+/target
+/Cargo.lock
+test-ledger
+.idea

Cargo.toml

@@ -0,0 +1,35 @@
+[package]
+name = "sologger-geyser-plugin"
+version = "1.0.0"
+edition = "2021"
+authors = ["Will Kennedy"]
+description = "Solana Geyser Plugin to parse raw logs emitted from a Solana RPC into structured logs and transport Solana logs to either a LogStash or OpenTelemetry endpoint via TCP"
+repository = "https://github.com/brytelands/sologger-geyser-plugin"
+categories = ["development-tools::debugging"]
+license = "Apache-2.0"
+keywords = ["solana", "geyser", "logging", "sologger", "otel"]
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+solana-geyser-plugin-interface = "=1.17.34"
+solana-logger = "=1.17.34"
+solana-sdk = "=1.17.34"
+solana-transaction-status = "=1.17.34"
+sologger_log_context = "0.1.2"
+sologger_log_transformer = "0.1.2"
+sologger_log_transport = "0.1.2"
+log = "0.4.20"
+bs58 = "0.5.1"
+log4rs = "1.2.0"
+anyhow = "1.0.71"
+serde_json = "1.0.117"
+serde = { version = "1.0.195", features = ["derive"] }
+crossbeam-deque = "0.8.5"
+
+tokio = { version = "1.0", features = ["sync", "rt"], optional = true }
+
+[features]
+enable_otel = ["sologger_log_transport/otel", "tokio"]
+enable_logstash = ["sologger_log_transport/logstash"]

Dockerfile-logstash

@@ -0,0 +1,84 @@
+# syntax=docker/dockerfile:1
+
+# Comments are provided throughout this file to help you get started.
+# If you need more help, visit the Dockerfile reference guide at
+# https://docs.docker.com/engine/reference/builder/
+
+################################################################################
+# Create a stage for building the application.
+
+ARG RUST_VERSION=1.73.0
+ARG APP_NAME=libsologger_geyser_plugin.so
+FROM rust:${RUST_VERSION}-slim-bookworm AS build
+ARG APP_NAME
+WORKDIR /app
+
+RUN apt update && apt install -y pkg-config && apt install -y libssl-dev && apt install -y openssl && apt install -y ca-certificates
+
+# Build the application.
+# Leverage a cache mount to /usr/local/cargo/registry/
+# for downloaded dependencies and a cache mount to /app/target/ for
+# compiled dependencies which will speed up subsequent builds.
+# Leverage a bind mount to the src directory to avoid having to copy the
+# source code into the container. Once built, copy the executable to an
+# output directory before the cache mounted /app/target is unmounted.
+RUN --mount=type=bind,source=src,target=src \
+    --mount=type=bind,source=Cargo.toml,target=Cargo.toml \
+    --mount=type=bind,source=Cargo.lock,target=Cargo.lock \
+    --mount=type=bind,source=dockerfile-config/,target=config/ \
+    --mount=type=cache,target=/app/target/ \
+    --mount=type=cache,target=/usr/local/cargo/registry/ \
+    <<EOF
+set -e
+cargo build --release --features 'enable_logstash'
+cp -r ./config /config
+cp ./target/release/$APP_NAME /bin/plugin.so
+EOF
+
+################################################################################
+# Create a new stage for running the application that contains the minimal
+# runtime dependencies for the application. This often uses a different base
+# image from the build stage where the necessary files are copied from the build
+# stage.
+#
+# The example below uses the debian bullseye image as the foundation for running the app.
+# By specifying the "bullseye-slim" tag, it will also use whatever happens to be the
+# most recent version of that tag when you build your Dockerfile. If
+# reproducability is important, consider using a digest
+# (e.g., debian@sha256:ac707220fbd7b67fc19b112cee8170b41a9e97f703f588b2cdbbcdcecdd8af57).
+FROM debian:stable-slim AS final
+
+RUN apt -y update && apt install -y pkg-config && apt install -y libssl-dev && apt install -y openssl && apt install -y ca-certificates
+
+RUN apt -y install curl bzip2
+RUN sh -c "$(curl -sSfL https://release.solana.com/v1.17.35/install)"
+#ENV PATH="/root/.local/share/solana/install/active_release/bin:${PATH}"
+RUN #solana-keygen new --no-bip39-passphrase
+
+ENV SOLOGGER_APP_CONFIG_LOCATION=/config/local/sologger-config.json
+
+# Create a non-privileged user that the app will run under.
+# See https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#user
+ARG UID=10001
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    --home "/nonexistent" \
+    --shell "/sbin/nologin" \
+    --no-create-home \
+    --uid "${UID}" \
+    appuser
+USER appuser
+
+RUN mkdir /tmp/test-ledger
+
+COPY --from=build /config /config
+
+# Copy the executable from the "build" stage.
+COPY --from=build /bin/plugin.so /usr/lib/plugin.so
+
+# Expose the port that the application listens on.
+EXPOSE 8899
+
+# What the container should run when it is started.
+CMD ["solana-test-validator", "--geyser-plugin-config", "/config/sologger-geyser-plugin-config.json", "-l", "/tmp/test-ledger"]

Dockerfile-otel

@@ -0,0 +1,81 @@
+# syntax=docker/dockerfile:1
+
+# Comments are provided throughout this file to help you get started.
+# If you need more help, visit the Dockerfile reference guide at
+# https://docs.docker.com/engine/reference/builder/
+
+################################################################################
+# Create a stage for building the application.
+
+ARG RUST_VERSION=1.73.0
+ARG APP_NAME=libsologger_geyser_plugin.so
+FROM rust:${RUST_VERSION}-slim-bookworm AS build
+ARG APP_NAME
+WORKDIR /app
+
+# Build the application.
+# Leverage a cache mount to /usr/local/cargo/registry/
+# for downloaded dependencies and a cache mount to /app/target/ for 
+# compiled dependencies which will speed up subsequent builds.
+# Leverage a bind mount to the src directory to avoid having to copy the
+# source code into the container. Once built, copy the executable to an
+# output directory before the cache mounted /app/target is unmounted.
+RUN --mount=type=bind,source=src,target=src \
+    --mount=type=bind,source=Cargo.toml,target=Cargo.toml \
+    --mount=type=bind,source=Cargo.lock,target=Cargo.lock \
+    --mount=type=bind,source=dockerfile-config/,target=config/ \
+    --mount=type=cache,target=/app/target/ \
+    --mount=type=cache,target=/usr/local/cargo/registry/ \
+    <<EOF
+set -e
+cargo build --release --features 'enable_otel'
+cp -r ./config /config
+cp ./target/release/$APP_NAME /bin/plugin.so
+EOF
+
+################################################################################
+# Create a new stage for running the application that contains the minimal
+# runtime dependencies for the application. This often uses a different base
+# image from the build stage where the necessary files are copied from the build
+# stage.
+#
+# The example below uses the debian bullseye image as the foundation for running the app.
+# By specifying the "bullseye-slim" tag, it will also use whatever happens to be the
+# most recent version of that tag when you build your Dockerfile. If
+# reproducability is important, consider using a digest
+# (e.g., debian@sha256:ac707220fbd7b67fc19b112cee8170b41a9e97f703f588b2cdbbcdcecdd8af57).
+FROM debian:stable-slim AS final
+
+FROM debian:stable-slim AS final
+
+RUN apt -y update && apt install -y pkg-config && apt install -y libssl-dev && apt install -y openssl && apt install -y ca-certificates
+
+RUN apt -y install curl bzip2
+
+RUN sh -c "$(curl -sSfL https://release.solana.com/v1.17.35/install)"
+#ENV PATH="/root/.local/share/solana/install/active_release/bin:${PATH}"
+RUN #solana-keygen new --no-bip39-passphrase
+
+ENV SOLOGGER_APP_CONFIG_LOCATION=/config/local/sologger-config.json
+
+# Create a non-privileged user that the app will run under.
+# See https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#user
+ARG UID=10001
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    --home "/nonexistent" \
+    --shell "/sbin/nologin" \
+    --no-create-home \
+    --uid "${UID}" \
+    appuser
+USER appuser
+
+RUN mkdir /tmp/test-ledger
+
+COPY --from=build /config /config
+
+COPY --from=build /bin/plugin.so /usr/lib/plugin.so
+
+# What the container should run when it is started.
+CMD ["solana-test-validator", "--geyser-plugin-config", "/config/sologger-geyser-plugin-config.json", "-l", "/tmp/test-ledger"]

README.md

@@ -0,0 +1,98 @@
+[![Rust](https://github.com/brytelands/sologger-geyser-plugin/actions/workflows/rust.yml/badge.svg)](https://github.com/brytelands/sologger-geyser-plugin/actions/workflows/rust.yml)
+
+# sologger-geyser-plugin
+
+Configurable Solana Geyser plugin that uses [Sologger](https://github.com/brytelands/sologger) to structure Solana validator logs and send them to either Logstash or an OpenTelemetry collector.
+Sologger parses raw logs ingested by the geyser plugin into structured logs and transports Solana logs to either a LogStash or OpenTelemetry endpoint via TCP. This helps improve the observability of your programs running on chain.
+
+### Quick Start
+
+If you just want to run a Solana test validator with the Sologger Geyser plugin, then you can use one of the following docker compose files to get up and running quickly.
+
+- [sologger with Parseable](./docker-examples/docker-parseable/) This is the easiest way to get up and running with Sologger. If you want to monitor specific programs, all you need to do is update the program IDs in the sologger-config.json file.
+- [sologger with Signoz](./docker-examples/docker-signoz/) This is an example using OpenTelemetry. It's a bit more involved and takes a little bit of time to startup Signoz. If you want to monitor specific programs, all you need to do is update the program IDs in the sologger-config.json file.
+
+
+**Building the source**
+There are two main features that can be enabled when building the plugin binaries. The first is the Logstash feature which will enable the Logstash transport. The second is the OpenTelemetry feature which will enable the OpenTelemetry transport. Technically you can build with both features enabled, but this is not recommended. If you need both LogStash and OTel support, the recommended approach is to build two binaries with each feature enabled, and run each separately.
+
+- Logstash: `enable_logstash`
+- OpenTelemetry: `enable_otel`
+
+```shell
+#If you want to build the binaries with Logstash support, then run the following command:
+cargo build --features 'enable_logstash'
+
+#If you want to build the binaries with OpenTelemetry support, then run the following command:
+cargo build --features 'enable_otel'
+```
+
+**Building the docker image**
+
+These images are basically just the Solana CLI that starts a test-validator with the Sologger Geyser Plugin installed.
+
+```shell
+#If you want to build the image with Logstash support, then run the following command:
+docker build -f 'Dockerfile-logstash' --tag sologger-logstash-geyser-plugin .
+
+#If you want to build the image with OpenTelemetry support, then run the following command:
+docker build --file 'Dockerfile-otel' --tag sologger-otel-geyser-plugin .
+```
+
+### Configure
+
+There are two configuration files that you will need to configure to get up and running with Sologger.
+The first is the sologger-config file. This file is used to configure the sologger binary.
+The second is the log4rs-config file. This file is used to configure the log4rs logger OR the opentelemetry-config file. This file is used to configure the logstash binary.
+
+By default, sologger will look for a config file named `sologger-config.json` in ./config/local/ directory. You can override this by setting the `SOLOGGER_APP_CONFIG_LOCATION` environment variable to the path of your config file. For example:
+
+Here is an example sologger-config.json. See [sologger_config.rs](src/sologger_config.rs) for documentation specific to each field.
+```json
+{
+    "log4rsConfigLocation": "../config/local/log4rs-config.yml",
+    "opentelemetryConfigLocation": "../config/local/opentelemetry-config.json",
+    "rpcUrl": "wss://api.devnet.solana.com",
+    "programsSelector" : {
+        "programs" : ["BPFLoaderUpgradeab1e11111111111111111111111", "Ed25519SigVerify111111111111111111111111111", "KeccakSecp256k11111111111111111111111111111"]
+    },
+    "accountDataNotificationsEnabled": false,
+    "transactionNotificationsEnabled": true,
+    "logProcessorWorkerThreadCount": 2
+}
+```
+
+For log4rs configurations please see: [log4rs](https://github.com/estk/log4rs)
+
+
+### Run
+
+You can also take a look at the run scripts in [scripts](./scripts)
+
+```shell
+solana-test-validator --geyser-plugin-config ../config/sologger-geyser-plugin-config.json
+
+#Or if you want to specify a location of the sologger-config.json
+SOLOGGER_APP_CONFIG_LOCATION=./config/sologger-config.json solana-test-validator --geyser-plugin-config ../config/sologger-geyser-plugin-config.json
+```
+
+Or
+
+```shell
+#If you just want to run the docker image
+sudo docker run --name solana-test-validator-sologger-otel -p 1024:1024 -p 9900:9900 -p 8900:8900 -p 8899:8899 sologger-otel-geyser-plugin
+```
+
+Or
+
+Start up the Parseable stack using docker-compose
+
+```shell
+docker-compose -f docker-examples/docker-parseable/docker-compose.yml up
+```
+
+You can deploy your program and see the logs here
+
+- url: http://localhost:8000
+- user: admin
+- password: admin