From 04041baebcd347e96c2d4585a14f58b438e140eb Mon Sep 17 00:00:00 2001 From: Arie Heinrich Date: Sat, 13 Sep 2025 15:31:08 +0200 Subject: [PATCH 1/2] Spelling fixes, Markdown standard --- README.md | 13 +++--- app/vtselect/traces/query/query.go | 2 +- dashboards/README.md | 2 +- deployment/docker/README.md | 35 ++++++++++----- docs/victoriatraces/README.md | 28 ++++++------ docs/victoriatraces/changelog/CHANGELOG.md | 4 +- docs/victoriatraces/cluster.md | 45 ++++++++++--------- .../data-ingestion/opentelemetry.md | 15 ++++--- docs/victoriatraces/keyConcepts.md | 24 +++++----- docs/victoriatraces/querying/README.md | 17 +++++-- docs/victoriatraces/querying/grafana.md | 5 ++- .../querying/jaeger-frontend.md | 9 ++-- docs/victoriatraces/quick-start.md | 23 ++++++++-- docs/victoriatraces/roadmap.md | 6 ++- docs/victoriatraces/troubleshooting.md | 7 ++- docs/victoriatraces/vmalert.md | 16 +++---- 16 files changed, 152 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index eee4a0551..9aeccca1f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ 🚧 This project is currently a work in progress. Therefore, the following issues may exist: + - On-disk data structures and API endpoints may change and may not be backward compatible. - Some links point to incorrect locations. - Some static files are unavailable. @@ -33,12 +34,12 @@ Both the single-node and the cluster versions of VictoriaTraces are open source Feel free asking any questions regarding VictoriaTraces: -* [Slack Inviter](https://slack.victoriametrics.com/) and [Slack channel](https://victoriametrics.slack.com/) -* [X (Twitter)](https://x.com/VictoriaMetrics/) -* [Linkedin](https://www.linkedin.com/company/victoriametrics/) -* [Reddit](https://www.reddit.com/r/VictoriaMetrics/) -* [Telegram](https://t.me/VictoriaTraces) -* [Mastodon](https://mastodon.social/@victoriametrics/) +- [Slack Inviter](https://slack.victoriametrics.com/) and [Slack channel](https://victoriametrics.slack.com/) +- [X (Twitter)](https://x.com/VictoriaMetrics/) +- [Linkedin](https://www.linkedin.com/company/victoriametrics/) +- [Reddit](https://www.reddit.com/r/VictoriaMetrics/) +- [Telegram](https://t.me/VictoriaTraces) +- [Mastodon](https://mastodon.social/@victoriametrics/) If you like VictoriaTraces and want to contribute, then please [read these docs](https://docs.victoriametrics.com/victoriametrics/contributing/). diff --git a/app/vtselect/traces/query/query.go b/app/vtselect/traces/query/query.go index 22796b693..7d8cbdc17 100644 --- a/app/vtselect/traces/query/query.go +++ b/app/vtselect/traces/query/query.go @@ -58,7 +58,7 @@ func (cp *CommonParams) UpdatePerQueryStatsMetrics() { func GetCommonParams(r *http.Request) (*CommonParams, error) { tenantID, err := logstorage.GetTenantIDFromRequest(r) if err != nil { - return nil, fmt.Errorf("cannot obtain tenanID: %w", err) + return nil, fmt.Errorf("cannot obtain tenantID: %w", err) } tenantIDs := []logstorage.TenantID{tenantID} cp := &CommonParams{ diff --git a/dashboards/README.md b/dashboards/README.md index 43934e21b..089008720 100644 --- a/dashboards/README.md +++ b/dashboards/README.md @@ -1,7 +1,7 @@ # VictoriaTraces dashboards The directory contains the official list of Grafana dashboards for VictoriaTraces components. -The `vm` folder contains copies of the listed dashboards but alternated to use +The `vm` folder contains copies of the listed dashboards but alternated to use [VictoriaMetrics datasource](https://github.com/VictoriaMetrics/victoriametrics-datasource). The listed dashboards can be found on [Grafana website](https://grafana.com/orgs/victoriametrics/dashboards). diff --git a/deployment/docker/README.md b/deployment/docker/README.md index a7a65f760..731cf5aa5 100644 --- a/deployment/docker/README.md +++ b/deployment/docker/README.md @@ -3,7 +3,7 @@ Docker compose environment for VictoriaTraces includes VictoriaTraces components and [Grafana](https://grafana.com/). For starting the docker-compose environment ensure that you have docker installed and running, and that you have access -to the Internet. +to the Internet. **All commands should be executed from the root directory of [the VictoriaTraces repo](https://github.com/VictoriaMetrics/VictoriaTraces).** * Traces: @@ -16,6 +16,7 @@ to the Internet. ## VictoriaTraces server To spin-up environment with [VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/), run the following command: + ```sh # clone VictoriaTraces git clone https://github.com/VictoriaMetrics/VictoriaTraces.git @@ -24,11 +25,13 @@ cd VictoriaTraces # start docker compose make docker-vt-single-up ``` + _See [compose-vt-single.yml](https://github.com/VictoriaMetrics/VictoriaTraces/blob/master/deployment/docker/compose-vt-single.yml)_ VictoriaTraces will be accessible on the `--httpListenAddr=:10428` port. In addition to VictoriaTraces server, the docker compose contains the following components: + * [HotROD](https://hub.docker.com/r/jaegertracing/example-hotrod) application to generate trace data. * `VictoriaMetrics single-node` to collect metrics from all the components. * [Grafana](#grafana) is configured with [VictoriaMetrics](https://github.com/VictoriaMetrics/victoriametrics-datasource) and Jaeger datasource pointing to VictoriaTraces server. @@ -45,17 +48,20 @@ To access [VictoriaTraces UI](https://docs.victoriametrics.com/victoriatraces/qu use link [http://localhost:10428/select/vmui](http://localhost:10428/select/vmui). To shut down environment execute the following command: -``` + +```sh make docker-vt-single-down ``` ## VictoriaTraces cluster To spin-up environment with [VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) **cluster**, run the following command: + ```sh # start docker compose make docker-vt-cluster-up ``` + _See [compose-vt-cluster.yml](https://github.com/VictoriaMetrics/VictoriaTraces/blob/master/deployment/docker/compose-vt-cluster.yml)_ VictoriaTraces cluster environment consists of `vtinsert`, `vtstorage` and `vtselect` components. @@ -64,6 +70,7 @@ For example, `HotROD` pushes trace spans via `http://vmauth:8427/insert/opentele and Grafana queries `http://vmauth:8427/select/jaeger/` for datasource queries. In addition to VictoriaTraces cluster, the docker compose contains the following components: + * [HotROD](https://hub.docker.com/r/jaegertracing/example-hotrod) application to generate trace data. * `VictoriaMetrics single-node` to collect metrics from all the components. * [Grafana](#grafana) is configured with [VictoriaMetrics](https://github.com/VictoriaMetrics/victoriametrics-datasource) and Jaeger datasource pointing to VictoriaTraces cluster (vmauth). @@ -79,7 +86,8 @@ To access [VictoriaTraces UI](https://docs.victoriametrics.com/victoriatraces/qu use link [http://localhost:8427/select/vmui](http://localhost:8427/select/vmui). To shut down environment execute the following command: -``` + +```sh make docker-vt-cluster-down ``` @@ -103,8 +111,9 @@ vmalert-traces evaluates [alerting rules](https://github.com/VictoriaMetrics/Vic They are connected with AlertManager for firing alerts. Web interface link: -- vmalert-metrics: [http://localhost:8880/](http://localhost:8880/). -- vmalert-traces: [http://localhost:8881/](http://localhost:8881/). + +* vmalert-metrics: [http://localhost:8880/](http://localhost:8880/). +* vmalert-traces: [http://localhost:8881/](http://localhost:8881/). ## alertmanager @@ -118,6 +127,7 @@ Web interface link [http://localhost:9093/](http://localhost:9093/). Web interface link [http://localhost:3000](http://localhost:3000). Default credentials: + * login: `admin` * password: `admin` @@ -126,30 +136,33 @@ Grafana is provisioned with default dashboards and datasources. ## Troubleshooting This environment has the following requirements: + * installed [docker compose](https://docs.docker.com/compose/); * access to the Internet for downloading docker images; * **All commands should be executed from the root directory of [the VictoriaTraces repo](https://github.com/VictoriaMetrics/VictoriaTraces).** The expected output of running a command like `make docker-vm-single-up` is the following: + ```sh make docker-vm-single-up :( docker compose -f deployment/docker/compose-vm-single.yml up -d [+] Running 9/9 - βœ” Network docker_default Created 0.0s - βœ” Volume "docker_vmagentdata" Created 0.0s - βœ” Container docker-alertmanager-1 Started 0.3s - βœ” Container docker-victoriametrics-1 Started 0.3s -... + βœ” Network docker_default Created 0.0s + βœ” Volume "docker_vmagentdata" Created 0.0s + βœ” Container docker-alertmanager-1 Started 0.3s + βœ” Container docker-victoriametrics-1 Started 0.3s +... ``` Containers are started in [--detach mode](https://docs.docker.com/reference/cli/docker/compose/up/), meaning they run in the background. As a result, you won't see their logs or exit status directly in the terminal. If something isn’t working as expected, try the following troubleshooting steps: + 1. Run from the correct directory. Make sure you're running the command from the root of the [VictoriaTraces repository](https://github.com/VictoriaMetrics/VictoriaTraces). 2. Check container status. Run `docker ps -a` to list all containers and their status. Healthy and running containers should have `STATUS` set to `Up`. 3. View container logs. To inspect logs for a specific container, get its container ID from step p2 and run: `docker logs -f `. 4. Read the logs carefully and follow any suggested actions. 5. Check for port conflicts. Some containers (e.g., Grafana) expose HTTP ports. If a port (like `:3000`) is already in use, the container may fail to start. Stop the conflicting process or change the exposed port in the Docker Compose file. 6. Shut down the deployment. To tear down the environment, run: `make -down` (i.e. `make docker-vm-single-down`). - Note, this command also removes all attached volumes, so all the temporary created data will be removed too (i.e. Grafana dashboards or collected metrics). \ No newline at end of file + Note, this command also removes all attached volumes, so all the temporary created data will be removed too (i.e. Grafana dashboards or collected metrics). diff --git a/docs/victoriatraces/README.md b/docs/victoriatraces/README.md index b7e3473b1..ac3da726f 100644 --- a/docs/victoriatraces/README.md +++ b/docs/victoriatraces/README.md @@ -7,17 +7,18 @@ sitemap: disable: true --- -VictoriaTraces is an open-source, user-friendly database designed for storing and querying distributed [tracing data](https://en.wikipedia.org/wiki/Tracing_(software)), +VictoriaTraces is an open-source, user-friendly database designed for storing and querying distributed [tracing data](https://en.wikipedia.org/wiki/Tracing_(software)), built by the [VictoriaMetrics](https://github.com/VictoriaMetrics) team. ## Prominent features VictoriaTraces provides the following prominent features: + - It is resource-efficient and fast. It uses up to [**3.7x less RAM and up to 2.6x less CPU**](https://victoriametrics.com/blog/dev-note-distributed-tracing-with-victorialogs/) than other solutions such as Grafana Tempo. - VictoriaTraces' capacity and performance scales linearly with the available resources (CPU, RAM, disk IO, disk space). Also, it can scale horizontally to many nodes in [cluster mode](https://docs.victoriametrics.com/victoriatraces/cluster/). - It has no additional storage dependencies (such as object storage or external databases like ClickHouse and Elasticsearch) for production readiness. - It accepts trace spans in the popular [OpenTelemetry protocol](https://opentelemetry.io/docs/specs/otel/protocol/)(OTLP). -- It provides [Jaeger Query Service JSON APIs](https://www.jaegertracing.io/docs/2.6/apis/#internal-http-json) +- It provides [Jaeger Query Service JSON APIs](https://www.jaegertracing.io/docs/2.6/apis/#internal-http-json) to integrate with [Grafana](https://grafana.com/docs/grafana/latest/datasources/jaeger/) or [Jaeger Frontend](https://www.jaegertracing.io/docs/2.6/frontend-ui/). - It supports alerting - see [these docs](https://docs.victoriametrics.com/victoriatraces/vmalert/). @@ -33,8 +34,9 @@ Building from sources is reasonable when developing additional features specific {{% collapse name="How to build from sources" %}} -Clone VictoriaTraces repository: -```bash +Clone VictoriaTraces repository: + +```bash git clone https://github.com/VictoriaMetrics/VictoriaTraces.git; cd VictoriaTraces; ``` @@ -42,13 +44,13 @@ cd VictoriaTraces; #### Build binary with go build 1. [Install Go](https://golang.org/doc/install). -2. Run `make victoria-traces` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaTraces). +2. Run `make victoria-traces` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaTraces). It builds `victoria-traces` binary and puts it into the `bin` folder. #### Build binary with Docker 1. [Install docker](https://docs.docker.com/install/). -2. Run `make victoria-traces-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaTraces). +2. Run `make victoria-traces-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaTraces). It builds `victoria-traces-prod` binary and puts it into the `bin` folder. #### Building docker images @@ -69,8 +71,8 @@ ROOT_IMAGE=scratch make package-victoria-traces ### Configure VictoriaTraces -VictoriaTraces is configured via command-line flags. -All the command-line flags have sane defaults, so there is no need in tuning them in general case. +VictoriaTraces is configured via command-line flags. +All the command-line flags have sane defaults, so there is no need in tuning them in general case. VictoriaTraces runs smoothly in most environments without additional configuration. Pass `-help` to VictoriaTraces in order to see the list of supported command-line flags with their description and default values: @@ -81,10 +83,10 @@ Pass `-help` to VictoriaTraces in order to see the list of supported command-lin The following command-line flags are used the most: -* `-storageDataPath` - VictoriaTraces stores all the data in this directory. The default path is `victoria-traces-data` in the current working directory. -* `-retentionPeriod` - retention for stored data. Older data is automatically deleted. Default retention is 7 days. +- `-storageDataPath` - VictoriaTraces stores all the data in this directory. The default path is `victoria-traces-data` in the current working directory. +- `-retentionPeriod` - retention for stored data. Older data is automatically deleted. Default retention is 7 days. -You can find the list of the command-line flags can be found [here](#list-of-command-line-flags). +You can find the list of the [command-line flags](#list-of-command-line-flags). ## High Availability @@ -255,7 +257,7 @@ For example, the following command starts VictoriaTraces, which stores the data ``` VictoriaTraces automatically creates the `-storageDataPath` directory on the first run if it is missing. VictoriaTraces stores trace spans -per every day into a seperated subdirectory (aka per-day partition). See [partitions lifecycle](#partitions-lifecycle) for details. +per every day into a separated subdirectory (aka per-day partition). See [partitions lifecycle](#partitions-lifecycle) for details. VictoriaTraces switches to cluster mode if `-storageNode` command-line flag is specified: @@ -287,7 +289,7 @@ These endpoints can be protected from unauthorized access via `-partitionManageA These endpoints can be used for building a flexible per-partition backup / restore schemes as described [in these docs](#backup-and-restore). These endpoints can be used also for setting up automated multi-tier storage schemes where recently ingested data is stored to VictoriaTraces instances -with fast NVMe (SSD) disks, while historical data is gradully migrated to VictoriaTraces instances with slower, but bigger and less expensive HDD disks. +with fast NVMe (SSD) disks, while historical data is gradually migrated to VictoriaTraces instances with slower, but bigger and less expensive HDD disks. This scheme can be implemented with the following simple cron job, which must run once per day: 1. To copy per-day partition for the older day stored at NVMe from NVMe to HDD, with the help of [`rsync`](https://en.wikipedia.org/wiki/Rsync). diff --git a/docs/victoriatraces/changelog/CHANGELOG.md b/docs/victoriatraces/changelog/CHANGELOG.md index e58aedddb..9073a5d80 100644 --- a/docs/victoriatraces/changelog/CHANGELOG.md +++ b/docs/victoriatraces/changelog/CHANGELOG.md @@ -18,7 +18,7 @@ The following `tip` changes can be tested by building VictoriaTraces components * FEATURE: [docker compose](https://github.com/VictoriaMetrics/VictoriaTraces/tree/master/deployment/docker): add cluster docker compose environment. * FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaTraces/blob/master/dashboards): update dashboard for VictoriaTraces single-node and cluster to provide more charts. -* BUGFIX: [Single-node VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) and vtinsert in [VictoriaTraces cluster](https://docs.victoriametrics.com/victoriatraces/cluster/): Rename various [HTTP headers](https://docs.victoriametrics.com/victoriatraces/data-ingestion/#http-headers) prefix from `VL-` to `VT-`. These headers help with debugging and customizing stream fields. Thank @JayiceZ for [the pull request](https://github.com/VictoriaMetrics/VictoriaTraces/pull/56). +* BUGFIX: [Single-node VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) and vtinsert in [VictoriaTraces cluster](https://docs.victoriametrics.com/victoriatraces/cluster/): Rename various [HTTP headers](https://docs.victoriametrics.com/victoriatraces/data-ingestion/#http-headers) prefix from `VL-` to `VT-`. These headers help with debugging and customizing stream fields. Thank @JayiceZ for [the pull request](https://github.com/VictoriaMetrics/VictoriaTraces/pull/56). * BUGFIX: all components: properly expose metadata for summaries and histograms in VictoriaMetrics components with enabled `-metrics.exposeMetadata` cmd-line flag. See [metrics#98](https://github.com/VictoriaMetrics/metrics/issues/98) for details. ## [v0.2.0](https://github.com/VictoriaMetrics/VictoriaTraces/releases/tag/v0.2.0) @@ -29,7 +29,7 @@ Released at 2025-09-01 * SECURITY: upgrade base docker image (Alpine) from 3.22.0 to 3.22.1. See [Alpine 3.22.1 release notes](https://www.alpinelinux.org/posts/Alpine-3.19.8-3.20.7-3.21.4-3.22.1-released.html). * FEATURE: [logstorage](https://docs.victoriametrics.com/victorialogs/): Upgrade VictoriaLogs dependency from [v1.25.1 to v1.27.0](https://github.com/VictoriaMetrics/VictoriaLogs/compare/v1.25.1...v1.27.0). -* FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaTraces/blob/master/dashboards): add dashboard for VictoriaTraces single-node and cluster. +* FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaTraces/blob/master/dashboards): add dashboard for VictoriaTraces single-node and cluster. ## [v0.1.0](https://github.com/VictoriaMetrics/VictoriaTraces/releases/tag/v0.1.0) diff --git a/docs/victoriatraces/cluster.md b/docs/victoriatraces/cluster.md index 3fe813694..2514865c3 100644 --- a/docs/victoriatraces/cluster.md +++ b/docs/victoriatraces/cluster.md @@ -40,13 +40,13 @@ sequenceDiagram participant VS2 as vtstorage-2 participant VT as vtselect participant QC as Query Client - + Note over TS,VS2: Trace Ingestion Flow TS->>VI: Send spans via OTLP VI->>VS1: POST /internal/insert (HTTP) VI->>VS2: POST /internal/insert (HTTP) Note right of VI: Distributes spans evenly
across vtstorage nodes - + Note over VS1,QC: Query Flow QC->>VT: Query via HTTP endpoints VT->>VS1: GET /internal/select/* (HTTP) @@ -56,23 +56,23 @@ sequenceDiagram VT->>QC: Processed & aggregated results ``` -- `vtinsert` handles trace spans ingestion via [the OpenTelemetry protocol (OTLP)](https://opentelemetry.io/docs/specs/otlp/). +- `vtinsert` handles trace spans ingestion via [the OpenTelemetry protocol (OTLP)](https://opentelemetry.io/docs/specs/otlp/). It distributes incoming trace spans evenly across `vtstorage` nodes, as specified by the `-storageNode` command-line flag. -- `vtselect` receives queries through [all supported HTTP query endpoints](https://docs.victoriametrics.com/victoriatraces/querying/). +- `vtselect` receives queries through [all supported HTTP query endpoints](https://docs.victoriametrics.com/victoriatraces/querying/). It fetches the required data from the configured `vtstorage` nodes, processes the queries, and returns the results. - `vtstorage` performs two key roles: - - It stores trace spans received from `vtinsert` at the directory defined by the `-storageDataPath` flag. + - It stores trace spans received from `vtinsert` at the directory defined by the `-storageDataPath` flag. See [storage configuration docs](https://docs.victoriametrics.com/victoriatraces/#storage) for details. - - It handles queries from `vtselect` by retrieving and transforming the requested data locally before returning results. + - It handles queries from `vtselect` by retrieving and transforming the requested data locally before returning results. -Each `vtstorage` node operates as a self-contained VictoriaTraces instance. -Refer to the [single-node and cluster mode duality](#single-node-and-cluster-mode-duality) documentation for more information. +Each `vtstorage` node operates as a self-contained VictoriaTraces instance. +Refer to the [single-node and cluster mode duality](#single-node-and-cluster-mode-duality) documentation for more information. This design allows you to reuse existing single-node VictoriaTraces instances by listing them in the `-storageNode` flag for `vtselect`, enabling unified querying across all nodes. -All VictoriaTraces components are horizontally scalable and can be deployed on hardware best suited to their respective workloads. -`vtinsert` and `vtselect` can be run on the same node, which allows the minimal cluster to consist of just one `vtstorage` node and one node acting as both `vtinsert` and `vtselect`. +All VictoriaTraces components are horizontally scalable and can be deployed on hardware best suited to their respective workloads. +`vtinsert` and `vtselect` can be run on the same node, which allows the minimal cluster to consist of just one `vtstorage` node and one node acting as both `vtinsert` and `vtselect`. However, for production environments, it is recommended to separate `vtinsert` and `vtselect` roles to avoid resource contention β€” for example, to prevent heavy queries from interfering with trace ingestion. Communication between `vtinsert` / `vtselect` and `vtstorage` is done via HTTP over the port specified by the `-httpListenAddr` flag: @@ -80,13 +80,12 @@ Communication between `vtinsert` / `vtselect` and `vtstorage` is done via HTTP o - `vtinsert` sends data to the `/internal/insert` endpoint on `vtstorage`. - `vtselect` sends queries to endpoints under `/internal/select/` on `vtstorage`. -This HTTP-based communication model allows you to use reverse proxies for authorization, routing, and encryption between components. +This HTTP-based communication model allows you to use reverse proxies for authorization, routing, and encryption between components. Use of [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) is recommended for managing access control. See [Security docs](https://docs.victoriametrics.com/victoriatraces/#security) for details. For advanced setups, refer to the [multi-level cluster setup](#multi-level-cluster-setup) documentation. - ## High availability In the cluster setup, the following rules apply: @@ -97,8 +96,8 @@ In the cluster setup, the following rules apply: - The `vtinsert` component continues to function normally when some vtstorage nodes are unavailable. It automatically routes new trace spans to the remaining available nodes to ensure that data ingestion remains uninterrupted and newly received spans are not lost. -> [!NOTE] Insight -> In most real-world cases, `vtstorage` nodes become unavailable during planned maintenance such as upgrades, config changes, or rolling restarts. These are typically infrequent (weekly or monthly) and brief (a few minutes). +> [!NOTE] Insight +> In most real-world cases, `vtstorage` nodes become unavailable during planned maintenance such as upgrades, config changes, or rolling restarts. These are typically infrequent (weekly or monthly) and brief (a few minutes). > A short period of query downtime during such events is acceptable and fits well within most SLAs. For example, 60 minutes of downtime per month still provides around 99.86% availability, which often outperforms complex HA setups that rely on opaque auto-recovery and may fail unpredictably. VictoriaTraces itself does not handle replication at the storage level. Instead, it relies on an external trace shipper, such as [the OpenTelemetry collector](https://opentelemetry.io/docs/collector/), to send the same trace spans to multiple independent VictoriaTraces instances: @@ -110,30 +109,30 @@ graph TD LS["Trace Sources
(Applications)"] VECTOR["Trace Collector
β€’ Buffering
β€’ Replication
β€’ Delivery Guarantees"] end - + subgraph "Storage Layer" subgraph "Zone A" VLA["VictoriaTraces Cluster A"] end - + subgraph "Zone B" VLB["VictoriaTraces Cluster B"] end end - + subgraph "Query Layer" LB["Load Balancer
(vmauth)
β€’ Health Checks
β€’ Failover
β€’ Query Distribution"] QC["Query Clients
(Grafana, API)"] end - + LS --> VECTOR VECTOR -->|"Replicate spans to
Zone A cluster"| VLA VECTOR -->|"Replicate spans to
Zone B cluster"| VLB - + VLA -->|"Serve queries from
Zone A cluster"| LB VLB -->|"Serve queries from
Zone B cluster"| LB LB --> QC - + style VECTOR fill:#e8f5e8 style VLA fill:#d5e8d4 style VLB fill:#d5e8d4 @@ -302,10 +301,11 @@ Their roles depend on whether the `-storageNode` command-line flag is set - if t Otherwise, it runs in `vtstorage` mode, which is identical to a [single-node VictoriaTraces mode](https://docs.victoriametrics.com/victoriatraces/). Let's **ingest some trace spans** (aka [wide events](https://jeremymorrell.dev/blog/a-practitioners-guide-to-wide-events/)) -from [GitHub archive](https://www.gharchive.org/) into the VictoriaTraces cluster. +from [GitHub archive](https://www.gharchive.org/) into the VictoriaTraces cluster. -If you don't have instrumented application, one simple way is to use [vtgen](https://github.com/VictoriaMetrics/VictoriaTraces/tree/master/app/vtgen), a trace data generator. +If you don't have instrumented application, one simple way is to use [vtgen](https://github.com/VictoriaMetrics/VictoriaTraces/tree/master/app/vtgen), a trace data generator. It needs to be built from source with Go or docker: + ```shell # clone the source code git clone https://github.com/VictoriaMetrics/VictoriaTraces.git @@ -318,6 +318,7 @@ make vtgen-prod ``` To generate and send data to VictoriaTraces cluster(`vtinsert`), run: + ```shell ./bin/vtgen -addrs=http://localhost:10481/insert/opentelemetry/v1/traces ``` diff --git a/docs/victoriatraces/data-ingestion/opentelemetry.md b/docs/victoriatraces/data-ingestion/opentelemetry.md index 81b3c915b..10e7c03e8 100644 --- a/docs/victoriatraces/data-ingestion/opentelemetry.md +++ b/docs/victoriatraces/data-ingestion/opentelemetry.md @@ -18,6 +18,7 @@ VictoriaTraces supports both client open-telemetry [SDK](https://opentelemetry.i ## Client SDK The OpenTelemetry provides detailed document and examples for various programming languages: + - [C++](https://opentelemetry.io/docs/languages/cpp/) - [C#/.NET](https://opentelemetry.io/docs/languages/dotnet/) - [Erlang/Elixir](https://opentelemetry.io/docs/languages/erlang/) @@ -36,14 +37,14 @@ Consider the following example for Go SDK: ```go traceExporter, err := otlptracehttp.New(ctx, - otlptracehttp.WithEndpointURL("http://:10428/insert/opentelemetry/v1/traces"), + otlptracehttp.WithEndpointURL("http://:10428/insert/opentelemetry/v1/traces"), ) ``` VictoriaTraces automatically use `service.name` in **resource attributes** and `name` in **span** as [stream fields](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#stream-fields). While the remaining data (including [resource](https://opentelemetry.io/docs/specs/otel/overview/#resources), [instrumentation scope](https://opentelemetry.io/docs/specs/otel/common/instrumentation-scope/), and fields in [span](https://opentelemetry.io/docs/specs/otel/trace/api/#span), like `trace_id`, `span_id`, span `attributes` and more) are stored as [regular fields](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#data-model): -VictoriaTraces supports other HTTP headers - see the list [here](https://docs.victoriametrics.com/victoriatraces/data-ingestion/#http-headers). +VictoriaTraces supports other [HTTP headers](https://docs.victoriametrics.com/victoriatraces/data-ingestion/#http-headers). The ingested trace spans can be queried according to [these docs](https://docs.victoriametrics.com/victoriatraces/querying/). @@ -51,7 +52,7 @@ The ingested trace spans can be queried according to [these docs](https://docs.v VictoriaTraces supports receiving traces from the following OpenTelemetry collector: -* [OpenTelemetry](#opentelemetry) +- [OpenTelemetry](#opentelemetry) ### OpenTelemetry @@ -63,7 +64,7 @@ exporters: traces_endpoint: http://:10428/insert/opentelemetry/v1/traces ``` -VictoriaTraces supports various HTTP headers, which can be used during data ingestion - see the list [here](https://docs.victoriametrics.com/victoriatraces/data-ingestion/#http-headers). +VictoriaTraces supports various HTTP headers, which can be used during data ingestion - see the list of [HTTP headers](https://docs.victoriametrics.com/victoriatraces/data-ingestion/#http-headers). These headers can be passed to OpenTelemetry exporter config via `headers` options. For example, the following configs add (or overwrites) `foo: bar` field to each trace span during data ingestion: ```yaml @@ -76,6 +77,6 @@ exporters: See also: -* [Data ingestion troubleshooting](https://docs.victoriametrics.com/victoriatraces/data-ingestion/#troubleshooting). -* [How to query VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/querying/). -* [Docker-compose demo for HotROD application integration with VictoriaTraces](https://github.com/VictoriaMetrics/VictoriaTraces/blob/master/deployment/docker/compose-vt-single.yml). +- [Data ingestion troubleshooting](https://docs.victoriametrics.com/victoriatraces/data-ingestion/#troubleshooting). +- [How to query VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/querying/). +- [Docker-compose demo for HotROD application integration with VictoriaTraces](https://github.com/VictoriaMetrics/VictoriaTraces/blob/master/deployment/docker/compose-vt-single.yml). diff --git a/docs/victoriatraces/keyConcepts.md b/docs/victoriatraces/keyConcepts.md index ea4fd701c..686c3c5fd 100644 --- a/docs/victoriatraces/keyConcepts.md +++ b/docs/victoriatraces/keyConcepts.md @@ -158,8 +158,9 @@ And here's how this trace span looks like in VictoriaTraces: ### Special mappings There are some special mappings when transforming a trace span into VictoriaTraces data model: + 1. Empty attribute values in trace spans are replaced with `-`. -2. Resource, scope and span attributes are stored with corresponding prefixes `resource_attr`, `scope_attr` and `span_attr:` accordingly. +2. Resource, scope and span attributes are stored with corresponding prefixes `resource_attr`, `scope_attr` and `span_attr:` accordingly. 3. For some attributes within a list (event list, link list in span), a corresponding prefix and index (such as `event:0:` and `event:0:event_attr:`) is added. 4. The `duration` field does not exist in the OTLP request, but for query efficiency, it's calculated during ingestion and stored as a separated field. @@ -186,21 +187,24 @@ as the stream fields. VictoriaTraces optimizes storing and [querying](https://docs.victoriametrics.com/victorialogs/logsql/#stream-filter) of individual trace span streams. This provides the following benefits: -- Reduced disk space usage, since a trace span stream from a single application instance is usually compressed better + +* Reduced disk space usage, since a trace span stream from a single application instance is usually compressed better than a mixed trace span stream from multiple distinct applications. -- Increased query performance, since VictoriaTraces needs to scan lower amounts of data +* Increased query performance, since VictoriaTraces needs to scan lower amounts of data when [searching by stream fields](https://docs.victoriametrics.com/victorialogs/logsql/#stream-filter). Every ingested trace span is associated with a trace span stream. Every trace span stream consists of the following special fields: -- `_stream_id` - this is an unique identifier for the trace span stream. All the trace spans for the particular stream can be selected +* `_stream_id` - this is an unique identifier for the trace span stream. All the trace spans for the particular stream can be selected via [`_stream_id:...` filter](https://docs.victoriametrics.com/victorialogs/logsql/#_stream_id-filter). -- `_stream` - this field contains stream labels in the format similar to [labels in Prometheus metrics](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#labels): +* `_stream` - this field contains stream labels in the format similar to [labels in Prometheus metrics](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#labels): + ``` {resource_attr:service_name="svc name", name="span name"} ``` + The `_stream` field can be searched with [stream filters](https://docs.victoriametrics.com/victorialogs/logsql/#stream-filter). #### High cardinality @@ -211,12 +215,12 @@ VictoriaTraces works perfectly with such fields unless they are associated with **Never** associate high-cardinality fields with [trace span streams](#stream-fields), since this may lead to the following issues: -- Performance degradation during [data ingestion](https://docs.victoriametrics.com/victoriatraces/data-ingestion/) +* Performance degradation during [data ingestion](https://docs.victoriametrics.com/victoriatraces/data-ingestion/) and [querying](https://docs.victoriametrics.com/victoriatraces/querying/) -- Increased memory usage -- Increased CPU usage -- Increased disk space usage -- Increased disk read / write IO +* Increased memory usage +* Increased CPU usage +* Increased disk space usage +* Increased disk read / write IO VictoriaTraces exposes `vt_streams_created_total` [metric](https://docs.victoriametrics.com/victorialogs/#monitoring), which shows the number of created streams since the last VictoriaTraces restart. If this metric grows at a rapid rate diff --git a/docs/victoriatraces/querying/README.md b/docs/victoriatraces/querying/README.md index 8f0fd3e9a..069ed8e7c 100644 --- a/docs/victoriatraces/querying/README.md +++ b/docs/victoriatraces/querying/README.md @@ -8,20 +8,22 @@ sitemap: --- -[VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) can be queried with [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/) +[VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) can be queried with [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/) via the following ways: + - [Web UI](#web-ui) - a web-based UI for querying trace spans - [HTTP API](#http-api) And on top of the LogsQL, VictoriaTraces also implemented [Jaeger Query Service JSON APIs](https://www.jaegertracing.io/docs/2.6/apis/#internal-http-json). These APIs can be used with: + - [Grafana Jaeger datasource](https://docs.victoriametrics.com/victoriatraces/querying/grafana/) - [Jaeger UI](https://github.com/jaegertracing/jaeger-ui) ## Web UI VictoriaTraces provides Web UI for trace spans [querying](https://docs.victoriametrics.com/victorialogs/logsql/) and exploration -at `http://:10428/select/vmui`. +at `http://:10428/select/vmui`. There are three modes of displaying query results: @@ -31,7 +33,7 @@ There are three modes of displaying query results: ## HTTP API -VictoriaTraces provides the same [HTTP endpoints](https://docs.victoriametrics.com/victorialogs/querying/#http-api) that VictoriaLogs provides. +VictoriaTraces provides the same [HTTP endpoints](https://docs.victoriametrics.com/victorialogs/querying/#http-api) that VictoriaLogs provides. Additionally, the following Jaeger HTTP endpoints are available: - `/select/jaeger/api/services` for querying all the services @@ -43,6 +45,7 @@ Additionally, the following Jaeger HTTP endpoints are available: Trace spans in VictoriaTraces can be queried at the The `/select/jaeger/api/traces` HTTP endpoint. This endpoint provides the following params: + - `service`: the service name. - `operation`: the span name (also known as the operation name in Jaeger). - `tags`: the attributes (also known as tags) filter, example: `{"key":"value"}` @@ -55,11 +58,13 @@ This endpoint provides the following params: For example, the following queries are typically how users try to find a specific trace: 1. List all the services: + ```sh curl http://:10428/select/jaeger/api/services ``` Here's a response example: + ```json { "data": [ @@ -77,11 +82,13 @@ Here's a response example: ``` 2. List all the span names of a service: + ```sh curl http://:10428/select/jaeger/api/services/checkout/operations ``` Here's a response example: + ```json { "data": [ @@ -99,21 +106,25 @@ Here's a response example: ``` 3. Filter traces by conditions: + ```sh curl http://:10428/select/jaeger/api/traces?service=checkout&operation=oteldemo&tags=%7B%22rpc.method%22%3A%22Convert%22%7D&minDuration=1ms&maxDuration=10ms&limit=5&start=1749969952453000&end=1750056352453000 ``` Here's a response example: + ```json {"data":[{"processes":{"p1":{"serviceName":"email","tags":[{"key":"process.command","type":"string","value":"email_server.rb"},{"key":"process.pid","type":"string","value":"1"},{"key":"process.runtime.description","type":"string","value":"ruby 3.4.4 (2025-05-14 revision a38531fd3f) +PRISM [aarch64-linux-musl]"},{"key":"process.runtime.name","type":"string","value":"ruby"},{"key":"process.runtime.version","type":"string","value":"3.4.4"},{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"},{"key":"telemetry.sdk.language","type":"string","value":"ruby"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"1.8.0"}]},"p10":{"serviceName":"cart","tags":[{"key":"container.id","type":"string","value":"5603ff989877ecf311403b6ea81fda10734846a0cbdad3a09c39fb068e4a07fc"},{"key":"host.name","type":"string","value":"5603ff989877"},{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"},{"key":"telemetry.sdk.language","type":"string","value":"dotnet"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"1.11.2"}]},"p11":{"serviceName":"product-catalog","tags":[{"key":"host.name","type":"string","value":"3dabfcfe8381"},{"key":"os.description","type":"string","value":"Debian GNU/Linux Debian GNU/Linux 12 (bookworm) (Linux 3dabfcfe8381 6.10.14-linuxkit #1 SMP Tue Apr 15 16:00:54 UTC 2025 aarch64)"},{"key":"os.type","type":"string","value":"linux"},{"key":"process.command_args","type":"string","value":"[\"./product-catalog\"]"},{"key":"process.executable.name","type":"string","value":"product-catalog"},{"key":"process.executable.path","type":"string","value":"/usr/src/app/product-catalog"},{"key":"process.owner","type":"string","value":"nonroot"},{"key":"process.pid","type":"string","value":"1"},{"key":"process.runtime.description","type":"string","value":"go version go1.24.4 linux/arm64"},{"key":"process.runtime.name","type":"string","value":"go"},{"key":"process.runtime.version","type":"string","value":"go1.24.4"},{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"},{"key":"telemetry.sdk.language","type":"string","value":"go"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"1.36.0"}]},"p12":{"serviceName":"currency","tags":[{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"},{"key":"telemetry.sdk.language","type":"string","value":"cpp"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"1.20.0"}]},"p2":{"serviceName":"quote","tags":[{"key":"container.id","type":"string","value":"759183873eeb1328f16df8ea5b5a10932506af136a6537c6a365131c04f1645c"},{"key":"host.arch","type":"string","value":"aarch64"},{"key":"host.name","type":"string","value":"759183873eeb"},{"key":"os.description","type":"string","value":"6.10.14-linuxkit"},{"key":"os.name","type":"string","value":"Linux"},{"key":"os.type","type":"string","value":"linux"},{"key":"os.version","type":"string","value":"#1 SMP Tue Apr 15 16:00:54 UTC 2025"},{"key":"process.command","type":"string","value":"public/index.php"},{"key":"process.command_args","type":"string","value":"[\"public/index.php\"]"},{"key":"process.executable.path","type":"string","value":"/usr/local/bin/php"},{"key":"process.owner","type":"string","value":"www-data"},{"key":"process.pid","type":"string","value":"1"},{"key":"process.runtime.name","type":"string","value":"cli"},{"key":"process.runtime.version","type":"string","value":"8.3.22"},{"key":"service.instance.id","type":"string","value":"9dc0abaa-c408-483e-9fed-8375a73efb91"},{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"},{"key":"telemetry.distro.name","type":"string","value":"opentelemetry-php-instrumentation"},{"key":"telemetry.distro.version","type":"string","value":"1.1.3"},{"key":"telemetry.sdk.language","type":"string","value":"php"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"1.5.0"}]},"p3":{"serviceName":"checkout","tags":[{"key":"host.name","type":"string","value":"cbdb5e0808c2"},{"key":"os.description","type":"string","value":"Debian GNU/Linux Debian GNU/Linux 12 (bookworm) (Linux cbdb5e0808c2 6.10.14-linuxkit #1 SMP Tue Apr 15 16:00:54 UTC 2025 aarch64)"},{"key":"os.type","type":"string","value":"linux"},{"key":"process.command_args","type":"string","value":"[\"./checkout\"]"},{"key":"process.executable.name","type":"string","value":"checkout"},{"key":"process.executable.path","type":"string","value":"/usr/src/app/checkout"},{"key":"process.owner","type":"string","value":"nonroot"},{"key":"process.pid","type":"string","value":"1"},{"key":"process.runtime.description","type":"string","value":"go version go1.24.4 linux/arm64"},{"key":"process.runtime.name","type":"string","value":"go"},{"key":"process.runtime.version","type":"string","value":"go1.24.4"},{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"},{"key":"telemetry.sdk.language","type":"string","value":"go"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"1.36.0"}]},"p4":{"serviceName":"frontend","tags":[{"key":"container.id","type":"string","value":"2d395f01353040612a00252cf6e8c32f00ab94ae06f82f143a3ea9c742072674"},{"key":"host.arch","type":"string","value":"arm64"},{"key":"host.name","type":"string","value":"2d395f013530"},{"key":"os.type","type":"string","value":"linux"},{"key":"os.version","type":"string","value":"6.10.14-linuxkit"},{"key":"process.command","type":"string","value":"/app/server.js"},{"key":"process.command_args","type":"string","value":"[\"/usr/local/bin/node\",\"--require\",\"./Instrumentation.js\",\"/app/server.js\"]"},{"key":"process.executable.name","type":"string","value":"node"},{"key":"process.executable.path","type":"string","value":"/usr/local/bin/node"},{"key":"process.owner","type":"string","value":"nextjs"},{"key":"process.pid","type":"string","value":"17"},{"key":"process.runtime.description","type":"string","value":"Node.js"},{"key":"process.runtime.name","type":"string","value":"nodejs"},{"key":"process.runtime.version","type":"string","value":"22.16.0"},{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"},{"key":"telemetry.sdk.language","type":"string","value":"nodejs"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"1.30.1"}]},"p5":{"serviceName":"shipping","tags":[{"key":"os.type","type":"string","value":"linux"},{"key":"process.command_args","type":"string","value":"[\"/app/shipping\"]"},{"key":"process.pid","type":"string","value":"1"},{"key":"process.runtime.description","type":"string","value":"rustc 1.82.0 (f6e511eec 2024-10-15)"},{"key":"process.runtime.name","type":"string","value":"rustc"},{"key":"process.runtime.version","type":"string","value":"1.82.0"},{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"},{"key":"telemetry.sdk.language","type":"string","value":"rust"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"0.30.0"}]},"p6":{"serviceName":"payment","tags":[{"key":"container.id","type":"string","value":"18ee03279d38ed0e0eedad037c260df78dfc3323aa662ca14a2d38fcc8bf3762"},{"key":"host.arch","type":"string","value":"arm64"},{"key":"host.name","type":"string","value":"18ee03279d38"},{"key":"os.type","type":"string","value":"linux"},{"key":"os.version","type":"string","value":"6.10.14-linuxkit"},{"key":"process.command","type":"string","value":"/usr/src/app/index.js"},{"key":"process.command_args","type":"string","value":"[\"/usr/local/bin/node\",\"--require\",\"./opentelemetry.js\",\"/usr/src/app/index.js\"]"},{"key":"process.executable.name","type":"string","value":"node"},{"key":"process.executable.path","type":"string","value":"/usr/local/bin/node"},{"key":"process.owner","type":"string","value":"node"},{"key":"process.pid","type":"string","value":"17"},{"key":"process.runtime.description","type":"string","value":"Node.js"},{"key":"process.runtime.name","type":"string","value":"nodejs"},{"key":"process.runtime.version","type":"string","value":"22.16.0"},{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"},{"key":"telemetry.sdk.language","type":"string","value":"nodejs"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"1.30.1"}]},"p7":{"serviceName":"flagd","tags":[{"key":"host.name","type":"string","value":"1f315d8a0f78"},{"key":"os.description","type":"string","value":"Debian GNU/Linux Debian GNU/Linux 12 (bookworm) (Linux 1f315d8a0f78 6.10.14-linuxkit #1 SMP Tue Apr 15 16:00:54 UTC 2025 aarch64)"},{"key":"os.type","type":"string","value":"linux"},{"key":"process.runtime.version","type":"string","value":"go1.24.1"},{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"v0.12.3"},{"key":"telemetry.sdk.language","type":"string","value":"go"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"1.35.0"}]},"p8":{"serviceName":"load-generator","tags":[{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"},{"key":"telemetry.sdk.language","type":"string","value":"python"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"1.34.0"}]},"p9":{"serviceName":"frontend-proxy","tags":[{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"}]}},"spans":[{"duration":4935,"logs":[],"operationName":"send_email","processID":"p1","references":[{"refType":"CHILD_OF","spanID":"739cd04d718779ae","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"032bf7007e123e8d","startTime":1750044449769690,"tags":[{"key":"span.kind","type":"string","value":"internal"},{"key":"otel.scope.name","type":"string","value":"email"},{"key":"error","type":"string","value":"unset"},{"key":"app.email.recipient","type":"string","value":"reed@example.com"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":283,"logs":[],"operationName":"sinatra.render_template","processID":"p1","references":[{"refType":"CHILD_OF","spanID":"1fd5f529c2dd316b","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"bc5f262c2f7d9bb5","startTime":1750044449770317,"tags":[{"key":"span.kind","type":"string","value":"internal"},{"key":"otel.scope.name","type":"string","value":"OpenTelemetry::Instrumentation::Sinatra"},{"key":"otel.scope.version","type":"string","value":"0.25.0"},{"key":"error","type":"string","value":"unset"},{"key":"sinatra.template_name","type":"string","value":"layout"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":961,"logs":[],"operationName":"sinatra.render_template","processID":"p1","references":[{"refType":"CHILD_OF","spanID":"032bf7007e123e8d","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"1fd5f529c2dd316b","startTime":1750044449769761,"tags":[{"key":"span.kind","type":"string","value":"internal"},{"key":"otel.scope.name","type":"string","value":"OpenTelemetry::Instrumentation::Sinatra"},{"key":"otel.scope.version","type":"string","value":"0.25.0"},{"key":"error","type":"string","value":"unset"},{"key":"sinatra.template_name","type":"string","value":"confirmation"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":3339,"logs":[{"timestamp":1750044449717803,"fields":[{"key":"event","type":"string","value":"Received get quote request, processing it"}]},{"timestamp":1750044449718100,"fields":[{"key":"event","type":"string","value":"Quote processed, response sent back"},{"key":"app.quote.cost.total","type":"string","value":"227.5"}]}],"operationName":"{closure}","processID":"p2","references":[{"refType":"CHILD_OF","spanID":"aaf29afb62662d95","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"ea80042fbe6e5887","startTime":1750044449717692,"tags":[{"key":"span.kind","type":"string","value":"internal"},{"key":"otel.scope.name","type":"string","value":"io.opentelemetry.contrib.php.slim"},{"key":"code.file.path","type":"string","value":"/var/www/vendor/php-di/slim-bridge/src/ControllerInvoker.php"},{"key":"code.function.name","type":"string","value":"DI\\Bridge\\Slim\\ControllerInvoker::__invoke"},{"key":"code.line.number","type":"string","value":"29"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":6755,"logs":[],"operationName":"oteldemo.PaymentService/Charge","processID":"p3","references":[{"refType":"CHILD_OF","spanID":"7683762fa74ffd1c","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"530667cc212dd6ed","startTime":1750044449739280,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"Charge"},{"key":"rpc.service","type":"string","value":"oteldemo.PaymentService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"server.address","type":"string","value":"172.18.0.14"},{"key":"server.port","type":"string","value":"50051"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":6544,"logs":[],"operationName":"POST /getquote","processID":"p2","references":[{"refType":"CHILD_OF","spanID":"09b03b9b5481c29c","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"aaf29afb62662d95","startTime":1750044449717102,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"io.opentelemetry.contrib.php.slim"},{"key":"code.file.path","type":"string","value":"/var/www/vendor/slim/slim/Slim/App.php"},{"key":"code.function.name","type":"string","value":"Slim\\App::handle"},{"key":"code.line.number","type":"string","value":"207"},{"key":"http.request.body.size","type":"string","value":"19"},{"key":"http.request.method","type":"string","value":"POST"},{"key":"http.response.body.size","type":"string","value":"-"},{"key":"http.response.status_code","type":"string","value":"200"},{"key":"http.route","type":"string","value":"/getquote"},{"key":"network.protocol.version","type":"string","value":"1.1"},{"key":"server.address","type":"string","value":"quote"},{"key":"server.port","type":"string","value":"8090"},{"key":"url.full","type":"string","value":"http://quote:8090/getquote"},{"key":"url.path","type":"string","value":"/getquote"},{"key":"url.scheme","type":"string","value":"http"},{"key":"user_agent.original","type":"string","value":"-"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":77220,"logs":[],"operationName":"executing api route (pages) /api/checkout","processID":"p4","references":[{"refType":"CHILD_OF","spanID":"01468af9419620f5","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"6b73da57ebca1b82","startTime":1750044449702000,"tags":[{"key":"span.kind","type":"string","value":"internal"},{"key":"otel.scope.name","type":"string","value":"next.js"},{"key":"otel.scope.version","type":"string","value":"0.0.1"},{"key":"http.status_code","type":"string","value":"200"},{"key":"next.span_name","type":"string","value":"executing api route (pages) /api/checkout"},{"key":"next.span_type","type":"string","value":"Node.runHandler"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":1831,"logs":[],"operationName":"oteldemo.CartService/GetCart","processID":"p3","references":[{"refType":"CHILD_OF","spanID":"96f2298052cc3fda","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"111cb151fdd9a915","startTime":1750044449708652,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"GetCart"},{"key":"rpc.service","type":"string","value":"oteldemo.CartService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"server.address","type":"string","value":"172.18.0.10"},{"key":"server.port","type":"string","value":"7070"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":46,"logs":[],"operationName":"/ship-order","processID":"p5","references":[{"refType":"CHILD_OF","spanID":"92345ad5d7cb4190","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"d1253691f90f5b95","startTime":1750044449746781,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"opentelemetry-instrumentation-actix-web"},{"key":"otel.scope.version","type":"string","value":"0.22.0"},{"key":"client.address","type":"string","value":"172.18.0.23"},{"key":"http.request.method","type":"string","value":"POST"},{"key":"http.response.status_code","type":"string","value":"200"},{"key":"http.route","type":"string","value":"/ship-order"},{"key":"network.protocol.version","type":"string","value":"1.1"},{"key":"server.address","type":"string","value":"shipping"},{"key":"server.port","type":"string","value":"50050"},{"key":"url.path","type":"string","value":"/ship-order"},{"key":"url.scheme","type":"string","value":"http"},{"key":"user_agent.original","type":"string","value":"Go-http-client/1.1"},{"key":"error","type":"string","value":"unset"},{"key":"messaging.message.body.size","type":"string","value":"182"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":78153,"logs":[],"operationName":"POST","processID":"p4","references":[{"refType":"CHILD_OF","spanID":"df1b3d5c8e0ab6be","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"47c48aa63a0c5a3d","startTime":1750044449701000,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"@opentelemetry/instrumentation-http"},{"key":"otel.scope.version","type":"string","value":"0.57.1"},{"key":"http.flavor","type":"string","value":"1.1"},{"key":"http.host","type":"string","value":"frontend-proxy:8080"},{"key":"http.method","type":"string","value":"POST"},{"key":"http.scheme","type":"string","value":"http"},{"key":"http.status_code","type":"string","value":"200"},{"key":"http.user_agent","type":"string","value":"python-requests/2.32.4"},{"key":"net.host.name","type":"string","value":"frontend-proxy"},{"key":"net.peer.ip","type":"string","value":"172.18.0.26"},{"key":"net.transport","type":"string","value":"ip_tcp"},{"key":"error","type":"string","value":"unset"},{"key":"http.request_content_length_uncompressed","type":"string","value":"388"},{"key":"http.status_text","type":"string","value":"OK"},{"key":"http.target","type":"string","value":"/api/checkout"},{"key":"http.url","type":"string","value":"http://frontend-proxy:8080/api/checkout"},{"key":"net.host.ip","type":"string","value":"172.18.0.24"},{"key":"net.host.port","type":"string","value":"8080"},{"key":"net.peer.port","type":"string","value":"35632"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":1988,"logs":[],"operationName":"charge","processID":"p6","references":[{"refType":"CHILD_OF","spanID":"df89f1712cb9fdec","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"f30e92001c694787","startTime":1750044449743000,"tags":[{"key":"span.kind","type":"string","value":"internal"},{"key":"otel.scope.name","type":"string","value":"payment"},{"key":"app.payment.card_type","type":"string","value":"visa"},{"key":"app.payment.card_valid","type":"string","value":"true"},{"key":"app.payment.charged","type":"string","value":"false"},{"key":"error","type":"string","value":"unset"},{"key":"app.loyalty.level","type":"string","value":"silver"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":128,"logs":[{"timestamp":1750044449717887,"fields":[{"key":"event","type":"string","value":"Calculating quote"}]},{"timestamp":1750044449717919,"fields":[{"key":"event","type":"string","value":"Quote calculated, returning its value"}]}],"operationName":"calculate-quote","processID":"p2","references":[{"refType":"CHILD_OF","spanID":"ea80042fbe6e5887","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"0b119b964828c67b","startTime":1750044449717886,"tags":[{"key":"span.kind","type":"string","value":"internal"},{"key":"otel.scope.name","type":"string","value":"manual-instrumentation"},{"key":"error","type":"string","value":"unset"},{"key":"app.quote.cost.total","type":"string","value":"227.5"},{"key":"app.quote.items.count","type":"string","value":"5"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":6,"logs":[],"operationName":"resolveBoolean","processID":"p7","references":[{"refType":"CHILD_OF","spanID":"3af2ca071042ef47","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"ab8c870e76bbe57f","startTime":1750044449753032,"tags":[{"key":"error","type":"string","value":"unset"},{"key":"span.kind","type":"string","value":"internal"},{"key":"otel.scope.name","type":"string","value":"jsonEvaluator"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":70,"logs":[],"operationName":"resolveBoolean","processID":"p7","references":[{"refType":"CHILD_OF","spanID":"9d054ff4aeb2b518","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"3af2ca071042ef47","startTime":1750044449753027,"tags":[{"key":"error","type":"string","value":"unset"},{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"flagd.evaluation.v1"},{"key":"feature_flag.key","type":"string","value":"cartFailure"},{"key":"feature_flag.provider_name","type":"string","value":"flagd"},{"key":"feature_flag.variant","type":"string","value":"off"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":69871,"logs":[{"timestamp":1750044449737830,"fields":[{"key":"event","type":"string","value":"prepared"}]},{"timestamp":1750044449739261,"fields":[{"key":"feature_flag.key","type":"string","value":"paymentUnreachable"},{"key":"feature_flag.provider_name","type":"string","value":"flagd"},{"key":"feature_flag.variant","type":"string","value":"off"},{"key":"event","type":"string","value":"feature_flag"}]},{"timestamp":1750044449746517,"fields":[{"key":"event","type":"string","value":"charged"},{"key":"app.payment.transaction.id","type":"string","value":"bbf912fe-0a55-4704-8eb9-02d43f60297d"}]},{"timestamp":1750044449746988,"fields":[{"key":"event","type":"string","value":"shipped"},{"key":"app.shipping.tracking.id","type":"string","value":"4668b5f9-17e2-4311-8b20-c7cf3b08ab39"}]},{"timestamp":1750044449776318,"fields":[{"key":"feature_flag.key","type":"string","value":"kafkaQueueProblems"},{"key":"feature_flag.provider_name","type":"string","value":"flagd"},{"key":"feature_flag.variant","type":"string","value":"off"},{"key":"event","type":"string","value":"feature_flag"}]}],"operationName":"oteldemo.CheckoutService/PlaceOrder","processID":"p3","references":[{"refType":"CHILD_OF","spanID":"b1cf4a62984b9984","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"7683762fa74ffd1c","startTime":1750044449706551,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"app.order.items.count","type":"string","value":"1"},{"key":"app.user.currency","type":"string","value":"USD"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"PlaceOrder"},{"key":"rpc.service","type":"string","value":"oteldemo.CheckoutService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"server.address","type":"string","value":"172.18.0.24"},{"key":"server.port","type":"string","value":"38682"},{"key":"error","type":"string","value":"unset"},{"key":"app.order.amount","type":"string","value":"1102"},{"key":"app.order.id","type":"string","value":"d52a1b43-4a61-11f0-9e2b-96226e8767f9"},{"key":"app.shipping.amount","type":"string","value":"227"},{"key":"app.shipping.tracking.id","type":"string","value":"4668b5f9-17e2-4311-8b20-c7cf3b08ab39"},{"key":"app.user.id","type":"string","value":"d526648e-4a61-11f0-8b6b-b20e5443dfb5"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":19817,"logs":[{"timestamp":1750044449735392,"fields":[{"key":"event","type":"string","value":"Received Quote"},{"key":"app.shipping.cost.total","type":"string","value":"227.50"}]}],"operationName":"/get-quote","processID":"p5","references":[{"refType":"CHILD_OF","spanID":"7b92ebafc9a2a0f1","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"599cbbf8e81ddaca","startTime":1750044449715635,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"opentelemetry-instrumentation-actix-web"},{"key":"otel.scope.version","type":"string","value":"0.22.0"},{"key":"client.address","type":"string","value":"172.18.0.23"},{"key":"http.request.method","type":"string","value":"POST"},{"key":"http.response.status_code","type":"string","value":"200"},{"key":"http.route","type":"string","value":"/get-quote"},{"key":"network.protocol.version","type":"string","value":"1.1"},{"key":"server.address","type":"string","value":"shipping"},{"key":"server.port","type":"string","value":"50050"},{"key":"url.path","type":"string","value":"/get-quote"},{"key":"url.scheme","type":"string","value":"http"},{"key":"user_agent.original","type":"string","value":"Go-http-client/1.1"},{"key":"error","type":"string","value":"unset"},{"key":"app.shipping.cost.total","type":"string","value":"227.50"},{"key":"messaging.message.body.size","type":"string","value":"182"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":1733,"logs":[],"operationName":"grpc.oteldemo.ProductCatalogService/GetProduct","processID":"p4","references":[{"refType":"CHILD_OF","spanID":"6b73da57ebca1b82","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"394722a3d65e5bee","startTime":1750044449777000,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"@opentelemetry/instrumentation-grpc"},{"key":"otel.scope.version","type":"string","value":"0.57.1"},{"key":"net.peer.name","type":"string","value":"product-catalog"},{"key":"net.peer.port","type":"string","value":"3550"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"GetProduct"},{"key":"rpc.service","type":"string","value":"oteldemo.ProductCatalogService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":805,"logs":[],"operationName":"orders publish","processID":"p3","references":[{"refType":"CHILD_OF","spanID":"7683762fa74ffd1c","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"842ad77105e18d23","startTime":1750044449775517,"tags":[{"key":"span.kind","type":"string","value":"producer"},{"key":"otel.scope.name","type":"string","value":"checkout"},{"key":"messaging.destination.name","type":"string","value":"orders"},{"key":"messaging.kafka.destination.partition","type":"string","value":"0"},{"key":"messaging.kafka.message.offset","type":"string","value":"0"},{"key":"messaging.kafka.producer.success","type":"string","value":"true"},{"key":"messaging.operation","type":"string","value":"publish"},{"key":"messaging.system","type":"string","value":"kafka"},{"key":"network.transport","type":"string","value":"tcp"},{"key":"peer.service","type":"string","value":"kafka"},{"key":"error","type":"string","value":"unset"},{"key":"messaging.kafka.producer.duration_ms","type":"string","value":"0"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":22024,"logs":[],"operationName":"HTTP POST","processID":"p3","references":[{"refType":"CHILD_OF","spanID":"96f2298052cc3fda","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"7b92ebafc9a2a0f1","startTime":1750044449713664,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"http.request.method","type":"string","value":"POST"},{"key":"http.response.status_code","type":"string","value":"200"},{"key":"network.protocol.version","type":"string","value":"1.1"},{"key":"error","type":"string","value":"unset"},{"key":"server.address","type":"string","value":"shipping"},{"key":"server.port","type":"string","value":"50050"},{"key":"url.full","type":"string","value":"http://shipping:50050/get-quote"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":391,"logs":[],"operationName":"HTTP POST","processID":"p3","references":[{"refType":"CHILD_OF","spanID":"7683762fa74ffd1c","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"92345ad5d7cb4190","startTime":1750044449746559,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"http.request.method","type":"string","value":"POST"},{"key":"http.response.status_code","type":"string","value":"200"},{"key":"network.protocol.version","type":"string","value":"1.1"},{"key":"error","type":"string","value":"unset"},{"key":"server.address","type":"string","value":"shipping"},{"key":"server.port","type":"string","value":"50050"},{"key":"url.full","type":"string","value":"http://shipping:50050/ship-order"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":15663,"logs":[],"operationName":"HTTP POST","processID":"p3","references":[{"refType":"CHILD_OF","spanID":"7683762fa74ffd1c","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"d96adf1246ad7d75","startTime":1750044449759771,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"http.request.method","type":"string","value":"POST"},{"key":"http.response.status_code","type":"string","value":"200"},{"key":"network.protocol.version","type":"string","value":"1.1"},{"key":"error","type":"string","value":"unset"},{"key":"server.address","type":"string","value":"email"},{"key":"server.port","type":"string","value":"6060"},{"key":"url.full","type":"string","value":"http://email:6060/send_order_confirmation"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":79737,"logs":[],"operationName":"POST","processID":"p8","references":[],"spanID":"10d27d153c44c541","startTime":1750044449700847,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"opentelemetry.instrumentation.requests"},{"key":"otel.scope.version","type":"string","value":"0.55b0"},{"key":"http.method","type":"string","value":"POST"},{"key":"http.status_code","type":"string","value":"200"},{"key":"http.url","type":"string","value":"http://frontend-proxy:8080/api/checkout"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":136,"logs":[{"timestamp":1750044449752991,"fields":[{"key":"message.id","type":"string","value":"1"},{"key":"message.type","type":"string","value":"RECEIVED"},{"key":"event","type":"string","value":"message"},{"key":"message.uncompressed_size","type":"string","value":"15"}]},{"timestamp":1750044449753111,"fields":[{"key":"message.id","type":"string","value":"1"},{"key":"message.type","type":"string","value":"SENT"},{"key":"message.uncompressed_size","type":"string","value":"15"},{"key":"event","type":"string","value":"message"}]}],"operationName":"flagd.evaluation.v1.Service/ResolveBoolean","processID":"p7","references":[{"refType":"CHILD_OF","spanID":"31d9931c1b054f86","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"9d054ff4aeb2b518","startTime":1750044449752984,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"connectrpc.com/otelconnect"},{"key":"otel.scope.version","type":"string","value":"semver:0.6.0-dev"},{"key":"rpc.method","type":"string","value":"ResolveBoolean"},{"key":"rpc.service","type":"string","value":"flagd.evaluation.v1.Service"},{"key":"error","type":"string","value":"unset"},{"key":"net.peer.name","type":"string","value":"172.18.0.10"},{"key":"net.peer.port","type":"string","value":"46838"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.system","type":"string","value":"grpc"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":2157,"logs":[],"operationName":"oteldemo.CurrencyService/Convert","processID":"p3","references":[{"refType":"CHILD_OF","spanID":"96f2298052cc3fda","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"34a9d7aa3afe1688","startTime":1750044449711310,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"Convert"},{"key":"rpc.service","type":"string","value":"oteldemo.CurrencyService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"server.address","type":"string","value":"172.18.0.18"},{"key":"server.port","type":"string","value":"7001"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":2021,"logs":[],"operationName":"oteldemo.CurrencyService/Convert","processID":"p3","references":[{"refType":"CHILD_OF","spanID":"96f2298052cc3fda","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"11295d69d0e661dd","startTime":1750044449735781,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"Convert"},{"key":"rpc.service","type":"string","value":"oteldemo.CurrencyService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"server.address","type":"string","value":"172.18.0.18"},{"key":"server.port","type":"string","value":"7001"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":19397,"logs":[],"operationName":"POST quote","processID":"p5","references":[{"refType":"CHILD_OF","spanID":"599cbbf8e81ddaca","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"09b03b9b5481c29c","startTime":1750044449715774,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"opentelemetry-instrumentation-actix-web"},{"key":"otel.scope.version","type":"string","value":"0.22.0"},{"key":"http.request.method","type":"string","value":"POST"},{"key":"http.response.status_code","type":"string","value":"200"},{"key":"server.address","type":"string","value":"quote"},{"key":"server.port","type":"string","value":"8090"},{"key":"url.full","type":"string","value":"http://quote:8090/getquote"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":78545,"logs":[],"operationName":"router frontend egress","processID":"p9","references":[{"refType":"CHILD_OF","spanID":"d66da216bedd159f","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"df1b3d5c8e0ab6be","startTime":1750044449701376,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"component","type":"string","value":"proxy"},{"key":"http.protocol","type":"string","value":"HTTP/1.1"},{"key":"peer.address","type":"string","value":"172.18.0.24:8080"},{"key":"upstream_address","type":"string","value":"172.18.0.24:8080"},{"key":"upstream_cluster","type":"string","value":"frontend"},{"key":"upstream_cluster.name","type":"string","value":"frontend"},{"key":"error","type":"string","value":"unset"},{"key":"http.status_code","type":"string","value":"200"},{"key":"response_flags","type":"string","value":"-"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":710,"logs":[],"operationName":"oteldemo.ProductCatalogService/GetProduct","processID":"p3","references":[{"refType":"CHILD_OF","spanID":"96f2298052cc3fda","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"7e5e7c2f1ea9cb0b","startTime":1750044449710565,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"GetProduct"},{"key":"rpc.service","type":"string","value":"oteldemo.ProductCatalogService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"server.address","type":"string","value":"172.18.0.19"},{"key":"server.port","type":"string","value":"3550"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":915,"logs":[{"timestamp":1750044449709335,"fields":[{"key":"event","type":"string","value":"Fetch cart"}]}],"operationName":"POST /oteldemo.CartService/GetCart","processID":"p10","references":[{"refType":"CHILD_OF","spanID":"111cb151fdd9a915","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"fefa4832f9254043","startTime":1750044449709238,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"Microsoft.AspNetCore"},{"key":"grpc.method","type":"string","value":"/oteldemo.CartService/GetCart"},{"key":"grpc.status_code","type":"string","value":"0"},{"key":"http.request.method","type":"string","value":"POST"},{"key":"http.response.status_code","type":"string","value":"200"},{"key":"http.route","type":"string","value":"/oteldemo.CartService/GetCart"},{"key":"network.protocol.version","type":"string","value":"2"},{"key":"server.address","type":"string","value":"cart"},{"key":"server.port","type":"string","value":"7070"},{"key":"url.path","type":"string","value":"/oteldemo.CartService/GetCart"},{"key":"url.scheme","type":"string","value":"http"},{"key":"error","type":"string","value":"unset"},{"key":"app.cart.items.count","type":"string","value":"5"},{"key":"app.user.id","type":"string","value":"d526648e-4a61-11f0-8b6b-b20e5443dfb5"},{"key":"user_agent.original","type":"string","value":"grpc-go/1.72.2"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":8349,"logs":[],"operationName":"POST /send_order_confirmation","processID":"p1","references":[{"refType":"CHILD_OF","spanID":"d96adf1246ad7d75","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"739cd04d718779ae","startTime":1750044449766969,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"OpenTelemetry::Instrumentation::Rack"},{"key":"otel.scope.version","type":"string","value":"0.26.0"},{"key":"http.host","type":"string","value":"email:6060"},{"key":"http.method","type":"string","value":"POST"},{"key":"http.route","type":"string","value":"/send_order_confirmation"},{"key":"http.scheme","type":"string","value":"http"},{"key":"http.status_code","type":"string","value":"200"},{"key":"http.target","type":"string","value":"/send_order_confirmation"},{"key":"http.user_agent","type":"string","value":"Go-http-client/1.1"},{"key":"error","type":"string","value":"unset"},{"key":"app.order.id","type":"string","value":"d52a1b43-4a61-11f0-9e2b-96226e8767f9"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":11927,"logs":[{"timestamp":1750044449747830,"fields":[{"key":"event","type":"string","value":"Empty cart"}]},{"timestamp":1750044449755100,"fields":[{"key":"feature_flag.key","type":"string","value":"cartFailure"},{"key":"feature_flag.provider_name","type":"string","value":"flagd Provider"},{"key":"feature_flag.variant","type":"string","value":"off"},{"key":"event","type":"string","value":"feature_flag"}]}],"operationName":"POST /oteldemo.CartService/EmptyCart","processID":"p10","references":[{"refType":"CHILD_OF","spanID":"4e08d386db6de0e6","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"d8802687844ff0da","startTime":1750044449747360,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"Microsoft.AspNetCore"},{"key":"app.user.id","type":"string","value":"d526648e-4a61-11f0-8b6b-b20e5443dfb5"},{"key":"feature_flag.key","type":"string","value":"cartFailure"},{"key":"feature_flag.provider_name","type":"string","value":"flagd Provider"},{"key":"feature_flag.variant","type":"string","value":"off"},{"key":"grpc.method","type":"string","value":"/oteldemo.CartService/EmptyCart"},{"key":"grpc.status_code","type":"string","value":"0"},{"key":"http.request.method","type":"string","value":"POST"},{"key":"http.response.status_code","type":"string","value":"200"},{"key":"http.route","type":"string","value":"/oteldemo.CartService/EmptyCart"},{"key":"network.protocol.version","type":"string","value":"2"},{"key":"server.address","type":"string","value":"cart"},{"key":"server.port","type":"string","value":"7070"},{"key":"url.path","type":"string","value":"/oteldemo.CartService/EmptyCart"},{"key":"url.scheme","type":"string","value":"http"},{"key":"user_agent.original","type":"string","value":"grpc-go/1.72.2"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":74743,"logs":[],"operationName":"grpc.oteldemo.CheckoutService/PlaceOrder","processID":"p4","references":[{"refType":"CHILD_OF","spanID":"6b73da57ebca1b82","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"b1cf4a62984b9984","startTime":1750044449702000,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"@opentelemetry/instrumentation-grpc"},{"key":"otel.scope.version","type":"string","value":"0.57.1"},{"key":"net.peer.name","type":"string","value":"checkout"},{"key":"net.peer.port","type":"string","value":"5050"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"PlaceOrder"},{"key":"rpc.service","type":"string","value":"oteldemo.CheckoutService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":12631,"logs":[],"operationName":"oteldemo.CartService/EmptyCart","processID":"p3","references":[{"refType":"CHILD_OF","spanID":"7683762fa74ffd1c","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"4e08d386db6de0e6","startTime":1750044449747019,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"EmptyCart"},{"key":"rpc.service","type":"string","value":"oteldemo.CartService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"server.address","type":"string","value":"172.18.0.10"},{"key":"server.port","type":"string","value":"7070"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":352,"logs":[{"timestamp":1750044449709386,"fields":[{"key":"event","type":"string","value":"Enqueued"}]},{"timestamp":1750044449709400,"fields":[{"key":"event","type":"string","value":"Sent"}]},{"timestamp":1750044449709718,"fields":[{"key":"event","type":"string","value":"ResponseReceived"}]}],"operationName":"HGET","processID":"p10","references":[{"refType":"CHILD_OF","spanID":"fefa4832f9254043","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"1c6fa81981e4960c","startTime":1750044449709366,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"OpenTelemetry.Instrumentation.StackExchangeRedis"},{"key":"otel.scope.version","type":"string","value":"1.11.0-beta.2"},{"key":"db.redis.database_index","type":"string","value":"0"},{"key":"db.redis.flags","type":"string","value":"None"},{"key":"db.system","type":"string","value":"redis"},{"key":"server.address","type":"string","value":"valkey-cart"},{"key":"server.port","type":"string","value":"6379"},{"key":"error","type":"string","value":"unset"},{"key":"db.statement","type":"string","value":"HGET d526648e-4a61-11f0-8b6b-b20e5443dfb5"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":30309,"logs":[],"operationName":"prepareOrderItemsAndShippingQuoteFromCart","processID":"p3","references":[{"refType":"CHILD_OF","spanID":"7683762fa74ffd1c","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"96f2298052cc3fda","startTime":1750044449707511,"tags":[{"key":"span.kind","type":"string","value":"internal"},{"key":"otel.scope.name","type":"string","value":"checkout"},{"key":"app.order.items.count","type":"string","value":"1"},{"key":"error","type":"string","value":"unset"},{"key":"app.cart.items.count","type":"string","value":"5"},{"key":"app.shipping.amount","type":"string","value":"227"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":4711,"logs":[],"operationName":"POST","processID":"p10","references":[{"refType":"CHILD_OF","spanID":"64e503f233846241","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"31d9931c1b054f86","startTime":1750044449749545,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"System.Net.Http"},{"key":"http.request.method","type":"string","value":"POST"},{"key":"http.response.status_code","type":"string","value":"200"},{"key":"network.protocol.version","type":"string","value":"2"},{"key":"server.address","type":"string","value":"flagd"},{"key":"server.port","type":"string","value":"8013"},{"key":"url.full","type":"string","value":"http://flagd:8013/flagd.evaluation.v1.Service/ResolveBoolean"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":3076,"logs":[],"operationName":"grpc.oteldemo.PaymentService/Charge","processID":"p6","references":[{"refType":"CHILD_OF","spanID":"530667cc212dd6ed","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"df89f1712cb9fdec","startTime":1750044449742000,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"@opentelemetry/instrumentation-grpc"},{"key":"otel.scope.version","type":"string","value":"0.57.1"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"Charge"},{"key":"rpc.service","type":"string","value":"oteldemo.PaymentService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"error","type":"string","value":"unset"},{"key":"app.payment.amount","type":"string","value":"1102.50"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":421,"logs":[{"timestamp":1750044449755249,"fields":[{"key":"event","type":"string","value":"Enqueued"}]},{"timestamp":1750044449755262,"fields":[{"key":"event","type":"string","value":"Sent"}]},{"timestamp":1750044449755655,"fields":[{"key":"event","type":"string","value":"ResponseReceived"}]}],"operationName":"HMSET","processID":"p10","references":[{"refType":"CHILD_OF","spanID":"d8802687844ff0da","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"5f78a21a81d1a9a3","startTime":1750044449755233,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"OpenTelemetry.Instrumentation.StackExchangeRedis"},{"key":"otel.scope.version","type":"string","value":"1.11.0-beta.2"},{"key":"db.redis.database_index","type":"string","value":"0"},{"key":"db.redis.flags","type":"string","value":"DemandMaster"},{"key":"db.system","type":"string","value":"redis"},{"key":"server.address","type":"string","value":"valkey-cart"},{"key":"server.port","type":"string","value":"6379"},{"key":"error","type":"string","value":"unset"},{"key":"db.statement","type":"string","value":"HMSET d526648e-4a61-11f0-8b6b-b20e5443dfb5"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":77796,"logs":[],"operationName":"POST /api/checkout","processID":"p4","references":[{"refType":"CHILD_OF","spanID":"47c48aa63a0c5a3d","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"01468af9419620f5","startTime":1750044449701000,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"next.js"},{"key":"otel.scope.version","type":"string","value":"0.0.1"},{"key":"http.method","type":"string","value":"POST"},{"key":"http.status_code","type":"string","value":"200"},{"key":"http.target","type":"string","value":"/api/checkout"},{"key":"next.rsc","type":"string","value":"false"},{"key":"next.span_name","type":"string","value":"POST /api/checkout"},{"key":"next.span_type","type":"string","value":"BaseServer.handleRequest"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":5855,"logs":[],"operationName":"flagd.evaluation.v1.Service/ResolveBoolean","processID":"p10","references":[{"refType":"CHILD_OF","spanID":"d8802687844ff0da","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"64e503f233846241","startTime":1750044449749012,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"OpenTelemetry.Instrumentation.GrpcNetClient"},{"key":"otel.scope.version","type":"string","value":"1.11.0-beta.2"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"ResolveBoolean"},{"key":"rpc.service","type":"string","value":"flagd.evaluation.v1.Service"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"server.address","type":"string","value":"flagd"},{"key":"server.port","type":"string","value":"8013"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":877,"logs":[{"timestamp":1750044449755696,"fields":[{"key":"event","type":"string","value":"Enqueued"}]},{"timestamp":1750044449755708,"fields":[{"key":"event","type":"string","value":"Sent"}]},{"timestamp":1750044449756563,"fields":[{"key":"event","type":"string","value":"ResponseReceived"}]}],"operationName":"EXPIRE","processID":"p10","references":[{"refType":"CHILD_OF","spanID":"d8802687844ff0da","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"4a42b7a5fa81bdfb","startTime":1750044449755686,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"OpenTelemetry.Instrumentation.StackExchangeRedis"},{"key":"otel.scope.version","type":"string","value":"1.11.0-beta.2"},{"key":"db.redis.database_index","type":"string","value":"0"},{"key":"db.redis.flags","type":"string","value":"DemandMaster"},{"key":"db.system","type":"string","value":"redis"},{"key":"server.address","type":"string","value":"valkey-cart"},{"key":"server.port","type":"string","value":"6379"},{"key":"error","type":"string","value":"unset"},{"key":"db.statement","type":"string","value":"EXPIRE d526648e-4a61-11f0-8b6b-b20e5443dfb5"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":75,"logs":[{"timestamp":1750044449711020,"fields":[{"key":"event","type":"string","value":"Product Found"}]}],"operationName":"oteldemo.ProductCatalogService/GetProduct","processID":"p11","references":[{"refType":"CHILD_OF","spanID":"7e5e7c2f1ea9cb0b","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"5b997902f830009b","startTime":1750044449710969,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"GetProduct"},{"key":"rpc.service","type":"string","value":"oteldemo.ProductCatalogService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"error","type":"string","value":"unset"},{"key":"app.product.id","type":"string","value":"0PUK6V6EV0"},{"key":"app.product.name","type":"string","value":"Solar System Color Imager"},{"key":"server.address","type":"string","value":"172.18.0.23"},{"key":"server.port","type":"string","value":"56058"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":78,"logs":[{"timestamp":1750044449778775,"fields":[{"key":"event","type":"string","value":"Product Found"}]}],"operationName":"oteldemo.ProductCatalogService/GetProduct","processID":"p11","references":[{"refType":"CHILD_OF","spanID":"394722a3d65e5bee","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"212f00429ff724f5","startTime":1750044449778734,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"GetProduct"},{"key":"rpc.service","type":"string","value":"oteldemo.ProductCatalogService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"error","type":"string","value":"unset"},{"key":"app.product.id","type":"string","value":"0PUK6V6EV0"},{"key":"app.product.name","type":"string","value":"Solar System Color Imager"},{"key":"server.address","type":"string","value":"172.18.0.24"},{"key":"server.port","type":"string","value":"47538"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":597,"logs":[{"timestamp":1750044449711719,"fields":[{"key":"event","type":"string","value":"Processing currency conversion request"}]},{"timestamp":1750044449711741,"fields":[{"key":"event","type":"string","value":"Conversion successful, response sent back"}]}],"operationName":"Currency/Convert","processID":"p12","references":[{"refType":"CHILD_OF","spanID":"34a9d7aa3afe1688","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"42e4324fcb045b99","startTime":1750044449711715,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"currency"},{"key":"app.currency.conversion.from","type":"string","value":"USD"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"Convert"},{"key":"rpc.service","type":"string","value":"oteldemo.CurrencyService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"error","type":"string","value":"false"},{"key":"app.currency.conversion.to","type":"string","value":"USD"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":655,"logs":[{"timestamp":1750044449736390,"fields":[{"key":"event","type":"string","value":"Processing currency conversion request"}]},{"timestamp":1750044449736414,"fields":[{"key":"event","type":"string","value":"Conversion successful, response sent back"}]}],"operationName":"Currency/Convert","processID":"p12","references":[{"refType":"CHILD_OF","spanID":"11295d69d0e661dd","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"adb556f3c99b633d","startTime":1750044449736386,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"currency"},{"key":"app.currency.conversion.from","type":"string","value":"USD"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"Convert"},{"key":"rpc.service","type":"string","value":"oteldemo.CurrencyService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"error","type":"string","value":"false"},{"key":"app.currency.conversion.to","type":"string","value":"USD"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":78648,"logs":[],"operationName":"ingress","processID":"p9","references":[{"refType":"CHILD_OF","spanID":"10d27d153c44c541","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"d66da216bedd159f","startTime":1750044449701298,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"component","type":"string","value":"proxy"},{"key":"downstream_cluster","type":"string","value":"-"},{"key":"http.protocol","type":"string","value":"HTTP/1.1"},{"key":"node_id","type":"string","value":"-"},{"key":"peer.address","type":"string","value":"172.18.0.25"},{"key":"zone","type":"string","value":"-"},{"key":"guid:x-request-id","type":"string","value":"347edd6d-e273-953e-87f6-7ba07f352331"},{"key":"http.method","type":"string","value":"POST"},{"key":"http.status_code","type":"string","value":"200"},{"key":"http.url","type":"string","value":"http://frontend-proxy:8080/api/checkout"},{"key":"request_size","type":"string","value":"388"},{"key":"response_flags","type":"string","value":"-"},{"key":"response_size","type":"string","value":"857"},{"key":"upstream_cluster","type":"string","value":"frontend"},{"key":"upstream_cluster.name","type":"string","value":"frontend"},{"key":"user_agent","type":"string","value":"python-requests/2.32.4"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null}],"errors":null,"limit":0,"offset":0,"total":1} ``` 4. Find a trace by `trace_id` + ```sh curl http://:10428/select/jaeger/api/traces/9e06226196051d9c3c10dfab343791ad ``` Here's a response example: + ```json {"data":[{"processes":{"p1":{"serviceName":"email","tags":[{"key":"process.command","type":"string","value":"email_server.rb"},{"key":"process.pid","type":"string","value":"1"},{"key":"process.runtime.description","type":"string","value":"ruby 3.4.4 (2025-05-14 revision a38531fd3f) +PRISM [aarch64-linux-musl]"},{"key":"process.runtime.name","type":"string","value":"ruby"},{"key":"process.runtime.version","type":"string","value":"3.4.4"},{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"},{"key":"telemetry.sdk.language","type":"string","value":"ruby"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"1.8.0"}]},"p10":{"serviceName":"load-generator","tags":[{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"},{"key":"telemetry.sdk.language","type":"string","value":"python"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"1.34.0"}]},"p11":{"serviceName":"product-catalog","tags":[{"key":"host.name","type":"string","value":"3dabfcfe8381"},{"key":"os.description","type":"string","value":"Debian GNU/Linux Debian GNU/Linux 12 (bookworm) (Linux 3dabfcfe8381 6.10.14-linuxkit #1 SMP Tue Apr 15 16:00:54 UTC 2025 aarch64)"},{"key":"os.type","type":"string","value":"linux"},{"key":"process.command_args","type":"string","value":"[\"./product-catalog\"]"},{"key":"process.executable.name","type":"string","value":"product-catalog"},{"key":"process.executable.path","type":"string","value":"/usr/src/app/product-catalog"},{"key":"process.owner","type":"string","value":"nonroot"},{"key":"process.pid","type":"string","value":"1"},{"key":"process.runtime.description","type":"string","value":"go version go1.24.4 linux/arm64"},{"key":"process.runtime.name","type":"string","value":"go"},{"key":"process.runtime.version","type":"string","value":"go1.24.4"},{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"},{"key":"telemetry.sdk.language","type":"string","value":"go"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"1.36.0"}]},"p12":{"serviceName":"currency","tags":[{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"},{"key":"telemetry.sdk.language","type":"string","value":"cpp"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"1.20.0"}]},"p2":{"serviceName":"quote","tags":[{"key":"container.id","type":"string","value":"759183873eeb1328f16df8ea5b5a10932506af136a6537c6a365131c04f1645c"},{"key":"host.arch","type":"string","value":"aarch64"},{"key":"host.name","type":"string","value":"759183873eeb"},{"key":"os.description","type":"string","value":"6.10.14-linuxkit"},{"key":"os.name","type":"string","value":"Linux"},{"key":"os.type","type":"string","value":"linux"},{"key":"os.version","type":"string","value":"#1 SMP Tue Apr 15 16:00:54 UTC 2025"},{"key":"process.command","type":"string","value":"public/index.php"},{"key":"process.command_args","type":"string","value":"[\"public/index.php\"]"},{"key":"process.executable.path","type":"string","value":"/usr/local/bin/php"},{"key":"process.owner","type":"string","value":"www-data"},{"key":"process.pid","type":"string","value":"1"},{"key":"process.runtime.name","type":"string","value":"cli"},{"key":"process.runtime.version","type":"string","value":"8.3.22"},{"key":"service.instance.id","type":"string","value":"9dc0abaa-c408-483e-9fed-8375a73efb91"},{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"},{"key":"telemetry.distro.name","type":"string","value":"opentelemetry-php-instrumentation"},{"key":"telemetry.distro.version","type":"string","value":"1.1.3"},{"key":"telemetry.sdk.language","type":"string","value":"php"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"1.5.0"}]},"p3":{"serviceName":"frontend","tags":[{"key":"container.id","type":"string","value":"2d395f01353040612a00252cf6e8c32f00ab94ae06f82f143a3ea9c742072674"},{"key":"host.arch","type":"string","value":"arm64"},{"key":"host.name","type":"string","value":"2d395f013530"},{"key":"os.type","type":"string","value":"linux"},{"key":"os.version","type":"string","value":"6.10.14-linuxkit"},{"key":"process.command","type":"string","value":"/app/server.js"},{"key":"process.command_args","type":"string","value":"[\"/usr/local/bin/node\",\"--require\",\"./Instrumentation.js\",\"/app/server.js\"]"},{"key":"process.executable.name","type":"string","value":"node"},{"key":"process.executable.path","type":"string","value":"/usr/local/bin/node"},{"key":"process.owner","type":"string","value":"nextjs"},{"key":"process.pid","type":"string","value":"17"},{"key":"process.runtime.description","type":"string","value":"Node.js"},{"key":"process.runtime.name","type":"string","value":"nodejs"},{"key":"process.runtime.version","type":"string","value":"22.16.0"},{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"},{"key":"telemetry.sdk.language","type":"string","value":"nodejs"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"1.30.1"}]},"p4":{"serviceName":"payment","tags":[{"key":"container.id","type":"string","value":"18ee03279d38ed0e0eedad037c260df78dfc3323aa662ca14a2d38fcc8bf3762"},{"key":"host.arch","type":"string","value":"arm64"},{"key":"host.name","type":"string","value":"18ee03279d38"},{"key":"os.type","type":"string","value":"linux"},{"key":"os.version","type":"string","value":"6.10.14-linuxkit"},{"key":"process.command","type":"string","value":"/usr/src/app/index.js"},{"key":"process.command_args","type":"string","value":"[\"/usr/local/bin/node\",\"--require\",\"./opentelemetry.js\",\"/usr/src/app/index.js\"]"},{"key":"process.executable.name","type":"string","value":"node"},{"key":"process.executable.path","type":"string","value":"/usr/local/bin/node"},{"key":"process.owner","type":"string","value":"node"},{"key":"process.pid","type":"string","value":"17"},{"key":"process.runtime.description","type":"string","value":"Node.js"},{"key":"process.runtime.name","type":"string","value":"nodejs"},{"key":"process.runtime.version","type":"string","value":"22.16.0"},{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"},{"key":"telemetry.sdk.language","type":"string","value":"nodejs"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"1.30.1"}]},"p5":{"serviceName":"flagd","tags":[{"key":"host.name","type":"string","value":"1f315d8a0f78"},{"key":"os.description","type":"string","value":"Debian GNU/Linux Debian GNU/Linux 12 (bookworm) (Linux 1f315d8a0f78 6.10.14-linuxkit #1 SMP Tue Apr 15 16:00:54 UTC 2025 aarch64)"},{"key":"os.type","type":"string","value":"linux"},{"key":"process.runtime.version","type":"string","value":"go1.24.1"},{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"v0.12.3"},{"key":"telemetry.sdk.language","type":"string","value":"go"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"1.35.0"}]},"p6":{"serviceName":"shipping","tags":[{"key":"os.type","type":"string","value":"linux"},{"key":"process.command_args","type":"string","value":"[\"/app/shipping\"]"},{"key":"process.pid","type":"string","value":"1"},{"key":"process.runtime.description","type":"string","value":"rustc 1.82.0 (f6e511eec 2024-10-15)"},{"key":"process.runtime.name","type":"string","value":"rustc"},{"key":"process.runtime.version","type":"string","value":"1.82.0"},{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"},{"key":"telemetry.sdk.language","type":"string","value":"rust"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"0.30.0"}]},"p7":{"serviceName":"checkout","tags":[{"key":"host.name","type":"string","value":"cbdb5e0808c2"},{"key":"os.description","type":"string","value":"Debian GNU/Linux Debian GNU/Linux 12 (bookworm) (Linux cbdb5e0808c2 6.10.14-linuxkit #1 SMP Tue Apr 15 16:00:54 UTC 2025 aarch64)"},{"key":"os.type","type":"string","value":"linux"},{"key":"process.command_args","type":"string","value":"[\"./checkout\"]"},{"key":"process.executable.name","type":"string","value":"checkout"},{"key":"process.executable.path","type":"string","value":"/usr/src/app/checkout"},{"key":"process.owner","type":"string","value":"nonroot"},{"key":"process.pid","type":"string","value":"1"},{"key":"process.runtime.description","type":"string","value":"go version go1.24.4 linux/arm64"},{"key":"process.runtime.name","type":"string","value":"go"},{"key":"process.runtime.version","type":"string","value":"go1.24.4"},{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"},{"key":"telemetry.sdk.language","type":"string","value":"go"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"1.36.0"}]},"p8":{"serviceName":"frontend-proxy","tags":[{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"}]},"p9":{"serviceName":"cart","tags":[{"key":"container.id","type":"string","value":"5603ff989877ecf311403b6ea81fda10734846a0cbdad3a09c39fb068e4a07fc"},{"key":"host.name","type":"string","value":"5603ff989877"},{"key":"service.namespace","type":"string","value":"opentelemetry-demo"},{"key":"service.version","type":"string","value":"2.0.2"},{"key":"telemetry.sdk.language","type":"string","value":"dotnet"},{"key":"telemetry.sdk.name","type":"string","value":"opentelemetry"},{"key":"telemetry.sdk.version","type":"string","value":"1.11.2"}]}},"spans":[{"duration":4935,"logs":[],"operationName":"send_email","processID":"p1","references":[{"refType":"CHILD_OF","spanID":"739cd04d718779ae","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"032bf7007e123e8d","startTime":1750044449769690,"tags":[{"key":"span.kind","type":"string","value":"internal"},{"key":"otel.scope.name","type":"string","value":"email"},{"key":"error","type":"string","value":"unset"},{"key":"app.email.recipient","type":"string","value":"reed@example.com"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":3339,"logs":[{"timestamp":1750044449717803,"fields":[{"key":"event","type":"string","value":"Received get quote request, processing it"}]},{"timestamp":1750044449718100,"fields":[{"key":"event","type":"string","value":"Quote processed, response sent back"},{"key":"app.quote.cost.total","type":"string","value":"227.5"}]}],"operationName":"{closure}","processID":"p2","references":[{"refType":"CHILD_OF","spanID":"aaf29afb62662d95","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"ea80042fbe6e5887","startTime":1750044449717692,"tags":[{"key":"span.kind","type":"string","value":"internal"},{"key":"otel.scope.name","type":"string","value":"io.opentelemetry.contrib.php.slim"},{"key":"code.file.path","type":"string","value":"/var/www/vendor/php-di/slim-bridge/src/ControllerInvoker.php"},{"key":"code.function.name","type":"string","value":"DI\\Bridge\\Slim\\ControllerInvoker::__invoke"},{"key":"code.line.number","type":"string","value":"29"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":6544,"logs":[],"operationName":"POST /getquote","processID":"p2","references":[{"refType":"CHILD_OF","spanID":"09b03b9b5481c29c","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"aaf29afb62662d95","startTime":1750044449717102,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"io.opentelemetry.contrib.php.slim"},{"key":"code.file.path","type":"string","value":"/var/www/vendor/slim/slim/Slim/App.php"},{"key":"code.function.name","type":"string","value":"Slim\\App::handle"},{"key":"code.line.number","type":"string","value":"207"},{"key":"http.request.body.size","type":"string","value":"19"},{"key":"http.request.method","type":"string","value":"POST"},{"key":"http.response.body.size","type":"string","value":"-"},{"key":"http.response.status_code","type":"string","value":"200"},{"key":"http.route","type":"string","value":"/getquote"},{"key":"network.protocol.version","type":"string","value":"1.1"},{"key":"server.address","type":"string","value":"quote"},{"key":"server.port","type":"string","value":"8090"},{"key":"url.full","type":"string","value":"http://quote:8090/getquote"},{"key":"url.path","type":"string","value":"/getquote"},{"key":"url.scheme","type":"string","value":"http"},{"key":"user_agent.original","type":"string","value":"-"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":77220,"logs":[],"operationName":"executing api route (pages) /api/checkout","processID":"p3","references":[{"refType":"CHILD_OF","spanID":"01468af9419620f5","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"6b73da57ebca1b82","startTime":1750044449702000,"tags":[{"key":"span.kind","type":"string","value":"internal"},{"key":"otel.scope.name","type":"string","value":"next.js"},{"key":"otel.scope.version","type":"string","value":"0.0.1"},{"key":"http.status_code","type":"string","value":"200"},{"key":"next.span_name","type":"string","value":"executing api route (pages) /api/checkout"},{"key":"next.span_type","type":"string","value":"Node.runHandler"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":78153,"logs":[],"operationName":"POST","processID":"p3","references":[{"refType":"CHILD_OF","spanID":"df1b3d5c8e0ab6be","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"47c48aa63a0c5a3d","startTime":1750044449701000,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"@opentelemetry/instrumentation-http"},{"key":"otel.scope.version","type":"string","value":"0.57.1"},{"key":"http.flavor","type":"string","value":"1.1"},{"key":"http.host","type":"string","value":"frontend-proxy:8080"},{"key":"http.method","type":"string","value":"POST"},{"key":"http.scheme","type":"string","value":"http"},{"key":"http.status_code","type":"string","value":"200"},{"key":"http.user_agent","type":"string","value":"python-requests/2.32.4"},{"key":"net.host.name","type":"string","value":"frontend-proxy"},{"key":"net.peer.ip","type":"string","value":"172.18.0.26"},{"key":"net.transport","type":"string","value":"ip_tcp"},{"key":"error","type":"string","value":"unset"},{"key":"http.request_content_length_uncompressed","type":"string","value":"388"},{"key":"http.status_text","type":"string","value":"OK"},{"key":"http.target","type":"string","value":"/api/checkout"},{"key":"http.url","type":"string","value":"http://frontend-proxy:8080/api/checkout"},{"key":"net.host.ip","type":"string","value":"172.18.0.24"},{"key":"net.host.port","type":"string","value":"8080"},{"key":"net.peer.port","type":"string","value":"35632"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":1988,"logs":[],"operationName":"charge","processID":"p4","references":[{"refType":"CHILD_OF","spanID":"df89f1712cb9fdec","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"f30e92001c694787","startTime":1750044449743000,"tags":[{"key":"span.kind","type":"string","value":"internal"},{"key":"otel.scope.name","type":"string","value":"payment"},{"key":"app.payment.card_type","type":"string","value":"visa"},{"key":"app.payment.card_valid","type":"string","value":"true"},{"key":"app.payment.charged","type":"string","value":"false"},{"key":"error","type":"string","value":"unset"},{"key":"app.loyalty.level","type":"string","value":"silver"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":6,"logs":[],"operationName":"resolveBoolean","processID":"p5","references":[{"refType":"CHILD_OF","spanID":"3af2ca071042ef47","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"ab8c870e76bbe57f","startTime":1750044449753032,"tags":[{"key":"error","type":"string","value":"unset"},{"key":"span.kind","type":"string","value":"internal"},{"key":"otel.scope.name","type":"string","value":"jsonEvaluator"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":70,"logs":[],"operationName":"resolveBoolean","processID":"p5","references":[{"refType":"CHILD_OF","spanID":"9d054ff4aeb2b518","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"3af2ca071042ef47","startTime":1750044449753027,"tags":[{"key":"error","type":"string","value":"unset"},{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"flagd.evaluation.v1"},{"key":"feature_flag.key","type":"string","value":"cartFailure"},{"key":"feature_flag.provider_name","type":"string","value":"flagd"},{"key":"feature_flag.variant","type":"string","value":"off"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":19817,"logs":[{"timestamp":1750044449735392,"fields":[{"key":"event","type":"string","value":"Received Quote"},{"key":"app.shipping.cost.total","type":"string","value":"227.50"}]}],"operationName":"/get-quote","processID":"p6","references":[{"refType":"CHILD_OF","spanID":"7b92ebafc9a2a0f1","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"599cbbf8e81ddaca","startTime":1750044449715635,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"opentelemetry-instrumentation-actix-web"},{"key":"otel.scope.version","type":"string","value":"0.22.0"},{"key":"client.address","type":"string","value":"172.18.0.23"},{"key":"http.request.method","type":"string","value":"POST"},{"key":"http.response.status_code","type":"string","value":"200"},{"key":"http.route","type":"string","value":"/get-quote"},{"key":"network.protocol.version","type":"string","value":"1.1"},{"key":"server.address","type":"string","value":"shipping"},{"key":"server.port","type":"string","value":"50050"},{"key":"url.path","type":"string","value":"/get-quote"},{"key":"url.scheme","type":"string","value":"http"},{"key":"user_agent.original","type":"string","value":"Go-http-client/1.1"},{"key":"error","type":"string","value":"unset"},{"key":"app.shipping.cost.total","type":"string","value":"227.50"},{"key":"messaging.message.body.size","type":"string","value":"182"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":283,"logs":[],"operationName":"sinatra.render_template","processID":"p1","references":[{"refType":"CHILD_OF","spanID":"1fd5f529c2dd316b","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"bc5f262c2f7d9bb5","startTime":1750044449770317,"tags":[{"key":"span.kind","type":"string","value":"internal"},{"key":"otel.scope.name","type":"string","value":"OpenTelemetry::Instrumentation::Sinatra"},{"key":"otel.scope.version","type":"string","value":"0.25.0"},{"key":"error","type":"string","value":"unset"},{"key":"sinatra.template_name","type":"string","value":"layout"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":961,"logs":[],"operationName":"sinatra.render_template","processID":"p1","references":[{"refType":"CHILD_OF","spanID":"032bf7007e123e8d","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"1fd5f529c2dd316b","startTime":1750044449769761,"tags":[{"key":"span.kind","type":"string","value":"internal"},{"key":"otel.scope.name","type":"string","value":"OpenTelemetry::Instrumentation::Sinatra"},{"key":"otel.scope.version","type":"string","value":"0.25.0"},{"key":"error","type":"string","value":"unset"},{"key":"sinatra.template_name","type":"string","value":"confirmation"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":6755,"logs":[],"operationName":"oteldemo.PaymentService/Charge","processID":"p7","references":[{"refType":"CHILD_OF","spanID":"7683762fa74ffd1c","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"530667cc212dd6ed","startTime":1750044449739280,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"Charge"},{"key":"rpc.service","type":"string","value":"oteldemo.PaymentService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"server.address","type":"string","value":"172.18.0.14"},{"key":"server.port","type":"string","value":"50051"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":1831,"logs":[],"operationName":"oteldemo.CartService/GetCart","processID":"p7","references":[{"refType":"CHILD_OF","spanID":"96f2298052cc3fda","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"111cb151fdd9a915","startTime":1750044449708652,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"GetCart"},{"key":"rpc.service","type":"string","value":"oteldemo.CartService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"server.address","type":"string","value":"172.18.0.10"},{"key":"server.port","type":"string","value":"7070"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":46,"logs":[],"operationName":"/ship-order","processID":"p6","references":[{"refType":"CHILD_OF","spanID":"92345ad5d7cb4190","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"d1253691f90f5b95","startTime":1750044449746781,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"opentelemetry-instrumentation-actix-web"},{"key":"otel.scope.version","type":"string","value":"0.22.0"},{"key":"client.address","type":"string","value":"172.18.0.23"},{"key":"http.request.method","type":"string","value":"POST"},{"key":"http.response.status_code","type":"string","value":"200"},{"key":"http.route","type":"string","value":"/ship-order"},{"key":"network.protocol.version","type":"string","value":"1.1"},{"key":"server.address","type":"string","value":"shipping"},{"key":"server.port","type":"string","value":"50050"},{"key":"url.path","type":"string","value":"/ship-order"},{"key":"url.scheme","type":"string","value":"http"},{"key":"user_agent.original","type":"string","value":"Go-http-client/1.1"},{"key":"error","type":"string","value":"unset"},{"key":"messaging.message.body.size","type":"string","value":"182"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":128,"logs":[{"timestamp":1750044449717887,"fields":[{"key":"event","type":"string","value":"Calculating quote"}]},{"timestamp":1750044449717919,"fields":[{"key":"event","type":"string","value":"Quote calculated, returning its value"}]}],"operationName":"calculate-quote","processID":"p2","references":[{"refType":"CHILD_OF","spanID":"ea80042fbe6e5887","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"0b119b964828c67b","startTime":1750044449717886,"tags":[{"key":"span.kind","type":"string","value":"internal"},{"key":"otel.scope.name","type":"string","value":"manual-instrumentation"},{"key":"error","type":"string","value":"unset"},{"key":"app.quote.cost.total","type":"string","value":"227.5"},{"key":"app.quote.items.count","type":"string","value":"5"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":78545,"logs":[],"operationName":"router frontend egress","processID":"p8","references":[{"refType":"CHILD_OF","spanID":"d66da216bedd159f","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"df1b3d5c8e0ab6be","startTime":1750044449701376,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"component","type":"string","value":"proxy"},{"key":"http.protocol","type":"string","value":"HTTP/1.1"},{"key":"peer.address","type":"string","value":"172.18.0.24:8080"},{"key":"upstream_address","type":"string","value":"172.18.0.24:8080"},{"key":"upstream_cluster","type":"string","value":"frontend"},{"key":"upstream_cluster.name","type":"string","value":"frontend"},{"key":"error","type":"string","value":"unset"},{"key":"http.status_code","type":"string","value":"200"},{"key":"response_flags","type":"string","value":"-"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":915,"logs":[{"timestamp":1750044449709335,"fields":[{"key":"event","type":"string","value":"Fetch cart"}]}],"operationName":"POST /oteldemo.CartService/GetCart","processID":"p9","references":[{"refType":"CHILD_OF","spanID":"111cb151fdd9a915","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"fefa4832f9254043","startTime":1750044449709238,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"Microsoft.AspNetCore"},{"key":"grpc.method","type":"string","value":"/oteldemo.CartService/GetCart"},{"key":"grpc.status_code","type":"string","value":"0"},{"key":"http.request.method","type":"string","value":"POST"},{"key":"http.response.status_code","type":"string","value":"200"},{"key":"http.route","type":"string","value":"/oteldemo.CartService/GetCart"},{"key":"network.protocol.version","type":"string","value":"2"},{"key":"server.address","type":"string","value":"cart"},{"key":"server.port","type":"string","value":"7070"},{"key":"url.path","type":"string","value":"/oteldemo.CartService/GetCart"},{"key":"url.scheme","type":"string","value":"http"},{"key":"error","type":"string","value":"unset"},{"key":"app.cart.items.count","type":"string","value":"5"},{"key":"app.user.id","type":"string","value":"d526648e-4a61-11f0-8b6b-b20e5443dfb5"},{"key":"user_agent.original","type":"string","value":"grpc-go/1.72.2"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":710,"logs":[],"operationName":"oteldemo.ProductCatalogService/GetProduct","processID":"p7","references":[{"refType":"CHILD_OF","spanID":"96f2298052cc3fda","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"7e5e7c2f1ea9cb0b","startTime":1750044449710565,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"GetProduct"},{"key":"rpc.service","type":"string","value":"oteldemo.ProductCatalogService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"server.address","type":"string","value":"172.18.0.19"},{"key":"server.port","type":"string","value":"3550"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":69871,"logs":[{"timestamp":1750044449737830,"fields":[{"key":"event","type":"string","value":"prepared"}]},{"timestamp":1750044449739261,"fields":[{"key":"feature_flag.key","type":"string","value":"paymentUnreachable"},{"key":"feature_flag.provider_name","type":"string","value":"flagd"},{"key":"feature_flag.variant","type":"string","value":"off"},{"key":"event","type":"string","value":"feature_flag"}]},{"timestamp":1750044449746517,"fields":[{"key":"event","type":"string","value":"charged"},{"key":"app.payment.transaction.id","type":"string","value":"bbf912fe-0a55-4704-8eb9-02d43f60297d"}]},{"timestamp":1750044449746988,"fields":[{"key":"event","type":"string","value":"shipped"},{"key":"app.shipping.tracking.id","type":"string","value":"4668b5f9-17e2-4311-8b20-c7cf3b08ab39"}]},{"timestamp":1750044449776318,"fields":[{"key":"feature_flag.key","type":"string","value":"kafkaQueueProblems"},{"key":"feature_flag.provider_name","type":"string","value":"flagd"},{"key":"feature_flag.variant","type":"string","value":"off"},{"key":"event","type":"string","value":"feature_flag"}]}],"operationName":"oteldemo.CheckoutService/PlaceOrder","processID":"p7","references":[{"refType":"CHILD_OF","spanID":"b1cf4a62984b9984","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"7683762fa74ffd1c","startTime":1750044449706551,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"app.order.items.count","type":"string","value":"1"},{"key":"app.user.currency","type":"string","value":"USD"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"PlaceOrder"},{"key":"rpc.service","type":"string","value":"oteldemo.CheckoutService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"server.address","type":"string","value":"172.18.0.24"},{"key":"server.port","type":"string","value":"38682"},{"key":"error","type":"string","value":"unset"},{"key":"app.order.amount","type":"string","value":"1102"},{"key":"app.order.id","type":"string","value":"d52a1b43-4a61-11f0-9e2b-96226e8767f9"},{"key":"app.shipping.amount","type":"string","value":"227"},{"key":"app.shipping.tracking.id","type":"string","value":"4668b5f9-17e2-4311-8b20-c7cf3b08ab39"},{"key":"app.user.id","type":"string","value":"d526648e-4a61-11f0-8b6b-b20e5443dfb5"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":8349,"logs":[],"operationName":"POST /send_order_confirmation","processID":"p1","references":[{"refType":"CHILD_OF","spanID":"d96adf1246ad7d75","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"739cd04d718779ae","startTime":1750044449766969,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"OpenTelemetry::Instrumentation::Rack"},{"key":"otel.scope.version","type":"string","value":"0.26.0"},{"key":"http.host","type":"string","value":"email:6060"},{"key":"http.method","type":"string","value":"POST"},{"key":"http.route","type":"string","value":"/send_order_confirmation"},{"key":"http.scheme","type":"string","value":"http"},{"key":"http.status_code","type":"string","value":"200"},{"key":"http.target","type":"string","value":"/send_order_confirmation"},{"key":"http.user_agent","type":"string","value":"Go-http-client/1.1"},{"key":"error","type":"string","value":"unset"},{"key":"app.order.id","type":"string","value":"d52a1b43-4a61-11f0-9e2b-96226e8767f9"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":74743,"logs":[],"operationName":"grpc.oteldemo.CheckoutService/PlaceOrder","processID":"p3","references":[{"refType":"CHILD_OF","spanID":"6b73da57ebca1b82","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"b1cf4a62984b9984","startTime":1750044449702000,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"@opentelemetry/instrumentation-grpc"},{"key":"otel.scope.version","type":"string","value":"0.57.1"},{"key":"net.peer.name","type":"string","value":"checkout"},{"key":"net.peer.port","type":"string","value":"5050"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"PlaceOrder"},{"key":"rpc.service","type":"string","value":"oteldemo.CheckoutService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":12631,"logs":[],"operationName":"oteldemo.CartService/EmptyCart","processID":"p7","references":[{"refType":"CHILD_OF","spanID":"7683762fa74ffd1c","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"4e08d386db6de0e6","startTime":1750044449747019,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"EmptyCart"},{"key":"rpc.service","type":"string","value":"oteldemo.CartService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"server.address","type":"string","value":"172.18.0.10"},{"key":"server.port","type":"string","value":"7070"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":11927,"logs":[{"timestamp":1750044449747830,"fields":[{"key":"event","type":"string","value":"Empty cart"}]},{"timestamp":1750044449755100,"fields":[{"key":"feature_flag.key","type":"string","value":"cartFailure"},{"key":"feature_flag.provider_name","type":"string","value":"flagd Provider"},{"key":"feature_flag.variant","type":"string","value":"off"},{"key":"event","type":"string","value":"feature_flag"}]}],"operationName":"POST /oteldemo.CartService/EmptyCart","processID":"p9","references":[{"refType":"CHILD_OF","spanID":"4e08d386db6de0e6","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"d8802687844ff0da","startTime":1750044449747360,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"Microsoft.AspNetCore"},{"key":"app.user.id","type":"string","value":"d526648e-4a61-11f0-8b6b-b20e5443dfb5"},{"key":"feature_flag.key","type":"string","value":"cartFailure"},{"key":"feature_flag.provider_name","type":"string","value":"flagd Provider"},{"key":"feature_flag.variant","type":"string","value":"off"},{"key":"grpc.method","type":"string","value":"/oteldemo.CartService/EmptyCart"},{"key":"grpc.status_code","type":"string","value":"0"},{"key":"http.request.method","type":"string","value":"POST"},{"key":"http.response.status_code","type":"string","value":"200"},{"key":"http.route","type":"string","value":"/oteldemo.CartService/EmptyCart"},{"key":"network.protocol.version","type":"string","value":"2"},{"key":"server.address","type":"string","value":"cart"},{"key":"server.port","type":"string","value":"7070"},{"key":"url.path","type":"string","value":"/oteldemo.CartService/EmptyCart"},{"key":"url.scheme","type":"string","value":"http"},{"key":"user_agent.original","type":"string","value":"grpc-go/1.72.2"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":1733,"logs":[],"operationName":"grpc.oteldemo.ProductCatalogService/GetProduct","processID":"p3","references":[{"refType":"CHILD_OF","spanID":"6b73da57ebca1b82","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"394722a3d65e5bee","startTime":1750044449777000,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"@opentelemetry/instrumentation-grpc"},{"key":"otel.scope.version","type":"string","value":"0.57.1"},{"key":"net.peer.name","type":"string","value":"product-catalog"},{"key":"net.peer.port","type":"string","value":"3550"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"GetProduct"},{"key":"rpc.service","type":"string","value":"oteldemo.ProductCatalogService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":30309,"logs":[],"operationName":"prepareOrderItemsAndShippingQuoteFromCart","processID":"p7","references":[{"refType":"CHILD_OF","spanID":"7683762fa74ffd1c","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"96f2298052cc3fda","startTime":1750044449707511,"tags":[{"key":"span.kind","type":"string","value":"internal"},{"key":"otel.scope.name","type":"string","value":"checkout"},{"key":"app.order.items.count","type":"string","value":"1"},{"key":"error","type":"string","value":"unset"},{"key":"app.cart.items.count","type":"string","value":"5"},{"key":"app.shipping.amount","type":"string","value":"227"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":805,"logs":[],"operationName":"orders publish","processID":"p7","references":[{"refType":"CHILD_OF","spanID":"7683762fa74ffd1c","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"842ad77105e18d23","startTime":1750044449775517,"tags":[{"key":"span.kind","type":"string","value":"producer"},{"key":"otel.scope.name","type":"string","value":"checkout"},{"key":"messaging.destination.name","type":"string","value":"orders"},{"key":"messaging.kafka.destination.partition","type":"string","value":"0"},{"key":"messaging.kafka.message.offset","type":"string","value":"0"},{"key":"messaging.kafka.producer.success","type":"string","value":"true"},{"key":"messaging.operation","type":"string","value":"publish"},{"key":"messaging.system","type":"string","value":"kafka"},{"key":"network.transport","type":"string","value":"tcp"},{"key":"peer.service","type":"string","value":"kafka"},{"key":"error","type":"string","value":"unset"},{"key":"messaging.kafka.producer.duration_ms","type":"string","value":"0"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":352,"logs":[{"timestamp":1750044449709386,"fields":[{"key":"event","type":"string","value":"Enqueued"}]},{"timestamp":1750044449709400,"fields":[{"key":"event","type":"string","value":"Sent"}]},{"timestamp":1750044449709718,"fields":[{"key":"event","type":"string","value":"ResponseReceived"}]}],"operationName":"HGET","processID":"p9","references":[{"refType":"CHILD_OF","spanID":"fefa4832f9254043","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"1c6fa81981e4960c","startTime":1750044449709366,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"OpenTelemetry.Instrumentation.StackExchangeRedis"},{"key":"otel.scope.version","type":"string","value":"1.11.0-beta.2"},{"key":"db.redis.database_index","type":"string","value":"0"},{"key":"db.redis.flags","type":"string","value":"None"},{"key":"db.system","type":"string","value":"redis"},{"key":"server.address","type":"string","value":"valkey-cart"},{"key":"server.port","type":"string","value":"6379"},{"key":"error","type":"string","value":"unset"},{"key":"db.statement","type":"string","value":"HGET d526648e-4a61-11f0-8b6b-b20e5443dfb5"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":22024,"logs":[],"operationName":"HTTP POST","processID":"p7","references":[{"refType":"CHILD_OF","spanID":"96f2298052cc3fda","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"7b92ebafc9a2a0f1","startTime":1750044449713664,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"http.request.method","type":"string","value":"POST"},{"key":"http.response.status_code","type":"string","value":"200"},{"key":"network.protocol.version","type":"string","value":"1.1"},{"key":"error","type":"string","value":"unset"},{"key":"server.address","type":"string","value":"shipping"},{"key":"server.port","type":"string","value":"50050"},{"key":"url.full","type":"string","value":"http://shipping:50050/get-quote"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":391,"logs":[],"operationName":"HTTP POST","processID":"p7","references":[{"refType":"CHILD_OF","spanID":"7683762fa74ffd1c","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"92345ad5d7cb4190","startTime":1750044449746559,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"http.request.method","type":"string","value":"POST"},{"key":"http.response.status_code","type":"string","value":"200"},{"key":"network.protocol.version","type":"string","value":"1.1"},{"key":"error","type":"string","value":"unset"},{"key":"server.address","type":"string","value":"shipping"},{"key":"server.port","type":"string","value":"50050"},{"key":"url.full","type":"string","value":"http://shipping:50050/ship-order"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":4711,"logs":[],"operationName":"POST","processID":"p9","references":[{"refType":"CHILD_OF","spanID":"64e503f233846241","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"31d9931c1b054f86","startTime":1750044449749545,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"System.Net.Http"},{"key":"http.request.method","type":"string","value":"POST"},{"key":"http.response.status_code","type":"string","value":"200"},{"key":"network.protocol.version","type":"string","value":"2"},{"key":"server.address","type":"string","value":"flagd"},{"key":"server.port","type":"string","value":"8013"},{"key":"url.full","type":"string","value":"http://flagd:8013/flagd.evaluation.v1.Service/ResolveBoolean"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":15663,"logs":[],"operationName":"HTTP POST","processID":"p7","references":[{"refType":"CHILD_OF","spanID":"7683762fa74ffd1c","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"d96adf1246ad7d75","startTime":1750044449759771,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"http.request.method","type":"string","value":"POST"},{"key":"http.response.status_code","type":"string","value":"200"},{"key":"network.protocol.version","type":"string","value":"1.1"},{"key":"error","type":"string","value":"unset"},{"key":"server.address","type":"string","value":"email"},{"key":"server.port","type":"string","value":"6060"},{"key":"url.full","type":"string","value":"http://email:6060/send_order_confirmation"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":3076,"logs":[],"operationName":"grpc.oteldemo.PaymentService/Charge","processID":"p4","references":[{"refType":"CHILD_OF","spanID":"530667cc212dd6ed","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"df89f1712cb9fdec","startTime":1750044449742000,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"@opentelemetry/instrumentation-grpc"},{"key":"otel.scope.version","type":"string","value":"0.57.1"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"Charge"},{"key":"rpc.service","type":"string","value":"oteldemo.PaymentService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"error","type":"string","value":"unset"},{"key":"app.payment.amount","type":"string","value":"1102.50"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":79737,"logs":[],"operationName":"POST","processID":"p10","references":[],"spanID":"10d27d153c44c541","startTime":1750044449700847,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"opentelemetry.instrumentation.requests"},{"key":"otel.scope.version","type":"string","value":"0.55b0"},{"key":"http.method","type":"string","value":"POST"},{"key":"http.status_code","type":"string","value":"200"},{"key":"http.url","type":"string","value":"http://frontend-proxy:8080/api/checkout"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":421,"logs":[{"timestamp":1750044449755249,"fields":[{"key":"event","type":"string","value":"Enqueued"}]},{"timestamp":1750044449755262,"fields":[{"key":"event","type":"string","value":"Sent"}]},{"timestamp":1750044449755655,"fields":[{"key":"event","type":"string","value":"ResponseReceived"}]}],"operationName":"HMSET","processID":"p9","references":[{"refType":"CHILD_OF","spanID":"d8802687844ff0da","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"5f78a21a81d1a9a3","startTime":1750044449755233,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"OpenTelemetry.Instrumentation.StackExchangeRedis"},{"key":"otel.scope.version","type":"string","value":"1.11.0-beta.2"},{"key":"db.redis.database_index","type":"string","value":"0"},{"key":"db.redis.flags","type":"string","value":"DemandMaster"},{"key":"db.system","type":"string","value":"redis"},{"key":"server.address","type":"string","value":"valkey-cart"},{"key":"server.port","type":"string","value":"6379"},{"key":"error","type":"string","value":"unset"},{"key":"db.statement","type":"string","value":"HMSET d526648e-4a61-11f0-8b6b-b20e5443dfb5"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":5855,"logs":[],"operationName":"flagd.evaluation.v1.Service/ResolveBoolean","processID":"p9","references":[{"refType":"CHILD_OF","spanID":"d8802687844ff0da","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"64e503f233846241","startTime":1750044449749012,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"OpenTelemetry.Instrumentation.GrpcNetClient"},{"key":"otel.scope.version","type":"string","value":"1.11.0-beta.2"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"ResolveBoolean"},{"key":"rpc.service","type":"string","value":"flagd.evaluation.v1.Service"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"server.address","type":"string","value":"flagd"},{"key":"server.port","type":"string","value":"8013"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":136,"logs":[{"timestamp":1750044449752991,"fields":[{"key":"message.id","type":"string","value":"1"},{"key":"message.type","type":"string","value":"RECEIVED"},{"key":"event","type":"string","value":"message"},{"key":"message.uncompressed_size","type":"string","value":"15"}]},{"timestamp":1750044449753111,"fields":[{"key":"message.id","type":"string","value":"1"},{"key":"message.type","type":"string","value":"SENT"},{"key":"message.uncompressed_size","type":"string","value":"15"},{"key":"event","type":"string","value":"message"}]}],"operationName":"flagd.evaluation.v1.Service/ResolveBoolean","processID":"p5","references":[{"refType":"CHILD_OF","spanID":"31d9931c1b054f86","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"9d054ff4aeb2b518","startTime":1750044449752984,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"connectrpc.com/otelconnect"},{"key":"otel.scope.version","type":"string","value":"semver:0.6.0-dev"},{"key":"rpc.method","type":"string","value":"ResolveBoolean"},{"key":"rpc.service","type":"string","value":"flagd.evaluation.v1.Service"},{"key":"error","type":"string","value":"unset"},{"key":"net.peer.name","type":"string","value":"172.18.0.10"},{"key":"net.peer.port","type":"string","value":"46838"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.system","type":"string","value":"grpc"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":877,"logs":[{"timestamp":1750044449755696,"fields":[{"key":"event","type":"string","value":"Enqueued"}]},{"timestamp":1750044449755708,"fields":[{"key":"event","type":"string","value":"Sent"}]},{"timestamp":1750044449756563,"fields":[{"key":"event","type":"string","value":"ResponseReceived"}]}],"operationName":"EXPIRE","processID":"p9","references":[{"refType":"CHILD_OF","spanID":"d8802687844ff0da","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"4a42b7a5fa81bdfb","startTime":1750044449755686,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"OpenTelemetry.Instrumentation.StackExchangeRedis"},{"key":"otel.scope.version","type":"string","value":"1.11.0-beta.2"},{"key":"db.redis.database_index","type":"string","value":"0"},{"key":"db.redis.flags","type":"string","value":"DemandMaster"},{"key":"db.system","type":"string","value":"redis"},{"key":"server.address","type":"string","value":"valkey-cart"},{"key":"server.port","type":"string","value":"6379"},{"key":"error","type":"string","value":"unset"},{"key":"db.statement","type":"string","value":"EXPIRE d526648e-4a61-11f0-8b6b-b20e5443dfb5"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":2157,"logs":[],"operationName":"oteldemo.CurrencyService/Convert","processID":"p7","references":[{"refType":"CHILD_OF","spanID":"96f2298052cc3fda","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"34a9d7aa3afe1688","startTime":1750044449711310,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"Convert"},{"key":"rpc.service","type":"string","value":"oteldemo.CurrencyService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"server.address","type":"string","value":"172.18.0.18"},{"key":"server.port","type":"string","value":"7001"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":2021,"logs":[],"operationName":"oteldemo.CurrencyService/Convert","processID":"p7","references":[{"refType":"CHILD_OF","spanID":"96f2298052cc3fda","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"11295d69d0e661dd","startTime":1750044449735781,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"Convert"},{"key":"rpc.service","type":"string","value":"oteldemo.CurrencyService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"server.address","type":"string","value":"172.18.0.18"},{"key":"server.port","type":"string","value":"7001"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":77796,"logs":[],"operationName":"POST /api/checkout","processID":"p3","references":[{"refType":"CHILD_OF","spanID":"47c48aa63a0c5a3d","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"01468af9419620f5","startTime":1750044449701000,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"next.js"},{"key":"otel.scope.version","type":"string","value":"0.0.1"},{"key":"http.method","type":"string","value":"POST"},{"key":"http.status_code","type":"string","value":"200"},{"key":"http.target","type":"string","value":"/api/checkout"},{"key":"next.rsc","type":"string","value":"false"},{"key":"next.span_name","type":"string","value":"POST /api/checkout"},{"key":"next.span_type","type":"string","value":"BaseServer.handleRequest"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":19397,"logs":[],"operationName":"POST quote","processID":"p6","references":[{"refType":"CHILD_OF","spanID":"599cbbf8e81ddaca","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"09b03b9b5481c29c","startTime":1750044449715774,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"otel.scope.name","type":"string","value":"opentelemetry-instrumentation-actix-web"},{"key":"otel.scope.version","type":"string","value":"0.22.0"},{"key":"http.request.method","type":"string","value":"POST"},{"key":"http.response.status_code","type":"string","value":"200"},{"key":"server.address","type":"string","value":"quote"},{"key":"server.port","type":"string","value":"8090"},{"key":"url.full","type":"string","value":"http://quote:8090/getquote"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":75,"logs":[{"timestamp":1750044449711020,"fields":[{"key":"event","type":"string","value":"Product Found"}]}],"operationName":"oteldemo.ProductCatalogService/GetProduct","processID":"p11","references":[{"refType":"CHILD_OF","spanID":"7e5e7c2f1ea9cb0b","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"5b997902f830009b","startTime":1750044449710969,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"GetProduct"},{"key":"rpc.service","type":"string","value":"oteldemo.ProductCatalogService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"error","type":"string","value":"unset"},{"key":"app.product.id","type":"string","value":"0PUK6V6EV0"},{"key":"app.product.name","type":"string","value":"Solar System Color Imager"},{"key":"server.address","type":"string","value":"172.18.0.23"},{"key":"server.port","type":"string","value":"56058"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":78,"logs":[{"timestamp":1750044449778775,"fields":[{"key":"event","type":"string","value":"Product Found"}]}],"operationName":"oteldemo.ProductCatalogService/GetProduct","processID":"p11","references":[{"refType":"CHILD_OF","spanID":"394722a3d65e5bee","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"212f00429ff724f5","startTime":1750044449778734,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"},{"key":"otel.scope.version","type":"string","value":"0.61.0"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"GetProduct"},{"key":"rpc.service","type":"string","value":"oteldemo.ProductCatalogService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"error","type":"string","value":"unset"},{"key":"app.product.id","type":"string","value":"0PUK6V6EV0"},{"key":"app.product.name","type":"string","value":"Solar System Color Imager"},{"key":"server.address","type":"string","value":"172.18.0.24"},{"key":"server.port","type":"string","value":"47538"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":597,"logs":[{"timestamp":1750044449711719,"fields":[{"key":"event","type":"string","value":"Processing currency conversion request"}]},{"timestamp":1750044449711741,"fields":[{"key":"event","type":"string","value":"Conversion successful, response sent back"}]}],"operationName":"Currency/Convert","processID":"p12","references":[{"refType":"CHILD_OF","spanID":"34a9d7aa3afe1688","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"42e4324fcb045b99","startTime":1750044449711715,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"currency"},{"key":"app.currency.conversion.from","type":"string","value":"USD"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"Convert"},{"key":"rpc.service","type":"string","value":"oteldemo.CurrencyService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"error","type":"string","value":"false"},{"key":"app.currency.conversion.to","type":"string","value":"USD"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":655,"logs":[{"timestamp":1750044449736390,"fields":[{"key":"event","type":"string","value":"Processing currency conversion request"}]},{"timestamp":1750044449736414,"fields":[{"key":"event","type":"string","value":"Conversion successful, response sent back"}]}],"operationName":"Currency/Convert","processID":"p12","references":[{"refType":"CHILD_OF","spanID":"11295d69d0e661dd","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"adb556f3c99b633d","startTime":1750044449736386,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"otel.scope.name","type":"string","value":"currency"},{"key":"app.currency.conversion.from","type":"string","value":"USD"},{"key":"rpc.grpc.status_code","type":"string","value":"0"},{"key":"rpc.method","type":"string","value":"Convert"},{"key":"rpc.service","type":"string","value":"oteldemo.CurrencyService"},{"key":"rpc.system","type":"string","value":"grpc"},{"key":"error","type":"string","value":"false"},{"key":"app.currency.conversion.to","type":"string","value":"USD"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null},{"duration":78648,"logs":[],"operationName":"ingress","processID":"p8","references":[{"refType":"CHILD_OF","spanID":"10d27d153c44c541","traceID":"9e06226196051d9c3c10dfab343791ad"}],"spanID":"d66da216bedd159f","startTime":1750044449701298,"tags":[{"key":"span.kind","type":"string","value":"server"},{"key":"component","type":"string","value":"proxy"},{"key":"downstream_cluster","type":"string","value":"-"},{"key":"http.protocol","type":"string","value":"HTTP/1.1"},{"key":"node_id","type":"string","value":"-"},{"key":"peer.address","type":"string","value":"172.18.0.25"},{"key":"zone","type":"string","value":"-"},{"key":"guid:x-request-id","type":"string","value":"347edd6d-e273-953e-87f6-7ba07f352331"},{"key":"http.method","type":"string","value":"POST"},{"key":"http.status_code","type":"string","value":"200"},{"key":"http.url","type":"string","value":"http://frontend-proxy:8080/api/checkout"},{"key":"request_size","type":"string","value":"388"},{"key":"response_flags","type":"string","value":"-"},{"key":"response_size","type":"string","value":"857"},{"key":"upstream_cluster","type":"string","value":"frontend"},{"key":"upstream_cluster.name","type":"string","value":"frontend"},{"key":"user_agent","type":"string","value":"python-requests/2.32.4"},{"key":"error","type":"string","value":"unset"}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null}],"traceID":"9e06226196051d9c3c10dfab343791ad","warnings":null}],"errors":null,"limit":0,"offset":0,"total":1} ``` diff --git a/docs/victoriatraces/querying/grafana.md b/docs/victoriatraces/querying/grafana.md index 4cbecd1d1..44cc949da 100644 --- a/docs/victoriatraces/querying/grafana.md +++ b/docs/victoriatraces/querying/grafana.md @@ -16,11 +16,12 @@ aliases: ![Visualization with Grafana](grafana-jaeger.webp) -Simply click "Add new data source" on Grafana, and then fill your VictoriaTraces URL to "Connection.URL". +Simply click "Add new data source" on Grafana, and then fill your VictoriaTraces URL to "Connection.URL". The URL format for VictoriaTraces single-node is: + ``` http://:10428/select/jaeger ``` -Finally, click "Save & Test" at the bottom to complete the process. \ No newline at end of file +Finally, click "Save & Test" at the bottom to complete the process. diff --git a/docs/victoriatraces/querying/jaeger-frontend.md b/docs/victoriatraces/querying/jaeger-frontend.md index ab7a371e9..465c95fba 100644 --- a/docs/victoriatraces/querying/jaeger-frontend.md +++ b/docs/victoriatraces/querying/jaeger-frontend.md @@ -17,17 +17,19 @@ and visualizes the response trace data. ## Deploy Jaeger UI -You can get Jaeger UI from [release page](https://github.com/jaegertracing/jaeger-ui/releases/tag/v1.70.0). +You can get Jaeger UI from [release page](https://github.com/jaegertracing/jaeger-ui/releases/tag/v1.70.0). -As it provides only assets and source code, an HTTP server is needed for serving requests. +As it provides only assets and source code, an HTTP server is needed for serving requests. ### Nginx Example Here's an example where we use Nginx to: + - Serve static content of Jaeger UI. - Forward query requests to VictoriaTraces. Assume you already have: + 1. VictoriaTraces running locally and listening on port `:10428`. 2. Jaeger UI assets (`index.html` and `/static`) located under `/path/to/jaeger-ui/build/`. 3. Nginx Installed. @@ -52,6 +54,7 @@ server { ``` Here are some common paths of Nginx config folder: + ```sh # Ubuntu & Install with apt cd /etc/nginx/sites-available/ @@ -60,4 +63,4 @@ cd /etc/nginx/sites-available/ cd /opt/homebrew/etc/nginx/servers/ ``` -After reloading Nginx, you should be able to visit Jaeger UI on: [http://127.0.0.1:8080/](http://127.0.0.1:8080/). \ No newline at end of file +After reloading Nginx, you should be able to visit Jaeger UI on: [http://127.0.0.1:8080/](http://127.0.0.1:8080/). diff --git a/docs/victoriatraces/quick-start.md b/docs/victoriatraces/quick-start.md index 31508819e..b127c9e14 100644 --- a/docs/victoriatraces/quick-start.md +++ b/docs/victoriatraces/quick-start.md @@ -18,21 +18,26 @@ aliases: ### How to install VictoriaTraces is available in the following distributions: + - Single-server-VictoriaTraces - all-in-one binary that is easy to run and maintain. VictoriaMetrics is available as: + - docker images at [Docker Hub](https://hub.docker.com/r/victoriametrics/victoria-traces) and [Quay](https://quay.io/repository/victoriametrics/victoria-traces). - [Binary releases](https://github.com/VictoriaMetrics/VictoriaTraces/releases/) - [Source code](https://github.com/VictoriaMetrics/VictoriaTraces). See [How to build from sources](https://docs.victoriametrics.com/victoriatraces/#how-to-build-from-sources) #### Starting VictoriaTraces Single Node via Docker + Run the newest available [VictoriaTraces release](https://docs.victoriametrics.com/victoriatraces/changelog/) from [Docker Hub](https://hub.docker.com/r/victoriametrics/victoria-traces) or [Quay](https://quay.io/repository/victoriametrics/victoria-traces): + ```shell docker run --rm -it -p 10428:10428 -v ./victoria-traces-data:/victoria-traces-data \ docker.io/victoriametrics/victoria-traces:v0.2.0 ``` This command will make VictoriaTraces run in the foreground, and store the ingested data to the `victoria-traces-data` directory. You should see the following logs: + ``` 2025-08-08T07:33:13.532Z info VictoriaTraces/app/victoria-traces/main.go:44 starting VictoriaTraces at "[:10428]"... 2025-08-08T07:33:13.532Z info VictoriaTraces/app/vtstorage/main.go:111 opening storage at -storageDataPath=victoria-traces-data @@ -48,17 +53,23 @@ See how to [write](#write-data) or [read](#read-data) from VictoriaTraces. #### Starting VictoriaTraces Single Node from a Binary - Download the correct binary for your OS and architecture from [GitHub](https://github.com/VictoriaMetrics/VictoriaTraces/releases/). Here's an example for `Linux/amd64`: + ```sh curl -L -O https://github.com/VictoriaMetrics/VictoriaTraces/releases/download/v0.2.0/victoria-traces-linux-amd64-v0.2.0.tar.gz ``` + - Extract the archive by running: + ```sh tar -xvf victoria-traces-linux-amd64-v0.2.0.tar.gz ``` + - Go to the binary's folder and start VictoriaTraces: + ```sh ./victoria-traces-prod ``` + This command will make VictoriaTraces run in the foreground, and store the ingested data to the `victoria-traces-data` directory by default. After VictoriaTraces is running, verify VMUI is working by going to `http://:10428/vmui`. @@ -66,11 +77,14 @@ After VictoriaTraces is running, verify VMUI is working by going to `http://:10428/vmui`. -[VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) also provides [Jaeger Query Service JSON APIs](https://www.jaegertracing.io/docs/2.6/apis/#internal-http-json). +[VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) also provides [Jaeger Query Service JSON APIs](https://www.jaegertracing.io/docs/2.6/apis/#internal-http-json). It allows users to visualize trace data on Grafana, by simply adding a [Jaeger datasource](https://grafana.com/docs/grafana/latest/datasources/jaeger/) with VictoriaTraces URL: + ``` http://:10428/select/jaeger ``` @@ -111,4 +126,4 @@ We recommend installing Grafana dashboard for [VictoriaTraces single-node](https We recommend setting up [alerts](https://github.com/VictoriaMetrics/VictoriaTraces/blob/master/deployment/docker/rules/alerts-vtraces.yml) via [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) or via Prometheus. -VictoriaTraces emits its own logs to stdout. It is recommended to investigate these logs during troubleshooting. \ No newline at end of file +VictoriaTraces emits its own logs to stdout. It is recommended to investigate these logs during troubleshooting. diff --git a/docs/victoriatraces/roadmap.md b/docs/victoriatraces/roadmap.md index 792db2e11..c413c3717 100644 --- a/docs/victoriatraces/roadmap.md +++ b/docs/victoriatraces/roadmap.md @@ -14,13 +14,15 @@ aliases: --- The following items need to be completed before general availability (GA) version: + - [ ] Finalize the data structure and commit to backward compatibility. - [ ] Finalize the data distribution algorithm in the cluster version. The following functionality is planned in the future versions of VictoriaTraces after GA: + - [ ] Provide web UI to visualize traces. - [ ] Provide [HTTP APIs](https://grafana.com/docs/tempo/latest/api_docs/) of Tempo Query-frontend. - [ ] Support tail-based sampling/downsampling. -Refer to [the Roadmap of VictoriaLogs](https://docs.victoriametrics.com/victorialogs/roadmap/#) as well for information -about object storage and retention filters. \ No newline at end of file +Refer to [the Roadmap of VictoriaLogs](https://docs.victoriametrics.com/victorialogs/roadmap/#) as well for information +about object storage and retention filters. diff --git a/docs/victoriatraces/troubleshooting.md b/docs/victoriatraces/troubleshooting.md index 47570f5c3..266290970 100644 --- a/docs/victoriatraces/troubleshooting.md +++ b/docs/victoriatraces/troubleshooting.md @@ -16,6 +16,7 @@ aliases: ## Troubleshooting This document contains troubleshooting guides for most common issues when working with VictoriaTraces: + - [Data ingestion](#data-ingestion) - [Querying](#querying) @@ -48,15 +49,13 @@ VictoriaTraces exposes various metrics, which may help debugging data ingestion - `vt_rows_ingested_total` - the number of ingested [trace span entries](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#data-model) since the last VictoriaTraces restart. If this number increases over time, then trace spans are successfully ingested into VictoriaTraces. The ingested trace spans can be inspected in the following ways: - - By passing `debug=1` parameter to every request to [data ingestion APIs](#http-apis). The ingested spans aren't stored in VictoriaTraces + - By passing `debug=1` parameter to every request to [data ingestion APIs](#http-apis). The ingested spans aren't stored in VictoriaTraces in this case. Instead, they are logged, so they can be investigated later. The `vt_rows_dropped_total` metric is incremented for each logged row. - - By passing `-logIngestedRows` command-line flag to VictoriaTraces. In this case it traces all the ingested data, so it can be investigated later. + - By passing `-logIngestedRows` command-line flag to VictoriaTraces. In this case it traces all the ingested data, so it can be investigated later. - `vt_streams_created_total` - the number of created [trace streams](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#stream-fields) since the last VictoriaTraces restart. If this metric grows rapidly during extended periods of time, then this may lead to [high cardinality issues](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#high-cardinality). The newly created trace streams can be inspected in traces by passing `-logNewStreams` command-line flag to VictoriaTraces. - - ### Querying diff --git a/docs/victoriatraces/vmalert.md b/docs/victoriatraces/vmalert.md index 73448e6cd..56e9be3cc 100644 --- a/docs/victoriatraces/vmalert.md +++ b/docs/victoriatraces/vmalert.md @@ -19,7 +19,7 @@ and [`/select/logsql/stats_query_range`](https://docs.victoriametrics.com/victor These endpoints return the trace spans stats in a format compatible with [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries). It allows using VictoriaTraces as the datasource in vmalert, creating alerting and recording rules via [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/). -_Note: This page provides only integration instructions for vmalert and VictoriaTraces. See the full textbook for vmalert [here](https://docs.victoriametrics.com/victoriametrics/vmalert/). +_Note: This page provides only integration instructions for vmalert and VictoriaTraces. See the full textbook for [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/). ## Quick Start @@ -91,11 +91,11 @@ The following are key flags related to integration with VictoriaTraces: Since there is no intentional search delay in VictoriaTraces, `-rule.evalDelay` can be reduced to a few seconds to accommodate network and ingestion time. ``` -See full list of configuration options [here](https://docs.victoriametrics.com/victoriametrics/vmalert/#configuration). +See full list of [configuration options](https://docs.victoriametrics.com/victoriametrics/vmalert/#configuration). ### Groups -Check the complete group attributes [here](https://docs.victoriametrics.com/victoriametrics/vmalert/#groups). +Check the complete [group attributes](https://docs.victoriametrics.com/victoriametrics/vmalert/#groups). #### Alerting rules @@ -177,7 +177,7 @@ vmalert supports alerting and recording rules backfilling (aka replay) against V -replay.timeTo=2021-05-29T18:40:43Z # to finish replay by, optional. By default, set to the current time ``` -See more details about backfilling [here](https://docs.victoriametrics.com/victoriametrics/vmalert/#rules-backfilling). +See more details about [backfilling](https://docs.victoriametrics.com/victoriametrics/vmalert/#rules-backfilling). ## Performance tip @@ -242,13 +242,13 @@ To persist different rule results to different tenants in VictoriaMetrics, there For example, run vmalert with: - ``` + ```sh ./bin/vmalert -datasource.url=http://localhost:9428 -remoteWrite.url=http://vminsert:8480/insert/multitenant/prometheus ... ``` With the rules below, `recordingTenant123` will be queried from VictoriaTraces tenant `123` and persisted to tenant `123` in VictoriaMetrics, while `recordingTenant123-456:789` will be queried from VictoriaTraces tenant `124` and persisted to tenant `456:789` in VictoriaMetrics. - ``` + ```yaml groups: - name: recordingTenant123 type: vlogs @@ -275,13 +275,13 @@ To persist different rule results to different tenants in VictoriaMetrics, there For example, run vmalert with: - ``` + ```sh ./bin/vmalert -datasource.url=http://localhost:9428 -clusterMode=true -remoteWrite.url=http://vminsert:8480/ ... ``` With the rules below, `recordingTenant123` will be queried from VictoriaTraces tenant `123` and persisted to tenant `123` in VictoriaMetrics, while `recordingTenant123-456:789` will be queried from VictoriaTraces tenant `124` and persisted to tenant `456:789` in VictoriaMetrics. - ``` + ```yaml groups: - name: recordingTenant123 type: vlogs From a205e8e04417473353cf886c2e2a56b7cf1c2b98 Mon Sep 17 00:00:00 2001 From: Arie Heinrich Date: Sat, 13 Sep 2025 16:12:48 +0200 Subject: [PATCH 2/2] Spelling and Markdown --- app/vtselect/logsql/logsql.go | 2 +- apptest/README.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/vtselect/logsql/logsql.go b/app/vtselect/logsql/logsql.go index 1c974cdb1..acfd524b0 100644 --- a/app/vtselect/logsql/logsql.go +++ b/app/vtselect/logsql/logsql.go @@ -543,7 +543,7 @@ func ProcessStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.R WriteValuesWithHitsJSON(w, streams) } -// ProcessLiveTailRequest processes live tailing request to /select/logsq/tail +// ProcessLiveTailRequest processes live tailing request to /select/logsql/tail // // See https://docs.victoriametrics.com/victorialogs/querying/#live-tailing func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) { diff --git a/apptest/README.md b/apptest/README.md index 70acd0b5d..cc1e197fe 100644 --- a/apptest/README.md +++ b/apptest/README.md @@ -19,11 +19,11 @@ work together as a system. The package provides a collection of helpers to start applications and make queries to them: -- `app.go` - contains the generic code for staring an application and should +- `app.go` - contains the generic code for staring an application and should not be used by integration tests directly. -- `{vtsingle,etc}.go` - build on top of `app.go` and provide the +- `{vtsingle,etc}.go` - build on top of `app.go` and provide the code for staring a specific application. -- `client.go` - provides helper functions for sending HTTP requests to +- `client.go` - provides helper functions for sending HTTP requests to applications. The integration tests themselves reside in `tests/*_test.go` files. Apart from having