Image Render: Fix Tracing

.gitignore

@@ -40,4 +40,6 @@ scripts/tmp
 
 tests/testdata/diff_*
 
-cache
+cache
+
+/devenv/docker/tracing/tempo-data/

README.md

@@ -8,6 +8,8 @@ A Grafana backend plugin that handles rendering panels and dashboards to PNGs us
 - Windows (x64)
 - Mac OS X (x64)
 
+For Mac ARM64, you need to [build the plugin from source](https://github.com/grafana/grafana-image-renderer/blob/master/docs/building_from_source.md) or use the [remote rendering installation](https://github.com/grafana/grafana-image-renderer?tab=readme-ov-file#remote-rendering-service-installation).
+
 ### Dependencies
 
 This plugin is packaged in a single executable with [Node.js](https://nodejs.org/) runtime and [Chromium browser](https://www.chromium.org/Home).
@@ -132,45 +134,4 @@ For available configuration settings, please refer to [Grafana Image Rendering d
 ## Troubleshooting
 
 For troubleshooting help, refer to
-[Grafana Image Rendering troubleshooting documentation](https://grafana.com/docs/grafana/latest/image-rendering/troubleshooting/).
-
-## Testing
-
-In order to run the image-renderer automated test suites, you need to run the following command from the root folder:
-
-```
-yarn test
-```
-
-This will launch a Grafana instance in Docker and, then, run the test suites.
-
-_Notes:_
-
-If there are some expected changes in the reference image files (located in `/tests/testdata`), run `yarn test-update` and push the updated references.
-
-If the tests are failing and you want to see the difference between the image you get and the reference image, run `yarn test-diff`. This will generate images (called `diff_<test case>.png`) containing the differences in the `/tests/testdata` folder.
-
-### Fixing Drone issues
-
-If tests are successful in your local environement but fail in Drone. You can follow these steps to run the tests in an environment similar to the Drone pipeline. This will mount your local files of the `grafana-image-renderer` repo in the Docker image so any change that happens in the Docker image will be available in your local environment. This allows you to run `yarn test-diff` and `yarn test-update` in Docker and see the results locally. 
-
-1. Run the Drone environment in Docker:
-
-```
-cd ./devenv/docker/drone
-docker-compose up
-```
-
-2. Open a terminal within the `drone-docker-puppeteer` container and run the following commands:
-
-```
-cd /drone/src
-PUPPETEER_CACHE_DIR=/drone/src/cache yarn install --frozen-lockfile --no-progress
-PUPPETEER_CACHE_DIR=/drone/src/cache CI=true yarn test-ci
-```
-
-_Notes:_
-The tests might take longer in the Docker container. If you run into timeout issues, you can run the test command with the `--testTimeout option`:
-```
-PUPPETEER_CACHE_DIR=/drone/src/cache CI=true yarn test-ci --testTimeout=10000
-```
+[Grafana Image Rendering troubleshooting documentation](https://grafana.com/docs/grafana/latest/image-rendering/troubleshooting/).

default.json

@@ -51,6 +51,11 @@
 
     "verboseLogging": false,
     "dumpio": false,
-    "timingMetrics": false
+    "timingMetrics": false,
+
+    "tracing": {
+      "url": "",
+      "serviceName": ""
+    }
   }
 }

dev.json

@@ -50,6 +50,11 @@
 
     "verboseLogging": true,
     "dumpio": false,
-    "timingMetrics": true
+    "timingMetrics": true,
+
+    "tracing": {
+      "url": "http://localhost:4318/v1/traces",
+      "serviceName": ""
+    }
   }
 }

devenv/docker/custom-config/config.json

@@ -41,6 +41,11 @@
     },
 
     "verboseLogging": false,
-    "dumpio": false
+    "dumpio": false,
+
+    "tracing": {
+      "url": "",
+      "serviceName": ""
+    }
   }
 }

devenv/docker/tracing/config.json

@@ -0,0 +1,51 @@
+{
+  "service": {
+    "host": null,
+    "port": 8081,
+
+    "metrics": {
+      "enabled": true,
+      "collectDefaultMetrics": true,
+      "requestDurationBuckets": [1, 5, 7, 9, 11, 13, 15, 20, 30]
+    },
+
+    "logging": {
+      "level": "debug",
+      "console": {
+        "json": true,
+        "colorize": false
+      }
+    }
+  },
+  "rendering": {
+    "chromeBin": null,
+    "args": [
+      "--no-sandbox"
+    ],
+    "ignoresHttpsErrors": false,
+
+    "timezone": null,
+    "acceptLanguage": null,
+    "width": 1000,
+    "height": 500,
+    "deviceScaleFactor": 1,
+    "maxWidth": 3080,
+    "maxHeight": 3000,
+    "maxDeviceScaleFactor": 4,
+
+    "mode": "clustered",
+    "clustering": {
+      "mode": "context",
+      "maxConcurrency": 5,
+      "timeout": 30
+    },
+
+    "verboseLogging": false,
+    "dumpio": false,
+
+    "tracing": {
+      "url": " http://tempo:4318/v1/traces",
+      "serviceName": ""
+    }
+  }
+}

devenv/docker/tracing/docker-compose.yaml

@@ -0,0 +1,71 @@
+services:
+
+  # Tempo runs as user 10001, and docker compose creates the volume as root.
+  # As such, we need to chown the volume in order for Tempo to start correctly.
+  init:
+    image: &tempoImage grafana/tempo:latest
+    user: root
+    entrypoint:
+      - "chown"
+      - "10001:10001"
+      - "/var/tempo"
+    volumes:
+      - ./tempo-data:/var/tempo
+
+  memcached:
+    image: memcached:1.6.29
+    container_name: memcached
+    ports:
+      - "11211:11211"
+    environment:
+      - MEMCACHED_MAX_MEMORY=64m  # Set the maximum memory usage
+      - MEMCACHED_THREADS=4       # Number of threads to use
+
+  tempo:
+    image: *tempoImage
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    volumes:
+      - ./tempo.yaml:/etc/tempo.yaml
+      - ./tempo-data:/var/tempo
+    ports:
+      - "14268:14268"  # jaeger ingest
+      - "3200:3200"   # tempo
+      - "9095:9095" # tempo grpc
+      - "4317:4317"  # otlp grpc
+      - "4318:4318"  # otlp http
+      - "9411:9411"   # zipkin
+    depends_on:
+      - init
+      - memcached
+
+  prometheus:
+    image: prom/prometheus:latest
+    command:
+      - --config.file=/etc/prometheus.yaml
+      - --web.enable-remote-write-receiver
+      - --enable-feature=exemplar-storage
+      - --enable-feature=native-histograms
+    volumes:
+      - ./prometheus.yaml:/etc/prometheus.yaml
+    ports:
+      - "9090:9090"
+
+  grafana:
+    image: grafana/grafana:latest
+    volumes:
+      - ./grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+    environment:
+      GF_RENDERING_SERVER_URL: http://renderer:8081/render
+      GF_RENDERING_CALLBACK_URL: http://grafana:3000/
+      GF_RENDERING_TRACING_URL: http://tempo:4318/v1/traces
+      GF_LOG_FILTERS: rendering:debug
+      GF_INSTALL_PLUGINS: https://storage.googleapis.com/integration-artifacts/grafana-exploretraces-app/grafana-exploretraces-app-latest.zip;grafana-traces-app
+      GF_TRACING_OPENTELEMETRY_OTLP_ADDRESS: tempo:4317
+    ports:
+      - "3000:3000"
+  renderer:
+    image: grafana/grafana-image-renderer:latest
+    ports:
+      - 8081:8081
+    volumes:
+      - ./config.json:/usr/src/app/config.json

devenv/docker/tracing/grafana-datasources.yaml

@@ -0,0 +1,33 @@
+apiVersion: 1
+
+datasources:
+- name: Prometheus
+  type: prometheus
+  uid: prometheus
+  access: proxy
+  orgId: 1
+  url: http://prometheus:9090
+  basicAuth: false
+  isDefault: false
+  version: 1
+  editable: false
+  jsonData:
+    httpMethod: GET
+- name: Tempo
+  type: tempo
+  access: proxy
+  orgId: 1
+  url: http://tempo:3200
+  basicAuth: false
+  isDefault: true
+  version: 1
+  editable: false
+  apiVersion: 1
+  uid: tempo
+  jsonData:
+    httpMethod: GET
+    serviceMap:
+      datasourceUid: prometheus
+    streamingEnabled:
+      search: true
+

devenv/docker/tracing/prometheus.yaml

@@ -0,0 +1,11 @@
+global:
+  scrape_interval:     15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: [ 'localhost:9090' ]
+  - job_name: 'tempo'
+    static_configs:
+      - targets: [ 'tempo:3200' ]

devenv/docker/tracing/tempo.yaml

@@ -0,0 +1,91 @@
+stream_over_http_enabled: true
+server:
+  http_listen_port: 3200
+  log_level: info
+
+
+cache:
+  background:
+    writeback_goroutines: 5
+  caches:
+  - roles:
+    - frontend-search  
+    memcached: 
+      addresses: dns+memcached:11211
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+    metadata_slo:
+        duration_slo: 5s
+        throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 100ms
+  metrics:
+    max_duration: 120h                # maximum duration of a metrics query, increase for local setups
+    query_backend_after: 5m
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+
+distributor:
+  receivers:                           # this configuration will listen on all ports and protocols that tempo is capable of.
+    jaeger:                            # the receives all come from the OpenTelemetry collector.  more configuration information can
+      protocols:                       # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver
+        thrift_http:                   #
+          endpoint: "tempo:14268"      # for a production deployment you should only enable the receivers you need!
+        grpc:
+          endpoint: "tempo:14250"
+        thrift_binary:
+          endpoint: "tempo:6832"
+        thrift_compact:
+          endpoint: "tempo:6831"
+    zipkin:
+      endpoint: "tempo:9411"
+    otlp:
+      protocols:
+        grpc:
+          endpoint: "tempo:4317"
+        http:
+          endpoint: "tempo:4318"
+    opencensus:
+      endpoint: "tempo:55678"
+
+ingester:
+  max_block_duration: 5m               # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+
+compactor:
+  compaction:
+    block_retention: 24h                # overall Tempo trace retention. set for demo purposes
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: docker-compose
+  storage:
+    path: /var/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+  traces_storage:
+    path: /var/tempo/generator/traces
+  processor:
+    local_blocks:
+      filter_server_spans: false
+      flush_to_storage: true
+
+storage:
+  trace:
+    backend: local                     # backend configuration to use
+    wal:
+      path: /var/tempo/wal             # where to store the wal locally
+    local:
+      path: /var/tempo/blocks
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics, local-blocks] # enables metrics generator
+      generate_native_histograms: both
+

docs/building_from_source.md

@@ -24,6 +24,9 @@ cd grafana-image-renderer
     # build and package for Darwin x64
     make build_package ARCH=darwin-x64-unknown
 
+    # build and package for Mac ARM64
+    make build_package ARCH=darwin-arm64-unknown
+
     # build and package without including Chromium
     make build_package ARCH=<ARCH> SKIP_CHROMIUM=true OUT=plugin-<ARCH>-no-chromium
     ```

docs/index.md

@@ -3,4 +3,5 @@ Use the following links for instructions on how to:
 
 - [Build the image renderer from source](building_from_source.md)
 - [Package plugin as a single executable](package_plugin_as_single_executable.md)
-- [Release and publish a new version](release_new_version.md)
+- [Release and publish a new version](release_new_version.md)
+- [Testing instructions](testing.md)