feat: emit metrics about entity imports

CHANGELOG.md

@@ -1,5 +1,8 @@
 # api
 
+## 10x.14.0 - 12 August 2024
+- Add Prometheus metrics regarding Entity Import feature
+
 ## 10x.13.0 - 08 august 2024
 - Fix typo in WikiController and test
 

Dockerfile

@@ -17,8 +17,9 @@ FROM php:8.2-apache
 RUN apt-get update \
 	# Needed for the imagick php extension install
 	&& apt-get install -y --no-install-recommends libmagickwand-dev libpq-dev \
-	&& echo "" | pecl install imagick \
+	&& echo "" | pecl install imagick redis \
 	&& docker-php-ext-enable imagick \
+	&& docker-php-ext-enable redis \
 	&& docker-php-ext-configure pcntl --enable-pcntl \
 	&& docker-php-ext-install pcntl \
 	&& docker-php-ext-enable pcntl \

app/Http/Controllers/WikiEntityImportController.php

@@ -9,9 +9,27 @@
 use App\WikiEntityImport;
 use App\Jobs\WikiEntityImportJob;
 use Carbon\Carbon;
+use Prometheus\CollectorRegistry;
+use Prometheus\Counter;
 
 class WikiEntityImportController extends Controller
 {
+    private Counter $successfulCounter;
+    private Counter $failedCounter;
+
+    public function __construct(CollectorRegistry $registry)
+    {
+        $this->successfulCounter = $registry->getOrRegisterCounter(
+            config('horizon-exporter.namespace'),
+            'wiki_entity_imports_successful',
+            'The number of successful Entity import records.',
+        );
+        $this->failedCounter = $registry->getOrRegisterCounter(
+            config('horizon-exporter.namespace'),
+            'wiki_entity_imports_failed',
+            'The number of failed Entity import records.',
+        );
+    }
     public function get(Request $request): \Illuminate\Http\JsonResponse
     {
         $validatedInput = $request->validate([
@@ -95,6 +113,13 @@ public function update(Request $request): \Illuminate\Http\JsonResponse
             abort(400, 'Import has to be pending if updated');
         }
 
+        if ($validatedInput['status'] === WikiEntityImportStatus::Failed->value) {
+            $this->failedCounter->inc();
+        }
+        if ($validatedInput['status'] === WikiEntityImportStatus::Success->value) {
+            $this->successfulCounter->inc();
+        }
+
         $import->update([
             'status' => $validatedInput['status'],
             'finished_at' => Carbon::now(),

app/Metrics/WikiEntityImports.php

@@ -0,0 +1,31 @@
+<?php
+
+
+namespace App\Metrics;
+
+use LKDevelopment\HorizonPrometheusExporter\Contracts\Exporter;
+use Prometheus\CollectorRegistry;
+use App\WikiEntityImport;
+use App\WikiEntityImportStatus;
+
+class WikiEntityImports implements Exporter
+{
+    protected $pending;
+    public function metrics(CollectorRegistry $collectorRegistry)
+    {
+        $this->pending = $collectorRegistry->getOrRegisterGauge(
+            config('horizon-exporter.namespace'),
+            'wiki_entity_imports_pending',
+            'The number of pending Entity imports currently being processed.',
+        );
+    }
+
+
+    public function collect()
+    {
+        // counters for failed / success are incremented in the HTTP controller
+        $this->pending->set(
+            WikiEntityImport::where(['status' => WikiEntityImportStatus::Pending])->count()
+        );
+    }
+}

app/Providers/CollectorRegistryProvider.php

@@ -0,0 +1,41 @@
+<?php
+
+namespace App\Providers;
+
+use Illuminate\Support\ServiceProvider;
+use Illuminate\Support\Facades\Config;
+use Prometheus\CollectorRegistry;
+use Prometheus\Storage\Redis;
+use Illuminate\Contracts\Foundation\Application;
+use LKDevelopment\HorizonPrometheusExporter\Repository\ExporterRepository;
+
+
+class CollectorRegistryProvider extends ServiceProvider
+{
+    /**
+     * Register services.
+     */
+    public function register(): void
+    {
+        $this->app->bind(CollectorRegistry::class, function (Application $app) {
+            return new CollectorRegistry(new Redis([
+                'host' => Config::get('database.redis.metrics.host'),
+                'port' => Config::get('database.redis.metrics.port'),
+                'password' => Config::get('database.redis.metrics.password'),
+                'timeout' => 0.1, // in seconds
+                'read_timeout' => '10', // in seconds
+                'persistent_connections' => false
+            ]));
+        }, true);
+    }
+
+    /**
+     * Bootstrap services.
+     */
+    public function boot(): void
+    {
+        ExporterRepository::setRegistry(
+            $this->app->make(CollectorRegistry::class),
+        );
+    }
+}

config/app.php

@@ -179,6 +179,7 @@
         App\Providers\KubernetesClientServiceProvider::class,
         App\Providers\RouteServiceProvider::class,
         App\Providers\ReCaptchaServiceProvider::class,
+        App\Providers\CollectorRegistryProvider::class,
 
         /*
          * Imported from LUMEN

config/database.php

@@ -157,6 +157,11 @@
             'database' => env('REDIS_CACHE_DB', 1),
         ],
 
+        'metrics' => [
+            'host' => env('REDIS_HOST', '127.0.0.1'),
+            'password' => env('REDIS_PASSWORD', null),
+            'port' => env('REDIS_PORT', 6379),
+        ],
     ],
 
 ];

config/horizon-exporter.php

@@ -23,6 +23,7 @@
         \LKDevelopment\HorizonPrometheusExporter\Exporter\RecentJobs::class,
         \App\Metrics\FailedQsBatches::class,
         \App\Metrics\PendingQsBatches::class,
+        \App\Metrics\WikiEntityImports::class,
     ],
 
     /**