Update docs & fix broken features

.github/workflows/instance-deploy-prod.yml

@@ -143,7 +143,7 @@ jobs:
           location: 'East US'
           resourceGroup: ${{ env.resource_group_name }}
           targetPort: 8080
-          environmentVariables: "OTLPEXPORTER__ENDPOINT=http://${{ env.container_instance_monitoring_fqdn }}:18889"
+          environmentVariables: "OTEL_EXPORTER_OTLP_ENDPOINT=http://${{ env.container_instance_monitoring_fqdn }}:18889"
       - name: Deploy website
         uses: azure/container-apps-deploy-action@v1
         with:

docs/Telemetry/DistributedMonitoring.md

@@ -0,0 +1,150 @@
+# Distributed monitoring in dotnet
+
+I want a cheap centralised monitoring solution as I start building out more services and more infrastructure for hobby projects.
+
+After some research, this is achievable using two open source components:
+1. **OpenTelemetry** - libraries that collect telemetry data and export it in a standardised format. The libraries are free, open-source, and generally available across [multiple languages](https://opentelemetry.io/docs/instrumentation/).
+2. **.NET Aspire** - an app that accepts OpenTelemetry formatted data, stores it in memory, and presents it via a dashboard.
+
+## Data generation (OpenTelemetry)
+
+The OpenTelemetry libraries do all of the heavy lifting, although they do need to be configured correctly. For my app, I used the following:
+
+```
+OpenTelemetry.Api.ProviderBuilderExtensions
+OpenTelemetry.Exporter.OpenTelemetryProtocol
+OpenTelemetry.Extensions.Hosting
+OpenTelemetry.Instrumentation.AspNetCore
+OpenTelemetry.Instrumentation.EntityFrameworkCore
+OpenTelemetry.Instrumentation.Runtime
+```
+
+### Logs
+
+Add OpenTelemetry to your logging when configuring your app:
+
+```csharp
+builder.Logging.AddOpenTelemetry(logging =>
+{
+    logging.IncludeFormattedMessage = true;
+    logging.IncludeScopes = true;
+});
+```
+
+### Traces & Metrics
+
+**Traces** are required to create flame graphs, visualising the call stack of a request / process. **Metrics** are important to monitor the health of your app, for example http client requests, open connections, GC heap size.
+
+##### Automatic instrumentation
+
+Automatic instrumentation is simple to set up, and there are already libraries for most of the things you want to monitor. An example of how you can configure metrics & tracing is as follows:
+
+```csharp
+var serviceName = Assembly.GetExecutingAssembly().GetName().Name.ToString();
+var serviceVersion = Assembly.GetExecutingAssembly().GetName().Version.ToString();
+var appResourceBuilder = ResourceBuilder.CreateDefault()
+    .AddService(serviceName: serviceName, serviceVersion: serviceVersion);
+builder.Services
+    .AddMetrics()
+    .AddOpenTelemetry()
+    .ConfigureResource(c => c.AddService(serviceName))
+    .WithMetrics(metricProviderBuilder => metricProviderBuilder
+        .AddMeter(
+            "Microsoft.AspNetCore.Hosting",
+            "Microsoft.AspNetCore.Server.Kestrel",
+            "System.Net.Http",
+            serviceName)
+        .SetResourceBuilder(appResourceBuilder)
+        .AddAspNetCoreInstrumentation()
+        .AddRuntimeInstrumentation())
+    .WithTracing(tracerProviderBuilder => tracerProviderBuilder
+        .AddSource(serviceName)
+        .SetResourceBuilder(appResourceBuilder)
+        .AddAspNetCoreInstrumentation()
+        .AddEntityFrameworkCoreInstrumentation());
+```
+
+Further reading: https://opentelemetry.io/docs/instrumentation/net/automatic/
+
+##### Manual instrumentation
+
+If you have any operations you need to monitor that are not covered in the automatic instrumentation, for example some in memory processing, you can create your own ActivitySource and create custom spans.
+
+```csharp
+public class Filter
+{
+    private static readonly ActivitySource MyActivitySource = new(nameof(Filter), "1.0.0");
+
+    public ICollection<T> ApplyFilter(ICollection<T> input, Func<T, bool> filter)
+    {
+        using var activity = MyActivitySource.StartActivity("ApplyFilter");
+        # your filtering code
+    }
+}
+```
+
+When adding an activity source, ensure you register it accordingly, else the activity monitor won't subscribe to it and you won't get your custom traces:
+
+```csharp
+    .WithTracing(tracerProviderBuilder => tracerProviderBuilder
+        .AddSource(serviceName, nameof(Filter)) // add all classes with an activity source here
+```
+
+Further reading: https://opentelemetry.io/docs/instrumentation/net/manual/
+
+## Exporting data (OpenTelemetry)
+
+Depending on your exporter the configuration will be different. Whilst getting things off the ground I used the console exporter (NuGet package OpenTelemetry.Exporter.Console) as follows:
+
+```csharp
+builder.Services.AddOpenTelemetry()
+    .WithTracing(
+        tracerProviderBuilder => tracerProviderBuilder.AddConsoleExporter()
+    );
+```
+
+Further reading: https://opentelemetry.io/docs/instrumentation/net/exporters/
+
+## Data collection (.NET Aspire)
+
+> .NET Aspire is an opinionated, cloud ready stack for building observable, production ready, distributed applications.
+
+Running the dashboard is simple, and can be done with the following command:
+
+```cmd
+docker run --rm -it -p 18888:18888 -p 18889:18889  mcr.microsoft.com/dotnet/aspire-dashboard:8.0
+```
+
+The two mapped ports are for the following:
+`18888` - Dashboard endpoint
+`18889` - OTLP endpoint
+
+.NET Aspire docs: https://learn.microsoft.com/en-us/dotnet/aspire/get-started/aspire-overview
+
+Once the dashboard is running, you need to configure your app to send data to it. This is done by setting `OTEL_EXPORTER_OTLP_ENDPOINT` (e.g. http://localhost:18889), and then configuring the OTLP exporter as follows:
+
+```csharp
+builder.Services.Configure<OpenTelemetryLoggerOptions>(options => options.AddOtlpExporter())
+    .ConfigureOpenTelemetryMeterProvider(metrics => metrics.AddOtlpExporter())
+    .ConfigureOpenTelemetryTracerProvider(tracing => tracing.AddOtlpExporter());
+```
+
+### CICD
+
+I copy the .NET Aspire Dashboard image to my Azure Container Registry [here](../../.github/workflows/permanent-image-import.yml), and deploy it using terraform [here](../../terraform/instance/container_instances.tf).
+
+## Data visualisation (.NET Aspire)
+
+All data sent from your app to the aspire dashboard is available through the dashboard (e.g. http://localhost:18888)
+
+### Logs
+
+![.NET Aspire Logs](Aspire_logs.png)
+
+### Traces
+
+![.NET Aspire Traces](Aspire_traces.png)
+
+### Metrics
+
+![.NET Aspire Metrics](Aspire_metrics.png)

src/Api/Company.Api/Company.Api.csproj

@@ -16,6 +16,7 @@
     <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.9.0" />
     <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.9.0" />
     <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.9.0" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.EntityFrameworkCore" Version="1.0.0-beta.12" />
     <PackageReference Include="OpenTelemetry.Instrumentation.Runtime" Version="1.9.0" />
     <PackageReference Include="Serilog.AspNetCore" Version="8.0.1" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />

src/Api/Company.Api/Products/ProductsController.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Diagnostics;
 using System.Linq;
 using System.Threading.Tasks;
 using Company.Api.Products.Dtos;
@@ -12,6 +13,8 @@ namespace Company.Api.Products;
 [Route("products")]
 public class ProductsController : ControllerBase
 {
+    private static readonly ActivitySource MyActivitySource = new(nameof(ProductsController), "1.0.0");
+
     private readonly ILogger<ProductsController> _logger;
     private readonly ProductContext _context;
 
@@ -24,6 +27,7 @@ public ProductsController(ILogger<ProductsController> logger, ProductContext con
     [HttpGet("")]
     public async Task<IActionResult> GetProducts() {
         var productDtos = _context.Products;
+        using var activity = MyActivitySource.StartActivity("GetProductsActivity");
         var products = await productDtos.Select(p => p.FromProductDto()).ToListAsync();
         return Ok(products);
     }

src/Api/Company.Api/Program.cs

@@ -1,17 +1,14 @@
 using System;
-using System.Diagnostics.Metrics;
 using System.Reflection;
 using Company.Api.Products;
 using Microsoft.AspNetCore.Builder;
-using Microsoft.AspNetCore.Hosting;
 using Microsoft.Data.Sqlite;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.OpenApi.Models;
-using OpenTelemetry;
 using OpenTelemetry.Logs;
 using OpenTelemetry.Metrics;
 using OpenTelemetry.Resources;
@@ -49,11 +46,10 @@
         .AddAspNetCoreInstrumentation()
         .AddRuntimeInstrumentation())
     .WithTracing(tracerProviderBuilder => tracerProviderBuilder
-        .AddSource(serviceName)
-        .SetResourceBuilder(
-            ResourceBuilder.CreateDefault()
-                .AddService(serviceName: serviceName, serviceVersion: serviceVersion))
-        .AddAspNetCoreInstrumentation());
+        .AddSource(serviceName, nameof(ProductsController))
+        .SetResourceBuilder(appResourceBuilder)
+        .AddAspNetCoreInstrumentation()
+        .AddEntityFrameworkCoreInstrumentation());
 
 var otlpExporterEndpoint = builder.Configuration.GetValue<string>("OTEL_EXPORTER_OTLP_ENDPOINT");
 if (Uri.TryCreate(otlpExporterEndpoint, UriKind.Absolute, out _))