Skip to content
This repository has been archived by the owner on Oct 11, 2023. It is now read-only.

Commit

Permalink
Adding diagnostics instrumentation around CosmosDB failures and proce…
Browse files Browse the repository at this point in the history 
…ss termination. (#341)
  • Loading branch information
@peterfelts @saixiaohui
peterfelts authored and saixiaohui committed Mar 29, 2019
1 parent 7a9dc7f commit b3de8ff
Show file tree
Hide file tree
Showing 4 changed files with 64 additions and 24 deletions.
6 changes: 4 additions & 2 deletions Services.Test/Storage/CosmosDbSql/EngineTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ public class EngineTest

private readonly Mock<IFactory> factory;
private readonly Mock<ILogger> logger;
private readonly Mock<IDiagnosticsLogger> mockDiagnosticsLogger;
private readonly Mock<IInstance> instance;
private readonly Mock<ISDKWrapper> cosmosDbSql;
private readonly Mock<IDocumentClient> cosmosDbSqlClient;
Expand All @@ -39,6 +40,7 @@ public EngineTest()
{
this.factory = new Mock<IFactory>();
this.logger = new Mock<ILogger>();
this.mockDiagnosticsLogger = new Mock<IDiagnosticsLogger>();
this.instance = new Mock<IInstance>();

this.storageConfig = new Config { CosmosDbSqlDatabase = "db", CosmosDbSqlCollection = "coll" };
Expand All @@ -49,7 +51,7 @@ public EngineTest()
this.cosmosDbSqlClient = new Mock<IDocumentClient>();
this.cosmosDbSql.Setup(x => x.GetClientAsync(this.storageConfig)).ReturnsAsync(this.cosmosDbSqlClient.Object);

this.target = new Engine(this.factory.Object, this.logger.Object, this.instance.Object);
this.target = new Engine(this.factory.Object, this.logger.Object, this.mockDiagnosticsLogger.Object, this.instance.Object);

this.target.Init(this.storageConfig);
this.instance.Invocations.Clear();
Expand All @@ -59,7 +61,7 @@ public EngineTest()
public void ItCanBeInitializedOnlyOnce()
{
// Act
var engine = new Engine(this.factory.Object, this.logger.Object, this.instance.Object);
var engine = new Engine(this.factory.Object, this.logger.Object, this.mockDiagnosticsLogger.Object, this.instance.Object);
engine.Init(this.storageConfig);

// Assert
Expand Down
3 changes: 2 additions & 1 deletion Services.Test/Storage/EnginesTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,9 @@ public EnginesTest()
// implement the same interface and don't have a parameterless ctor
this.instance = new Mock<IInstance>();
var logger = new Mock<ILogger>();
var mockDiagnosticsLogger = new Mock<IDiagnosticsLogger>();
this.factory.Setup(x => x.Resolve<CosmosDbSqlEngine>())
.Returns(new CosmosDbSqlEngine(this.factory.Object, logger.Object, this.instance.Object));
.Returns(new CosmosDbSqlEngine(this.factory.Object, logger.Object, mockDiagnosticsLogger.Object, this.instance.Object));
this.factory.Setup(x => x.Resolve<TableStorageEngine>())
.Returns(new TableStorageEngine(this.factory.Object, logger.Object, this.instance.Object));
}
Expand Down
58 changes: 43 additions & 15 deletions Services/Storage/CosmosDbSql/Engine.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using Microsoft.Azure.Documents;
Expand All @@ -19,6 +18,7 @@ public class Engine : IEngine, IDisposable
private readonly ILogger log;
private readonly IInstance instance;
private readonly IFactory factory;
private readonly IDiagnosticsLogger diagnosticsLogger;

private Config storageConfig;

Expand All @@ -34,10 +34,12 @@ public class Engine : IEngine, IDisposable
public Engine(
IFactory factory,
ILogger logger,
IDiagnosticsLogger diagnosticsLogger,
IInstance instance)
{
this.log = logger;
this.instance = instance;
this.diagnosticsLogger = diagnosticsLogger;
this.factory = factory;

this.disposedValue = false;
Expand Down Expand Up @@ -118,7 +120,11 @@ public async Task<IEnumerable<IDataRecord>> GetAllAsync()
}
catch (Exception e)
{
this.log.Error("Unexpected error while reading from Cosmos DB SQL", () => new { this.storageName, e });
const string MSG = "Unexpected error while reading from Cosmos DB SQL";
var errorData = new { this.storageName, e };

this.log.Error(MSG, () => errorData);
this.diagnosticsLogger.LogServiceError(MSG, errorData);
throw new ExternalDependencyException(e);
}
}
Expand Down Expand Up @@ -146,8 +152,11 @@ public async Task<IDataRecord> CreateAsync(IDataRecord input)
}
catch (Exception e)
{
this.log.Error("Unexpected error while writing to Cosmos DB SQL",
() => new { this.storageName, Id = input.GetId(), e });
const string MSG = "Unexpected error while writing to Cosmos DB SQL";
var errorData = new { this.storageName, Id = input.GetId(), e };

this.log.Error(MSG, () => errorData);
this.diagnosticsLogger.LogServiceError(MSG, errorData);
throw new ExternalDependencyException(e);
}
}
Expand Down Expand Up @@ -182,8 +191,11 @@ public async Task<IDataRecord> UpsertAsync(IDataRecord input, string eTag)
}
catch (Exception e)
{
this.log.Error("Unexpected error while writing to Cosmos DB SQL",
() => new { this.storageName, Id = input.GetId(), eTag, e });
const string MSG = "Unexpected error while writing to Cosmos DB SQL";
var errorData = new { this.storageName, Id = input.GetId(), eTag, e };

this.log.Error(MSG, () => errorData);
this.diagnosticsLogger.LogServiceError(MSG, errorData);
throw new ExternalDependencyException(e);
}
}
Expand All @@ -207,8 +219,11 @@ public async Task DeleteAsync(string id)
}
catch (Exception e)
{
this.log.Error("Unexpected error while writing to Cosmos DB SQL",
() => new { this.storageName, id, e });
const string MSG = "Unexpected error while writing to Cosmos DB SQL";
var errorData = new { this.storageName, id, e };

this.log.Error(MSG, () => errorData);
this.diagnosticsLogger.LogServiceError(MSG, errorData);
throw new ExternalDependencyException(e);
}
}
Expand Down Expand Up @@ -274,8 +289,11 @@ public async Task<bool> TryToLockAsync(
}
catch (Exception e)
{
this.log.Error("Unexpected error while writing to Cosmos DB SQL",
() => new { this.storageName, id, ownerId, ownerType, lockDurationSecs = durationSeconds, e });
const string MSG = "Unexpected error while writing to Cosmos DB SQL";
var errorData = new { this.storageName, id, ownerId, ownerType, lockDurationSecs = durationSeconds, e };

this.log.Error(MSG, () => errorData);
this.diagnosticsLogger.LogServiceError(MSG, errorData);
}

return false;
Expand Down Expand Up @@ -321,8 +339,11 @@ public async Task<bool> TryToUnlockAsync(string id, string ownerId, string owner
}
catch (Exception e)
{
this.log.Error("Unexpected error while writing to Cosmos DB SQL",
() => new { this.storageName, id, ownerId, ownerType, e });
const string MSG = "Unexpected error while writing to Cosmos DB SQL";
var errorData = new { this.storageName, id, ownerId, ownerType, e };

this.log.Error(MSG, () => errorData);
this.diagnosticsLogger.LogServiceError(MSG, errorData);
}

return false;
Expand Down Expand Up @@ -397,8 +418,11 @@ private async Task SetupStorageAsync()
}
catch (Exception e)
{
this.log.Error("Unexpected error while reading from Cosmos DB SQL",
() => new { this.storageName, id, e });
const string MSG = "Unexpected error while reading from Cosmos DB SQL";
var errorData = new { this.storageName, id, e};

this.log.Error(MSG, () => errorData);
this.diagnosticsLogger.LogServiceError(MSG, errorData);
throw new ExternalDependencyException(e);
}
}
Expand Down Expand Up @@ -436,7 +460,11 @@ private async Task TryToDeleteExpiredRecord(string id)
catch (Exception e)
{
// Log and do not throw, we're just trying to delete and will retry automatically later
this.log.Warn("Unexpected error while writing to Cosmos DB SQL", () => new { this.storageName, id, e });
const string MSG = "Unexpected error while writing to Cosmos DB SQL";
var errorData = new { this.storageName, id, e };

this.log.Error(MSG, () => errorData);
this.diagnosticsLogger.LogServiceError(MSG, errorData);
}
}
}
Expand Down
21 changes: 15 additions & 6 deletions WebService/Startup.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using Microsoft.AspNetCore.Hosting;
using Microsoft.Azure.IoTSolutions.DeviceSimulation.PartitioningAgent;
using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services;
using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Diagnostics;
using Microsoft.Azure.IoTSolutions.DeviceSimulation.SimulationAgent;
using Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.Auth;
using Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.Runtime;
Expand Down Expand Up @@ -158,6 +159,8 @@ private void StopAgents()

private Task MonitorThreadsAsync(IApplicationLifetime appLifetime)
{
const string MSG = "Part of the service is not running";

return Task.Run(() =>
{
while (!this.appStopToken.IsCancellationRequested)
Expand All @@ -173,12 +176,18 @@ private Task MonitorThreadsAsync(IApplicationLifetime appLifetime)
|| this.partitioningAgentTask.Status == TaskStatus.RanToCompletion)
{
var log = this.ApplicationContainer.Resolve<ILogger>();
log.Error("Part of the service is not running",
() => new
{
SimulationAgent = this.simulationAgentTask.Status.ToString(),
PartitioningAgent = this.partitioningAgentTask.Status.ToString()
});
var diagnosticsLogger = this.ApplicationContainer.Resolve<IDiagnosticsLogger>();

var errorData = new
{
SimulationAgent = this.simulationAgentTask.Status.ToString(),
PartitioningAgent = this.partitioningAgentTask.Status.ToString()
};

log.Error(MSG, () => errorData);

// Send diagnostics information
diagnosticsLogger.LogServiceError(MSG, errorData);

// Allow few seconds to flush logs
Thread.Sleep(5000);
Expand Down

0 comments on commit b3de8ff

Please sign in to comment.