Skip to content

Commit 2b661dc

Browse files
hakenmthakenmt
authored
Update README.md
1 parent 7c8e5b1 commit 2b661dc

File tree

1 file changed

+175
-1
lines changed

1 file changed

+175
-1
lines changed

‎README.md

+175-1
Original file line numberDiff line numberDiff line change
@@ -1 +1,175 @@
1-
# replace this
1+
# multi-az-observability
2+
This is a CDK construct for multi-AZ observability to help detect single-AZ impairments. This is currently an `alpha` version, but is being used in the AWS [Advanced Multi-AZ Resilience Patterns](https://catalog.workshops.aws/multi-az-gray-failures/en-US) workshop.
3+
4+
There is a lot of available information to think through and combine to provide signals about single-AZ impact. To simplify the setup and use reasonable defaults, this construct (available in TypeScript, Go, Python, and .NET [Java coming soon]) sets up the necessary observability. To use the CDK construct, you first define your service like this:
5+
6+
```csharp
7+
var wildRydesService = new Service(new ServiceProps(){
8+
ServiceName = "WildRydes",
9+
BaseUrl = "http://www.example.com",
10+
FaultCountThreshold = 25,
11+
AvailabilityZoneNames = vpc.AvailabilityZones,
12+
Period = Duration.Seconds(60),
13+
LoadBalancer = loadBalancer,
14+
DefaultAvailabilityMetricDetails = new ServiceMetricDetails(new ServiceMetricDetailsProps() {
15+
AlarmStatistic = "Sum",
16+
DatapointsToAlarm = 3,
17+
EvaluationPeriods = 5,
18+
FaultAlarmThreshold = 1,
19+
FaultMetricNames = new string[] { "Fault", "Error" },
20+
GraphedFaultStatistics = new string[] { "Sum" },
21+
GraphedSuccessStatistics = new string[] { "Sum" },
22+
MetricNamespace = metricsNamespace,
23+
Period = Duration.Seconds(60),
24+
SuccessAlarmThreshold = 99,
25+
SuccessMetricNames = new string[] {"Success"},
26+
Unit = Unit.COUNT,
27+
}),
28+
DefaultLatencyMetricDetails = new ServiceMetricDetails(new ServiceMetricDetailsProps(){
29+
AlarmStatistic = "p99",
30+
DatapointsToAlarm = 3,
31+
EvaluationPeriods = 5,
32+
FaultAlarmThreshold = 1,
33+
FaultMetricNames = new string[] { "FaultLatency" },
34+
GraphedFaultStatistics = new string[] { "p50" },
35+
GraphedSuccessStatistics = new string[] { "p50", "p99", "tm50", "tm99" },
36+
MetricNamespace = metricsNamespace,
37+
Period = Duration.Seconds(60),
38+
SuccessAlarmThreshold = 100,
39+
SuccessMetricNames = new string[] {"SuccessLatency"},
40+
Unit = Unit.MILLISECONDS,
41+
}),
42+
DefaultContributorInsightRuleDetails = new ContributorInsightRuleDetails(new ContributorInsightRuleDetailsProps() {
43+
AvailabilityZoneIdJsonPath = azIdJsonPath,
44+
FaultMetricJsonPath = faultMetricJsonPath,
45+
InstanceIdJsonPath = instanceIdJsonPath,
46+
LogGroups = serverLogGroups,
47+
OperationNameJsonPath = operationNameJsonPath,
48+
SuccessLatencyMetricJsonPath = successLatencyMetricJsonPath
49+
}),
50+
CanaryTestProps = new AddCanaryTestProps() {
51+
RequestCount = 10,
52+
LoadBalancer = loadBalancer,
53+
Schedule = "rate(1 minute)",
54+
NetworkConfiguration = new NetworkConfigurationProps() {
55+
Vpc = vpc,
56+
SubnetSelection = new SubnetSelection() { SubnetType = SubnetType.PRIVATE_ISOLATED }
57+
}
58+
}
59+
});
60+
wildRydesService.AddOperation(new Operation(new OperationProps() {
61+
OperationName = "Signin",
62+
Path = "/signin",
63+
Service = wildRydesService,
64+
Critical = true,
65+
HttpMethods = new string[] { "GET" },
66+
ServerSideAvailabilityMetricDetails = new OperationMetricDetails(new OperationMetricDetailsProps() {
67+
OperationName = "Signin",
68+
MetricDimensions = new MetricDimensions(new Dictionary<string, string> {{ "Operation", "Signin"}}, "AZ-ID", "Region")
69+
}, wildRydesService.DefaultAvailabilityMetricDetails),
70+
ServerSideLatencyMetricDetails = new OperationMetricDetails(new OperationMetricDetailsProps() {
71+
OperationName = "Signin",
72+
SuccessAlarmThreshold = 150,
73+
MetricDimensions = new MetricDimensions(new Dictionary<string, string> {{ "Operation", "Signin"}}, "AZ-ID", "Region")
74+
}, wildRydesService.DefaultLatencyMetricDetails),
75+
CanaryTestLatencyMetricsOverride = new CanaryTestMetricsOverride(new CanaryTestMetricsOverrideProps() {
76+
SuccessAlarmThreshold = 250
77+
})
78+
}));
79+
wildRydesService.AddOperation(new Operation(new OperationProps() {
80+
OperationName = "Pay",
81+
Path = "/pay",
82+
Service = wildRydesService,
83+
HttpMethods = new string[] { "GET" },
84+
Critical = true,
85+
ServerSideAvailabilityMetricDetails = new OperationMetricDetails(new OperationMetricDetailsProps() {
86+
OperationName = "Pay",
87+
MetricDimensions = new MetricDimensions(new Dictionary<string, string> {{ "Operation", "Pay"}}, "AZ-ID", "Region")
88+
}, wildRydesService.DefaultAvailabilityMetricDetails),
89+
ServerSideLatencyMetricDetails = new OperationMetricDetails(new OperationMetricDetailsProps() {
90+
OperationName = "Pay",
91+
SuccessAlarmThreshold = 200,
92+
MetricDimensions = new MetricDimensions(new Dictionary<string, string> {{ "Operation", "Pay"}}, "AZ-ID", "Region")
93+
}, wildRydesService.DefaultLatencyMetricDetails),
94+
CanaryTestLatencyMetricsOverride = new CanaryTestMetricsOverride(new CanaryTestMetricsOverrideProps() {
95+
SuccessAlarmThreshold = 300
96+
})
97+
}));
98+
wildRydesService.AddOperation(new Operation(new OperationProps() {
99+
OperationName = "Ride",
100+
Path = "/ride",
101+
Service = wildRydesService,
102+
HttpMethods = new string[] { "GET" },
103+
Critical = true,
104+
ServerSideAvailabilityMetricDetails = new OperationMetricDetails(new OperationMetricDetailsProps() {
105+
OperationName = "Ride",
106+
MetricDimensions = new MetricDimensions(new Dictionary<string, string> {{ "Operation", "Ride"}}, "AZ-ID", "Region")
107+
}, wildRydesService.DefaultAvailabilityMetricDetails),
108+
ServerSideLatencyMetricDetails = new OperationMetricDetails(new OperationMetricDetailsProps() {
109+
OperationName = "Ride",
110+
SuccessAlarmThreshold = 350,
111+
MetricDimensions = new MetricDimensions(new Dictionary<string, string> {{ "Operation", "Ride"}}, "AZ-ID", "Region")
112+
}, wildRydesService.DefaultLatencyMetricDetails),
113+
CanaryTestLatencyMetricsOverride = new CanaryTestMetricsOverride(new CanaryTestMetricsOverrideProps() {
114+
SuccessAlarmThreshold = 550
115+
})
116+
}));
117+
wildRydesService.AddOperation(new Operation(new OperationProps() {
118+
OperationName = "Home",
119+
Path = "/home",
120+
Service = wildRydesService,
121+
HttpMethods = new string[] { "GET" },
122+
Critical = true,
123+
ServerSideAvailabilityMetricDetails = new OperationMetricDetails(new OperationMetricDetailsProps() {
124+
OperationName = "Home",
125+
MetricDimensions = new MetricDimensions(new Dictionary<string, string> {{ "Operation", "Ride"}}, "AZ-ID", "Region")
126+
}, wildRydesService.DefaultAvailabilityMetricDetails),
127+
ServerSideLatencyMetricDetails = new OperationMetricDetails(new OperationMetricDetailsProps() {
128+
OperationName = "Home",
129+
SuccessAlarmThreshold = 100,
130+
MetricDimensions = new MetricDimensions(new Dictionary<string, string> {{ "Operation", "Ride"}}, "AZ-ID", "Region")
131+
}, wildRydesService.DefaultLatencyMetricDetails),
132+
CanaryTestLatencyMetricsOverride = new CanaryTestMetricsOverride(new CanaryTestMetricsOverrideProps() {
133+
SuccessAlarmThreshold = 200
134+
})
135+
}));
136+
```
137+
138+
Then you provide that service definition to the CDK construct.
139+
140+
```csharp
141+
InstrumentedServiceMultiAZObservability multiAvailabilityZoneObservability = new InstrumentedServiceMultiAZObservability(this, "MultiAZObservability", new InstrumentedServiceMultiAZObservabilityProps() {
142+
Service = wildRydesService,
143+
CreateDashboards = true,
144+
Interval = Duration.Minutes(60), // The interval for the dashboard
145+
OutlierDetectionAlgorithm = OutlierDetectionAlgorithm.STATIC
146+
});
147+
```
148+
149+
You define some characteristics of the service, default values for metrics and alarms, and then add operations as well as any overrides for default values that you need. The construct can also automatically create synthetic canaries that test each operation with a very simple HTTP check, or you can configure your own synthetics and just tell the construct about the metric details and optionally log files. This creates metrics, alarms, and dashboards that can be used to detect single-AZ impact.
150+
151+
If you don't have service specific logs and custom metrics with per-AZ dimensions, you can still use the construct to evaluate ALB and NAT Gateway metrics to find single AZ faults.
152+
153+
```csharp
154+
BasicServiceMultiAZObservability multiAvailabilityZoneObservability = new BasicServiceMultiAZObservability(this, "MultiAZObservability", new BasicServiceMultiAZObservabilityProps() {
155+
ApplicationLoadBalancers = new IApplicationLoadBalancer[] { loadBalancer },
156+
NatGateways = new Dictionary<string, CfnNatGateway>() {
157+
{ "us-east-1a", natGateway1},
158+
{ "us-east-1b", natGateway2},
159+
{ "us-east-1c", natGateway3},
160+
},
161+
CreateDashboard = true,
162+
OutlierDetectionAlgorithm = OutlierDetectionAlgorithm.STATIC,
163+
FaultCountPercentageThreshold = 1.0, // The fault rate to alarm on for errors seen from the ALBs in the same AZ
164+
PacketLossImpactPercentageThreshold = 0.01, // The percentage of packet loss to alarm on for the NAT Gateways in the same AZ
165+
ServiceName = "WildRydes",
166+
Period = Duration.Seconds(60), // The period for metric evaluation
167+
Interval = Duration.Minutes(60) // The interval for the dashboards
168+
EvaluationPeriods = 5,
169+
DatapointsToAlarm = 3
170+
});
171+
```
172+
173+
If you provide a load balancer, the construct assumes it is deployed in each AZ of the VPC the load balancer is associated with and will look for HTTP metrics using those AZs as dimensions.
174+
175+
Both options support running workloads on EC2, ECS, Lambda, and EKS.

0 commit comments

Comments
 (0)