From 321228e8cd2d752f84de434dd869b8b36f938d55 Mon Sep 17 00:00:00 2001 From: Jan David Date: Wed, 11 Sep 2024 09:53:59 -0400 Subject: [PATCH 1/5] Update log format for release service on Fastly The log format of the Fastly service for releases has been slightly changed to set Datadog's unified tags[^1]. This makes it easier to work with the logs in Datadog's UI. [^1]: https://docs.datadoghq.com/getting_started/tagging/unified_service_tagging --- .../modules/release-distribution/fastly-log-format.tftpl | 1 + terragrunt/modules/release-distribution/fastly-static.tf | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 terragrunt/modules/release-distribution/fastly-log-format.tftpl diff --git a/terragrunt/modules/release-distribution/fastly-log-format.tftpl b/terragrunt/modules/release-distribution/fastly-log-format.tftpl new file mode 100644 index 000000000..1db96d055 --- /dev/null +++ b/terragrunt/modules/release-distribution/fastly-log-format.tftpl @@ -0,0 +1 @@ +{ "ddsource": "fastly", "ddtags": "app:${dd_app},env:${dd_env}", "service": "${service_name}", "date": "%%{begin:%Y-%m-%dT%H:%M:%S%Z}t", "time_start": "%%{begin:%Y-%m-%dT%H:%M:%S%Z}t", "time_end": "%%{end:%Y-%m-%dT%H:%M:%S%Z}t", "http": { "request_time_ms": %D, "method": "%m", "url": "%%{json.escape(req.url)}V", "useragent": "%%{User-Agent}i", "referer": "%%{Referer}i", "protocol": "%H", "request_x_forwarded_for": "%%{X-Forwarded-For}i", "status_code": "%s" }, "network": { "client": { "ip": "%h", "name": "%%{client.as.name}V", "number": "%%{client.as.number}V", "connection_speed": "%%{client.geo.conn_speed}V" }, "destination": { "ip": "%A" }, "geoip": { "geo_city": "%%{client.geo.city.utf8}V", "geo_country_code": "%%{client.geo.country_code}V", "geo_continent_code": "%%{client.geo.continent_code}V", "geo_region": "%%{client.geo.region}V" }, "bytes_written": %B, "bytes_read": %%{req.body_bytes_read}V }, "host": "%%{Fastly-Orig-Host}i", "origin_host": "%v", "is_ipv6": %%{if(req.is_ipv6, "true", "false")}V, "is_tls": %%{if(req.is_ssl, "true", "false")}V, "tls_client_protocol": "%%{json.escape(tls.client.protocol)}V", "tls_client_servername": "%%{json.escape(tls.client.servername)}V", "tls_client_cipher": "%%{json.escape(tls.client.cipher)}V", "tls_client_cipher_sha": "%%{json.escape(tls.client.ciphers_sha)}V", "tls_client_tlsexts_sha": "%%{json.escape(tls.client.tlsexts_sha)}V", "is_h2": %%{if(fastly_info.is_h2, "true", "false")}V, "is_h2_push": %%{if(fastly_info.h2.is_push, "true", "false")}V, "h2_stream_id": "%%{fastly_info.h2.stream_id}V", "request_accept_content": "%%{Accept}i", "request_accept_language": "%%{Accept-Language}i", "request_accept_encoding": "%%{Accept-Encoding}i", "request_accept_charset": "%%{Accept-Charset}i", "request_connection": "%%{Connection}i", "request_dnt": "%%{DNT}i", "request_forwarded": "%%{Forwarded}i", "request_via": "%%{Via}i", "request_cache_control": "%%{Cache-Control}i", "request_x_requested_with": "%%{X-Requested-With}i", "request_x_att_device_id": "%%{X-ATT-Device-Id}i", "content_type": "%%{Content-Type}o", "is_cacheable": %%{if(fastly_info.state~"^(HIT|MISS)$", "true","false")}V, "response_age": "%%{Age}o", "response_cache_control": "%%{Cache-Control}o", "response_expires": "%%{Expires}o", "response_last_modified": "%%{Last-Modified}o", "response_tsv": "%%{TSV}o", "server_datacenter": "%%{server.datacenter}V", "req_header_size": %%{req.header_bytes_read}V, "resp_header_size": %%{resp.header_bytes_written}V, "socket_cwnd": %%{client.socket.cwnd}V, "socket_nexthop": "%%{client.socket.nexthop}V", "socket_tcpi_rcv_mss": %%{client.socket.tcpi_rcv_mss}V, "socket_tcpi_snd_mss": %%{client.socket.tcpi_snd_mss}V, "socket_tcpi_rtt": %%{client.socket.tcpi_rtt}V, "socket_tcpi_rttvar": %%{client.socket.tcpi_rttvar}V, "socket_tcpi_rcv_rtt": %%{client.socket.tcpi_rcv_rtt}V, "socket_tcpi_rcv_space": %%{client.socket.tcpi_rcv_space}V, "socket_tcpi_last_data_sent": %%{client.socket.tcpi_last_data_sent}V, "socket_tcpi_total_retrans": %%{client.socket.tcpi_total_retrans}V, "socket_tcpi_delta_retrans": %%{client.socket.tcpi_delta_retrans}V, "socket_ploss": %%{client.socket.ploss}V } diff --git a/terragrunt/modules/release-distribution/fastly-static.tf b/terragrunt/modules/release-distribution/fastly-static.tf index c5d4592b0..c4f056032 100644 --- a/terragrunt/modules/release-distribution/fastly-static.tf +++ b/terragrunt/modules/release-distribution/fastly-static.tf @@ -118,6 +118,12 @@ resource "fastly_service_vcl" "static" { logging_datadog { name = "datadog" token = data.aws_ssm_parameter.datadog_api_key.value + + format = templatefile("${path.module}/fastly-log-format.tftpl", { + dd_app = "releases", + dd_env = var.environment, + service_name = var.static_domain_name, + }) } logging_s3 { From 47567e5d03b9c0c1fde3cd5219f048da18aaf4e8 Mon Sep 17 00:00:00 2001 From: Jan David Date: Wed, 11 Sep 2024 16:13:26 -0400 Subject: [PATCH 2/5] Deploy new log format to production --- terragrunt/accounts/legacy/releases-prod/deployed-ref | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terragrunt/accounts/legacy/releases-prod/deployed-ref b/terragrunt/accounts/legacy/releases-prod/deployed-ref index 74cc47ba9..749472584 100644 --- a/terragrunt/accounts/legacy/releases-prod/deployed-ref +++ b/terragrunt/accounts/legacy/releases-prod/deployed-ref @@ -1 +1 @@ -781f7f29a1009bf7abaed8aa06c99abf28533f88 +321228e8cd2d752f84de434dd869b8b36f938d55 From 3f120c2eb194cc0d15a52d950f43ae9104b6dfd1 Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 12 Sep 2024 12:46:26 -0400 Subject: [PATCH 3/5] Hard-code service name for Datadog We use a hierarchy of tags on Datadog to provide different means for filtering metrics and logs. All logs from a particular piece of infrastructure share the same `service` tag and then have different tags for the `env`. In this specific case, we want to tag everything as coming from the `static.rust-lang.org` service and then differentiate between the environments using the `dev` and `prod` valus for the environment. --- terragrunt/modules/release-distribution/fastly-static.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terragrunt/modules/release-distribution/fastly-static.tf b/terragrunt/modules/release-distribution/fastly-static.tf index c4f056032..e3a21fe99 100644 --- a/terragrunt/modules/release-distribution/fastly-static.tf +++ b/terragrunt/modules/release-distribution/fastly-static.tf @@ -120,9 +120,9 @@ resource "fastly_service_vcl" "static" { token = data.aws_ssm_parameter.datadog_api_key.value format = templatefile("${path.module}/fastly-log-format.tftpl", { + service_name = "static.rust-lang.org" dd_app = "releases", dd_env = var.environment, - service_name = var.static_domain_name, }) } From e8a249616c13651c14c2f8c0241c466b436489c4 Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 12 Sep 2024 13:08:03 -0400 Subject: [PATCH 4/5] Populate host attribute in Fastly logs The `host` attribute in the logs was not set properly, but is now being populated properly based in different information available in Fastly. --- terragrunt/modules/release-distribution/fastly-log-format.tftpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terragrunt/modules/release-distribution/fastly-log-format.tftpl b/terragrunt/modules/release-distribution/fastly-log-format.tftpl index 1db96d055..411dae6f3 100644 --- a/terragrunt/modules/release-distribution/fastly-log-format.tftpl +++ b/terragrunt/modules/release-distribution/fastly-log-format.tftpl @@ -1 +1 @@ -{ "ddsource": "fastly", "ddtags": "app:${dd_app},env:${dd_env}", "service": "${service_name}", "date": "%%{begin:%Y-%m-%dT%H:%M:%S%Z}t", "time_start": "%%{begin:%Y-%m-%dT%H:%M:%S%Z}t", "time_end": "%%{end:%Y-%m-%dT%H:%M:%S%Z}t", "http": { "request_time_ms": %D, "method": "%m", "url": "%%{json.escape(req.url)}V", "useragent": "%%{User-Agent}i", "referer": "%%{Referer}i", "protocol": "%H", "request_x_forwarded_for": "%%{X-Forwarded-For}i", "status_code": "%s" }, "network": { "client": { "ip": "%h", "name": "%%{client.as.name}V", "number": "%%{client.as.number}V", "connection_speed": "%%{client.geo.conn_speed}V" }, "destination": { "ip": "%A" }, "geoip": { "geo_city": "%%{client.geo.city.utf8}V", "geo_country_code": "%%{client.geo.country_code}V", "geo_continent_code": "%%{client.geo.continent_code}V", "geo_region": "%%{client.geo.region}V" }, "bytes_written": %B, "bytes_read": %%{req.body_bytes_read}V }, "host": "%%{Fastly-Orig-Host}i", "origin_host": "%v", "is_ipv6": %%{if(req.is_ipv6, "true", "false")}V, "is_tls": %%{if(req.is_ssl, "true", "false")}V, "tls_client_protocol": "%%{json.escape(tls.client.protocol)}V", "tls_client_servername": "%%{json.escape(tls.client.servername)}V", "tls_client_cipher": "%%{json.escape(tls.client.cipher)}V", "tls_client_cipher_sha": "%%{json.escape(tls.client.ciphers_sha)}V", "tls_client_tlsexts_sha": "%%{json.escape(tls.client.tlsexts_sha)}V", "is_h2": %%{if(fastly_info.is_h2, "true", "false")}V, "is_h2_push": %%{if(fastly_info.h2.is_push, "true", "false")}V, "h2_stream_id": "%%{fastly_info.h2.stream_id}V", "request_accept_content": "%%{Accept}i", "request_accept_language": "%%{Accept-Language}i", "request_accept_encoding": "%%{Accept-Encoding}i", "request_accept_charset": "%%{Accept-Charset}i", "request_connection": "%%{Connection}i", "request_dnt": "%%{DNT}i", "request_forwarded": "%%{Forwarded}i", "request_via": "%%{Via}i", "request_cache_control": "%%{Cache-Control}i", "request_x_requested_with": "%%{X-Requested-With}i", "request_x_att_device_id": "%%{X-ATT-Device-Id}i", "content_type": "%%{Content-Type}o", "is_cacheable": %%{if(fastly_info.state~"^(HIT|MISS)$", "true","false")}V, "response_age": "%%{Age}o", "response_cache_control": "%%{Cache-Control}o", "response_expires": "%%{Expires}o", "response_last_modified": "%%{Last-Modified}o", "response_tsv": "%%{TSV}o", "server_datacenter": "%%{server.datacenter}V", "req_header_size": %%{req.header_bytes_read}V, "resp_header_size": %%{resp.header_bytes_written}V, "socket_cwnd": %%{client.socket.cwnd}V, "socket_nexthop": "%%{client.socket.nexthop}V", "socket_tcpi_rcv_mss": %%{client.socket.tcpi_rcv_mss}V, "socket_tcpi_snd_mss": %%{client.socket.tcpi_snd_mss}V, "socket_tcpi_rtt": %%{client.socket.tcpi_rtt}V, "socket_tcpi_rttvar": %%{client.socket.tcpi_rttvar}V, "socket_tcpi_rcv_rtt": %%{client.socket.tcpi_rcv_rtt}V, "socket_tcpi_rcv_space": %%{client.socket.tcpi_rcv_space}V, "socket_tcpi_last_data_sent": %%{client.socket.tcpi_last_data_sent}V, "socket_tcpi_total_retrans": %%{client.socket.tcpi_total_retrans}V, "socket_tcpi_delta_retrans": %%{client.socket.tcpi_delta_retrans}V, "socket_ploss": %%{client.socket.ploss}V } +{ "ddsource": "fastly", "ddtags": "app:${dd_app},env:${dd_env}", "service": "${service_name}", "date": "%%{begin:%Y-%m-%dT%H:%M:%S%Z}t", "time_start": "%%{begin:%Y-%m-%dT%H:%M:%S%Z}t", "time_end": "%%{end:%Y-%m-%dT%H:%M:%S%Z}t", "http": { "request_time_ms": %D, "method": "%m", "url": "%%{json.escape(req.url)}V", "useragent": "%%{User-Agent}i", "referer": "%%{Referer}i", "protocol": "%H", "request_x_forwarded_for": "%%{X-Forwarded-For}i", "status_code": "%s" }, "network": { "client": { "ip": "%h", "name": "%%{client.as.name}V", "number": "%%{client.as.number}V", "connection_speed": "%%{client.geo.conn_speed}V" }, "destination": { "ip": "%A" }, "geoip": { "geo_city": "%%{client.geo.city.utf8}V", "geo_country_code": "%%{client.geo.country_code}V", "geo_continent_code": "%%{client.geo.continent_code}V", "geo_region": "%%{client.geo.region}V" }, "bytes_written": %B, "bytes_read": %%{req.body_bytes_read}V }, "host": "%%{if(req.http.Fastly-Orig-Host, req.http.Fastly-Orig-Host, req.http.Host)}V", "origin_host": "%v", "is_ipv6": %%{if(req.is_ipv6, "true", "false")}V, "is_tls": %%{if(req.is_ssl, "true", "false")}V, "tls_client_protocol": "%%{json.escape(tls.client.protocol)}V", "tls_client_servername": "%%{json.escape(tls.client.servername)}V", "tls_client_cipher": "%%{json.escape(tls.client.cipher)}V", "tls_client_cipher_sha": "%%{json.escape(tls.client.ciphers_sha)}V", "tls_client_tlsexts_sha": "%%{json.escape(tls.client.tlsexts_sha)}V", "is_h2": %%{if(fastly_info.is_h2, "true", "false")}V, "is_h2_push": %%{if(fastly_info.h2.is_push, "true", "false")}V, "h2_stream_id": "%%{fastly_info.h2.stream_id}V", "request_accept_content": "%%{Accept}i", "request_accept_language": "%%{Accept-Language}i", "request_accept_encoding": "%%{Accept-Encoding}i", "request_accept_charset": "%%{Accept-Charset}i", "request_connection": "%%{Connection}i", "request_dnt": "%%{DNT}i", "request_forwarded": "%%{Forwarded}i", "request_via": "%%{Via}i", "request_cache_control": "%%{Cache-Control}i", "request_x_requested_with": "%%{X-Requested-With}i", "request_x_att_device_id": "%%{X-ATT-Device-Id}i", "content_type": "%%{Content-Type}o", "is_cacheable": %%{if(fastly_info.state~"^(HIT|MISS)$", "true","false")}V, "response_age": "%%{Age}o", "response_cache_control": "%%{Cache-Control}o", "response_expires": "%%{Expires}o", "response_last_modified": "%%{Last-Modified}o", "response_tsv": "%%{TSV}o", "server_datacenter": "%%{server.datacenter}V", "req_header_size": %%{req.header_bytes_read}V, "resp_header_size": %%{resp.header_bytes_written}V, "socket_cwnd": %%{client.socket.cwnd}V, "socket_nexthop": "%%{client.socket.nexthop}V", "socket_tcpi_rcv_mss": %%{client.socket.tcpi_rcv_mss}V, "socket_tcpi_snd_mss": %%{client.socket.tcpi_snd_mss}V, "socket_tcpi_rtt": %%{client.socket.tcpi_rtt}V, "socket_tcpi_rttvar": %%{client.socket.tcpi_rttvar}V, "socket_tcpi_rcv_rtt": %%{client.socket.tcpi_rcv_rtt}V, "socket_tcpi_rcv_space": %%{client.socket.tcpi_rcv_space}V, "socket_tcpi_last_data_sent": %%{client.socket.tcpi_last_data_sent}V, "socket_tcpi_total_retrans": %%{client.socket.tcpi_total_retrans}V, "socket_tcpi_delta_retrans": %%{client.socket.tcpi_delta_retrans}V, "socket_ploss": %%{client.socket.ploss}V } From 132eb89d1faad66ca62c228d3321ab44666f61f2 Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 12 Sep 2024 13:17:42 -0400 Subject: [PATCH 5/5] Deploy updated log format to production --- terragrunt/accounts/legacy/releases-prod/deployed-ref | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terragrunt/accounts/legacy/releases-prod/deployed-ref b/terragrunt/accounts/legacy/releases-prod/deployed-ref index 749472584..34cf023b1 100644 --- a/terragrunt/accounts/legacy/releases-prod/deployed-ref +++ b/terragrunt/accounts/legacy/releases-prod/deployed-ref @@ -1 +1 @@ -321228e8cd2d752f84de434dd869b8b36f938d55 +e8a249616c13651c14c2f8c0241c466b436489c4