Add metrics for the checksums of the HTTP body

Add metrics to monitor the integrity of an HTTP resource.
These are configured using:

- `fail_if_body_not_matches_hash` configures hash-based probe failures.
- `hash_algorithm` (`sha256` by default) configures the hash used.
- `export_hash` enables exporting the hashed body as a label.

This results in the following new metrics:

- `probe_http_content_checksum` contains the CRC32 of the page as a value.
  This is not cryptographically secure, but should work sufficiently well
  for monitoring changes in normal situations.
- `probe_http_content_hash` contains a configurable hash of the page in a label.
  This *is* cryptographically secure, but may lead to high cardinality when enabled.
  The hash is configurable.
- `probe_failed_due_to_hash` contains a metric that indicates if the probe failed
  because the content hash did not match the expected value.

Signed-off-by: Silke Hofstra <[email protected]>

Author: silkeh

CONFIGURATION.md

@@ -97,6 +97,11 @@ modules:
   fail_if_body_not_matches_regexp:
     [ - <regex>, ... ]
 
+  # Probe fails if the reponse body does not match one of the given hashes.
+  # Hashes must be valid hexadecimal encoded strings.
+  fail_if_body_not_matches_hash:
+    [ - <string>, ... ]
+
   # Probe fails if response header matches regex. For headers with multiple values, fails if *at least one* matches.
   fail_if_header_matches:
     [ - <http_header_match_spec>, ... ]
@@ -160,6 +165,13 @@ modules:
   # It is mutually exclusive with `body`.
   [ body_file: <filename> ]
 
+  # Hashing algorithm used for `fail_if_hash_not_matches` and `export_hash` (sha256, sha512)
+  [ hash_algorithm: <string> | default = "sha256" ]
+
+  # Export a hash of the response body.
+  # NOTE: only use this on resources that seldom change, as it may lead to high label cardinality.
+  [ export_hash: <bool> | default = false ]
+
 ```
 
 #### `<http_header_match_spec>`

config/config.go

@@ -215,13 +215,16 @@ type HTTPProbe struct {
 	Headers                      map[string]string       `yaml:"headers,omitempty"`
 	FailIfBodyMatchesRegexp      []Regexp                `yaml:"fail_if_body_matches_regexp,omitempty"`
 	FailIfBodyNotMatchesRegexp   []Regexp                `yaml:"fail_if_body_not_matches_regexp,omitempty"`
+	FailIfBodyNotMatchesHash     []string                `yaml:"fail_if_body_not_matches_hash,omitempty"`
 	FailIfHeaderMatchesRegexp    []HeaderMatch           `yaml:"fail_if_header_matches,omitempty"`
 	FailIfHeaderNotMatchesRegexp []HeaderMatch           `yaml:"fail_if_header_not_matches,omitempty"`
 	Body                         string                  `yaml:"body,omitempty"`
 	BodyFile                     string                  `yaml:"body_file,omitempty"`
 	HTTPClientConfig             config.HTTPClientConfig `yaml:"http_client_config,inline"`
 	Compression                  string                  `yaml:"compression,omitempty"`
 	BodySizeLimit                units.Base2Bytes        `yaml:"body_size_limit,omitempty"`
+	HashAlgorithm                string                  `yaml:"hash_algorithm,omitempty"`
+	ExportHash                   bool                    `yaml:"export_hash,omitempty"`
 }
 
 type GRPCProbe struct {

prober/http.go

@@ -17,9 +17,14 @@ import (
 	"compress/flate"
 	"compress/gzip"
 	"context"
+	"crypto/sha256"
+	"crypto/sha512"
 	"crypto/tls"
+	"encoding/hex"
 	"errors"
 	"fmt"
+	"hash"
+	"hash/crc32"
 	"io"
 	"log/slog"
 	"net"
@@ -29,6 +34,7 @@ import (
 	"net/textproto"
 	"net/url"
 	"os"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -245,6 +251,14 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
 			Name: "probe_http_content_length",
 			Help: "Length of http content response",
 		})
+		contentChecksumGauge = prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "probe_http_content_checksum",
+			Help: "Contains the CRC32 checksum of the page body as a value",
+		})
+		contentHashGaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "probe_http_content_hash",
+			Help: "Contains the cryptographic hash of the page body as a label",
+		}, []string{module.HTTP.HashAlgorithm})
 		bodyUncompressedLengthGauge = prometheus.NewGauge(prometheus.GaugeOpts{
 			Name: "probe_http_uncompressed_body_length",
 			Help: "Length of uncompressed response body",
@@ -295,6 +309,10 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
 			Name: "probe_failed_due_to_regex",
 			Help: "Indicates if probe failed due to regex",
 		})
+		probeFailedDueToHash = prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "probe_failed_due_to_hash",
+			Help: "Indicates if probe failed due to a hash mismatch",
+		})
 
 		probeHTTPLastModified = prometheus.NewGauge(prometheus.GaugeOpts{
 			Name: "probe_http_last_modified_timestamp_seconds",
@@ -310,6 +328,7 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
 	registry.MustRegister(statusCodeGauge)
 	registry.MustRegister(probeHTTPVersionGauge)
 	registry.MustRegister(probeFailedDueToRegex)
+	registry.MustRegister(probeFailedDueToHash)
 
 	httpConfig := module.HTTP
 
@@ -548,7 +567,9 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
 		}
 
 		if !requestErrored {
-			_, err = io.Copy(io.Discard, byteCounter)
+			enableHash := len(httpConfig.FailIfBodyNotMatchesHash) > 0 || httpConfig.ExportHash
+
+			hashStr, crc, err := hashContent(byteCounter, enableHash, httpConfig.HashAlgorithm)
 			if err != nil {
 				logger.Info("Failed to read HTTP response body", "err", err)
 				success = false
@@ -562,6 +583,25 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
 				// case it contains useful information as to what's the problem.
 				logger.Info("Error while closing response from server", "error", err.Error())
 			}
+
+			if success {
+				registry.MustRegister(contentChecksumGauge)
+				contentChecksumGauge.Set(float64(crc))
+
+				if len(httpConfig.FailIfBodyNotMatchesHash) > 0 {
+					success = slices.Contains(httpConfig.FailIfBodyNotMatchesHash, hashStr)
+					if success {
+						probeFailedDueToHash.Set(0)
+					} else {
+						probeFailedDueToHash.Set(1)
+					}
+				}
+
+				if httpConfig.ExportHash {
+					registry.MustRegister(contentHashGaugeVec)
+					contentHashGaugeVec.WithLabelValues(hashStr).Set(1)
+				}
+			}
 		}
 
 		// At this point body is fully read and we can write end time.
@@ -682,3 +722,34 @@ func getDecompressionReader(algorithm string, origBody io.ReadCloser) (io.ReadCl
 		return nil, errors.New("unsupported compression algorithm")
 	}
 }
+
+func hashContent(src io.Reader, hashBody bool, useHash string) (hashStr string, crc uint32, err error) {
+	crcHash := crc32.New(crc32.MakeTable(crc32.IEEE))
+	cryptoHash := hash.Hash(nil)
+
+	if hashBody {
+		switch useHash {
+		case "", "sha256":
+			cryptoHash = sha256.New()
+		case "sha512":
+			cryptoHash = sha512.New()
+		default:
+			return "", 0, errors.New("unsupported hash algorithm")
+		}
+	}
+
+	if cryptoHash != nil {
+		src = io.TeeReader(src, cryptoHash)
+	}
+
+	_, err = io.Copy(crcHash, src)
+	if err != nil {
+		return "", 0, err
+	}
+
+	if cryptoHash != nil {
+		return hex.EncodeToString(cryptoHash.Sum(nil)), crcHash.Sum32(), nil
+	}
+
+	return "", crcHash.Sum32(), nil
+}

prober/http_test.go

@@ -1554,3 +1554,117 @@ func TestBody(t *testing.T) {
 		}
 	}
 }
+
+func TestProbeHTTP_checksums(t *testing.T) {
+	tests := map[string]struct {
+		Body        []byte
+		Hash        string
+		Match       []string
+		Export      bool
+		ExpCRC      float64
+		ExpSuccess  bool
+		ExpHashFail float64
+		ExpHash     string
+	}{
+		"no body, no hash": {
+			Body:        nil,
+			Export:      false,
+			Hash:        "no",
+			ExpCRC:      0,
+			ExpSuccess:  true,
+			ExpHashFail: 0,
+		},
+		"export sha256": {
+			Body:        []byte("testVector"),
+			Hash:        "sha256",
+			Export:      true,
+			ExpCRC:      float64(uint32(0x9D2407FA)),
+			ExpSuccess:  true,
+			ExpHashFail: 0,
+			ExpHash:     "9a85c8667798425f82e41d72a4bf3c5901ccfb726f62868048ddbc1934ab18ad",
+		},
+		"export sha512": {
+			Body:        []byte("testVector"),
+			Hash:        "sha512",
+			Export:      true,
+			ExpCRC:      float64(uint32(0x9D2407FA)),
+			ExpSuccess:  true,
+			ExpHashFail: 0,
+			ExpHash: "d32b14dd7cc9bf27a18037c057b27bebe05eb536f9035324a64d598bfd642f32" +
+				"d7732d1855be0a8ec7c464cc6b9a4cc74a69883e74875105c7203b751170121e",
+		},
+		"match sha256": {
+			Body:        []byte("testVector"),
+			Hash:        "sha256",
+			Match:       []string{"9a85c8667798425f82e41d72a4bf3c5901ccfb726f62868048ddbc1934ab18ad"},
+			ExpCRC:      float64(uint32(0x9D2407FA)),
+			ExpSuccess:  true,
+			ExpHashFail: 0,
+		},
+		"no match sha256": {
+			Body:        []byte("tampered body"),
+			Hash:        "sha256",
+			Export:      false,
+			Match:       []string{"9a85c8667798425f82e41d72a4bf3c5901ccfb726f62868048ddbc1934ab18ad"},
+			ExpCRC:      float64(uint32(0x2280A9F2)),
+			ExpSuccess:  false,
+			ExpHashFail: 1,
+		},
+		"invalid hash": {
+			Body:       []byte("testVector"),
+			Hash:       "invalid",
+			Export:     true,
+			ExpSuccess: false,
+		},
+	}
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Write(test.Body)
+			}))
+			defer ts.Close()
+
+			module := config.Module{
+				Timeout: time.Second,
+				HTTP: config.HTTPProbe{
+					IPProtocolFallback:       true,
+					FailIfBodyNotMatchesHash: test.Match,
+					HashAlgorithm:            test.Hash,
+					ExportHash:               test.Export,
+				},
+			}
+			registry := prometheus.NewRegistry()
+			testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+
+			if res := ProbeHTTP(testCTX, ts.URL, module, registry, promslog.NewNopLogger()); res != test.ExpSuccess {
+				t.Errorf("Expected result %t, got %t", test.ExpSuccess, res)
+			}
+
+			mfs, err := registry.Gather()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if test.Hash == "invalid" {
+				return
+			}
+
+			if test.Export {
+				expectedLabels := map[string]map[string]string{
+					"probe_http_content_hash": {
+						test.Hash: test.ExpHash,
+					},
+				}
+				checkRegistryLabels(expectedLabels, mfs, t)
+			}
+
+			expectedResults := map[string]float64{
+				"probe_http_content_length":   float64(len(test.Body)),
+				"probe_http_content_checksum": test.ExpCRC,
+				"probe_failed_due_to_hash":    float64(test.ExpHashFail),
+			}
+			checkRegistryResults(expectedResults, mfs, t)
+		})
+	}
+}