SEPE-1177: Add pdagent-receiver, the inbound webhook half

.github/workflows/build.yml

@@ -92,9 +92,14 @@ jobs:
         run: |
           set -euo pipefail
           unit_path=$(grep -oP '(?<=^ExecStart=)\S+' init/pdagent.service)
-          echo "Unit invokes: $unit_path"
-
-          docker run --rm -v "$PWD/dist:/dist:ro" -e unit_path="$unit_path" \
+          # The receiver's ExecStart spans continuation lines, so take the
+          # first field rather than the whole invocation.
+          recv_path=$(grep -oP '(?<=^ExecStart=)\S+' init/pdagent-receiver.service)
+          echo "Agent unit invokes:    $unit_path"
+          echo "Receiver unit invokes: $recv_path"
+
+          docker run --rm -v "$PWD/dist:/dist:ro" \
+            -e unit_path="$unit_path" -e recv_path="$recv_path" \
             rockylinux:8 bash -euo pipefail -c '
               shopt -s nullglob
               rpms=(/dist/*.rpm)
@@ -110,13 +115,25 @@ jobs:
                 files=$(rpm -qpl "$rpm")
                 echo "$files"
 
-                for f in "$unit_path" /usr/local/bin/pd-send /usr/local/bin/pd-queue \
-                         /var/lib/pdagent/scripts/pdagent.service; do
+                for f in "$unit_path" "$recv_path" \
+                         /usr/local/bin/pd-send /usr/local/bin/pd-queue \
+                         /var/lib/pdagent/scripts/pdagent.service \
+                         /var/lib/pdagent/scripts/pdagent-receiver.service; do
                   if ! grep -qx "$f" <<<"$files"; then
                     echo "::error::$rpm is missing expected path $f"
                     exit 1
                   fi
                 done
+
+                # The receiver unit must ship staged, not installed. Dropping it
+                # straight into /lib/systemd/system would present a unit that
+                # cannot start until configuration management supplies a signing
+                # secret, and a failed start there would abort the postinstall
+                # scriptlet before the paging daemon is started.
+                if grep -qx "/lib/systemd/system/pdagent-receiver.service" <<<"$files"; then
+                  echo "::error::$rpm installs the receiver unit directly; it must be staged under /var/lib/pdagent/scripts"
+                  exit 1
+                fi
               done
               echo "Package layout OK across ${#rpms[@]} RPM(s)"
             '

.goreleaser.yml

@@ -6,7 +6,8 @@ before:
     - go generate ./...
 
 builds:
-  - binary: pdagent
+  - id: pdagent
+    binary: pdagent
     env:
       - CGO_ENABLED=0
     # Matches the platform set the project has historically released.
@@ -26,10 +27,45 @@ builds:
       - -X 'github.com/PagerDuty/go-pdagent/pkg/common.Commit={{.ShortCommit}}'
       - -X 'github.com/PagerDuty/go-pdagent/pkg/common.Date={{.Date}}'
 
+  # The inbound webhook receiver. Built from the same module and shipped in the
+  # same package as the agent, so the two can never disagree about the dedup key
+  # format they share -- one is the writer and the other the reader, and a
+  # version skew between them would break acknowledgements silently.
+  #
+  # Linux only. It exists to write Naemon's command pipe on a monitoring host,
+  # which is not a thing anyone does on a Mac.
+  - id: pdagent-receiver
+    binary: pdagent-receiver
+    main: ./cmd/receiver
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+    # The same Linux architectures the agent ships. Not because anyone runs a
+    # monitoring host on i386, but because a package that claims to be this
+    # product and silently lacks one of its two binaries on one architecture is
+    # the kind of inconsistency that is discovered at install time.
+    goarch:
+      - amd64
+      - arm64
+      - "386"
+    ldflags:
+      - -s -w
+      - -X 'github.com/PagerDuty/go-pdagent/pkg/common.Version={{.Version}}'
+      - -X 'github.com/PagerDuty/go-pdagent/pkg/common.Commit={{.ShortCommit}}'
+      - -X 'github.com/PagerDuty/go-pdagent/pkg/common.Date={{.Date}}'
+
 # `replacements` was removed in goreleaser v2; the same artifact names are
 # now produced by templating the OS and architecture directly.
+#
+# Scoped to the agent alone. The receiver is a Linux-only server component
+# deployed from the rpm or deb, so including it here would make the archive
+# hold two binaries on Linux and one on macOS -- which goreleaser rejects, and
+# rightly, as confusing to anyone downloading it.
 archives:
-  - name_template: >-
+  - ids:
+      - pdagent
+    name_template: >-
       {{ .ProjectName }}_{{ .Version }}_
       {{- title .Os }}_
       {{- if eq .Arch "amd64" }}x86_64
@@ -74,8 +110,19 @@ nfpms:
         dst: "/var/lib/pdagent/scripts/pdagent.init"
       - src: "init/pdagent.service"
         dst: "/var/lib/pdagent/scripts/pdagent.service"
+      # Staged, not installed into /lib/systemd/system by the package. The
+      # receiver needs a signing secret the package cannot supply, so it is
+      # enabled by configuration management once that is in place. This also
+      # keeps a receiver that cannot start from aborting the postinstall
+      # scriptlet and leaving the paging daemon down.
+      - src: "init/pdagent-receiver.service"
+        dst: "/var/lib/pdagent/scripts/pdagent-receiver.service"
       - src: "scripts/pd-*"
         dst: "/usr/local/bin/"
+      - dst: /etc/pdagent-receiver
+        type: dir
+        file_info:
+          mode: 0750
       # `empty_folders` was removed in goreleaser v2; directories the agent
       # needs at runtime are now declared as dir-type contents so the package
       # still owns them and the postinstall chown has something to act on.

cmd/integrations/nagios/nagios_enqueue.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/PagerDuty/go-pdagent/pkg/cmdutil"
 	"github.com/PagerDuty/go-pdagent/pkg/common"
+	"github.com/PagerDuty/go-pdagent/pkg/dedupkey"
 	"github.com/PagerDuty/go-pdagent/pkg/eventsapi"
 	"github.com/spf13/cobra"
 )
@@ -287,14 +288,17 @@ func buildEventDescription(cmdInputs nagiosEnqueueInput) string {
 // for a given object, so the recovery resolves the incident the problem opened.
 // It is deliberately identical to the v1 incident key format, so migrating an
 // existing installation from v1 to v2 does not orphan open incidents.
+//
+// The format itself lives in pkg/dedupkey because the inbound webhook receiver
+// has to parse these keys back into a host and service. Keeping the writer and
+// the reader in one package is what stops the two halves of the integration
+// drifting apart.
 func buildDedupKey(cmdInputs nagiosEnqueueInput) string {
-	if cmdInputs.sourceType == "host" {
-		return fmt.Sprintf("event_source=host;host_name=%v", cmdInputs.customFields["HOSTNAME"])
-	}
-	return fmt.Sprintf(
-		"event_source=service;host_name=%v;service_desc=%v",
-		cmdInputs.customFields["HOSTNAME"], cmdInputs.customFields["SERVICEDESC"],
-	)
+	return dedupkey.Build(dedupkey.Key{
+		IsHost:      cmdInputs.sourceType == "host",
+		HostName:    cmdInputs.customFields["HOSTNAME"],
+		ServiceDesc: cmdInputs.customFields["SERVICEDESC"],
+	})
 }
 
 func validateNagiosSendCommand(cmdInputs nagiosEnqueueInput) error {

cmd/receiver/main.go

@@ -0,0 +1,197 @@
+// Command pdagent-receiver accepts PagerDuty V3 webhook deliveries and turns
+// acknowledgements into Naemon external commands.
+//
+// It is a separate binary and a separate systemd unit from pdagent, and is
+// expected to run as its own system user. That is not stylistic: the controls
+// that contain a compromise of an internet-facing listener — a dedicated uid,
+// an nftables egress rule matched on that uid, a systemd sandbox, a write-only
+// ACL on the command FIFO — are all per-process. Sharing an address space with
+// the outbound agent would put every PagerDuty routing key in the same memory
+// as the code parsing untrusted input.
+//
+// On RHEL 8 specifically, systemd's IPAddressDeny is silently ineffective under
+// cgroups v1, so the uid-matched nftables rule is the only egress control
+// available. It cannot be written without a separate user.
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/PagerDuty/go-pdagent/pkg/common"
+	"github.com/PagerDuty/go-pdagent/pkg/naemoncmd"
+	"github.com/PagerDuty/go-pdagent/pkg/receiver"
+	"github.com/spf13/cobra"
+)
+
+const (
+	defaultAddress  = "127.0.0.1:49464"
+	defaultPath     = "/webhook"
+	shutdownTimeout = 10 * time.Second
+)
+
+type options struct {
+	address     string
+	path        string
+	secretFile  string
+	commandFile string
+	tokenFile   string
+	apiURL      string
+	capture     bool
+}
+
+func main() {
+	if err := newRootCommand().Execute(); err != nil {
+		// Cobra has already printed the error.
+		os.Exit(1)
+	}
+}
+
+func newRootCommand() *cobra.Command {
+	var opts options
+
+	cmd := &cobra.Command{
+		Use:           "pdagent-receiver",
+		Short:         "Receive PagerDuty webhooks and acknowledge Naemon problems",
+		Version:       common.Version,
+		SilenceUsage:  true,
+		SilenceErrors: false,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return run(cmd.Context(), opts)
+		},
+	}
+
+	f := cmd.Flags()
+	f.StringVarP(&opts.address, "address", "a", defaultAddress,
+		"Address to listen on. Bind to loopback and let a reverse proxy terminate TLS.")
+	f.StringVar(&opts.path, "path", defaultPath,
+		"Path to serve the webhook on. Every other path is refused.")
+	f.StringVar(&opts.secretFile, "secret-file", "",
+		"File containing PagerDuty webhook signing secrets, one per line, mode 0400 (required)")
+	f.StringVar(&opts.commandFile, "command-file", "",
+		"Naemon external command pipe. Without it the receiver observes and writes nothing.")
+	f.StringVar(&opts.tokenFile, "token-file", "",
+		"File containing a read-only PagerDuty REST API token, mode 0400. "+
+			"Required with --command-file: V3 incident payloads carry no dedup key, "+
+			"so the alert must be looked up to identify the Naemon object.")
+	f.StringVar(&opts.apiURL, "api-url", receiver.DefaultAPIURL,
+		"PagerDuty REST API base URL")
+	f.BoolVar(&opts.capture, "capture", false,
+		"Log authenticated payloads verbatim and submit nothing to Naemon")
+
+	return cmd
+}
+
+func run(ctx context.Context, opts options) error {
+	logger := common.Logger.Named("receiver")
+
+	if opts.secretFile == "" {
+		return errors.New("--secret-file is required")
+	}
+
+	secrets, err := receiver.LoadSecrets(opts.secretFile)
+	if err != nil {
+		return err
+	}
+
+	verifier, err := receiver.NewVerifier(secrets)
+	if err != nil {
+		return err
+	}
+
+	handlerOpts := []receiver.Option{receiver.WithCapture(opts.capture)}
+
+	if opts.commandFile != "" {
+		// The lookup is not optional alongside a command file. Without it the
+		// receiver cannot tell which Naemon object an incident refers to, and
+		// there is no safe fallback — acknowledging the wrong host is worse
+		// than acknowledging nothing.
+		if opts.tokenFile == "" {
+			return errors.New("--token-file is required with --command-file")
+		}
+		token, err := receiver.LoadSingleSecret(opts.tokenFile)
+		if err != nil {
+			return fmt.Errorf("reading API token: %w", err)
+		}
+		lookup, err := receiver.NewAlertLookup(token, opts.apiURL, nil)
+		if err != nil {
+			return err
+		}
+
+		writer := naemoncmd.NewFIFOWriter(opts.commandFile)
+		defer func() { _ = writer.Close() }()
+
+		handlerOpts = append(handlerOpts,
+			receiver.WithCommandWriter(writer),
+			receiver.WithKeyLookup(lookup))
+	}
+
+	handler, err := receiver.NewHandler(verifier, logger, handlerOpts...)
+	if err != nil {
+		return err
+	}
+
+	// A bare ServeMux, not the agent's router. The agent applies its auth
+	// middleware router-wide, and PagerDuty sends a signature rather than a
+	// bearer token, so sharing a router would mean carving a path exemption
+	// into shared security-critical code. Keeping the routers apart makes the
+	// agent's enqueue endpoints unreachable here by construction rather than by
+	// a matcher that has to stay correct forever.
+	mux := http.NewServeMux()
+	mux.Handle(opts.path, handler)
+
+	srv := &http.Server{
+		Addr:              opts.address,
+		Handler:           mux,
+		ReadHeaderTimeout: 5 * time.Second,
+		ReadTimeout:       15 * time.Second,
+		WriteTimeout:      15 * time.Second,
+		IdleTimeout:       60 * time.Second,
+		// The agent inherits 1 MiB here. An internet-facing listener has no
+		// reason to accept headers that large.
+		MaxHeaderBytes: 64 << 10,
+	}
+
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	errCh := make(chan error, 1)
+	go func() {
+		logger.Infow("listening",
+			"address", opts.address, "path", opts.path,
+			"secrets", len(secrets), "capture", opts.capture)
+
+		// Unlike the agent, a bind failure is fatal here rather than logged and
+		// ignored. A receiver that is running but bound to nothing looks healthy
+		// to systemd and to a process check while silently dropping every
+		// acknowledgement.
+		if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			errCh <- fmt.Errorf("listener stopped: %w", err)
+			return
+		}
+		errCh <- nil
+	}()
+
+	select {
+	case err := <-errCh:
+		return err
+	case <-ctx.Done():
+		logger.Info("shutting down")
+	}
+
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
+	defer cancel()
+	if err := srv.Shutdown(shutdownCtx); err != nil {
+		return fmt.Errorf("graceful shutdown failed: %w", err)
+	}
+	return <-errCh
+}