fix(grpc-proxy): record NATS startup failures

src/invocation-plane-services/grpc-proxy/proxy/invocation/BUILD.bazel

@@ -73,5 +73,7 @@ go_test(
         "@com_github_prometheus_client_golang//prometheus/testutil",
         "@com_github_stretchr_testify//assert",
         "@com_github_stretchr_testify//require",
+        "@org_uber_go_zap//:zap",
+        "@org_uber_go_zap//zaptest/observer",
     ],
 )

src/invocation-plane-services/grpc-proxy/proxy/invocation/nats.go

@@ -44,12 +44,14 @@ func NewNatsConnection(natsFqdn, nKeySeed, serviceName, ssaFqdn, secretsPath str
 		var err error
 		natsAuthOption, err = newNkeyAuthOption(nKeySeed)
 		if err != nil {
+			recordNatsStartupFailure("nats nkey auth setup failed", err)
 			return nil, err
 		}
 	} else {
 		var err error
 		natsAuthOption, err = newAuthCalloutAuthOption(ssaFqdn, secretsPath)
 		if err != nil {
+			recordNatsStartupFailure("nats auth callout setup failed", err)
 			return nil, err
 		}
 	}
@@ -78,6 +80,7 @@ func NewNatsConnection(natsFqdn, nKeySeed, serviceName, ssaFqdn, secretsPath str
 			zap.L().Info("connected to nats", zap.String("server", conn.ConnectedServerName()), zap.String("cluster", conn.ConnectedClusterName()))
 		}))
 	if err != nil {
+		recordNatsStartupFailure("nats initial connection failed", err)
 		return nil, err
 	}
 	metrics.SetNatsStatsConnection(nc)
@@ -104,6 +107,15 @@ func recordNatsFailure(message string, err error) {
 	metrics.NatsFailureCounter.WithLabelValues(reason).Inc()
 }
 
+// recordNatsStartupFailure records a failure that prevents the proxy from
+// starting. The process exits before the metrics endpoint is served, so the log
+// line, not the counter, is what leaves the pod.
+func recordNatsStartupFailure(message string, err error) {
+	reason := natsErrorReason(err)
+	zap.L().Error(message, zap.String("reason", reason), zap.Error(err))
+	metrics.NatsFailureCounter.WithLabelValues(reason).Inc()
+}
+
 func natsErrorReason(err error) string {
 	var certificateInvalid x509.CertificateInvalidError
 	if errors.As(err, &certificateInvalid) {

src/invocation-plane-services/grpc-proxy/proxy/invocation/nats_test.go

@@ -29,10 +29,14 @@ import (
 	"testing"
 
 	"github.com/nats-io/nats.go"
+	"net"
+
 	"github.com/nats-io/nkeys"
 	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zaptest/observer"
 
 	"nvcf-grpc-proxy/proxy/metrics"
 )
@@ -232,3 +236,63 @@ func TestAuthCalloutPluginTokenProviderMarshalling(t *testing.T) {
 		assert.Equal(t, testPayload, result["payload"])
 	})
 }
+
+func TestRecordNatsStartupFailure(t *testing.T) {
+	core, logs := observer.New(zap.DebugLevel)
+	defer zap.ReplaceGlobals(zap.New(core))()
+
+	failureCounter := metrics.NatsFailureCounter.WithLabelValues(metrics.NatsErrorReasonAuthentication)
+	before := testutil.ToFloat64(failureCounter)
+
+	recordNatsStartupFailure("nats initial connection failed", nats.ErrAuthorization)
+
+	assert.Equal(t, before+1, testutil.ToFloat64(failureCounter))
+
+	entries := logs.FilterMessage("nats initial connection failed").All()
+	require.Len(t, entries, 1)
+	// Startup failures are fatal to the process, so they must not be logged at warn.
+	assert.Equal(t, zap.ErrorLevel, entries[0].Level)
+	assert.Equal(t, metrics.NatsErrorReasonAuthentication, entries[0].ContextMap()["reason"])
+}
+
+// TestNewNatsConnectionRecordsInitialConnectFailure covers the gap this change
+// closes: nats.Connect returns the error synchronously and never invokes the
+// registered ErrorHandler, so nothing was recorded for a failed initial connect.
+func TestNewNatsConnectionRecordsInitialConnectFailure(t *testing.T) {
+	core, logs := observer.New(zap.DebugLevel)
+	defer zap.ReplaceGlobals(zap.New(core))()
+
+	// Bind and immediately release a port so the dial is refused rather than timing out.
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	unreachable := listener.Addr().String()
+	require.NoError(t, listener.Close())
+
+	kp, err := nkeys.CreateUser()
+	require.NoError(t, err)
+	seed, err := kp.Seed()
+	require.NoError(t, err)
+
+	failureCounter := metrics.NatsFailureCounter.WithLabelValues(metrics.NatsErrorReasonConnection)
+	before := testutil.ToFloat64(failureCounter)
+
+	nc, err := NewNatsConnection("nats://"+unreachable, string(seed), "test-service", "", "")
+
+	assert.Error(t, err)
+	assert.Nil(t, nc)
+	assert.Equal(t, before+1, testutil.ToFloat64(failureCounter))
+	assert.Len(t, logs.FilterMessage("nats initial connection failed").All(), 1)
+}
+
+func TestNewNatsConnectionRecordsNkeySetupFailure(t *testing.T) {
+	core, logs := observer.New(zap.DebugLevel)
+	defer zap.ReplaceGlobals(zap.New(core))()
+
+	nc, err := NewNatsConnection("nats://127.0.0.1:4222", "invalid-seed", "test-service", "", "")
+
+	assert.Error(t, err)
+	assert.Nil(t, nc)
+	assert.Len(t, logs.FilterMessage("nats nkey auth setup failed").All(), 1)
+	// The connect is never attempted, so no connection failure should be logged.
+	assert.Empty(t, logs.FilterMessage("nats initial connection failed").All())
+}