From 0bcda12d0e877c8cca3d6a0866f8afc070db4f9a Mon Sep 17 00:00:00 2001 From: jannfis Date: Wed, 30 Oct 2024 01:17:25 +0000 Subject: [PATCH] feat: Add informer metrics Signed-off-by: jannfis --- .github/workflows/ci.yaml | 10 - .../informer/appproject/projectinformer.go | 153 ++++++------ internal/informer/informer.go | 51 ++++ internal/informer/informer_test.go | 231 ++++++++++++++++++ internal/manager/application/application.go | 10 +- internal/manager/appproject/appproject.go | 4 +- internal/metrics/metrics.go | 70 ++++-- internal/metrics/server.go | 23 +- principal/server.go | 13 +- 9 files changed, 458 insertions(+), 107 deletions(-) create mode 100644 internal/informer/informer_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7cce1cf..350c762 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -83,11 +83,6 @@ jobs: uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 with: go-version: ${{ env.GOLANG_VERSION }} - - name: Restore go build cache - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 - with: - path: ~/.cache/go-build - key: ${{ runner.os }}-go-build-v1-${{ github.run_id }} - name: Download all Go modules run: | go mod download @@ -104,11 +99,6 @@ jobs: uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 with: go-version: ${{ env.GOLANG_VERSION }} - - name: Restore go build cache - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 - with: - path: ~/.cache/go-build - key: ${{ runner.os }}-go-build-v1-${{ github.run_id }} - name: Download all Go modules run: | go mod download diff --git a/internal/informer/appproject/projectinformer.go b/internal/informer/appproject/projectinformer.go index e8a6915..dcca71a 100644 --- a/internal/informer/appproject/projectinformer.go +++ b/internal/informer/appproject/projectinformer.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/watch" "github.com/argoproj-labs/argocd-agent/internal/informer" + "github.com/argoproj-labs/argocd-agent/internal/metrics" ) type AppProjectInformer struct { @@ -41,6 +42,8 @@ type AppProjectInformer struct { addFunc func(proj *v1alpha1.AppProject) updateFunc func(oldProj *v1alpha1.AppProject, newProj *v1alpha1.AppProject) deleteFunc func(proj *v1alpha1.AppProject) + + metrics *metrics.AppProjectWatcherMetrics } type AppProjectInformerOption func(pi *AppProjectInformer) error @@ -98,6 +101,15 @@ func WithNamespaces(namespaces ...string) AppProjectInformerOption { } } +// WithMetrics sets the AppProject watcher metrics to be used with this +// informer. +func WithMetrics(m *metrics.AppProjectWatcherMetrics) AppProjectInformerOption { + return func(pi *AppProjectInformer) error { + pi.metrics = m + return nil + } +} + // NewAppProjectInformer returns a new instance of a GenericInformer set up to // handle AppProjects. It will be configured with the given options, using the // given appclientset. @@ -114,77 +126,82 @@ func NewAppProjectInformer(ctx context.Context, client appclientset.Interface, n if pi.logger == nil { pi.logger = logrus.WithField("module", "AppProjectInformer") } + iopts := []informer.InformerOption{} + if pi.metrics != nil { + iopts = append(iopts, informer.WithMetrics(pi.metrics.ProjectsAdded, pi.metrics.ProjectsUpdated, pi.metrics.ProjectsRemoved, pi.metrics.ProjectsWatched)) + } i, err := informer.NewGenericInformer(&v1alpha1.AppProject{}, - informer.WithListCallback(func(options v1.ListOptions, namespace string) (runtime.Object, error) { - log().Infof("Listing AppProjects in namespace %s", namespace) - projects, err := client.ArgoprojV1alpha1().AppProjects(namespace).List(ctx, options) - log().Infof("Lister returned %d AppProjects", len(projects.Items)) - if pi.filterFunc != nil { - newItems := make([]v1alpha1.AppProject, 0) - for _, p := range projects.Items { - if pi.filterFunc(&p) { - newItems = append(newItems, p) + append([]informer.InformerOption{ + informer.WithListCallback(func(options v1.ListOptions, namespace string) (runtime.Object, error) { + log().Infof("Listing AppProjects in namespace %s", namespace) + projects, err := client.ArgoprojV1alpha1().AppProjects(namespace).List(ctx, options) + log().Infof("Lister returned %d AppProjects", len(projects.Items)) + if pi.filterFunc != nil { + newItems := make([]v1alpha1.AppProject, 0) + for _, p := range projects.Items { + if pi.filterFunc(&p) { + newItems = append(newItems, p) + } } + pi.logger.Debugf("Lister has %d AppProjects after filtering", len(newItems)) + projects.Items = newItems + } + return projects, err + }), + informer.WithNamespaces(pi.namespaces...), + informer.WithWatchCallback(func(options v1.ListOptions, namespace string) (watch.Interface, error) { + log().Info("Watching AppProjects") + return client.ArgoprojV1alpha1().AppProjects(namespace).Watch(ctx, options) + }), + informer.WithAddCallback(func(obj interface{}) { + log().Info("Add AppProject Callback") + proj, ok := obj.(*v1alpha1.AppProject) + if !ok { + pi.logger.Errorf("Received add event for unknown type %T", obj) + return + } + pi.logger.Debugf("AppProject add event: %s", proj.Name) + if pi.addFunc != nil { + pi.addFunc(proj) + } + }), + informer.WithUpdateCallback(func(oldObj, newObj interface{}) { + log().Info("Update AppProject Callback") + oldProj, oldProjOk := oldObj.(*v1alpha1.AppProject) + newProj, newProjOk := newObj.(*v1alpha1.AppProject) + if !newProjOk || !oldProjOk { + pi.logger.Errorf("Received update event for unknown type old:%T new:%T", oldObj, newObj) + return + } + pi.logger.Debugf("AppProject update event: old:%s new:%s", oldProj.Name, newProj.Name) + if pi.updateFunc != nil { + pi.updateFunc(oldProj, newProj) + } + }), + informer.WithDeleteCallback(func(obj interface{}) { + log().Info("Delete AppProject Callback") + proj, ok := obj.(*v1alpha1.AppProject) + if !ok { + pi.logger.Errorf("Received delete event for unknown type %T", obj) + return + } + pi.logger.Debugf("AppProject delete event: %s", proj.Name) + if pi.deleteFunc != nil { + pi.deleteFunc(proj) + } + }), + informer.WithFilterFunc(func(obj interface{}) bool { + if pi.filterFunc == nil { + return true + } + o, ok := obj.(*v1alpha1.AppProject) + if !ok { + pi.logger.Errorf("Failed type conversion for unknown type %T", obj) + return false } - pi.logger.Debugf("Lister has %d AppProjects after filtering", len(newItems)) - projects.Items = newItems - } - return projects, err - }), - informer.WithNamespaces(pi.namespaces...), - informer.WithWatchCallback(func(options v1.ListOptions, namespace string) (watch.Interface, error) { - log().Info("Watching AppProjects") - return client.ArgoprojV1alpha1().AppProjects(namespace).Watch(ctx, options) - }), - informer.WithAddCallback(func(obj interface{}) { - log().Info("Add AppProject Callback") - proj, ok := obj.(*v1alpha1.AppProject) - if !ok { - pi.logger.Errorf("Received add event for unknown type %T", obj) - return - } - pi.logger.Debugf("AppProject add event: %s", proj.Name) - if pi.addFunc != nil { - pi.addFunc(proj) - } - }), - informer.WithUpdateCallback(func(oldObj, newObj interface{}) { - log().Info("Update AppProject Callback") - oldProj, oldProjOk := oldObj.(*v1alpha1.AppProject) - newProj, newProjOk := newObj.(*v1alpha1.AppProject) - if !newProjOk || !oldProjOk { - pi.logger.Errorf("Received update event for unknown type old:%T new:%T", oldObj, newObj) - return - } - pi.logger.Debugf("AppProject update event: old:%s new:%s", oldProj.Name, newProj.Name) - if pi.updateFunc != nil { - pi.updateFunc(oldProj, newProj) - } - }), - informer.WithDeleteCallback(func(obj interface{}) { - log().Info("Delete AppProject Callback") - proj, ok := obj.(*v1alpha1.AppProject) - if !ok { - pi.logger.Errorf("Received delete event for unknown type %T", obj) - return - } - pi.logger.Debugf("AppProject delete event: %s", proj.Name) - if pi.deleteFunc != nil { - pi.deleteFunc(proj) - } - }), - informer.WithFilterFunc(func(obj interface{}) bool { - if pi.filterFunc == nil { - return true - } - o, ok := obj.(*v1alpha1.AppProject) - if !ok { - pi.logger.Errorf("Failed type conversion for unknown type %T", obj) - return false - } - return pi.filterFunc(o) - }), - ) + return pi.filterFunc(o) + }), + }, iopts...)...) pi.projectInformer = i pi.projectLister = applisters.NewAppProjectLister(i.Indexer()) return pi, err diff --git a/internal/informer/informer.go b/internal/informer/informer.go index 84511bd..48676a6 100644 --- a/internal/informer/informer.go +++ b/internal/informer/informer.go @@ -50,6 +50,30 @@ type GenericInformer struct { fieldSelector string // mutex should be owned before accessing namespaces namespaces map[string]interface{} + // metrics is the metrics provider for this informer + metrics metrics +} + +// metrics is a simplified and specialized metrics provider for the informer +type metrics struct { + added metricsCounter + updated metricsCounter + removed metricsCounter + watched metricsGauge +} + +// metricsCounter is an interface for a metrics implementation of type counter. +// The reason we use an oversimplified abstraction is for testing purposes. +type metricsCounter interface { + Inc() +} + +// metricsGauge is an interface for a metrics implementation of type gauge. +// The reason we use an oversimplified abstraction is for testing purposes. +type metricsGauge interface { + Inc() + Dec() + Set(float64) } type InformerOption func(i *GenericInformer) error @@ -124,6 +148,17 @@ func WithFieldSelector(sel string) InformerOption { } } +// WithMetrics sets the metrics functions to use with this informer. +func WithMetrics(added, updated, removed metricsCounter, watched metricsGauge) InformerOption { + return func(i *GenericInformer) error { + i.metrics.added = added + i.metrics.updated = updated + i.metrics.removed = removed + i.metrics.watched = watched + return nil + } +} + // WithNamespaces sets the namespaces for which the informer will process any // event. If an event is seen for an object in a namespace that is not in this // list, the event will be ignored. If either zero or multiple namespaces are @@ -166,6 +201,9 @@ func NewGenericInformer(objType runtime.Object, options ...InformerOption) (*Gen &cache.ListWatch{ ListFunc: func(options v1.ListOptions) (runtime.Object, error) { logCtx.Trace("Executing list") + if i.metrics.watched != nil { + i.metrics.watched.Set(0) + } return i.listFunc(options, i.watchAndListNamespace()) }, WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { @@ -202,6 +240,10 @@ func NewGenericInformer(objType runtime.Object, options ...InformerOption) (*Gen if i.addFunc != nil { i.addFunc(obj) } + if i.metrics.added != nil && i.metrics.watched != nil { + i.metrics.added.Inc() + i.metrics.watched.Inc() + } }, UpdateFunc: func(oldObj, newObj interface{}) { mobj, err := meta.Accessor(newObj) @@ -221,6 +263,9 @@ func NewGenericInformer(objType runtime.Object, options ...InformerOption) (*Gen if i.updateFunc != nil { i.updateFunc(oldObj, newObj) } + if i.metrics.updated != nil { + i.metrics.updated.Inc() + } }, DeleteFunc: func(obj interface{}) { mobj, err := meta.Accessor(obj) @@ -240,6 +285,10 @@ func NewGenericInformer(objType runtime.Object, options ...InformerOption) (*Gen if i.deleteFunc != nil { i.deleteFunc(obj) } + if i.metrics.removed != nil && i.metrics.watched != nil { + i.metrics.removed.Inc() + i.metrics.watched.Dec() + } }, }) if err != nil { @@ -342,6 +391,8 @@ func (i *GenericInformer) watchAndListNamespace() string { // isNamespaceAllowed returns whether the namespace of an event's object is // permitted. func (i *GenericInformer) isNamespaceAllowed(obj v1.Object) bool { + i.mutex.RLock() + defer i.mutex.RUnlock() if len(i.namespaces) == 0 { return true } diff --git a/internal/informer/informer_test.go b/internal/informer/informer_test.go new file mode 100644 index 0000000..10e51ea --- /dev/null +++ b/internal/informer/informer_test.go @@ -0,0 +1,231 @@ +package informer + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" +) + +func Test_Informer(t *testing.T) { + listFunc := func(options v1.ListOptions, namespace string) (runtime.Object, error) { + return &v1alpha1.ApplicationList{Items: []v1alpha1.Application{}}, nil + } + watcher := watch.NewFake() + nopWatchFunc := func(options v1.ListOptions, namespace string) (watch.Interface, error) { + return watcher, nil + } + t.Run("Error when no list function given", func(t *testing.T) { + i, err := NewGenericInformer(&v1alpha1.Application{}) + assert.Nil(t, i) + assert.ErrorContains(t, err, "without list function") + }) + t.Run("Error when no watch function given", func(t *testing.T) { + i, err := NewGenericInformer(&v1alpha1.Application{}, + WithListCallback(listFunc)) + assert.Nil(t, i) + assert.ErrorContains(t, err, "without watch function") + }) + + t.Run("Error on option", func(t *testing.T) { + opt := func(i *GenericInformer) error { + return errors.New("some error") + } + i, err := NewGenericInformer(&v1alpha1.Application{}, opt) + assert.Nil(t, i) + assert.ErrorContains(t, err, "some error") + }) + + t.Run("Instantiate generic informer without processing", func(t *testing.T) { + i, err := NewGenericInformer(&v1alpha1.Application{}, + WithListCallback(listFunc), + WithWatchCallback(nopWatchFunc), + ) + require.NotNil(t, i) + require.NoError(t, err) + err = i.Start(context.Background()) + assert.NoError(t, err) + assert.True(t, i.IsRunning()) + err = i.Stop() + assert.NoError(t, err) + assert.False(t, i.IsRunning()) + }) + + t.Run("Cannot start informer twice", func(t *testing.T) { + ctl := fakeGenericInformer(t) + err := ctl.i.Start(context.TODO()) + assert.NoError(t, err) + err = ctl.i.Start(context.TODO()) + assert.Error(t, err) + }) + + t.Run("Can stop informer only when running", func(t *testing.T) { + ctl := fakeGenericInformer(t) + // Not running yet + err := ctl.i.Stop() + assert.Error(t, err) + err = ctl.i.Start(context.TODO()) + assert.NoError(t, err) + // Running + err = ctl.i.Stop() + assert.NoError(t, err) + // Not running + err = ctl.i.Stop() + assert.Error(t, err) + }) + + t.Run("Run callbacks", func(t *testing.T) { + ctl := fakeGenericInformer(t) + err := ctl.i.Start(context.TODO()) + assert.NoError(t, err) + app := &v1alpha1.Application{} + ctl.watcher.Add(app) + ctl.watcher.Modify(app) + ctl.watcher.Delete(app) + added, updated, deleted := requireCallbacks(t, ctl) + assert.True(t, added) + assert.True(t, updated) + assert.True(t, deleted) + err = ctl.i.Stop() + assert.NoError(t, err) + }) +} + +type informerCtl struct { + add chan (bool) + upd chan (bool) + del chan (bool) + i *GenericInformer + watcher *watch.FakeWatcher +} + +// fakeGenericInformer returns a generic informer suitable for unit testing. +func fakeGenericInformer(t *testing.T, opts ...InformerOption) *informerCtl { + var err error + ctl := &informerCtl{ + add: make(chan bool), + upd: make(chan bool), + del: make(chan bool), + } + t.Helper() + listFunc := func(options v1.ListOptions, namespace string) (runtime.Object, error) { + return &v1alpha1.ApplicationList{Items: []v1alpha1.Application{}}, nil + } + ctl.watcher = watch.NewFake() + nopWatchFunc := func(options v1.ListOptions, namespace string) (watch.Interface, error) { + return ctl.watcher, nil + } + + ctl.i, err = NewGenericInformer(&v1alpha1.Application{}, + append([]InformerOption{ + WithListCallback(listFunc), + WithWatchCallback(nopWatchFunc), + WithAddCallback(func(obj interface{}) { + ctl.add <- true + }), + WithUpdateCallback(func(newObj, oldObj interface{}) { + ctl.upd <- true + }), + WithDeleteCallback(func(obj interface{}) { + ctl.del <- true + }), + }, opts...)...) + if err != nil { + t.Fatal(err) + } + return ctl +} + +func requireCallbacks(t *testing.T, ctl *informerCtl) (added, updated, deleted bool) { + t.Helper() + run := true + tick := time.NewTicker(1 * time.Second) + added = false + for run { + select { + case added = <-ctl.add: + case updated = <-ctl.upd: + case deleted = <-ctl.del: + case <-tick.C: + run = false + } + } + return +} + +func Test_NamespaceRestrictions(t *testing.T) { + app := &v1alpha1.Application{ + ObjectMeta: v1.ObjectMeta{ + Name: "test", + Namespace: "argocd", + }, + } + t.Run("Namespace is allowed", func(t *testing.T) { + ctl := fakeGenericInformer(t) + assert.True(t, ctl.i.isNamespaceAllowed(app)) + ctl = fakeGenericInformer(t, WithNamespaces("argocd")) + assert.True(t, ctl.i.isNamespaceAllowed(app)) + }) + t.Run("Namespace is not allowed", func(t *testing.T) { + ctl := fakeGenericInformer(t, WithNamespaces("argocd")) + app := app.DeepCopy() + app.Namespace = "foobar" + assert.False(t, ctl.i.isNamespaceAllowed(app)) + }) + t.Run("Adding a namespace works", func(t *testing.T) { + ctl := fakeGenericInformer(t, WithNamespaces("foobar")) + assert.False(t, ctl.i.isNamespaceAllowed(app)) + err := ctl.i.AddNamespace("argocd") + require.NoError(t, err) + assert.True(t, ctl.i.isNamespaceAllowed(app)) + // May not be added a second time + err = ctl.i.AddNamespace("argocd") + require.Error(t, err) + }) + t.Run("Removing a namespace works", func(t *testing.T) { + ctl := fakeGenericInformer(t, WithNamespaces("argocd", "foobar")) + assert.True(t, ctl.i.isNamespaceAllowed(app)) + err := ctl.i.RemoveNamespace("argocd") + require.NoError(t, err) + assert.False(t, ctl.i.isNamespaceAllowed(app)) + // May not be removed if not existing + err = ctl.i.RemoveNamespace("argocd") + require.Error(t, err) + }) + t.Run("Prevent resources in non-allowed namespaces to execute add handlers", func(t *testing.T) { + ctl := fakeGenericInformer(t, WithNamespaces("argocd")) + err := ctl.i.Start(context.TODO()) + require.NoError(t, err) + ctl.watcher.Add(&v1alpha1.Application{ + ObjectMeta: v1.ObjectMeta{ + Name: "test", + Namespace: "foobar", + }, + }) + added, _, _ := requireCallbacks(t, ctl) + assert.False(t, added) + }) +} + +func Test_watchAndListNamespaces(t *testing.T) { + t.Run("Watch namespace is empty when no namespace is given", func(t *testing.T) { + ctl := fakeGenericInformer(t) + assert.Empty(t, ctl.i.watchAndListNamespace()) + }) + t.Run("Watch namespace is same as single namespace constraint", func(t *testing.T) { + ctl := fakeGenericInformer(t, WithNamespaces("argocd")) + assert.Equal(t, "argocd", ctl.i.watchAndListNamespace()) + }) + + t.Run("Watch namespace is empty when multiple namespaces are given", func(t *testing.T) { + ctl := fakeGenericInformer(t, WithNamespaces("argocd", "foobar")) + assert.Empty(t, ctl.i.watchAndListNamespace()) + }) +} diff --git a/internal/manager/application/application.go b/internal/manager/application/application.go index 4c1c664..be8e495 100644 --- a/internal/manager/application/application.go +++ b/internal/manager/application/application.go @@ -160,7 +160,7 @@ func (m *ApplicationManager) Create(ctx context.Context, app *v1alpha1.Applicati } } else { if m.metrics != nil { - m.metrics.Errors.Inc() + m.metrics.AppClientErrors.Inc() } } @@ -242,7 +242,7 @@ func (m *ApplicationManager) UpdateManagedApp(ctx context.Context, incoming *v1a } } else { if m.metrics != nil { - m.metrics.Errors.Inc() + m.metrics.AppClientErrors.Inc() } } return updated, err @@ -327,7 +327,7 @@ func (m *ApplicationManager) UpdateAutonomousApp(ctx context.Context, namespace } } else { if m.metrics != nil { - m.metrics.Errors.Inc() + m.metrics.AppClientErrors.Inc() } } return updated, err @@ -408,7 +408,7 @@ func (m *ApplicationManager) UpdateStatus(ctx context.Context, namespace string, } } else { if m.metrics != nil { - m.metrics.Errors.Inc() + m.metrics.AppClientErrors.Inc() } } return updated, err @@ -472,7 +472,7 @@ func (m *ApplicationManager) UpdateOperation(ctx context.Context, incoming *v1al } } else { if m.metrics != nil { - m.metrics.Errors.Inc() + m.metrics.AppClientErrors.Inc() } } return updated, err diff --git a/internal/manager/appproject/appproject.go b/internal/manager/appproject/appproject.go index 4b468de..6b11b95 100644 --- a/internal/manager/appproject/appproject.go +++ b/internal/manager/appproject/appproject.go @@ -179,7 +179,7 @@ func createAppProject(ctx context.Context, m *AppProjectManager, project *v1alph return created, nil } else { if m.metrics != nil { - m.metrics.Errors.Inc() + m.metrics.ProjectClientErrors.Inc() } } return nil, nil @@ -244,7 +244,7 @@ func (m *AppProjectManager) UpdateAppProject(ctx context.Context, incoming *v1al } } else { if m.metrics != nil { - m.metrics.Errors.Inc() + m.metrics.ProjectClientErrors.Inc() } } return updated, err diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 2b6e094..e440344 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -20,27 +20,45 @@ import ( // "github.com/prometheus/client_golang/prometheus/promhttp" ) +type ApplicationMetrics struct { + ApplicationWatcherMetrics + ApplicationClientMetrics +} + +type AppProjectMetrics struct { + AppProjectClientMetrics + AppProjectWatcherMetrics +} + // ApplicationWatcherMetrics holds metrics about Applications watched by the agent type ApplicationWatcherMetrics struct { - AppsWatched prometheus.Gauge - AppsAdded prometheus.Counter - AppsUpdated prometheus.Counter - AppsRemoved prometheus.Counter - Errors prometheus.Counter + AppsWatched prometheus.Gauge + AppsAdded prometheus.Counter + AppsUpdated prometheus.Counter + AppsRemoved prometheus.Counter + AppWatcherErrors prometheus.Counter } type ApplicationClientMetrics struct { - AppsCreated *prometheus.CounterVec - AppsUpdated *prometheus.CounterVec - AppsDeleted *prometheus.CounterVec - Errors prometheus.Counter + AppsCreated *prometheus.CounterVec + AppsUpdated *prometheus.CounterVec + AppsDeleted *prometheus.CounterVec + AppClientErrors prometheus.Counter +} + +type AppProjectWatcherMetrics struct { + ProjectsWatched prometheus.Gauge + ProjectsAdded prometheus.Counter + ProjectsUpdated prometheus.Counter + ProjectsRemoved prometheus.Counter + ProjectErrors prometheus.Counter } type AppProjectClientMetrics struct { - AppProjectsCreated *prometheus.CounterVec - AppProjectsUpdated *prometheus.CounterVec - AppProjectsDeleted *prometheus.CounterVec - Errors prometheus.Counter + AppProjectsCreated *prometheus.CounterVec + AppProjectsUpdated *prometheus.CounterVec + AppProjectsDeleted *prometheus.CounterVec + ProjectClientErrors prometheus.Counter } // NewApplicationWatcherMetrics returns a new instance of ApplicationMetrics @@ -80,7 +98,7 @@ func NewApplicationClientMetrics() *ApplicationClientMetrics { Name: "argocd_agent_client_applications_deleted", Help: "The total number of applications deleted by the application client", }, []string{"namespace"}), - Errors: promauto.NewCounter(prometheus.CounterOpts{ + AppClientErrors: promauto.NewCounter(prometheus.CounterOpts{ Name: "argocd_agent_client_applications_errors", Help: "The total number of applications deleted by the application client", }), @@ -101,13 +119,35 @@ func NewAppProjectClientMetrics() *AppProjectClientMetrics { Name: "argocd_agent_client_appprojects_deleted", Help: "The total number of appprojects deleted by the appproject client", }, []string{"namespace"}), - Errors: promauto.NewCounter(prometheus.CounterOpts{ + ProjectClientErrors: promauto.NewCounter(prometheus.CounterOpts{ Name: "argocd_agent_client_appprojects_errors", Help: "The total number of appprojects deleted by the appproject client", }), } } +func NewAppProjectWatcherMetrics() *AppProjectWatcherMetrics { + am := &AppProjectWatcherMetrics{ + ProjectsWatched: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "argocd_agent_watcher_appprojects_watched", + Help: "The total number of AppProjects watched by the agent", + }), + ProjectsAdded: promauto.NewCounter(prometheus.CounterOpts{ + Name: "argocd_agent_watcher_appprojects_added", + Help: "The number of AppProjects that have been added to the agent", + }), + ProjectsUpdated: promauto.NewCounter(prometheus.CounterOpts{ + Name: "argocd_agent_watcher_appprojects_updated", + Help: "The number of AppProjects that have been updated", + }), + ProjectsRemoved: promauto.NewCounter(prometheus.CounterOpts{ + Name: "argocd_agent_watcher_appprojects_removed", + Help: "The number of AppProjects that have been removed from the agent", + }), + } + return am +} + // func (am *ApplicationWatcherMetrics) SetWatched(num int64) { // am.AppsWatched.Set(float64(num)) // } diff --git a/internal/metrics/server.go b/internal/metrics/server.go index 1f2188f..ce2861f 100644 --- a/internal/metrics/server.go +++ b/internal/metrics/server.go @@ -17,13 +17,16 @@ package metrics import ( "fmt" "net/http" + neturl "net/url" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/sirupsen/logrus" ) type MetricsServerOptions struct { host string port int + path string } type MetricsServerOption func(*MetricsServerOptions) @@ -32,11 +35,27 @@ func listener(o *MetricsServerOptions) string { l := "" if o.host != "" { l += o.host + } else { + l += "0.0.0.0" } l += fmt.Sprintf(":%d", o.port) return l } +func url(o *MetricsServerOptions) string { + u := neturl.URL{} + u.Scheme = "http" + h := "" + if o.host != "" { + h = o.host + } else { + h = "0.0.0.0" + } + u.Host = fmt.Sprintf("%s:%d", h, o.port) + u.Path = "/metrics" + return u.String() +} + func WithListener(hostname string, port int) MetricsServerOption { return func(o *MetricsServerOptions) { if hostname != "" { @@ -52,14 +71,16 @@ func StartMetricsServer(opts ...MetricsServerOption) chan error { config := &MetricsServerOptions{ host: "", port: 8080, + path: "/metrics", } for _, o := range opts { o(config) } errCh := make(chan error) + logrus.Infof("Starting metrics server on %s", url(config)) go func() { sm := http.NewServeMux() - sm.Handle("/metrics", promhttp.Handler()) + sm.Handle(config.path, promhttp.Handler()) errCh <- http.ListenAndServe(listener(config), sm) }() return errCh diff --git a/principal/server.go b/principal/server.go index 796ac27..665a7fc 100644 --- a/principal/server.go +++ b/principal/server.go @@ -143,6 +143,12 @@ func NewServer(ctx context.Context, kubeClient *kube.KubernetesClient, namespace application.WithRole(manager.ManagerRolePrincipal), } + appProjectInformerOptions := []appprojectinformer.AppProjectInformerOption{ + appprojectinformer.WithAddFunc(s.newAppProjectCallback), + appprojectinformer.WithUpdateFunc(s.updateAppProjectCallback), + appprojectinformer.WithDeleteFunc(s.deleteAppProjectCallback), + } + appProjectManagerOption := []appproject.AppProjectManagerOption{ appproject.WithAllowUpsert(true), appproject.WithRole(manager.ManagerRolePrincipal), @@ -151,6 +157,7 @@ func NewServer(ctx context.Context, kubeClient *kube.KubernetesClient, namespace if s.options.metricsPort > 0 { appInformerOptions = append(appInformerOptions, appinformer.WithMetrics(metrics.NewApplicationWatcherMetrics())) managerOpts = append(managerOpts, application.WithMetrics(metrics.NewApplicationClientMetrics())) + appProjectInformerOptions = append(appProjectInformerOptions, appprojectinformer.WithMetrics(metrics.NewAppProjectWatcherMetrics())) appProjectManagerOption = append(appProjectManagerOption, appproject.WithMetrics(metrics.NewAppProjectClientMetrics())) } @@ -166,12 +173,6 @@ func NewServer(ctx context.Context, kubeClient *kube.KubernetesClient, namespace return nil, err } - appProjectInformerOptions := []appprojectinformer.AppProjectInformerOption{ - appprojectinformer.WithAddFunc(s.newAppProjectCallback), - appprojectinformer.WithUpdateFunc(s.updateAppProjectCallback), - appprojectinformer.WithDeleteFunc(s.deleteAppProjectCallback), - } - projectInformer, err := appprojectinformer.NewAppProjectInformer(s.ctx, kubeClient.ApplicationsClientset, s.namespace, appProjectInformerOptions..., )