Skip to content

Commit

Permalink
feat: revisions info reporting with multi-sourced apps support (#333)
Browse files Browse the repository at this point in the history
* event-reporter: added utils methods to retrieve revisions metadata for application

* event-reporter: report all revisions metadata instead single to support multisourced apps

* event-reporter: added revision sha to reported value in anotations - "app.meta.revisions-metadata"

* event-reporter: added change revisions sha to reported value in anotations - app.meta.revisions-metadata

* event-reporter: updated changelog

* event-reporter: changes to anotations repoting - app.meta.revisions-metadata, report only revision in case of helm chart

* event-reporter: changes after pr review

* event-reporter: fixed unit tests

* event-reporter: fix lint issues

* event-reporter: changes after pr reviev, fixing typo, added dedicated func a.Spec.IsHelmSource(idx), removed legacy code

* event-reporter: refactoring of getApplicationLegacyRevisionDetails method

* event-reporter / app_revision_test.go: added some tests to AddCommitsDetailsToAnnotations, AddCommitsDetailsToAppAnnotations, getRevisions, getOperationSyncRevisions

* event-reporter / app_revision_test.go: added tests for GetRevisionsDetails method

* event-reporter: updated app client to support sourceIndex param in non-grpc mode

* event-reporter / app_revision.go: added sourceIndex param to applicationServiceClient.RevisionMetadata in order to properly support multisourced apps

* event-reporter: lint fix

* event-reporter: fix lint issues

* event-reporter: fix lint issues

* event-reporter: added back regacy logic with setting of commit details to labels as new runtimes should work on old on-prem versions

* event-reporter: added condition to not send empty array for ChangeRevisions metadata

---------

Co-authored-by: pashakostohrys <[email protected]>
  • Loading branch information
oleksandr-codefresh and pasha-codefresh authored Oct 5, 2024
1 parent 288e7d3 commit 66e1e38
Show file tree
Hide file tree
Showing 11 changed files with 763 additions and 69 deletions.
2 changes: 1 addition & 1 deletion changelog/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
### Features
- fix: change revision controller should verify that revision already exists
- feat: event-reporter: report change revisions metadata in app annotations
3 changes: 3 additions & 0 deletions event_reporter/application/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,9 @@ func (c *httpApplicationClient) RevisionMetadata(ctx context.Context, in *appcli
params := fmt.Sprintf("?appNamespace=%s&project=%s",
*in.AppNamespace,
*in.Project)
if in.SourceIndex != nil {
params += fmt.Sprintf("&sourceIndex=%d", *in.SourceIndex)
}
url := fmt.Sprintf("%s/api/v1/applications/%s/revisions/%s/metadata%s", c.baseUrl, *in.Name, *in.Revision, params)
revisionMetadata := &v1alpha1.RevisionMetadata{}
err := c.execute(ctx, url, revisionMetadata)
Expand Down
89 changes: 81 additions & 8 deletions event_reporter/reporter/app_revision.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,89 @@ package reporter
import (
"context"

"github.com/argoproj/argo-cd/v2/event_reporter/utils"
"github.com/argoproj/argo-cd/v2/pkg/apiclient/application"
appv1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
"github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"

log "github.com/sirupsen/logrus"
)

func (s *applicationEventReporter) getApplicationRevisionDetails(ctx context.Context, a *appv1.Application, revision string) (*appv1.RevisionMetadata, error) {
// treats multi-sourced apps as single source and gets first revision details
func getApplicationLegacyRevisionDetails(a *v1alpha1.Application, revisionsWithMetadata *utils.AppSyncRevisionsMetadata) *v1alpha1.RevisionMetadata {
if revisionsWithMetadata.SyncRevisions == nil || len(revisionsWithMetadata.SyncRevisions) == 0 {
return nil
}

sourceIdx := 0

if a.Spec.HasMultipleSources() {
_, sourceIdx = a.Spec.GetNonRefSource()
}

if revisionWithMetadata := revisionsWithMetadata.SyncRevisions[sourceIdx]; revisionWithMetadata != nil {
return revisionWithMetadata.Metadata
}

return nil
}

func (s *applicationEventReporter) getRevisionsDetails(ctx context.Context, a *v1alpha1.Application, revisions []string) ([]*utils.RevisionWithMetadata, error) {
project := a.Spec.GetProject()
return s.applicationServiceClient.RevisionMetadata(ctx, &application.RevisionMetadataQuery{
Name: &a.Name,
AppNamespace: &a.Namespace,
Revision: &revision,
Project: &project,
})
rms := make([]*utils.RevisionWithMetadata, 0)

for idx, revision := range revisions {
// report just revision for helm sources
if a.Spec.SourceUnderIdxIsHelm(idx) {
rms = append(rms, &utils.RevisionWithMetadata{
Revision: revision,
})
continue
}

sourceIndex := int32(idx)

rm, err := s.applicationServiceClient.RevisionMetadata(ctx, &application.RevisionMetadataQuery{
Name: &a.Name,
AppNamespace: &a.Namespace,
Revision: &revision,
Project: &project,
SourceIndex: &sourceIndex,
})
if err != nil {
return nil, err
}
rms = append(rms, &utils.RevisionWithMetadata{
Revision: revision,
Metadata: rm,
})
}

return rms, nil
}

func (s *applicationEventReporter) getApplicationRevisionsMetadata(ctx context.Context, logCtx *log.Entry, a *v1alpha1.Application) (*utils.AppSyncRevisionsMetadata, error) {
result := &utils.AppSyncRevisionsMetadata{}

if a.Status.Sync.Revision != "" || a.Status.Sync.Revisions != nil || (a.Status.History != nil && len(a.Status.History) > 0) {
// can be the latest revision of repository
operationSyncRevisionsMetadata, err := s.getRevisionsDetails(ctx, a, utils.GetOperationSyncRevisions(a))
if err != nil {
logCtx.WithError(err).Warnf("failed to get application(%s) sync revisions metadata, resuming", a.GetName())
}

if err == nil && operationSyncRevisionsMetadata != nil {
result.SyncRevisions = operationSyncRevisionsMetadata
}
// latest revision of repository where changes to app resource were actually made; empty if no changeRevision(-s) present
operationChangeRevisionsMetadata, err := s.getRevisionsDetails(ctx, a, utils.GetOperationChangeRevisions(a))
if err != nil {
logCtx.WithError(err).Warnf("failed to get application(%s) change revisions metadata, resuming", a.GetName())
}

if err == nil && operationChangeRevisionsMetadata != nil && len(operationChangeRevisionsMetadata) > 0 {
result.ChangeRevisions = operationChangeRevisionsMetadata
}
}

return result, nil
}
168 changes: 168 additions & 0 deletions event_reporter/reporter/app_revision_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
package reporter

import (
"context"
"testing"

"github.com/argoproj/argo-cd/v2/event_reporter/application/mocks"
"github.com/argoproj/argo-cd/v2/event_reporter/metrics"
"github.com/argoproj/argo-cd/v2/event_reporter/utils"
"github.com/argoproj/argo-cd/v2/pkg/apiclient/application"
"github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
"github.com/argoproj/argo-cd/v2/server/cache"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)

func TestGetRevisionsDetails(t *testing.T) {
t.Run("should return revisions for single source app", func(t *testing.T) {
expectedRevision := "expected-revision"
expectedResult := []*utils.RevisionWithMetadata{{
Revision: expectedRevision,
Metadata: &v1alpha1.RevisionMetadata{
Author: "Test Author",
Message: "first commit",
},
}}

app := v1alpha1.Application{
Spec: v1alpha1.ApplicationSpec{
Source: &v1alpha1.ApplicationSource{
RepoURL: "https://my-site.com",
TargetRevision: "HEAD",
Path: ".",
},
},
}

appServiceClient := mocks.NewApplicationClient(t)
project := app.Spec.GetProject()
sourceIdx1 := int32(0)

appServiceClient.On("RevisionMetadata", mock.Anything, &application.RevisionMetadataQuery{
Name: &app.Name,
AppNamespace: &app.Namespace,
Revision: &expectedResult[0].Revision,
Project: &project,
SourceIndex: &sourceIdx1,
}).Return(expectedResult[0].Metadata, nil)

reporter := &applicationEventReporter{
&cache.Cache{},
&MockCodefreshClient{},
newAppLister(),
appServiceClient,
&metrics.MetricsServer{},
}

result, _ := reporter.getRevisionsDetails(context.Background(), &app, []string{expectedRevision})

assert.Equal(t, expectedResult, result)
})

t.Run("should return revisions for multi sourced apps", func(t *testing.T) {
expectedRevision1 := "expected-revision-1"
expectedRevision2 := "expected-revision-2"
expectedResult := []*utils.RevisionWithMetadata{{
Revision: expectedRevision1,
Metadata: &v1alpha1.RevisionMetadata{
Author: "Repo1 Author",
Message: "first commit repo 1",
},
}, {
Revision: expectedRevision2,
Metadata: &v1alpha1.RevisionMetadata{
Author: "Repo2 Author",
Message: "first commit repo 2",
},
}}

app := v1alpha1.Application{
Spec: v1alpha1.ApplicationSpec{
Sources: []v1alpha1.ApplicationSource{{
RepoURL: "https://my-site.com/repo-1",
TargetRevision: "branch1",
Path: ".",
}, {
RepoURL: "https://my-site.com/repo-2",
TargetRevision: "branch2",
Path: ".",
}},
},
}

project := app.Spec.GetProject()

appServiceClient := mocks.NewApplicationClient(t)
sourceIdx1 := int32(0)
sourceIdx2 := int32(1)
appServiceClient.On("RevisionMetadata", mock.Anything, &application.RevisionMetadataQuery{
Name: &app.Name,
AppNamespace: &app.Namespace,
Revision: &expectedRevision1,
Project: &project,
SourceIndex: &sourceIdx1,
}).Return(expectedResult[0].Metadata, nil)
appServiceClient.On("RevisionMetadata", mock.Anything, &application.RevisionMetadataQuery{
Name: &app.Name,
AppNamespace: &app.Namespace,
Revision: &expectedRevision2,
Project: &project,
SourceIndex: &sourceIdx2,
}).Return(expectedResult[1].Metadata, nil)

reporter := &applicationEventReporter{
&cache.Cache{},
&MockCodefreshClient{},
newAppLister(),
appServiceClient,
&metrics.MetricsServer{},
}

result, _ := reporter.getRevisionsDetails(context.Background(), &app, []string{expectedRevision1, expectedRevision2})

assert.Equal(t, expectedResult, result)
})

t.Run("should return only revision because of helm single source app", func(t *testing.T) {
expectedRevision := "expected-revision"
expectedResult := []*utils.RevisionWithMetadata{{
Revision: expectedRevision,
}}

app := v1alpha1.Application{
Spec: v1alpha1.ApplicationSpec{
Source: &v1alpha1.ApplicationSource{
RepoURL: "https://my-site.com",
TargetRevision: "HEAD",
Path: ".",
},
},
}

appServiceClient := mocks.NewApplicationClient(t)
project := app.Spec.GetProject()
sourceIdx1 := int32(0)

appServiceClient.On("RevisionMetadata", mock.Anything, &application.RevisionMetadataQuery{
Name: &app.Name,
AppNamespace: &app.Namespace,
Revision: &expectedResult[0].Revision,
Project: &project,
SourceIndex: &sourceIdx1,
}).Return(expectedResult[0].Metadata, nil)

reporter := &applicationEventReporter{
&cache.Cache{},
&MockCodefreshClient{},
newAppLister(),
appServiceClient,
&metrics.MetricsServer{},
}

result, _ := reporter.getRevisionsDetails(context.Background(), &app, []string{expectedRevision})

assert.Equal(t, expectedResult, result)
})
}
32 changes: 16 additions & 16 deletions event_reporter/reporter/application_event_reporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -166,14 +166,13 @@ func (s *applicationEventReporter) StreamApplicationEvents(

// helm app hasnt revision
// TODO: add check if it helm application
parentOperationRevision := utils.GetOperationRevision(parentApplicationEntity)
parentRevisionMetadata, err := s.getApplicationRevisionDetails(ctx, parentApplicationEntity, parentOperationRevision)
parentAppSyncRevisionsMetadata, err := s.getApplicationRevisionsMetadata(ctx, logCtx, parentApplicationEntity)
if err != nil {
logCtx.WithError(err).Warn("failed to get parent application's revision metadata, resuming")
}

utils.SetHealthStatusIfMissing(rs)
err = s.processResource(ctx, *rs, parentApplicationEntity, logCtx, ts, parentDesiredManifests, appTree, manifestGenErr, a, parentRevisionMetadata, appInstanceLabelKey, trackingMethod, desiredManifests.ApplicationVersions)
err = s.processResource(ctx, *rs, parentApplicationEntity, logCtx, ts, parentDesiredManifests, appTree, manifestGenErr, a, parentAppSyncRevisionsMetadata, appInstanceLabelKey, trackingMethod, desiredManifests.ApplicationVersions)
if err != nil {
s.metricsServer.IncErroredEventsCounter(metrics.MetricChildAppEventType, metrics.MetricEventUnknownErrorType, a.Name)
return err
Expand Down Expand Up @@ -203,7 +202,7 @@ func (s *applicationEventReporter) StreamApplicationEvents(
s.metricsServer.ObserveEventProcessingDurationHistogramDuration(a.Name, metrics.MetricParentAppEventType, reconcileDuration)
}

revisionMetadata, _ := s.getApplicationRevisionDetails(ctx, a, utils.GetOperationRevision(a))
revisionsMetadata, _ := s.getApplicationRevisionsMetadata(ctx, logCtx, a)
// for each resource in the application get desired and actual state,
// then stream the event
for _, rs := range a.Status.Resources {
Expand All @@ -215,7 +214,7 @@ func (s *applicationEventReporter) StreamApplicationEvents(
s.metricsServer.IncCachedIgnoredEventsCounter(metrics.MetricResourceEventType, a.Name)
continue
}
err := s.processResource(ctx, rs, a, logCtx, ts, desiredManifests, appTree, manifestGenErr, nil, revisionMetadata, appInstanceLabelKey, trackingMethod, nil)
err := s.processResource(ctx, rs, a, logCtx, ts, desiredManifests, appTree, manifestGenErr, nil, revisionsMetadata, appInstanceLabelKey, trackingMethod, nil)
if err != nil {
s.metricsServer.IncErroredEventsCounter(metrics.MetricResourceEventType, metrics.MetricEventUnknownErrorType, a.Name)
return err
Expand All @@ -227,21 +226,22 @@ func (s *applicationEventReporter) StreamApplicationEvents(
func (s *applicationEventReporter) getAppForResourceReporting(
rs appv1.ResourceStatus,
ctx context.Context,
logCtx *log.Entry,
a *appv1.Application,
revisionMetadata *appv1.RevisionMetadata,
) (*appv1.Application, *appv1.RevisionMetadata) {
syncRevisionsMetadata *utils.AppSyncRevisionsMetadata,
) (*appv1.Application, *utils.AppSyncRevisionsMetadata) {
if rs.Kind != "Rollout" { // for rollout it's crucial to report always correct operationSyncRevision
return a, revisionMetadata
return a, syncRevisionsMetadata
}

latestAppStatus, err := s.appLister.Applications(a.Namespace).Get(a.Name)
if err != nil {
return a, revisionMetadata
return a, syncRevisionsMetadata
}

revisionMetadataToReport, err := s.getApplicationRevisionDetails(ctx, latestAppStatus, utils.GetOperationRevision(latestAppStatus))
revisionMetadataToReport, err := s.getApplicationRevisionsMetadata(ctx, logCtx, latestAppStatus)
if err != nil {
return a, revisionMetadata
return a, syncRevisionsMetadata
}

return latestAppStatus, revisionMetadataToReport
Expand All @@ -257,7 +257,7 @@ func (s *applicationEventReporter) processResource(
appTree *appv1.ApplicationTree,
manifestGenErr bool,
originalApplication *appv1.Application,
revisionMetadata *appv1.RevisionMetadata,
revisionsMetadata *utils.AppSyncRevisionsMetadata,
appInstanceLabelKey string,
trackingMethod appv1.TrackingMethod,
applicationVersions *apiclient.ApplicationVersions,
Expand All @@ -283,12 +283,12 @@ func (s *applicationEventReporter) processResource(
return nil
}

parentApplicationToReport, revisionMetadataToReport := s.getAppForResourceReporting(rs, ctx, parentApplication, revisionMetadata)
parentApplicationToReport, revisionMetadataToReport := s.getAppForResourceReporting(rs, ctx, logCtx, parentApplication, revisionsMetadata)

var originalAppRevisionMetadata *appv1.RevisionMetadata = nil
var originalAppRevisionMetadata *utils.AppSyncRevisionsMetadata = nil

if originalApplication != nil {
originalAppRevisionMetadata, _ = s.getApplicationRevisionDetails(ctx, originalApplication, utils.GetOperationRevision(originalApplication))
originalAppRevisionMetadata, _ = s.getApplicationRevisionsMetadata(ctx, logCtx, originalApplication)
}

ev, err := getResourceEventPayload(parentApplicationToReport, &rs, actualState, desiredState, appTree, manifestGenErr, ts, originalApplication, revisionMetadataToReport, originalAppRevisionMetadata, appInstanceLabelKey, trackingMethod, applicationVersions)
Expand All @@ -305,7 +305,7 @@ func (s *applicationEventReporter) processResource(
appName = appRes.Name
} else {
utils.LogWithResourceStatus(logCtx, rs).Info("streaming resource event")
appName = rs.Name
appName = parentApplication.Name
}

if err := s.codefreshClient.SendEvent(ctx, appName, ev); err != nil {
Expand Down
Loading

0 comments on commit 66e1e38

Please sign in to comment.