[ID-71]Use mongo db aggregation pipeline for get surveys and survey r…

…esponses (#73)

* set the Changelog.md

* make aggregation pipeline to get survey and surveyresponses

* set the mocks

* fix lint issue

---------

Co-authored-by: Stefan Vitanov <[email protected]>

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 ## [1.10.1] - 2024-08-01
 ### Fixed
+- Use mongo db aggregation pipeline for get surveys and survey responses [#71](https://github.com/rokwire/surveys-building-block/issues/71)
+### Fixed
 - Fix survey completed field [#69](https://github.com/rokwire/surveys-building-block/issues/69)
 
 ## [1.10.0] - 2024-007-30

core/app_shared.go

@@ -35,17 +35,8 @@ func (a appShared) getSurvey(id string, orgID string, appID string) (*model.Surv
 }
 
 func (a appShared) getSurveys(orgID string, appID string, userID *string, creatorID *string, surveyIDs []string, surveyTypes []string, calendarEventID string, limit *int, offset *int, filter *model.SurveyTimeFilter, public *bool, archived *bool, completed *bool) ([]model.Survey, []model.SurveyResponse, error) {
-	surveys, err := a.app.storage.GetSurveys(orgID, appID, creatorID, surveyIDs, surveyTypes, calendarEventID, limit, offset, filter, public, archived, nil)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	matchingSurveyIDs := make([]string, len(surveys))
-	for i, survey := range surveys {
-		matchingSurveyIDs[i] = survey.ID
-	}
-
-	surveysResponse, err := a.app.storage.GetSurveyResponses(&orgID, &appID, userID, matchingSurveyIDs, nil, nil, nil, nil, nil)
+	surveys, surveysResponse, err := a.app.storage.GetSurveysAndSurveyResponses(orgID, appID, creatorID, surveyIDs, surveyTypes, calendarEventID,
+		public, archived, limit, offset, userID, filter)
 	if err != nil {
 		return nil, nil, err
 	}

core/interfaces/driven.go

@@ -47,6 +47,8 @@ type Storage interface {
 	DeleteSurveyResponses(orgID string, appID string, userID string, surveyIDs []string, surveyTypes []string, startDate *time.Time, endDate *time.Time) error
 	DeleteSurveyResponsesWithIDs(orgID string, appID string, accountsIDs []string) error
 
+	GetSurveysAndSurveyResponses(orgID string, appID string, creatorID *string, surveyIDs []string, surveyTypes []string, calendarEventID string, public *bool, archived *bool, limit *int, offset *int, userID *string, filter *model.SurveyTimeFilter) ([]model.Survey, []model.SurveyResponse, error)
+
 	GetAlertContacts(orgID string, appID string) ([]model.AlertContact, error)
 	GetAlertContact(id string, orgID string, appID string) (*model.AlertContact, error)
 	GetAlertContactsByKey(key string, orgID string, appID string) ([]model.AlertContact, error)

driven/storage/adapter_surveys.go

@@ -22,6 +22,7 @@ import (
 	"github.com/rokwire/logging-library-go/v2/logutils"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
 )
 
@@ -190,3 +191,165 @@ func (a *Adapter) DeleteSurvey(id string, orgID string, appID string, creatorID
 
 	return nil
 }
+
+// GetSurveysAndSurveyResponses gets surveys and matching survey responses
+func (a *Adapter) GetSurveysAndSurveyResponses(orgID string, appID string, creatorID *string, surveyIDs []string, surveyTypes []string, calendarEventID string, public *bool, archived *bool, limit *int, offset *int, userID *string, timeFilter *model.SurveyTimeFilter) ([]model.Survey, []model.SurveyResponse, error) {
+	// Construct the survey filter
+	surveyFilter := bson.D{
+		{Key: "org_id", Value: orgID},
+		{Key: "app_id", Value: appID},
+	}
+
+	if creatorID != nil {
+		surveyFilter = append(surveyFilter, bson.E{Key: "creator_id", Value: *creatorID})
+	}
+	if len(surveyIDs) > 0 {
+		surveyFilter = append(surveyFilter, bson.E{Key: "_id", Value: bson.M{"$in": surveyIDs}})
+	}
+	if len(surveyTypes) > 0 {
+		surveyFilter = append(surveyFilter, bson.E{Key: "type", Value: bson.M{"$in": surveyTypes}})
+	}
+	if calendarEventID != "" {
+		surveyFilter = append(surveyFilter, bson.E{Key: "calendar_event_id", Value: calendarEventID})
+	}
+	if timeFilter.StartTimeAfter != nil {
+		surveyFilter = append(surveyFilter, bson.E{Key: "start_date", Value: bson.M{"$gte": *timeFilter.StartTimeAfter}})
+	}
+	if timeFilter.StartTimeBefore != nil {
+		surveyFilter = append(surveyFilter, bson.E{Key: "start_date", Value: bson.M{"$lte": *timeFilter.StartTimeBefore}})
+	}
+	if timeFilter.EndTimeAfter != nil {
+		surveyFilter = append(surveyFilter, bson.E{Key: "end_date", Value: bson.M{"$gte": *timeFilter.EndTimeAfter}})
+	}
+	if timeFilter.EndTimeBefore != nil {
+		surveyFilter = append(surveyFilter, bson.E{Key: "end_date", Value: bson.M{"$lte": *timeFilter.EndTimeBefore}})
+	}
+
+	if public != nil {
+		if *public {
+			surveyFilter = append(surveyFilter, bson.E{Key: "public", Value: true})
+		} else {
+			surveyFilter = append(surveyFilter, bson.E{Key: "$or", Value: bson.A{
+				bson.M{"public": false},
+				bson.M{"public": bson.M{"$exists": false}},
+				bson.M{"public": nil},
+			}})
+		}
+	}
+	if archived != nil {
+		if *archived {
+			surveyFilter = append(surveyFilter, bson.E{Key: "archived", Value: true})
+		} else {
+			surveyFilter = append(surveyFilter, bson.E{Key: "$or", Value: bson.A{
+				bson.M{"archived": false},
+				bson.M{"archived": bson.M{"$exists": false}},
+				bson.M{"archived": nil},
+			}})
+		}
+	}
+
+	// Create the aggregation pipeline
+	pipeline := mongo.Pipeline{
+		// Match stage to filter surveys
+		bson.D{{Key: "$match", Value: surveyFilter}},
+		// Lookup stage to join with survey_response and match survey_id
+		bson.D{
+			{Key: "$lookup", Value: bson.D{
+				{Key: "from", Value: "survey_responses"},
+				// Define the variable `surveyIdVar` to hold the value of the survey's _id
+				{Key: "let", Value: bson.D{{Key: "surveyIdVar", Value: "$_id"}}},
+				{Key: "pipeline", Value: bson.A{
+					// Use the variable `surveyIdVar` within the $match stage to filter survey responses
+					bson.D{{Key: "$match", Value: bson.M{
+						"$expr": bson.M{
+							"$and": bson.A{
+								bson.M{"$eq": bson.A{"$survey._id", "$$surveyIdVar"}}, // Match survey._id with the surveyIdVar variable
+								bson.M{"$eq": bson.A{"$org_id", orgID}},
+								bson.M{"$eq": bson.A{"$app_id", appID}},
+								bson.M{"$eq": bson.A{"$user_id", userID}},
+							},
+						},
+					}}},
+					bson.D{{Key: "$project", Value: bson.D{
+						{Key: "_id", Value: 1},
+						{Key: "user_id", Value: 1},
+						{Key: "date_created", Value: 1},
+						{Key: "survey", Value: 1},
+					}}},
+				}},
+				{Key: "as", Value: "responses"},
+			}},
+		},
+		// Project stage to include only required fields
+		bson.D{{Key: "$project", Value: bson.D{
+			{Key: "_id", Value: 1},
+			{Key: "creator_id", Value: 1},
+			{Key: "org_id", Value: 1},
+			{Key: "app_id", Value: 1},
+			{Key: "title", Value: 1},
+			{Key: "more_info", Value: 1},
+			{Key: "data", Value: 1},
+			{Key: "scored", Value: 1},
+			{Key: "result_rules", Value: 1},
+			{Key: "result_json", Value: 1},
+			{Key: "type", Value: 1},
+			{Key: "stats", Value: 1},
+			{Key: "sensitive", Value: 1},
+			{Key: "anonymous", Value: 1},
+			{Key: "default_data_key", Value: 1},
+			{Key: "default_data_key_rule", Value: 1},
+			{Key: "constants", Value: 1},
+			{Key: "strings", Value: 1},
+			{Key: "sub_rules", Value: 1},
+			{Key: "response_keys", Value: 1},
+			{Key: "date_created", Value: 1},
+			{Key: "date_updated", Value: 1},
+			{Key: "calendar_event_id", Value: 1},
+			{Key: "start_date", Value: 1},
+			{Key: "end_date", Value: 1},
+			{Key: "public", Value: 1},
+			{Key: "archived", Value: 1},
+			{Key: "estimated_completion_time", Value: 1},
+			{Key: "responses", Value: "$responses"},
+		}}},
+		// Sort stage if needed
+		bson.D{{Key: "$sort", Value: bson.D{{Key: "date_created", Value: -1}}}},
+	}
+
+	// Add pagination stages if limit is positive
+	if limit != nil && *limit > 0 {
+		pipeline = append(pipeline, bson.D{{Key: "$limit", Value: *limit}})
+	}
+	if offset != nil && *offset > 0 {
+		pipeline = append(pipeline, bson.D{{Key: "$skip", Value: *offset}})
+	}
+
+	var surveysWithResponses []bson.M
+	err := a.db.surveys.Aggregate(pipeline, &surveysWithResponses, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var surveys []model.Survey
+	var surveyResponses []model.SurveyResponse
+
+	// Process results
+	for _, item := range surveysWithResponses {
+		survey := model.Survey{}
+		responseDocs, _ := item["responses"].(primitive.A)
+		responses := make([]model.SurveyResponse, len(responseDocs))
+		for i, doc := range responseDocs {
+			var response model.SurveyResponse
+			bsonBytes, _ := bson.Marshal(doc)
+			bson.Unmarshal(bsonBytes, &response)
+			responses[i] = response
+		}
+
+		bsonBytes, _ := bson.Marshal(item)
+		bson.Unmarshal(bsonBytes, &survey)
+		surveys = append(surveys, survey)
+		surveyResponses = append(surveyResponses, responses...)
+	}
+
+	return surveys, surveyResponses, nil
+}

driven/storage/collection.go

@@ -236,7 +236,11 @@ func (collWrapper *collectionWrapper) CountDocuments(ctx context.Context, filter
 	return count, nil
 }
 
-func (collWrapper *collectionWrapper) Aggregate(ctx context.Context, pipeline interface{}, result interface{}, ops *options.AggregateOptions) error {
+func (collWrapper *collectionWrapper) Aggregate(pipeline interface{}, result interface{}, ops *options.AggregateOptions) error {
+	return collWrapper.AggregateWithContext(context.Background(), pipeline, result, ops)
+}
+
+func (collWrapper *collectionWrapper) AggregateWithContext(ctx context.Context, pipeline interface{}, result interface{}, ops *options.AggregateOptions) error {
 	if ctx == nil {
 		ctx = context.Background()
 	}