[ID-87] Events issues (#88)

* Update CHANGELOG.md

* Load items by source id = 0 (webtools)

* Keep the ids

* Remove by source id

* Compete the logic

* Use whitelist for the categories

* Revert the timer

---------

Co-authored-by: Petyo Stoyanov <[email protected]>

CHANGELOG.md

@@ -5,13 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Changed
+- Events issues [#87](https://github.com/rokwire/gateway-building-block/issues/87)
+
 [2.6.1] - 2024-04-18
 ### Fixed
 - Fix Legacy event import [#83](https://github.com/rokwire/gateway-building-block/issues/83)
 
 [2.6.0] - 2024-04-18
 ### Changed
-- Legacy event import issues [#80] (https://github.com/rokwire/gateway-building-block/issues/80)
+- Legacy event import issues [#80](https://github.com/rokwire/gateway-building-block/issues/80)
 
 [2.5.0] - 2024-04-18
 ### Changed

core/interfaces.go

@@ -126,8 +126,10 @@ type Storage interface {
 	FindLegacyLocations() (model.LegacyLocationsListType, error)
 
 	FindLegacyEventItems(context storage.TransactionContext) ([]model.LegacyEventItem, error)
+	FindLegacyEventItemsBySourceID(context storage.TransactionContext, sourceID string) ([]model.LegacyEventItem, error)
 	InsertLegacyEvents(context storage.TransactionContext, items []model.LegacyEventItem) ([]model.LegacyEventItem, error)
 	DeleteLegacyEventsByIDs(context storage.TransactionContext, Ids map[string]string) error
+	DeleteLegacyEventsBySourceID(context storage.TransactionContext, sourceID string) error
 	DeleteLegacyEventsByIDsAndCreator(context storage.TransactionContext, ids []string, accountID string) error
 	FindAllLegacyEvents() ([]model.LegacyEvent, error)
 

core/logic_events.go

@@ -259,38 +259,28 @@ func (e eventsLogic) processWebToolsEvents() {
 
 	//in transaction
 	err = e.app.storage.PerformTransaction(func(context storage.TransactionContext) error {
-		//1. first find which events are already in the database. You have to compare by dataSourceEventId field.
-		legacyEventItemFromStorage, err := e.app.storage.FindLegacyEventItems(context)
+		//1. first we must keep the events ids for the webtools events(sourceId = "0") because we will remove all of them and later recreated with the new ones
+		webtoolsItemsFromStorage, err := e.app.storage.FindLegacyEventItemsBySourceID(context, "0")
 		if err != nil {
-			e.logger.Errorf("error on loading events from the storage - %s", err)
+			e.logger.Errorf("error on loading webtools events from the storage - %s", err)
 			return err
 		}
 
-		var leExist []model.LegacyEventItem
-		for _, w := range allWebToolsEvents {
-			for _, l := range legacyEventItemFromStorage {
-				if w.EventID == l.Item.DataSourceEventID {
-					leExist = append(leExist, l)
-				}
-			}
-		}
-
-		//1.1 before to execute point 2(i.e. remove all of them) you must keep their IDs so that to put them again on point 3
 		existingLegacyIdsMap := make(map[string]string)
-		for _, w := range leExist {
-			if w.Item.DataSourceEventID != "" {
+		for _, w := range webtoolsItemsFromStorage {
+			if len(w.Item.DataSourceEventID) > 0 {
 				existingLegacyIdsMap[w.Item.DataSourceEventID] = w.Item.ID
 			}
 		}
 
-		//2. Once you know which are already in the datatabse then you must remove all of them
-		err = e.app.storage.DeleteLegacyEventsByIDs(context, existingLegacyIdsMap)
+		//2. once we already have the ids then we have to remove all webtools events from the database
+		err = e.app.storage.DeleteLegacyEventsBySourceID(context, "0")
 		if err != nil {
-			e.logger.Errorf("error on deleting events from the storage - %s", err)
+			e.logger.Errorf("error on deleting legacy events from the storage - %s", err)
 			return err
 		}
 
-		//at this moment the existing events are removed and we can add what comes from webtools
+		//at this moment the all webtools items are removed from the database and we can add what comes from webtools
 
 		//3. we have a requirement to ignore events or modify them before applying
 		modifiedWebToolsEvents, err := e.modifyWebtoolsEventsList(allWebToolsEvents)
@@ -299,7 +289,7 @@ func (e eventsLogic) processWebToolsEvents() {
 			return err
 		}
 
-		//4. Now you have to convert all allWebToolsEvents into legacy events
+		//4. now you have to convert all allWebToolsEvents into legacy events
 		newLegacyEvents := []model.LegacyEventItem{}
 		for _, wt := range modifiedWebToolsEvents {
 
@@ -310,16 +300,14 @@ func (e eventsLogic) processWebToolsEvents() {
 			newLegacyEvents = append(newLegacyEvents, le)
 		}
 
-		//5. Store all them in the database
+		//5. store all them in the database
 		_, err = e.app.storage.InsertLegacyEvents(context, newLegacyEvents)
 		if err != nil {
 			e.logger.Errorf("error on saving events to the storage - %s", err)
 			return err
 		}
 		// It is all!
 
-		//* keep the already exisiting events IDS THE SAME!
-
 		return nil
 	}, 180000)
 
@@ -336,7 +324,7 @@ func (e eventsLogic) modifyWebtoolsEventsList(allWebtoolsEvents []model.WebTools
 	ignored := 0
 	modified := 0
 
-	//map for category conversions
+	//whitelist with categories which we care + map for category conversions
 	categoryMap := map[string]string{
 		"exhibition":               "Exhibits",
 		"festival/celebration":     "Festivals and Celebrations",
@@ -366,22 +354,16 @@ func (e eventsLogic) modifyWebtoolsEventsList(allWebtoolsEvents []model.WebTools
 			continue
 		}
 
-		//ignore some categories
-		if lowerCategory == "informational" || lowerCategory == "meeting" ||
-			lowerCategory == "community service" || lowerCategory == "ceremony/service" ||
-			lowerCategory == "other" {
-
-			e.logger.Infof("skipping event as category is %s", category)
-			ignored++
-			continue
-		}
-
-		//modify some categories
+		//get only the events which have a category from the whitelist
 		if newCategory, ok := categoryMap[lowerCategory]; ok {
 			currentWte.EventType = newCategory
 			e.logger.Infof("modifying event category from %s to %s", category, newCategory)
 
 			modified++
+		} else {
+			e.logger.Infof("skipping event as category is %s", category)
+			ignored++
+			continue
 		}
 
 		//add it to the modified list

driven/storage/adapter.go

@@ -292,6 +292,19 @@ func (a *Adapter) FindLegacyEventItems(context TransactionContext) ([]model.Lega
 	return data, nil
 }
 
+// FindLegacyEventItemsBySourceID finds legacy events items by source id
+func (a *Adapter) FindLegacyEventItemsBySourceID(context TransactionContext, sourceID string) ([]model.LegacyEventItem, error) {
+	filter := bson.D{primitive.E{Key: "item.sourceId", Value: sourceID}}
+	var data []model.LegacyEventItem
+	timeout := 15 * time.Second //15 seconds timeout
+	err := a.db.legacyEvents.FindWithParams(context, filter, &data, nil, &timeout)
+	if err != nil {
+		return nil, errors.WrapErrorAction(logutils.ActionFind, model.TypeExample, filterArgs(nil), err)
+	}
+
+	return data, nil
+}
+
 // InsertLegacyEvents inserts legacy events
 func (a *Adapter) InsertLegacyEvents(context TransactionContext, items []model.LegacyEventItem) ([]model.LegacyEventItem, error) {
 
@@ -316,7 +329,7 @@ func (a *Adapter) DeleteLegacyEvents() error {
 	return err
 }
 
-// DeleteLegacyEventsByIDs deletes all items by dataSourceEventIds
+// DeleteLegacyEventsByIDs deletes all items by dataSourceEventIds ????
 func (a *Adapter) DeleteLegacyEventsByIDs(context TransactionContext, Ids map[string]string) error {
 
 	var valueIds []string
@@ -332,6 +345,16 @@ func (a *Adapter) DeleteLegacyEventsByIDs(context TransactionContext, Ids map[st
 	return err
 }
 
+// DeleteLegacyEventsBySourceID deletes all legacy events by source id
+func (a *Adapter) DeleteLegacyEventsBySourceID(context TransactionContext, sourceID string) error {
+	filter := bson.D{
+		primitive.E{Key: "item.sourceId", Value: sourceID},
+	}
+	timeout := 15 * time.Second //15 seconds timeout
+	_, err := a.db.legacyEvents.DeleteManyWithParams(context, filter, nil, &timeout)
+	return err
+}
+
 // DeleteLegacyEventsByIDsAndCreator deletes legacy events by ids and creator
 func (a *Adapter) DeleteLegacyEventsByIDsAndCreator(context TransactionContext, ids []string, accountID string) error {
 	var valueIds []string

driven/storage/database.go

@@ -175,6 +175,12 @@ func (d *database) applyLegacyEventsChecks(legacyEvents *collectionWrapper) erro
 		return err
 	}
 
+	//sourceId
+	err = legacyEvents.AddIndex(bson.D{primitive.E{Key: "item.sourceId", Value: 1}}, false)
+	if err != nil {
+		return err
+	}
+
 	//TODO - add other - source event id - calendar id?
 
 	d.logger.Info("legacy events passed")