FIx Finding Housekeeping Candidates and Modify Housekeeping Structure

server/backend/database/database.go

@@ -134,13 +134,19 @@ type Database interface {
 	// after handling PushPull.
 	UpdateClientInfoAfterPushPull(ctx context.Context, clientInfo *ClientInfo, docInfo *DocInfo) error
 
-	// FindDeactivateCandidates finds the housekeeping candidates.
-	FindDeactivateCandidates(
+	// FindNextNCyclingProjectInfos finds the next N cycling projects from the given projectID.
+	FindNextNCyclingProjectInfos(
 		ctx context.Context,
-		candidatesLimitPerProject int,
-		projectFetchSize int,
+		pageSize int,
 		lastProjectID types.ID,
-	) (types.ID, []*ClientInfo, error)
+	) ([]*ProjectInfo, error)
+
+	// FindDeactivateCandidatesPerProject finds the clients that need housekeeping per project.
+	FindDeactivateCandidatesPerProject(
+		ctx context.Context,
+		project *ProjectInfo,
+		candidatesLimit int,
+	) ([]*ClientInfo, error)
 
 	// FindDocInfoByKey finds the document of the given key.
 	FindDocInfoByKey(

server/backend/database/memory/database.go

@@ -224,38 +224,58 @@ func (d *DB) CreateProjectInfo(
 	return info, nil
 }
 
-// listProjectInfos returns all project infos rotationally.
-func (d *DB) listProjectInfos(
+// FindNextNCyclingProjectInfos finds the next N cycling projects from the given projectID.
+func (d *DB) FindNextNCyclingProjectInfos(
 	_ context.Context,
 	pageSize int,
-	housekeepingLastProjectID types.ID,
+	lastProjectID types.ID,
 ) ([]*database.ProjectInfo, error) {
 	txn := d.db.Txn(false)
 	defer txn.Abort()
 
 	iter, err := txn.LowerBound(
 		tblProjects,
 		"id",
-		housekeepingLastProjectID.String(),
+		lastProjectID.String(),
 	)
 	if err != nil {
 		return nil, fmt.Errorf("fetch projects: %w", err)
 	}
 
 	var infos []*database.ProjectInfo
+	isCircular := false
 
 	for i := 0; i < pageSize; i++ {
 		raw := iter.Next()
 		if raw == nil {
-			break
+			if isCircular {
+				break
+			}
+
+			iter, err = txn.LowerBound(
+				tblProjects,
+				"id",
+				database.DefaultProjectID.String(),
+			)
+			if err != nil {
+				return nil, fmt.Errorf("fetch projects: %w", err)
+			}
+
+			i--
+			isCircular = true
+			continue
 		}
 		info := raw.(*database.ProjectInfo).DeepCopy()
 
-		if i == 0 && info.ID == housekeepingLastProjectID {
+		if i == 0 && info.ID == lastProjectID {
 			pageSize++
 			continue
 		}
 
+		if len(infos) > 0 && infos[0].ID == info.ID {
+			break
+		}
+
 		infos = append(infos, info)
 	}
 
@@ -563,8 +583,8 @@ func (d *DB) UpdateClientInfoAfterPushPull(
 	return nil
 }
 
-// findDeactivateCandidatesPerProject finds the clients that need housekeeping per project.
-func (d *DB) findDeactivateCandidatesPerProject(
+// FindDeactivateCandidatesPerProject finds the clients that need housekeeping per project.
+func (d *DB) FindDeactivateCandidatesPerProject(
 	_ context.Context,
 	project *database.ProjectInfo,
 	candidatesLimit int,
@@ -599,41 +619,12 @@ func (d *DB) findDeactivateCandidatesPerProject(
 			info.UpdatedAt.After(offset) {
 			break
 		}
-		infos = append(infos, info)
-	}
-	return infos, nil
-}
 
-// FindDeactivateCandidates finds the clients that need housekeeping.
-func (d *DB) FindDeactivateCandidates(
-	ctx context.Context,
-	candidatesLimitPerProject int,
-	projectFetchSize int,
-	lastProjectID types.ID,
-) (types.ID, []*database.ClientInfo, error) {
-	projects, err := d.listProjectInfos(ctx, projectFetchSize, lastProjectID)
-	if err != nil {
-		return database.DefaultProjectID, nil, err
-	}
-
-	var candidates []*database.ClientInfo
-	for _, project := range projects {
-		infos, err := d.findDeactivateCandidatesPerProject(ctx, project, candidatesLimitPerProject)
-		if err != nil {
-			return database.DefaultProjectID, nil, err
+		if info.ProjectID == project.ID {
+			infos = append(infos, info)
 		}
-
-		candidates = append(candidates, infos...)
-	}
-
-	var topProjectID types.ID
-	if len(projects) < projectFetchSize {
-		topProjectID = database.DefaultProjectID
-	} else {
-		topProjectID = projects[len(projects)-1].ID
 	}
-
-	return topProjectID, candidates, nil
+	return infos, nil
 }
 
 // FindDocInfoByKeyAndOwner finds the document of the given key. If the

server/backend/database/memory/database_test.go

@@ -36,6 +36,14 @@ func TestDB(t *testing.T) {
 	db, err := memory.New()
 	assert.NoError(t, err)
 
+	t.Run("FindNextNCyclingProjectInfos test", func(t *testing.T) {
+		testcases.RunFindNextNCyclingProjectInfosTest(t, db)
+	})
+
+	t.Run("FindDeactivateCandidatesPerProject test", func(t *testing.T) {
+		testcases.RunFindDeactivateCandidatesPerProjectTest(t, db)
+	})
+
 	t.Run("RunFindDocInfo test", func(t *testing.T) {
 		testcases.RunFindDocInfoTest(t, db, projectID)
 	})

server/backend/database/mongo/client.go

@@ -240,13 +240,13 @@ func (c *Client) CreateProjectInfo(
 	return info, nil
 }
 
-// listProjectInfos returns all project infos rotationally.
-func (c *Client) listProjectInfos(
+// FindNextNCyclingProjectInfos finds the next N cycling projects from the given projectID.
+func (c *Client) FindNextNCyclingProjectInfos(
 	ctx context.Context,
 	pageSize int,
-	housekeepingLastProjectID types.ID,
+	lastProjectID types.ID,
 ) ([]*database.ProjectInfo, error) {
-	encodedID, err := encodeID(housekeepingLastProjectID)
+	encodedID, err := encodeID(lastProjectID)
 	if err != nil {
 		return nil, err
 	}
@@ -268,6 +268,25 @@ func (c *Client) listProjectInfos(
 		return nil, fmt.Errorf("fetch project infos: %w", err)
 	}
 
+	if len(infos) < pageSize {
+		opts.SetLimit(int64(pageSize - len(infos)))
+
+		cursor, err := c.collection(colProjects).Find(ctx, bson.M{
+			"_id": bson.M{
+				"$lte": encodedID,
+			},
+		}, opts)
+		if err != nil {
+			return nil, fmt.Errorf("find project infos: %w", err)
+		}
+
+		var newInfos []*database.ProjectInfo
+		if err := cursor.All(ctx, &newInfos); err != nil {
+			return nil, fmt.Errorf("fetch project infos: %w", err)
+		}
+		infos = append(infos, newInfos...)
+	}
+
 	return infos, nil
 }
 
@@ -633,8 +652,8 @@ func (c *Client) UpdateClientInfoAfterPushPull(
 	return nil
 }
 
-// findDeactivateCandidatesPerProject finds the clients that need housekeeping per project.
-func (c *Client) findDeactivateCandidatesPerProject(
+// FindDeactivateCandidatesPerProject finds the clients that need housekeeping per project.
+func (c *Client) FindDeactivateCandidatesPerProject(
 	ctx context.Context,
 	project *database.ProjectInfo,
 	candidatesLimit int,
@@ -669,37 +688,6 @@ func (c *Client) findDeactivateCandidatesPerProject(
 	return clientInfos, nil
 }
 
-// FindDeactivateCandidates finds the clients that need housekeeping.
-func (c *Client) FindDeactivateCandidates(
-	ctx context.Context,
-	candidatesLimitPerProject int,
-	projectFetchSize int,
-	lastProjectID types.ID,
-) (types.ID, []*database.ClientInfo, error) {
-	projects, err := c.listProjectInfos(ctx, projectFetchSize, lastProjectID)
-	if err != nil {
-		return database.DefaultProjectID, nil, err
-	}
-
-	var candidates []*database.ClientInfo
-	for _, project := range projects {
-		clientInfos, err := c.findDeactivateCandidatesPerProject(ctx, project, candidatesLimitPerProject)
-		if err != nil {
-			return database.DefaultProjectID, nil, err
-		}
-
-		candidates = append(candidates, clientInfos...)
-	}
-
-	var topProjectID types.ID
-	if len(projects) < projectFetchSize {
-		topProjectID = database.DefaultProjectID
-	} else {
-		topProjectID = projects[len(projects)-1].ID
-	}
-	return topProjectID, candidates, nil
-}
-
 // FindDocInfoByKeyAndOwner finds the document of the given key. If the
 // createDocIfNotExist condition is true, create the document if it does not
 // exist.