diff --git a/server/implementation/cicd.go b/server/implementation/cicd.go index 6a67fca..55f1b10 100644 --- a/server/implementation/cicd.go +++ b/server/implementation/cicd.go @@ -126,60 +126,95 @@ func (impl *DripStrictServerImplementation) GetGitcommit(ctx context.Context, re func (impl *DripStrictServerImplementation) GetGitcommitsummary(ctx context.Context, request drip.GetGitcommitsummaryRequestObject) (drip.GetGitcommitsummaryResponseObject, error) { log.Ctx(ctx).Info().Msg("Getting git commit summary") - // TODO: This is implementing as a dense processing at time of request, but this should really be preprocessed in advance - // probably even stored in the database - query := impl.Client.CIWorkflowResult.Query(). - WithGitcommit(). - WithStorageFile() - - // Apply filters - repoName := "comfyanonymous/ComfyUI" - if request.Params.RepoName != nil { - repoName = *request.Params.RepoName - } - repoName = strings.ToLower(repoName) - if request.Params.RepoName != nil { - log.Ctx(ctx).Info().Msgf("Filtering git commit by repo name %s", repoName) - query.Where(ciworkflowresult.HasGitcommitWith(gitcommit.RepoNameEQ(repoName))) + // Prep relevant vars + page := 1 + pageSize := 10 + if request.Params.Page != nil { + page = *request.Params.Page } - if request.Params.BranchName != nil { - log.Ctx(ctx).Info().Msgf("Filtering git commit by branch name %s", *request.Params.BranchName) - query.Where(ciworkflowresult.HasGitcommitWith(gitcommit.BranchNameEQ(*request.Params.BranchName))) + if request.Params.PageSize != nil { + pageSize = *request.Params.PageSize } - query.Order(ciworkflowresult.ByGitcommitField(gitcommit.FieldCommitTimestamp, sql.OrderDesc())) + targetCount := (page + 2) * pageSize // Note: +1 for current page, +1 more to avoid partial result at end + summaries := make(map[string]*drip.GitCommitSummary) + commitsThusFar := 0 + readCount := 4096 - // Execute the query to get all commits - commits, err := query.All(ctx) - if err != nil { - log.Ctx(ctx).Error().Msgf("Error retrieving git commits w/ err: %v", err) - message := fmt.Sprintf("Error retrieving git commits: %v", err) - return drip.GetGitcommitsummary500JSONResponse{ - Message: &message, - }, nil - } - log.Ctx(ctx).Info().Msgf("Retrieved %d commits to summarize", len(commits)) + for len(summaries) < targetCount { - // Create summaries - summaries := make(map[string]*drip.GitCommitSummary) - for _, commit := range commits { - summary, exists := summaries[commit.Edges.Gitcommit.CommitHash] - if !exists { - summary = &drip.GitCommitSummary{ - CommitHash: &commit.Edges.Gitcommit.CommitHash, - Timestamp: &commit.Edges.Gitcommit.CommitTimestamp, - Author: &commit.Edges.Gitcommit.Author, - CommitName: &commit.Edges.Gitcommit.CommitMessage, - BranchName: &commit.Edges.Gitcommit.BranchName, - StatusSummary: &map[string]string{}, + // TODO: This is implementing as a dense processing at time of request, but this should really be preprocessed in advance + // probably even stored in the database + query := impl.Client.CIWorkflowResult.Query(). + WithGitcommit(). + WithStorageFile() + + // Apply filters + repoName := "comfyanonymous/ComfyUI" + if request.Params.RepoName != nil { + repoName = *request.Params.RepoName + } + repoName = strings.ToLower(repoName) + if request.Params.RepoName != nil { + if commitsThusFar == 0 { + log.Ctx(ctx).Info().Msgf("Filtering git commit by repo name %s", repoName) + } + query.Where(ciworkflowresult.HasGitcommitWith(gitcommit.RepoNameEQ(repoName))) + } + if request.Params.BranchName != nil { + if commitsThusFar == 0 { + log.Ctx(ctx).Info().Msgf("Filtering git commit by branch name %s", *request.Params.BranchName) } - summaries[commit.Edges.Gitcommit.CommitHash] = summary + query.Where(ciworkflowresult.HasGitcommitWith(gitcommit.BranchNameEQ(*request.Params.BranchName))) + } + query.Order(ciworkflowresult.ByGitcommitField(gitcommit.FieldCommitTimestamp, sql.OrderDesc())) + + query.Offset(commitsThusFar).Limit(readCount) + + // Execute the query to get all commits + commits, err := query.All(ctx) + if err != nil { + log.Ctx(ctx).Error().Msgf("Error retrieving git commits w/ err: %v", err) + message := fmt.Sprintf("Error retrieving git commits: %v", err) + return drip.GetGitcommitsummary500JSONResponse{ + Message: &message, + }, nil + } + log.Ctx(ctx).Info().Msgf("Retrieved %d commits to summarize", len(commits)) + if len(commits) == 0 { + break } - _, exists = (*summary.StatusSummary)[commit.OperatingSystem] - if !exists { - (*summary.StatusSummary)[commit.OperatingSystem] = string(commit.Status) - } else if commit.Status == schema.WorkflowRunStatusTypeFailed { - (*summary.StatusSummary)[commit.OperatingSystem] = string(commit.Status) + + // Create summaries + for _, commit := range commits { + summary, exists := summaries[commit.Edges.Gitcommit.CommitHash] + if !exists { + summary = &drip.GitCommitSummary{ + CommitHash: &commit.Edges.Gitcommit.CommitHash, + Timestamp: &commit.Edges.Gitcommit.CommitTimestamp, + Author: &commit.Edges.Gitcommit.Author, + CommitName: &commit.Edges.Gitcommit.CommitMessage, + BranchName: &commit.Edges.Gitcommit.BranchName, + StatusSummary: &map[string]string{}, + } + summaries[commit.Edges.Gitcommit.CommitHash] = summary + } + _, exists = (*summary.StatusSummary)[commit.OperatingSystem] + if !exists { + (*summary.StatusSummary)[commit.OperatingSystem] = string(commit.Status) + } else if commit.Status == schema.WorkflowRunStatusTypeFailed { + (*summary.StatusSummary)[commit.OperatingSystem] = string(commit.Status) + } } + // TODO: This heuristic hack is technically valid, but scaling up the database transfer hurts + // And this is all just a placeholder hack anyway so meh + /* + if commitsThusFar == 0 { + // Approximation of how many need to avoid too many loops + heuristic := (targetCount * len(commits)) / len(summaries) + readCount = max(1, min(10, heuristic+1)) * 4096 + }*/ + commitsThusFar += len(commits) + log.Ctx(ctx).Info().Msgf("Retrieved %d commits to summarize", len(commits)) } // Convert map to slice for pagination @@ -196,15 +231,6 @@ func (impl *DripStrictServerImplementation) GetGitcommitsummary(ctx context.Cont }) // Pagination - page := 1 - pageSize := 10 - if request.Params.Page != nil { - page = *request.Params.Page - } - if request.Params.PageSize != nil { - pageSize = *request.Params.PageSize - } - start := (page - 1) * pageSize end := start + pageSize if end > len(summarySlice) {