From b55585811b6cce46dabbd3f6fe92803a8f516a75 Mon Sep 17 00:00:00 2001 From: yandu Date: Thu, 3 Apr 2025 18:23:20 -0400 Subject: [PATCH 1/7] Update version to 0.9.4-pre.5bdb556 --- js/package-lock.json | 4 ++-- js/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/js/package-lock.json b/js/package-lock.json index dc72d527..c549729b 100644 --- a/js/package-lock.json +++ b/js/package-lock.json @@ -1,12 +1,12 @@ { "name": "@gadgetinc/dateilager", - "version": "0.9.3", + "version": "0.9.4-pre.5bdb556", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@gadgetinc/dateilager", - "version": "0.9.3", + "version": "0.9.4-pre.5bdb556", "license": "MIT", "dependencies": { "@grpc/grpc-js": "^1.10.6", diff --git a/js/package.json b/js/package.json index b49a5ac7..7fec1059 100644 --- a/js/package.json +++ b/js/package.json @@ -1,6 +1,6 @@ { "name": "@gadgetinc/dateilager", - "version": "0.9.3", + "version": "0.9.4-pre.5bdb556", "homepage": "https://github.com/gadget-inc/dateilager", "bugs": "https://github.com/gadget-inc/dateilager/issues", "repository": { From a1905b0e48f4123cfccbf615170a70ad182e390e Mon Sep 17 00:00:00 2001 From: yandu Date: Thu, 3 Apr 2025 18:28:21 -0400 Subject: [PATCH 2/7] Add cpu profiling to rebuild --- pkg/cli/rebuild.go | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/pkg/cli/rebuild.go b/pkg/cli/rebuild.go index 0b4b96e2..bbdd26e4 100644 --- a/pkg/cli/rebuild.go +++ b/pkg/cli/rebuild.go @@ -3,7 +3,11 @@ package cli import ( "encoding/json" "fmt" + "os" + "path/filepath" + "runtime/pprof" "strings" + "time" "github.com/gadget-inc/dateilager/internal/files" "github.com/gadget-inc/dateilager/internal/key" @@ -12,6 +16,10 @@ import ( "github.com/spf13/cobra" ) +const ( + profileName = "dateilager-profile-%s.prof" +) + func NewCmdRebuild() *cobra.Command { var ( project int64 @@ -24,6 +32,7 @@ func NewCmdRebuild() *cobra.Command { cacheDir string fileMatchInclude string fileMatchExclude string + profilePath string ) cmd := &cobra.Command{ @@ -51,7 +60,18 @@ func NewCmdRebuild() *cobra.Command { return err } + if profilePath != "" { + profileName := fmt.Sprintf(profileName, time.Now().Format("2006-01-02-15-04-05")) + file, err := os.Create(filepath.Join(profilePath, profileName)) + if err != nil { + return fmt.Errorf("cannot open profile path %s: %w", profilePath, err) + } + _ = pprof.StartCPUProfile(file) + } result, err := client.Rebuild(ctx, project, prefix, to, dir, ignoreList, subpathList, cacheDir, matcher, summarize) + if profilePath != "" { + pprof.StopCPUProfile() + } if err != nil { return fmt.Errorf("could not rebuild project: %w", err) } @@ -84,8 +104,8 @@ func NewCmdRebuild() *cobra.Command { cmd.Flags().StringVar(&cacheDir, "cachedir", "", "Path where the cache folder is mounted") cmd.Flags().StringVar(&fileMatchInclude, "matchinclude", "", "Set fileMatch to true if the written files are matched by this glob pattern") cmd.Flags().StringVar(&fileMatchExclude, "matchexclude", "", "Set fileMatch to false if the written files are matched by this glob pattern") + cmd.Flags().StringVar(&profilePath, "profile", "", "Path to the file where the profile will be written") to = cmd.Flags().Int64("to", -1, "To version ID (optional)") - _ = cmd.MarkFlagRequired("project") return cmd From 9f786b17c1061e4a73ad6f98e100cb7584d89e1d Mon Sep 17 00:00:00 2001 From: yandu Date: Thu, 3 Apr 2025 18:44:54 -0400 Subject: [PATCH 3/7] Update version to 0.9.4-pre.a1905b0 --- js/package-lock.json | 4 ++-- js/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/js/package-lock.json b/js/package-lock.json index c549729b..5e816eed 100644 --- a/js/package-lock.json +++ b/js/package-lock.json @@ -1,12 +1,12 @@ { "name": "@gadgetinc/dateilager", - "version": "0.9.4-pre.5bdb556", + "version": "0.9.4-pre.a1905b0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@gadgetinc/dateilager", - "version": "0.9.4-pre.5bdb556", + "version": "0.9.4-pre.a1905b0", "license": "MIT", "dependencies": { "@grpc/grpc-js": "^1.10.6", diff --git a/js/package.json b/js/package.json index 7fec1059..dff76f52 100644 --- a/js/package.json +++ b/js/package.json @@ -1,6 +1,6 @@ { "name": "@gadgetinc/dateilager", - "version": "0.9.4-pre.5bdb556", + "version": "0.9.4-pre.a1905b0", "homepage": "https://github.com/gadget-inc/dateilager", "bugs": "https://github.com/gadget-inc/dateilager/issues", "repository": { From d4b5bcd12a96b80ba9e51e6c863ab9e5c80d3d04 Mon Sep 17 00:00:00 2001 From: yandu Date: Thu, 3 Apr 2025 18:53:14 -0400 Subject: [PATCH 4/7] WIP --- js/src/binary-client.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/js/src/binary-client.ts b/js/src/binary-client.ts index 33d6154a..908de3ef 100644 --- a/js/src/binary-client.ts +++ b/js/src/binary-client.ts @@ -86,6 +86,11 @@ export interface DateiLagerBinaryClientOptions { */ log: (level: "debug" | "info" | "warn" | "error", msg: string, fields: Record) => void; }; + + /** + * The path to the file where the profile will be written. + */ + profile?: string; } /** @@ -152,6 +157,7 @@ export class DateiLagerBinaryClient { }, tracing: options.tracing ?? false, logger: options.logger, + profile: options.profile, }; } @@ -259,6 +265,10 @@ export class DateiLagerBinaryClient { args.push(`--subpaths=${options.subpaths.join(",")}`); } + if (this._options.profile) { + args.push(`--profile=${this._options.profile}`); + } + args.push("--project", String(project), "--dir", directory); const result = await this._call("rebuild", args, directory, options); const parsed = JSON.parse(result.stdout) as { version: number; count: number; fileMatch: boolean }; From 6178f67aca3b09d2a842740b0165eb2856a59a3e Mon Sep 17 00:00:00 2001 From: yandu Date: Thu, 3 Apr 2025 18:53:32 -0400 Subject: [PATCH 5/7] Update version to 0.9.4-pre.d4b5bcd --- js/package-lock.json | 4 ++-- js/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/js/package-lock.json b/js/package-lock.json index 5e816eed..733eb88c 100644 --- a/js/package-lock.json +++ b/js/package-lock.json @@ -1,12 +1,12 @@ { "name": "@gadgetinc/dateilager", - "version": "0.9.4-pre.a1905b0", + "version": "0.9.4-pre.d4b5bcd", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@gadgetinc/dateilager", - "version": "0.9.4-pre.a1905b0", + "version": "0.9.4-pre.d4b5bcd", "license": "MIT", "dependencies": { "@grpc/grpc-js": "^1.10.6", diff --git a/js/package.json b/js/package.json index dff76f52..a5204cf6 100644 --- a/js/package.json +++ b/js/package.json @@ -1,6 +1,6 @@ { "name": "@gadgetinc/dateilager", - "version": "0.9.4-pre.a1905b0", + "version": "0.9.4-pre.d4b5bcd", "homepage": "https://github.com/gadget-inc/dateilager", "bugs": "https://github.com/gadget-inc/dateilager/issues", "repository": { From e02d2b0449217f548543f3a497e1e25947a3a77a Mon Sep 17 00:00:00 2001 From: yandu Date: Fri, 4 Apr 2025 10:40:01 -0400 Subject: [PATCH 6/7] Add base64 encoding --- pkg/cli/rebuild.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/pkg/cli/rebuild.go b/pkg/cli/rebuild.go index bbdd26e4..8ea87ef9 100644 --- a/pkg/cli/rebuild.go +++ b/pkg/cli/rebuild.go @@ -1,6 +1,7 @@ package cli import ( + "encoding/base64" "encoding/json" "fmt" "os" @@ -17,7 +18,7 @@ import ( ) const ( - profileName = "dateilager-profile-%s.prof" + PROFILE_NAME = "dateilager-profile-%s-%s.prof" ) func NewCmdRebuild() *cobra.Command { @@ -61,8 +62,24 @@ func NewCmdRebuild() *cobra.Command { } if profilePath != "" { - profileName := fmt.Sprintf(profileName, time.Now().Format("2006-01-02-15-04-05")) - file, err := os.Create(filepath.Join(profilePath, profileName)) + args := []string{ + fmt.Sprintf("project=%d", project), + fmt.Sprintf("prefix=%s", prefix), + fmt.Sprintf("dir=%s", dir), + fmt.Sprintf("ignores=%s", ignores), + fmt.Sprintf("subpaths=%s", subpaths), + fmt.Sprintf("summarize=%v", summarize), + fmt.Sprintf("cachedir=%s", cacheDir), + fmt.Sprintf("matchinclude=%s", fileMatchInclude), + fmt.Sprintf("matchexclude=%s", fileMatchExclude), + } + if to != nil { + args = append(args, fmt.Sprintf("to=%d", *to)) + } + + encoded := base64.StdEncoding.EncodeToString([]byte(strings.Join(args, ";"))) + fileName := fmt.Sprintf(PROFILE_NAME, time.Now().Format("2006-01-02-15-04-05"), encoded) + file, err := os.Create(filepath.Join(profilePath, fileName)) if err != nil { return fmt.Errorf("cannot open profile path %s: %w", profilePath, err) } From 3e9ac0d956ce7aa5977809ff88530d48e357d900 Mon Sep 17 00:00:00 2001 From: yandu Date: Fri, 4 Apr 2025 10:45:46 -0400 Subject: [PATCH 7/7] Update version to 0.9.4-pre.e02d2b0 --- js/package-lock.json | 4 ++-- js/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/js/package-lock.json b/js/package-lock.json index 733eb88c..f2625352 100644 --- a/js/package-lock.json +++ b/js/package-lock.json @@ -1,12 +1,12 @@ { "name": "@gadgetinc/dateilager", - "version": "0.9.4-pre.d4b5bcd", + "version": "0.9.4-pre.e02d2b0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@gadgetinc/dateilager", - "version": "0.9.4-pre.d4b5bcd", + "version": "0.9.4-pre.e02d2b0", "license": "MIT", "dependencies": { "@grpc/grpc-js": "^1.10.6", diff --git a/js/package.json b/js/package.json index a5204cf6..c461f2d0 100644 --- a/js/package.json +++ b/js/package.json @@ -1,6 +1,6 @@ { "name": "@gadgetinc/dateilager", - "version": "0.9.4-pre.d4b5bcd", + "version": "0.9.4-pre.e02d2b0", "homepage": "https://github.com/gadget-inc/dateilager", "bugs": "https://github.com/gadget-inc/dateilager/issues", "repository": {