WIP: S3 improvements (#16167)

Author: cirospaciari

Diff for: src/bun.js/api/server.zig

@@ -90,7 +90,7 @@ const linux = std.os.linux;
 const Async = bun.Async;
 const httplog = Output.scoped(.Server, false);
 const ctxLog = Output.scoped(.RequestContext, false);
-const AWS = @import("../../s3.zig").AWSCredentials;
+const S3 = bun.S3;
 const BlobFileContentResult = struct {
     data: [:0]const u8,
 
@@ -3182,7 +3182,7 @@ fn NewRequestContext(comptime ssl_enabled: bool, comptime debug_mode: bool, comp
             this.endWithoutBody(this.shouldCloseConnection());
             this.deref();
         }
-        pub fn onS3SizeResolved(result: AWS.S3StatResult, this: *RequestContext) void {
+        pub fn onS3SizeResolved(result: S3.S3StatResult, this: *RequestContext) void {
             defer {
                 this.deref();
             }
@@ -3254,7 +3254,7 @@ fn NewRequestContext(comptime ssl_enabled: bool, comptime debug_mode: bool, comp
                             const path = blob.store.?.data.s3.path();
                             const env = globalThis.bunVM().transpiler.env;
 
-                            credentials.s3Stat(path, @ptrCast(&onS3SizeResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null);
+                            S3.stat(credentials, path, @ptrCast(&onS3SizeResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null);
 
                             return;
                         }

Diff for: src/bun.js/bindings/JSS3Bucket.cpp

@@ -37,6 +37,7 @@ JSC_DECLARE_HOST_FUNCTION(functionS3Bucket_size);
 
 static const HashTableValue JSS3BucketPrototypeTableValues[] = {
     { "unlink"_s, static_cast<unsigned>(PropertyAttribute::Function | PropertyAttribute::ReadOnly), NoIntrinsic, { HashTableValue::NativeFunctionType, functionS3Bucket_unlink, 0 } },
+    { "delete"_s, static_cast<unsigned>(PropertyAttribute::Function | PropertyAttribute::ReadOnly), NoIntrinsic, { HashTableValue::NativeFunctionType, functionS3Bucket_unlink, 0 } },
     { "write"_s, static_cast<unsigned>(PropertyAttribute::Function | PropertyAttribute::ReadOnly), NoIntrinsic, { HashTableValue::NativeFunctionType, functionS3Bucket_write, 1 } },
     { "presign"_s, static_cast<unsigned>(PropertyAttribute::Function | PropertyAttribute::ReadOnly), NoIntrinsic, { HashTableValue::NativeFunctionType, functionS3Bucket_presign, 1 } },
     { "exists"_s, static_cast<unsigned>(PropertyAttribute::Function | PropertyAttribute::ReadOnly), NoIntrinsic, { HashTableValue::NativeFunctionType, functionS3Bucket_exists, 1 } },

Diff for: src/bun.js/event_loop.zig

@@ -18,9 +18,9 @@ const ReadFileTask = WebCore.Blob.ReadFile.ReadFileTask;
 const WriteFileTask = WebCore.Blob.WriteFile.WriteFileTask;
 const napi_async_work = JSC.napi.napi_async_work;
 const FetchTasklet = Fetch.FetchTasklet;
-const AWS = @import("../s3.zig").AWSCredentials;
-const S3HttpSimpleTask = AWS.S3HttpSimpleTask;
-const S3HttpDownloadStreamingTask = AWS.S3HttpDownloadStreamingTask;
+const S3 = bun.S3;
+const S3HttpSimpleTask = S3.S3HttpSimpleTask;
+const S3HttpDownloadStreamingTask = S3.S3HttpDownloadStreamingTask;
 
 const JSValue = JSC.JSValue;
 const js = JSC.C;

Diff for: src/bun.js/webcore/S3Bucket.zig

@@ -6,11 +6,11 @@ const PathOrBlob = JSC.Node.PathOrBlob;
 const ZigString = JSC.ZigString;
 const Method = bun.http.Method;
 const S3File = @import("./S3File.zig");
-const AWSCredentials = bun.AWSCredentials;
+const S3Credentials = bun.S3.S3Credentials;
 
 const S3BucketOptions = struct {
-    credentials: *AWSCredentials,
-    options: bun.S3.MultiPartUpload.MultiPartUploadOptions = .{},
+    credentials: *S3Credentials,
+    options: bun.S3.MultiPartUploadOptions = .{},
     acl: ?bun.S3.ACL = null,
     pub usingnamespace bun.New(@This());
 
@@ -20,7 +20,7 @@ const S3BucketOptions = struct {
     }
 };
 
-pub fn writeFormatCredentials(credentials: *AWSCredentials, options: bun.S3.MultiPartUpload.MultiPartUploadOptions, acl: ?bun.S3.ACL, comptime Formatter: type, formatter: *Formatter, writer: anytype, comptime enable_ansi_colors: bool) !void {
+pub fn writeFormatCredentials(credentials: *S3Credentials, options: bun.S3.MultiPartUploadOptions, acl: ?bun.S3.ACL, comptime Formatter: type, formatter: *Formatter, writer: anytype, comptime enable_ansi_colors: bool) !void {
     try writer.writeAll("\n");
 
     {
@@ -37,7 +37,7 @@ pub fn writeFormatCredentials(credentials: *AWSCredentials, options: bun.S3.Mult
         formatter.printComma(Writer, writer, enable_ansi_colors) catch bun.outOfMemory();
         try writer.writeAll("\n");
 
-        const region = if (credentials.region.len > 0) credentials.region else AWSCredentials.guessRegion(credentials.endpoint);
+        const region = if (credentials.region.len > 0) credentials.region else S3Credentials.guessRegion(credentials.endpoint);
         try formatter.writeIndent(Writer, writer);
         try writer.writeAll(comptime bun.Output.prettyFmt("<r>region<d>:<r> \"", enable_ansi_colors));
         try writer.print(comptime bun.Output.prettyFmt("<r><b>{s}<r>\"", enable_ansi_colors), .{region});
@@ -133,7 +133,7 @@ pub fn call(ptr: *S3BucketOptions, globalThis: *JSC.JSGlobalObject, callframe: *
     };
     errdefer path.deinit();
     const options = args.nextEat();
-    var blob = Blob.new(try S3File.constructS3FileWithAWSCredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl));
+    var blob = Blob.new(try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl));
     blob.allocator = bun.default_allocator;
     return blob.toJS(globalThis);
 }
@@ -151,7 +151,7 @@ pub fn presign(ptr: *S3BucketOptions, globalThis: *JSC.JSGlobalObject, callframe
     errdefer path.deinit();
 
     const options = args.nextEat();
-    var blob = try S3File.constructS3FileWithAWSCredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl);
+    var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl);
     defer blob.detach();
     return S3File.getPresignUrlFrom(&blob, globalThis, options);
 }
@@ -168,7 +168,7 @@ pub fn exists(ptr: *S3BucketOptions, globalThis: *JSC.JSGlobalObject, callframe:
     };
     errdefer path.deinit();
     const options = args.nextEat();
-    var blob = try S3File.constructS3FileWithAWSCredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl);
+    var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl);
     defer blob.detach();
     return S3File.S3BlobStatTask.exists(globalThis, &blob);
 }
@@ -185,7 +185,7 @@ pub fn size(ptr: *S3BucketOptions, globalThis: *JSC.JSGlobalObject, callframe: *
     };
     errdefer path.deinit();
     const options = args.nextEat();
-    var blob = try S3File.constructS3FileWithAWSCredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl);
+    var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl);
     defer blob.detach();
     return S3File.S3BlobStatTask.size(globalThis, &blob);
 }
@@ -203,7 +203,7 @@ pub fn write(ptr: *S3BucketOptions, globalThis: *JSC.JSGlobalObject, callframe:
     };
 
     const options = args.nextEat();
-    var blob = try S3File.constructS3FileWithAWSCredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl);
+    var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl);
     defer blob.detach();
     var blob_internal: PathOrBlob = .{ .blob = blob };
     return Blob.writeFileInternal(globalThis, &blob_internal, data, .{
@@ -221,7 +221,7 @@ pub fn unlink(ptr: *S3BucketOptions, globalThis: *JSC.JSGlobalObject, callframe:
     };
     errdefer path.deinit();
     const options = args.nextEat();
-    var blob = try S3File.constructS3FileWithAWSCredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl);
+    var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl);
     defer blob.detach();
     return blob.store.?.data.s3.unlink(blob.store.?, globalThis, options);
 }
@@ -235,7 +235,7 @@ pub fn construct(globalThis: *JSC.JSGlobalObject, callframe: *JSC.CallFrame) cal
     if (options.isEmptyOrUndefinedOrNull() or !options.isObject()) {
         globalThis.throwInvalidArguments("Expected S3 options to be passed", .{}) catch return null;
     }
-    var aws_options = AWSCredentials.getCredentialsWithOptions(globalThis.bunVM().transpiler.env.getAWSCredentials(), .{}, options, null, globalThis) catch return null;
+    var aws_options = S3Credentials.getCredentialsWithOptions(globalThis.bunVM().transpiler.env.getS3Credentials(), .{}, options, null, globalThis) catch return null;
     defer aws_options.deinit();
     return S3BucketOptions.new(.{
         .credentials = aws_options.credentials.dupe(),

Diff for: src/bun.js/webcore/S3File.zig

@@ -8,6 +8,7 @@ const Method = bun.http.Method;
 const strings = bun.strings;
 const Output = bun.Output;
 const S3Bucket = @import("./S3Bucket.zig");
+const S3 = bun.S3;
 
 pub fn writeFormat(s3: *Blob.S3Store, comptime Formatter: type, formatter: *Formatter, writer: anytype, comptime enable_ansi_colors: bool) !void {
     try writer.writeAll(comptime Output.prettyFmt("<r>S3Ref<r>", enable_ansi_colors));
@@ -214,19 +215,19 @@ fn constructS3FileInternalStore(
     options: ?JSC.JSValue,
 ) bun.JSError!Blob {
     // get credentials from env
-    const existing_credentials = globalObject.bunVM().transpiler.env.getAWSCredentials();
-    return constructS3FileWithAWSCredentials(globalObject, path, options, existing_credentials);
+    const existing_credentials = globalObject.bunVM().transpiler.env.getS3Credentials();
+    return constructS3FileWithS3Credentials(globalObject, path, options, existing_credentials);
 }
 /// if the credentials have changed, we need to clone it, if not we can just ref/deref it
-pub fn constructS3FileWithAWSCredentialsAndOptions(
+pub fn constructS3FileWithS3CredentialsAndOptions(
     globalObject: *JSC.JSGlobalObject,
     path: JSC.Node.PathLike,
     options: ?JSC.JSValue,
-    default_credentials: *AWS,
-    default_options: bun.S3.MultiPartUpload.MultiPartUploadOptions,
+    default_credentials: *S3.S3Credentials,
+    default_options: bun.S3.MultiPartUploadOptions,
     default_acl: ?bun.S3.ACL,
 ) bun.JSError!Blob {
-    var aws_options = try AWS.getCredentialsWithOptions(default_credentials.*, default_options, options, default_acl, globalObject);
+    var aws_options = try S3.S3Credentials.getCredentialsWithOptions(default_credentials.*, default_options, options, default_acl, globalObject);
     defer aws_options.deinit();
 
     const store = brk: {
@@ -268,13 +269,13 @@ pub fn constructS3FileWithAWSCredentialsAndOptions(
     return blob;
 }
 
-pub fn constructS3FileWithAWSCredentials(
+pub fn constructS3FileWithS3Credentials(
     globalObject: *JSC.JSGlobalObject,
     path: JSC.Node.PathLike,
     options: ?JSC.JSValue,
-    existing_credentials: AWS,
+    existing_credentials: S3.S3Credentials,
 ) bun.JSError!Blob {
-    var aws_options = try AWS.getCredentialsWithOptions(existing_credentials, .{}, options, null, globalObject);
+    var aws_options = try S3.S3Credentials.getCredentialsWithOptions(existing_credentials, .{}, options, null, globalObject);
     defer aws_options.deinit();
     const store = Blob.Store.initS3(path, null, aws_options.credentials, bun.default_allocator) catch bun.outOfMemory();
     errdefer store.deinit();
@@ -318,14 +319,12 @@ fn constructS3FileInternal(
     return ptr;
 }
 
-const AWS = bun.S3.AWSCredentials;
-
 pub const S3BlobStatTask = struct {
     promise: JSC.JSPromise.Strong,
     store: *Blob.Store,
     usingnamespace bun.New(S3BlobStatTask);
 
-    pub fn onS3ExistsResolved(result: AWS.S3StatResult, this: *S3BlobStatTask) void {
+    pub fn onS3ExistsResolved(result: S3.S3StatResult, this: *S3BlobStatTask) void {
         defer this.deinit();
         const globalThis = this.promise.globalObject().?;
         switch (result) {
@@ -346,7 +345,7 @@ pub const S3BlobStatTask = struct {
         }
     }
 
-    pub fn onS3SizeResolved(result: AWS.S3StatResult, this: *S3BlobStatTask) void {
+    pub fn onS3SizeResolved(result: S3.S3StatResult, this: *S3BlobStatTask) void {
         defer this.deinit();
         const globalThis = this.promise.globalObject().?;
 
@@ -371,7 +370,7 @@ pub const S3BlobStatTask = struct {
         const path = blob.store.?.data.s3.path();
         const env = globalThis.bunVM().transpiler.env;
 
-        credentials.s3Stat(path, @ptrCast(&S3BlobStatTask.onS3ExistsResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null);
+        S3.stat(credentials, path, @ptrCast(&S3BlobStatTask.onS3ExistsResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null);
         return promise;
     }
 
@@ -386,7 +385,7 @@ pub const S3BlobStatTask = struct {
         const path = blob.store.?.data.s3.path();
         const env = globalThis.bunVM().transpiler.env;
 
-        credentials.s3Stat(path, @ptrCast(&S3BlobStatTask.onS3SizeResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null);
+        S3.stat(credentials, path, @ptrCast(&S3BlobStatTask.onS3SizeResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null);
         return promise;
     }
 
@@ -405,7 +404,7 @@ pub fn getPresignUrlFrom(this: *Blob, globalThis: *JSC.JSGlobalObject, extra_opt
     var method: bun.http.Method = .GET;
     var expires: usize = 86400; // 1 day default
 
-    var credentialsWithOptions: AWS.AWSCredentialsWithOptions = .{
+    var credentialsWithOptions: S3.S3CredentialsWithOptions = .{
         .credentials = this.store.?.data.s3.getCredentials().*,
     };
     defer {
@@ -434,7 +433,7 @@ pub fn getPresignUrlFrom(this: *Blob, globalThis: *JSC.JSGlobalObject, extra_opt
         .method = method,
         .acl = credentialsWithOptions.acl,
     }, .{ .expires = expires }) catch |sign_err| {
-        return AWS.throwSignError(sign_err, globalThis);
+        return S3.throwSignError(sign_err, globalThis);
     };
     defer result.deinit();
     var str = bun.String.fromUTF8(result.url);