fix: listObject v1 s3 endpoint (#765)

Author: fenos

src/storage/object.ts

@@ -46,6 +46,7 @@ export interface ListObjectsV2Result {
   objects: Obj[]
   hasNext: boolean
   nextCursor?: string
+  nextCursorKey?: string
 }
 
 /**
@@ -661,6 +662,8 @@ export class ObjectStorage {
     })
 
     let nextContinuationToken: string | undefined
+    let nextCursorKey: string | undefined
+
     if (isTruncated) {
       const sortColumn = (cursor?.sortColumn || options?.sortBy?.column) as
         | 'name'
@@ -677,11 +680,13 @@ export class ObjectStorage {
             ? new Date(searchResult[searchResult.length - 1][sortColumn] || '').toISOString()
             : undefined,
       })
+      nextCursorKey = searchResult[searchResult.length - 1].name
     }
 
     return {
       hasNext: isTruncated,
       nextCursor: nextContinuationToken,
+      nextCursorKey,
       folders: folders,
       objects: objects,
     }

src/storage/protocols/s3/s3-handler.ts

@@ -164,15 +164,16 @@ export class S3ProtocolHandler {
       EncodingType: command.EncodingType,
       MaxKeys: command.MaxKeys,
       Prefix: command.Prefix,
-      ContinuationToken: command.Marker,
+      StartAfter: command.Marker,
+      cursorV1: true,
     })
 
     return {
       responseBody: {
         ListBucketResult: {
           Name: list.responseBody.ListBucketResult.Name,
           Prefix: list.responseBody.ListBucketResult.Prefix,
-          Marker: list.responseBody.ListBucketResult.ContinuationToken,
+          Marker: list.responseBody.ListBucketResult.NextContinuationToken,
           MaxKeys: list.responseBody.ListBucketResult.MaxKeys,
           IsTruncated: list.responseBody.ListBucketResult.IsTruncated,
           Contents: list.responseBody.ListBucketResult.Contents,
@@ -190,7 +191,7 @@ export class S3ProtocolHandler {
    *
    * @param command
    */
-  async listObjectsV2(command: ListObjectsV2CommandInput) {
+  async listObjectsV2(command: ListObjectsV2CommandInput & { cursorV1?: boolean }) {
     if (!command.Bucket) {
       throw ERRORS.MissingParameter('Bucket')
     }
@@ -249,7 +250,11 @@ export class S3ProtocolHandler {
     }
 
     if (results.nextCursor) {
-      response.ListBucketResult.NextContinuationToken = results.nextCursor
+      if (command.cursorV1) {
+        response.ListBucketResult.NextContinuationToken = results.nextCursorKey
+      } else {
+        response.ListBucketResult.NextContinuationToken = results.nextCursor
+      }
     }
 
     return {

src/test/s3-protocol.test.ts

@@ -233,6 +233,36 @@ describe('S3 Protocol', () => {
         const resp = await client.send(listBuckets)
         expect(resp.Contents?.length).toBe(3)
       })
+
+      it('list all keys with pagination', async () => {
+        const bucket = await createBucket(client)
+        const listObjects = new ListObjectsCommand({
+          Bucket: bucket,
+          Delimiter: '/',
+          MaxKeys: 1,
+        })
+
+        await Promise.all([
+          uploadFile(client, bucket, 'test-1.jpg', 1),
+          uploadFile(client, bucket, 'prefix-1/test-1.jpg', 1),
+          uploadFile(client, bucket, 'prefix-3/test-1.jpg', 1),
+          uploadFile(client, bucket, 'prefix-4/test-1.jpg', 1),
+        ])
+
+        const resp = await client.send(listObjects)
+        expect(resp.CommonPrefixes?.length).toBe(1)
+        expect(resp.IsTruncated).toBe(true)
+
+        const listObjects2 = new ListObjectsCommand({
+          Bucket: bucket,
+          Delimiter: '/',
+          MaxKeys: 3,
+          Marker: resp.Marker,
+        })
+        const resp2 = await client.send(listObjects2)
+        expect(resp2.CommonPrefixes?.length).toBe(2)
+        expect(resp2.Contents?.length).toBe(1)
+      })
     })
 
     describe('ListObjectsV2Command', () => {