Merge branch 'main' into release-please--branches--main--components--…

…did-mailto

.github/release-please-manifest.json

@@ -1,5 +1,5 @@
 {
-  "packages/access-client": "18.0.6",
+  "packages/access-client": "18.0.7",
   "packages/filecoin-api": "4.3.0",
   "packages/filecoin-client": "3.2.0",
   "packages/capabilities": "13.0.0",

packages/access-client/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## [18.0.7](https://github.com/web3-storage/w3up/compare/access-v18.0.6...access-v18.0.7) (2023-12-13)
+
+
+### Fixes
+
+* support storing ArrayBuffers in conf ([#1236](https://github.com/web3-storage/w3up/issues/1236)) ([9b1aafb](https://github.com/web3-storage/w3up/commit/9b1aafbcf241d268e4f365ed99005458dda1a05a))
+
 ## [18.0.6](https://github.com/web3-storage/w3up/compare/access-v18.0.5...access-v18.0.6) (2023-12-07)
 
 

packages/access-client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@web3-storage/access",
-  "version": "18.0.6",
+  "version": "18.0.7",
   "description": "w3access client",
   "homepage": "https://web3.storage",
   "repository": {

packages/upload-client/package.json

@@ -31,6 +31,7 @@
   "exports": {
     ".": "./dist/src/index.js",
     "./car": "./dist/src/car.js",
+    "./fetch-with-upload-progress": "./dist/src/fetch-with-upload-progress.js",
     "./sharding": "./dist/src/sharding.js",
     "./upload": "./dist/src/upload.js",
     "./store": "./dist/src/store.js",

packages/upload-client/src/fetch-with-upload-progress.js

@@ -0,0 +1,8 @@
+import ipfsUtilsFetch from 'ipfs-utils/src/http/fetch.js'
+
+/**
+ * @type {import('./types.js').FetchWithUploadProgress}
+ */
+export const fetchWithUploadProgress = (url, init) => {
+  return ipfsUtilsFetch.fetch(url, init)
+}

packages/upload-client/src/store.js

@@ -4,8 +4,6 @@ import { SpaceDID } from '@web3-storage/capabilities/utils'
 import retry, { AbortError } from 'p-retry'
 import { servicePrincipal, connection } from './service.js'
 import { REQUEST_RETRIES } from './constants.js'
-import fetchPkg from 'ipfs-utils/src/http/fetch.js'
-const { fetch } = fetchPkg
 
 /**
  *
@@ -90,10 +88,9 @@ export async function add(
   const responseAddUpload = result.out.ok
 
   const fetchWithUploadProgress =
-    /** @type {(url: string, init?: import('./types.js').FetchOptions) => Promise<Response>} */ (
-      fetch
-    )
+    options.fetchWithUploadProgress || options.fetch || globalThis.fetch
 
+  let fetchDidCallUploadProgressCb = false
   const res = await retry(
     async () => {
       try {
@@ -103,12 +100,14 @@ export async function add(
           body: car,
           headers: responseAddUpload.headers,
           signal: options.signal,
-          onUploadProgress: options.onUploadProgress
-            ? createUploadProgressHandler(
+          onUploadProgress: (status) => {
+            fetchDidCallUploadProgressCb = true
+            if (options.onUploadProgress)
+              createUploadProgressHandler(
                 responseAddUpload.url,
                 options.onUploadProgress
-              )
-            : undefined,
+              )(status)
+          },
           // @ts-expect-error - this is needed by recent versions of node - see https://github.com/bluesky-social/atproto/pull/470 for more info
           duplex: 'half',
         })
@@ -128,6 +127,16 @@ export async function add(
     }
   )
 
+  if (!fetchDidCallUploadProgressCb && options.onUploadProgress) {
+    // the fetch implementation didn't support onUploadProgress
+    const carBlob = new Blob([car])
+    options.onUploadProgress({
+      total: carBlob.size,
+      loaded: carBlob.size,
+      lengthComputable: false,
+    })
+  }
+
   if (!res.ok) {
     throw new Error(`upload failed: ${res.status}`)
   }

packages/upload-client/src/types.ts

@@ -1,5 +1,5 @@
 import type {
-  FetchOptions,
+  FetchOptions as IpfsUtilsFetchOptions,
   ProgressStatus as XHRProgressStatus,
 } from 'ipfs-utils/src/types.js'
 import { Link, UnknownLink, Version } from 'multiformats/link'
@@ -45,6 +45,16 @@ import {
   UsageReportFailure,
 } from '@web3-storage/capabilities/types'
 
+type Override<T, R> = Omit<T, keyof R> & R
+
+type FetchOptions = Override<
+  IpfsUtilsFetchOptions,
+  {
+    // `fetch` is a browser API and browsers don't have `Readable`
+    body: Exclude<IpfsUtilsFetchOptions['body'], import('node:stream').Readable>
+  }
+>
+
 export type {
   FetchOptions,
   StoreAdd,
@@ -186,7 +196,13 @@ export interface Connectable {
   connection?: ConnectionView<Service>
 }
 
+export type FetchWithUploadProgress = (
+  url: string,
+  init?: FetchOptions
+) => Promise<Response>
+
 export interface UploadProgressTrackable {
+  fetchWithUploadProgress?: FetchWithUploadProgress
   onUploadProgress?: ProgressFn
 }
 
@@ -210,7 +226,9 @@ export interface RequestOptions
   extends Retryable,
     Abortable,
     Connectable,
-    UploadProgressTrackable {}
+    UploadProgressTrackable {
+  fetch?: typeof globalThis.fetch
+}
 
 export interface ListRequestOptions extends RequestOptions, Pageable {}
 

packages/upload-client/test/store.test.js

@@ -10,6 +10,7 @@ import { serviceSigner } from './fixtures.js'
 import { randomCAR } from './helpers/random.js'
 import { mockService } from './helpers/mocks.js'
 import { validateAuthorization } from './helpers/utils.js'
+import { fetchWithUploadProgress } from '../src/fetch-with-upload-progress.js'
 
 describe('Store.add', () => {
   it('stores a DAG with the service', async () => {
@@ -62,25 +63,52 @@ describe('Store.add', () => {
       channel: server,
     })
 
-    let loaded = 0
+    /** @type {import('../src/types.js').ProgressStatus[]} */
+    const progress = []
     const carCID = await Store.add(
       { issuer: agent, with: space.did(), proofs, audience: serviceSigner },
       car,
       {
         connection,
         onUploadProgress: (status) => {
           assert(typeof status.loaded === 'number' && status.loaded > 0)
-          loaded = status.loaded
+          progress.push(status)
         },
+        fetchWithUploadProgress,
       }
     )
 
     assert(service.store.add.called)
     assert.equal(service.store.add.callCount, 1)
-    assert.equal(loaded, 225)
+    assert.equal(
+      progress.reduce((max, { loaded }) => Math.max(max, loaded), 0),
+      225
+    )
 
     assert(carCID)
     assert.equal(carCID.toString(), car.cid.toString())
+
+    // make sure it can also work without fetchWithUploadProgress
+    /** @type {import('../src/types.js').ProgressStatus[]} */
+    let progressWithoutUploadProgress = []
+    const addedWithoutUploadProgress = await Store.add(
+      { issuer: agent, with: space.did(), proofs, audience: serviceSigner },
+      car,
+      {
+        connection,
+        onUploadProgress: (status) => {
+          progressWithoutUploadProgress.push(status)
+        },
+      }
+    )
+    assert.equal(addedWithoutUploadProgress.toString(), car.cid.toString())
+    assert.equal(
+      progressWithoutUploadProgress.reduce(
+        (max, { loaded }) => Math.max(max, loaded),
+        0
+      ),
+      225
+    )
   })
 
   it('throws for bucket URL client error 4xx', async () => {

packages/w3up-client/package.json

@@ -4,7 +4,7 @@
   "description": "Client for the web3.storage w3up api",
   "license": "Apache-2.0 OR MIT",
   "type": "module",
-  "main": "src/index.js",
+  "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
   "typesVersions": {
     "*": {
@@ -25,56 +25,55 @@
   "exports": {
     ".": {
       "types": "./dist/src/index.d.ts",
-      "node": "./src/index.node.js",
-      "import": "./src/index.js"
+      "node": "./dist/src/index.node.js",
+      "import": "./dist/src/index.js"
     },
     "./account": {
       "types": "./dist/src/account.d.ts",
-      "import": "./src/account.js"
+      "import": "./dist/src/account.js"
     },
     "./space": {
       "types": "./dist/src/space.d.ts",
-      "import": "./src/space.js"
+      "import": "./dist/src/space.js"
     },
     "./result": {
       "types": "./dist/src/result.d.ts",
-      "import": "./src/result.js"
+      "import": "./dist/src/result.js"
     },
     "./client": {
       "types": "./dist/src/client.d.ts",
-      "import": "./src/client.js"
+      "import": "./dist/src/client.js"
     },
     "./capability/access": {
       "types": "./dist/src/capability/access.d.ts",
-      "import": "./src/capability/access.js"
+      "import": "./dist/src/capability/access.js"
     },
     "./capability/space": {
       "types": "./dist/src/capability/space.d.ts",
-      "import": "./src/capability/space.js"
+      "import": "./dist/src/capability/space.js"
     },
     "./capability/store": {
       "types": "./dist/src/capability/store.d.ts",
-      "import": "./src/capability/store.js"
+      "import": "./dist/src/capability/store.js"
     },
     "./capability/subscription": {
       "types": "./dist/src/capability/subscription.d.ts",
-      "import": "./src/capability/subscription.js"
+      "import": "./dist/src/capability/subscription.js"
     },
     "./capability/upload": {
       "types": "./dist/src/capability/upload.d.ts",
-      "import": "./src/capability/upload.js"
+      "import": "./dist/src/capability/upload.js"
     },
     "./capability/usage": {
       "types": "./dist/src/capability/usage.d.ts",
-      "import": "./src/capability/usage.js"
+      "import": "./dist/src/capability/usage.js"
     },
-    "./types": "./src/types.js"
+    "./types": "./dist/src/types.js"
   },
   "publishConfig": {
     "access": "public"
   },
   "files": [
-    "src",
     "dist"
   ],
   "scripts": {

packages/w3up-client/src/delegation.js

@@ -1,10 +1,12 @@
 import { Delegation as CoreDelegation } from '@ucanto/core/delegation'
 
+/* c8 ignore start */
 /**
  * @template {import('./types.js').Capabilities} C
  * @extends {CoreDelegation<C>}
  */
 export class Delegation extends CoreDelegation {
+  /* c8 ignore stop */
   /** @type {Record<string, any>} */
   #meta
 

packages/w3up-client/tsconfig.json

@@ -2,7 +2,6 @@
   "extends": "../../tsconfig.json",
   "compilerOptions": {
     "outDir": "dist",
-    "emitDeclarationOnly": true,
     "composite": true,
     "noUnusedParameters": false
   },