diff --git a/CHANGELOG.md b/CHANGELOG.md index f4a8c4e..73fb249 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [0.0.1-alpha.40](https://github.com/DIG-Network/dig-cli/compare/v0.0.1-alpha.39...v0.0.1-alpha.40) (2024-10-10) + ### [0.0.1-alpha.39](https://github.com/DIG-Network/dig-cli/compare/v0.0.1-alpha.38...v0.0.1-alpha.39) (2024-10-08) ### [0.0.1-alpha.38](https://github.com/DIG-Network/dig-cli/compare/v0.0.1-alpha.37...v0.0.1-alpha.38) (2024-10-03) diff --git a/package-lock.json b/package-lock.json index 9cd5731..6aa28bb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,15 @@ { "name": "@dignetwork/dig-chia-cli", - "version": "0.0.1-alpha.39", + "version": "0.0.1-alpha.40", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@dignetwork/dig-chia-cli", - "version": "0.0.1-alpha.39", + "version": "0.0.1-alpha.40", "license": "ISC", "dependencies": { - "@dignetwork/dig-sdk": "^0.0.1-alpha.167", + "@dignetwork/dig-sdk": "^0.0.1-alpha.169", "bip39": "^3.1.0", "datalayer-driver": "^0.1.21", "inquirer": "^10.1.8", @@ -243,9 +243,9 @@ } }, "node_modules/@dignetwork/dig-sdk": { - "version": "0.0.1-alpha.167", - "resolved": "https://registry.npmjs.org/@dignetwork/dig-sdk/-/dig-sdk-0.0.1-alpha.167.tgz", - "integrity": "sha512-t46nVGSRui9bLsmjx6LO/c8IOMDQI4Cy/S7vC4jRUT8CJconrHtGkL3BJ2v+NS8r/RbMHCJlnXDHdOm5c/T1sA==", + "version": "0.0.1-alpha.169", + "resolved": "https://registry.npmjs.org/@dignetwork/dig-sdk/-/dig-sdk-0.0.1-alpha.169.tgz", + "integrity": "sha512-lDrsTbNuDz1KXRkAmOy6D07S4pUpoqZPBdeYf94pKIOuFyOd79HgI2n+afpIJTdx6PjZbyKWCKpIQXDNQ258Fw==", "dependencies": { "@dignetwork/datalayer-driver": "^0.1.29", "@dignetwork/dig-sdk": "^0.0.1-alpha.158", diff --git a/package.json b/package.json index 7c1032f..6003686 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dignetwork/dig-chia-cli", - "version": "0.0.1-alpha.39", + "version": "0.0.1-alpha.40", "description": "", "type": "commonjs", "main": "./dist/cli.js", @@ -24,7 +24,7 @@ "LICENSE" ], "dependencies": { - "@dignetwork/dig-sdk": "^0.0.1-alpha.167", + "@dignetwork/dig-sdk": "^0.0.1-alpha.169", "bip39": "^3.1.0", "datalayer-driver": "^0.1.21", "inquirer": "^10.1.8", diff --git a/src/actions/commit.ts b/src/actions/commit.ts index 7a4b272..12b050f 100644 --- a/src/actions/commit.ts +++ b/src/actions/commit.ts @@ -1,7 +1,8 @@ import path from "path"; import fs from "fs"; import { - addDirectory, + listFilesRecursively, + asyncPool, calculateFolderSize, waitForPromise, DataStore, @@ -29,34 +30,51 @@ export const commit = async (): Promise => { const digConfig = await loadDigConfig(process.cwd()); - await addDirectory( - dataStore.Tree, - path.join(process.cwd(), digConfig.deploy_dir) + // Group into 1000 files per batch to make management and upload easier + const fileGroups = listFilesRecursively( + path.join(process.cwd(), digConfig.deploy_dir), + 1000 ); - const newRootHash = dataStore.Tree.commit(); + const concurrencyLimit = 10; - if (!newRootHash) { - return; - } + for (const files of fileGroups) { + await asyncPool(concurrencyLimit, files as string[], async (file) => { + const filePath = path.join(process.cwd(), digConfig.deploy_dir, file); + const relativePath = file.replace(/\\/g, "/"); + const stream = fs.createReadStream(filePath); + await dataStore.Tree.upsertKey( + stream, + Buffer.from(relativePath).toString("hex") + ); + }); - const totalBytes = calculateFolderSize( - path.resolve(STORE_PATH, dataStore.StoreId) - ); + const newRootHash = dataStore.Tree.commit(); - console.log( - `Updating store metadata with new root hash: ${newRootHash}, bytes: ${totalBytes}` - ); + if (!newRootHash) { + return; + } - const updatedStoreInfo = await dataStore.updateMetadata({ - ...latestStore.metadata, - rootHash: Buffer.from(newRootHash, "hex"), - bytes: totalBytes, - }); + const totalBytes = calculateFolderSize( + path.resolve(STORE_PATH, dataStore.StoreId) + ); - await FullNodePeer.waitForConfirmation( - updatedStoreInfo.coin.parentCoinInfo - ); + console.log( + `Updating store metadata with new root hash: ${newRootHash}, bytes: ${totalBytes}` + ); + + const updatedStoreInfo = await dataStore.updateMetadata({ + ...latestStore.metadata, + rootHash: Buffer.from(newRootHash, "hex"), + bytes: totalBytes, + }); + + await FullNodePeer.waitForConfirmation( + updatedStoreInfo.coin.parentCoinInfo + ); + + await dataStore.fetchCoinInfo(); + } await waitForPromise( () => dataStore.fetchCoinInfo(),