From 569673058ccbebdca0f0498c37f02d94bde3ec5b Mon Sep 17 00:00:00 2001 From: Benjamin Schmidt Date: Mon, 4 Nov 2024 10:05:31 -0500 Subject: [PATCH] add raw createIndex function with upstream types (#82) The types on createIndex are pretty vague and have some custom wrappings; adding a new function at createIndexRaw lets us use the API types directly without breaking back-compatibility. ---- > [!IMPORTANT] > Adds `createIndexRaw` function in `AtlasDataset` for direct API type usage in index creation, with tests and dependency updates. > > - **Behavior**: > - Adds `createIndexRaw` function in `AtlasDataset` class in `src/project.ts` to use upstream API types directly for index creation. > - Uses `createIndexTextDefaults` for default values, allowing user options to override them. > - Ensures `project_id` is set in the request. > - **Tests**: > - Adds test for `createIndexRaw` in `project.test.ts` to verify index creation. > - **Dependencies**: > - Updates `apache-arrow` to `^13.0.0` in `package.json`. > - Removes `@types/jasmine` from `devDependencies` in `package.json`. > > This description was created by [Ellipsis](https://www.ellipsis.dev?ref=nomic-ai%2Fts-nomic&utm_source=github&utm_medium=referral) for bb175abe7e2e224adbce37f333260f680afdb1c4. It will automatically update as commits are pushed. --- package-lock.json | 283 ++++++++++++++++++++++++++++++------------ package.json | 4 +- src/project.ts | 81 +++++++++++- tests/project.test.ts | 10 +- 4 files changed, 292 insertions(+), 86 deletions(-) diff --git a/package-lock.json b/package-lock.json index de30049..b4d14cd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,8 @@ "version": "0.11.0", "license": "MIT", "dependencies": { - "apache-arrow": "^12.0.1", + "apache-arrow": "^13.0.0", + "dotenv": "^16.0.3", "js-yaml": "^4.1.0" }, "devDependencies": { @@ -37,6 +38,28 @@ "vitest": "^2.1.4" } }, + "node_modules/@75lb/deep-merge": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@75lb/deep-merge/-/deep-merge-1.1.2.tgz", + "integrity": "sha512-08K9ou5VNbheZFxM5tDWoqjA3ImC50DiuuJ2tj1yEPRfkp8lLLg6XAaJ4On+a0yAXor/8ay5gHnAIshRM44Kpw==", + "license": "MIT", + "dependencies": { + "lodash": "^4.17.21", + "typical": "^7.1.1" + }, + "engines": { + "node": ">=12.17" + } + }, + "node_modules/@75lb/deep-merge/node_modules/typical": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.2.0.tgz", + "integrity": "sha512-W1+HdVRUl8fS3MZ9ogD51GOb46xMmhAZzR0WPw5jcgIZQJVvkddYzAl4YTU6g5w33Y1iRQLdIi2/1jhi2RNL0g==", + "license": "MIT", + "engines": { + "node": ">=12.17" + } + }, "node_modules/@ampproject/remapping": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz", @@ -2488,6 +2511,7 @@ "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, "dependencies": { "color-convert": "^1.9.0" }, @@ -2509,29 +2533,31 @@ } }, "node_modules/apache-arrow": { - "version": "12.0.1", - "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-12.0.1.tgz", - "integrity": "sha512-g17ARsc/KEAzViy8PEFsDBlL4ZLx3BesgQCplDLgUWtY0aFWNdEmfaZsbbXVRDfQ21D7vbUKtu0ZWNgcbxDrig==", + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-13.0.0.tgz", + "integrity": "sha512-3gvCX0GDawWz6KFNC28p65U+zGh/LZ6ZNKWNu74N6CQlKzxeoWHpi4CgEQsgRSEMuyrIIXi1Ea2syja7dwcHvw==", + "license": "Apache-2.0", "dependencies": { "@types/command-line-args": "5.2.0", "@types/command-line-usage": "5.0.2", - "@types/node": "18.14.5", + "@types/node": "20.3.0", "@types/pad-left": "2.1.1", "command-line-args": "5.2.1", - "command-line-usage": "6.1.3", - "flatbuffers": "23.3.3", + "command-line-usage": "7.0.1", + "flatbuffers": "23.5.26", "json-bignum": "^0.0.3", "pad-left": "^2.1.0", - "tslib": "^2.5.0" + "tslib": "^2.5.3" }, "bin": { "arrow2csv": "bin/arrow2csv.js" } }, "node_modules/apache-arrow/node_modules/@types/node": { - "version": "18.14.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.5.tgz", - "integrity": "sha512-CRT4tMK/DHYhw1fcCEBwME9CSaZNclxfzVMe7GsO6ULSwsttbj70wSiX6rZdIjGblu93sTJxLdhNIT85KKI7Qw==" + "version": "20.3.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz", + "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==", + "license": "MIT" }, "node_modules/arg": { "version": "4.1.3", @@ -2926,6 +2952,7 @@ "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, "dependencies": { "ansi-styles": "^3.2.1", "escape-string-regexp": "^1.0.5", @@ -2935,6 +2962,91 @@ "node": ">=4" } }, + "node_modules/chalk-template": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/chalk-template/-/chalk-template-0.4.0.tgz", + "integrity": "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==", + "license": "MIT", + "dependencies": { + "chalk": "^4.1.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/chalk-template?sponsor=1" + } + }, + "node_modules/chalk-template/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/chalk-template/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chalk-template/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/chalk-template/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/chalk-template/node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/chalk-template/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/char-regex": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz", @@ -3040,6 +3152,7 @@ "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, "dependencies": { "color-name": "1.1.3" } @@ -3047,7 +3160,8 @@ "node_modules/color-name": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true }, "node_modules/command-line-args": { "version": "5.2.1", @@ -3064,33 +3178,36 @@ } }, "node_modules/command-line-usage": { - "version": "6.1.3", - "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-6.1.3.tgz", - "integrity": "sha512-sH5ZSPr+7UStsloltmDh7Ce5fb8XPlHyoPzTpyyMuYCtervL65+ubVZ6Q61cFtFl62UyJlc8/JwERRbAFPUqgw==", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-7.0.1.tgz", + "integrity": "sha512-NCyznE//MuTjwi3y84QVUGEOT+P5oto1e1Pk/jFPVdPPfsG03qpTIl3yw6etR+v73d0lXsoojRpvbru2sqePxQ==", + "license": "MIT", "dependencies": { - "array-back": "^4.0.2", - "chalk": "^2.4.2", - "table-layout": "^1.0.2", - "typical": "^5.2.0" + "array-back": "^6.2.2", + "chalk-template": "^0.4.0", + "table-layout": "^3.0.0", + "typical": "^7.1.1" }, "engines": { - "node": ">=8.0.0" + "node": ">=12.20.0" } }, "node_modules/command-line-usage/node_modules/array-back": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz", - "integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz", + "integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==", + "license": "MIT", "engines": { - "node": ">=8" + "node": ">=12.17" } }, "node_modules/command-line-usage/node_modules/typical": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz", - "integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.2.0.tgz", + "integrity": "sha512-W1+HdVRUl8fS3MZ9ogD51GOb46xMmhAZzR0WPw5jcgIZQJVvkddYzAl4YTU6g5w33Y1iRQLdIi2/1jhi2RNL0g==", + "license": "MIT", "engines": { - "node": ">=8" + "node": ">=12.17" } }, "node_modules/concat-map": { @@ -3161,14 +3278,6 @@ "node": ">=6" } }, - "node_modules/deep-extend": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "engines": { - "node": ">=4.0.0" - } - }, "node_modules/deepmerge": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", @@ -3727,6 +3836,7 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, "engines": { "node": ">=0.8.0" } @@ -3893,9 +4003,10 @@ } }, "node_modules/flatbuffers": { - "version": "23.3.3", - "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-23.3.3.tgz", - "integrity": "sha512-jmreOaAT1t55keaf+Z259Tvh8tR/Srry9K8dgCgvizhKSEr6gLGgaOJI2WFL5fkOpGOGRZwxUrlFn0GCmXUy6g==" + "version": "23.5.26", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-23.5.26.tgz", + "integrity": "sha512-vE+SI9vrJDwi1oETtTIFldC/o9GsVKRM+s6EL0nQgxXlYV1Vc4Tk30hj4xGICftInKQKj1F3up2n8UbIVobISQ==", + "license": "SEE LICENSE IN LICENSE" }, "node_modules/fs.realpath": { "version": "1.0.0", @@ -4030,6 +4141,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, "engines": { "node": ">=4" } @@ -6105,6 +6217,12 @@ "node": ">=8" } }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "license": "MIT" + }, "node_modules/lodash.camelcase": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", @@ -6916,14 +7034,6 @@ "node": ">=8.10.0" } }, - "node_modules/reduce-flatten": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/reduce-flatten/-/reduce-flatten-2.0.0.tgz", - "integrity": "sha512-EJ4UNY/U1t2P/2k6oqotuX2Cc3T6nxJwsM0N0asT7dhrtH1ltUxDn4NalSYmPE2rCkVpcf/X6R0wDwcFpzhd4w==", - "engines": { - "node": ">=6" - } - }, "node_modules/repeat-string": { "version": "1.6.1", "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", @@ -7227,6 +7337,15 @@ "dev": true, "license": "MIT" }, + "node_modules/stream-read-all": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/stream-read-all/-/stream-read-all-3.0.1.tgz", + "integrity": "sha512-EWZT9XOceBPlVJRrYcykW8jyRSZYbkb/0ZK36uLEmoWVO5gxBOnntNTseNzfREsqxqdfEGQrD8SXQ3QWbBmq8A==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, "node_modules/string-length": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", @@ -7303,6 +7422,7 @@ "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, "dependencies": { "has-flag": "^3.0.0" }, @@ -7323,33 +7443,42 @@ } }, "node_modules/table-layout": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-1.0.2.tgz", - "integrity": "sha512-qd/R7n5rQTRFi+Zf2sk5XVVd9UQl6ZkduPFC3S7WEGJAmetDTjY3qPN50eSKzwuzEyQKy5TN2TiZdkIjos2L6A==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-3.0.2.tgz", + "integrity": "sha512-rpyNZYRw+/C+dYkcQ3Pr+rLxW4CfHpXjPDnG7lYhdRoUcZTUt+KEsX+94RGp/aVp/MQU35JCITv2T/beY4m+hw==", + "license": "MIT", "dependencies": { - "array-back": "^4.0.1", - "deep-extend": "~0.6.0", - "typical": "^5.2.0", - "wordwrapjs": "^4.0.0" + "@75lb/deep-merge": "^1.1.1", + "array-back": "^6.2.2", + "command-line-args": "^5.2.1", + "command-line-usage": "^7.0.0", + "stream-read-all": "^3.0.1", + "typical": "^7.1.1", + "wordwrapjs": "^5.1.0" + }, + "bin": { + "table-layout": "bin/cli.js" }, "engines": { - "node": ">=8.0.0" + "node": ">=12.17" } }, "node_modules/table-layout/node_modules/array-back": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz", - "integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz", + "integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==", + "license": "MIT", "engines": { - "node": ">=8" + "node": ">=12.17" } }, "node_modules/table-layout/node_modules/typical": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz", - "integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.2.0.tgz", + "integrity": "sha512-W1+HdVRUl8fS3MZ9ogD51GOb46xMmhAZzR0WPw5jcgIZQJVvkddYzAl4YTU6g5w33Y1iRQLdIi2/1jhi2RNL0g==", + "license": "MIT", "engines": { - "node": ">=8" + "node": ">=12.17" } }, "node_modules/test-exclude": { @@ -7656,9 +7785,10 @@ } }, "node_modules/tslib": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", - "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==" + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" }, "node_modules/type-detect": { "version": "4.0.8", @@ -8075,23 +8205,12 @@ } }, "node_modules/wordwrapjs": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-4.0.1.tgz", - "integrity": "sha512-kKlNACbvHrkpIw6oPeYDSmdCTu2hdMHoyXLTcUKala++lx5Y+wjJ/e474Jqv5abnVmwxw08DiTuHmw69lJGksA==", - "dependencies": { - "reduce-flatten": "^2.0.0", - "typical": "^5.2.0" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/wordwrapjs/node_modules/typical": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz", - "integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-5.1.0.tgz", + "integrity": "sha512-JNjcULU2e4KJwUNv6CHgI46UvDGitb6dGryHajXTDiLgg1/RiGoPSDw4kZfYnwGtEXf2ZMeIewDQgFGzkCB2Sg==", + "license": "MIT", "engines": { - "node": ">=8" + "node": ">=12.17" } }, "node_modules/wrap-ansi": { diff --git a/package.json b/package.json index 7726a88..d945618 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,6 @@ }, "devDependencies": { "@auth0/auth0-react": "^2.1.1", - "@types/jasmine": "^4.3.1", "@types/node": "^18.13.0", "dotenv": "^16.4.5", "env-cmd": "^10.1.0", @@ -46,7 +45,8 @@ "vitest": "^2.1.4" }, "dependencies": { - "apache-arrow": "^12.0.1", + "apache-arrow": "^13.0.0", + "dotenv": "^16.0.3", "js-yaml": "^4.1.0" } } diff --git a/src/project.ts b/src/project.ts index e66c59d..6d45384 100644 --- a/src/project.ts +++ b/src/project.ts @@ -52,6 +52,46 @@ type CreateAtlasIndexRequest = { duplicate_detection_hyperparameters: string | null; }; +const createIndexTextDefaults: Omit< + components['schemas']['CreateAtlasIndexRequest'], + 'project_id' +> = { + index_name: 'New index', + indexed_field: 'document', // nb upstream types are vague here. + colorable_fields: [], + atomizer_strategies: ['document', 'charchunk'], + geometry_strategies: [['document']], + model: 'nomic-embed-text-v1.5', + model_hyperparameters: JSON.stringify({ + dataset_buffer_size: 1000, + batch_size: 20, + polymerize_by: 'charchunk', + norm: 'both', + }), + nearest_neighbor_index: 'HNSWIndex', + nearest_neighbor_index_hyperparameters: JSON.stringify({ + space: 'l2', + ef_construction: 100, + M: 16, + }), + projection: 'NomicProject', + projection_hyperparameters: JSON.stringify({ + n_neighbors: 15, + n_epochs: 50, + spread: 1, + }), + topic_model_hyperparameters: JSON.stringify({ + build_topic_model: true, + community_description_target_field: null, + cluster_method: 'fast', + enforce_topic_hierarchy: false, + }), + duplicate_detection_hyperparameters: JSON.stringify({ + tag_duplicates: false, + duplicate_cutoff: 0.1, + }), +}; + /** * An AtlasDataset represents a single mutable dataset in Atlas. It provides an * interfaces to upload, update, and delete data, as well as create and delete @@ -202,12 +242,51 @@ export class AtlasDataset extends BaseAtlasClass< return response as Record>; } + /** + * This should be preferred over createIndex, as it uses the actual + * up-to-date types. Note that it is definitely possible to pass a + * malformed but type-compliant object. (E.g., specifying no embedding field + * but also specifying a model.) + * + * @param options See types for options. + * @returns an AtlasIndex object based on the new arguments. + */ + async createIndexRaw( + options: Partial & { + indexed_field: string; + } + ): Promise { + // The default options for this request are very long, so we use them. + // Then we override them with the user's options, and finally guarantee + // that the project_id is set. + + const fields: components['schemas']['CreateAtlasIndexRequest'] = { + ...createIndexTextDefaults, + ...options, + project_id: this.id, + }; + + const response = await this.apiCall( + '/v1/project/index/create', + 'POST', + fields + ); + const id = response as string; + return new AtlasIndex(id, this.viewer, { project: this }); + } + + /** + * Prefer createIndexRaw, which has better syncing of types + * @deprecated + * @param options + * @returns + */ async createIndex( options: Omit ): Promise { const info = await this.fetchAttributes(); const isText = info.modality === 'text'; - // TODO: Python version has a number of asserts here - should we replicate? + const fields: CreateAtlasIndexRequest = { project_id: this.id, index_name: options.index_name ?? 'New index', diff --git a/tests/project.test.ts b/tests/project.test.ts index 4b45868..01146df 100644 --- a/tests/project.test.ts +++ b/tests/project.test.ts @@ -55,12 +55,20 @@ describe('Project Flow Suite', () => { }); test('Create index on project and verify it', async () => { - const index = await project.createIndex({ + const index = await project.createIndexRaw({ index_name: 'test index', indexed_field: 'text', colorable_fields: [], }); }); + + test('Cleanup deletion', async () => { + async function getMeta() { + return project.fetchAttributes(true); + } + expect(getMeta).not.toThrow(); + const f = await project.delete(); + }); }); test('test_arrow_text', () => {