Improvements to COPY support, including to/from and url

packages/pglite/Makefile

@@ -14,7 +14,7 @@ build:
 	EMCC_CFLAGS="-s SIDE_MODULE=1 -sERROR_ON_UNDEFINED_SYMBOLS=0 -sWARN_ON_UNDEFINED_SYMBOLS=0 -sTOTAL_MEMORY=65536000 -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORTED_RUNTIME_METHODS='FS'" \
 		emmake make -C src/pl/plpgsql MAKELEVEL=0
 
-	EMCC_CFLAGS="-sALLOW_MEMORY_GROWTH=1 -sERROR_ON_UNDEFINED_SYMBOLS=0 -sWARN_ON_UNDEFINED_SYMBOLS=0 -sTOTAL_MEMORY=65536000 -sEMULATE_FUNCTION_POINTER_CASTS=1 -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORTED_FUNCTIONS=_main,_ExecProtocolMsg,_malloc,_free -sEXPORTED_RUNTIME_METHODS=ccall,cwrap,FS" \
+	EMCC_CFLAGS="-sALLOW_MEMORY_GROWTH=1 -sERROR_ON_UNDEFINED_SYMBOLS=0 -sWARN_ON_UNDEFINED_SYMBOLS=0 -sTOTAL_MEMORY=65536000 -sEMULATE_FUNCTION_POINTER_CASTS=1 -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORTED_FUNCTIONS=_main,_ExecProtocolMsg,_malloc,_free -sEXPORTED_RUNTIME_METHODS=ccall,cwrap,FS,stringToNewUTF8" \
 		emmake make -C src/backend MAKELEVEL=0
 
 	mkdir -p ../packages/pglite/release

packages/pglite/README.md

@@ -196,8 +196,8 @@ The `query` and `exec` methods take an optional `options` objects with the follo
   });
   ```
 
-- `blob: Blob | File`
-  Attach a `Blob` or `File` object to the query that can used with a `COPY FROM` command by using the virtual `/dev/blob` device, see [importing and exporting](#importing-and-exporting-with-copy-tofrom).
+- `blobs: {[name: string]: Blob | File}`
+  Attach a `Blob` or `File` object to the query that can used with a `COPY FROM` command by using the syntax `COPY table FROM 'blob:name'`, see [importing and exporting](#importing-and-exporting-with-copy-tofrom).
 
 #### `.exec(query: string, options?: QueryOptions): Promise<Array<Results>>`
 
@@ -307,7 +307,7 @@ Result objects have the following properties:
 - `rows: Row<T>[]` - The rows retuned by the query
 - `affectedRows?: number` - Count of the rows affected by the query. Note this is *not* the count of rows returned, it is the number or rows in the database changed by the query.
 - `fields: { name: string; dataTypeID: number }[]` - Field name and Postgres data type ID for each field returned.
-- `blob: Blob` - A `Blob` containing the data written to the virtual `/dev/blob/` device by a `COPY TO` command. See [importing and exporting](#importing-and-exporting-with-copy-tofrom).
+- `blobs: {[name: string]: Blob}` - A `Blob` containing the data written to `blob:name` by a `COPY TO` command. See [importing and exporting](#importing-and-exporting-with-copy-tofrom).
 
 
 ### Row<T> Objects:
@@ -336,21 +336,33 @@ await pg.exec(`
 
 ### Importing and exporting with `COPY TO/FROM`
 
-PGlite has support importing and exporting via `COPY TO/FROM` by using a virtual `/dev/blob` device.
+PGlite has support importing and exporting via `COPY TO/FROM` by attaching blobs to queries and results.
 
-To import a file pass the `File` or `Blob` in the query options as `blob`, and copy from the `/dev/blob` device.
+To import a file pass the `File` or `Blob` in the query `blobs` options:
 
 ```ts
-await pg.query("COPY my_table FROM '/dev/blob';", [], {
-  blob: MyBlob
+await pg.query("COPY my_table FROM 'blob:my_blob';", [], {
+  blob: { my_blob: MyBlob },
 })
 ```
 
-To export a table or query to a file you just have to write to the `/dev/blob` device, the file will be retied as `blob` on the query results:
+To export a table or query to a file you write to the `blob:blob_name`, the file will be returned on the query results:
 
 ```ts
-const ret = await pg.query("COPY my_table TO '/dev/blob';")
-// ret.blob is a `Blob` object with the data from the copy.
+const ret = await pg.query("COPY my_table TO 'blob:my_blob';")
+// ret.blobs['my_blob'] is a `Blob` object with the data from the copy.
+```
+
+It is also possible to copy from a URL:
+
+```ts
+await pg.query("COPY my_table FROM 'https://example.com/my_data.csv';")
+```
+
+and export to a URL, the request is sent a a HTTP PUT:
+
+```ts
+await pg.query("COPY my_table TO 'https://example.com/api/endpoint';")
 ```
 
 ## Extensions

packages/pglite/examples/copy.html

@@ -15,10 +15,10 @@
 
   // Copy the date to a file:
   console.log('Copying data to file...') // 'test.csv
-  const ret = await pg.query("COPY test TO '/dev/blob' WITH (FORMAT binary);");
+  const ret = await pg.query("COPY test TO 'blob:test.csv' WITH (FORMAT binary);");
 
   console.log('Data copied to blob:')
-  const blob = ret.blob;
+  const blob = ret.blobs['test.csv'];
   console.log(blob);
 
   // Download the file:
@@ -37,8 +37,8 @@
 
   // import the data from the file:
   console.log('Importing data from file...')
-  const ret2 = await pg.query("COPY test2 FROM '/dev/blob' WITH (FORMAT binary);", [], {
-    blob: blob
+  const ret2 = await pg.query("COPY test2 FROM 'blob:test.csv' WITH (FORMAT binary);", [], {
+    blobs: {'test.csv': blob}
   });
 
   console.log('Data imported from file:')

packages/pglite/release/postgres.d.ts

@@ -426,6 +426,9 @@ export interface EmPostgres extends EmscriptenModule {
   FS: FS;
   eventTarget: EventTarget;
   Event: typeof CustomEvent;
+  copyFrom: (fileName: string, isProgram: boolean) => string | null;
+  copyTo: (fileName: string, isProgram: boolean) => string | null;
+  copyToEnd: () => string | null;
   onRuntimeInitialized: (Module: EmPostgres) => Promise<void>;
 }
 

packages/pglite/src/interface.ts

@@ -14,7 +14,11 @@ export interface ParserOptions {
 export interface QueryOptions {
   rowMode?: RowMode;
   parsers?: ParserOptions;
-  blob?: Blob | File;
+  blobs?: QueryBlobs;
+}
+
+export interface QueryBlobs {
+  [name: string]: Blob | File;
 }
 
 export interface ExecProtocolOptions {
@@ -98,11 +102,15 @@ export type PGliteInterfaceExtensions<E> = E extends Extensions
 
 export type Row<T = { [key: string]: any }> = T;
 
+export type ResultBlobs = {
+  [name: string]: Blob | File;
+};
+
 export type Results<T = { [key: string]: any }> = {
   rows: Row<T>[];
   affectedRows?: number;
   fields: { name: string; dataTypeID: number }[];
-  blob?: Blob; // Only set when a file is returned, such as from a COPY command
+  blobs?: ResultBlobs;
 };
 
 export interface Transaction {

packages/pglite/src/parse.ts

@@ -4,7 +4,7 @@ import {
   DataRowMessage,
   CommandCompleteMessage,
 } from "pg-protocol/dist/messages.js";
-import type { Results, QueryOptions } from "./interface.js";
+import type { Results, QueryOptions, ResultBlobs } from "./interface.js";
 import { parseType } from "./types.js";
 
 /**
@@ -14,7 +14,7 @@ import { parseType } from "./types.js";
 export function parseResults(
   messages: Array<BackendMessage>,
   options?: QueryOptions,
-  blob?: Blob
+  blobs?: ResultBlobs,
 ): Array<Results> {
   const resultSets: Results[] = [];
   let currentResultSet: Results = { rows: [], fields: [] };
@@ -24,7 +24,7 @@ export function parseResults(
     (msg) =>
       msg instanceof RowDescriptionMessage ||
       msg instanceof DataRowMessage ||
-      msg instanceof CommandCompleteMessage
+      msg instanceof CommandCompleteMessage,
   );
 
   filteredMessages.forEach((msg, index) => {
@@ -40,9 +40,9 @@ export function parseResults(
             parseType(
               field,
               currentResultSet!.fields[i].dataTypeID,
-              options?.parsers
-            )
-          )
+              options?.parsers,
+            ),
+          ),
         );
       } else {
         // rowMode === "object"
@@ -53,10 +53,10 @@ export function parseResults(
               parseType(
                 field,
                 currentResultSet!.fields[i].dataTypeID,
-                options?.parsers
+                options?.parsers,
               ),
-            ])
-          )
+            ]),
+          ),
         );
       }
     } else if (msg instanceof CommandCompleteMessage) {
@@ -66,7 +66,7 @@ export function parseResults(
         resultSets.push({
           ...currentResultSet,
           affectedRows,
-          ...(blob ? { blob } : {}),
+          ...(blobs ? { blobs } : {}),
         });
       else resultSets.push(currentResultSet);