From 842528949445271fdb49519de171c5b42ba301d6 Mon Sep 17 00:00:00 2001 From: NathanosDev Date: Tue, 23 Apr 2024 00:09:54 +0200 Subject: [PATCH] feat: add http2 support --- docs/docs/guides/03-using-vitest.md | 6 +- docs/docs/guides/05-running-tests.md | 37 ++++ ...-the-nns.md => 06-working-with-the-nns.md} | 0 ...rations.md => 07-canister-declarations.md} | 0 examples/clock/tests/global-setup.ts | 4 +- .../nns_proxy/tests/src/nns-proxy.spec.ts | 2 +- packages/pic/src/http-client.ts | 55 ------ packages/pic/src/http2-client.ts | 187 ++++++++++++++++++ packages/pic/src/pocket-ic-client-types.ts | 12 +- packages/pic/src/pocket-ic-client.ts | 94 +++++---- packages/pic/src/pocket-ic-server-types.ts | 8 +- packages/pic/src/pocket-ic-server.ts | 4 +- packages/pic/src/pocket-ic-types.ts | 5 + 13 files changed, 288 insertions(+), 126 deletions(-) create mode 100644 docs/docs/guides/05-running-tests.md rename docs/docs/guides/{05-working-with-the-nns.md => 06-working-with-the-nns.md} (100%) rename docs/docs/guides/{06-canister-declarations.md => 07-canister-declarations.md} (100%) delete mode 100644 packages/pic/src/http-client.ts create mode 100644 packages/pic/src/http2-client.ts diff --git a/docs/docs/guides/03-using-vitest.md b/docs/docs/guides/03-using-vitest.md index 49dd56b..9fd2de2 100644 --- a/docs/docs/guides/03-using-vitest.md +++ b/docs/docs/guides/03-using-vitest.md @@ -91,7 +91,7 @@ export async function teardown(): Promise { To improve type-safety for `ctx.provide('PIC_URL')` and (later) `inject('PIC_URL')`, create a `types.d.ts` file: -```typescript title="types.d.ts" +```ts title="types.d.ts" export declare module 'vitest' { export interface ProvidedContext { PIC_URL: string; @@ -101,7 +101,7 @@ export declare module 'vitest' { Create a `vitest.config.ts` file: -```typescript title="vitest.config.ts" +```ts title="vitest.config.ts" import { defineConfig } from 'vitest/config'; export default defineConfig({ @@ -119,7 +119,7 @@ export default defineConfig({ The basic skeleton of all PicJS tests written with [Vitest](https://vitest.dev/) will look something like this: -```typescript title="tests/example.spec.ts" +```ts title="tests/example.spec.ts" import { describe, beforeEach, afterEach, it, expect, inject } from 'vitest'; // Import generated types for your canister diff --git a/docs/docs/guides/05-running-tests.md b/docs/docs/guides/05-running-tests.md new file mode 100644 index 0000000..420f8f5 --- /dev/null +++ b/docs/docs/guides/05-running-tests.md @@ -0,0 +1,37 @@ +# Running tests + +## Configuring logging + +### Canister logs + +Logs for canisters can be configured when running the PocketIC server using the `showCanisterLogs` option, for example: + +```ts +const pic = await PocketIcServer.start({ + showCanisterLogs: true, +}); +``` + +### Server logs + +Logs for the PocketIC server can be configured by setting the `POCKET_IC_LOG_DIR` and `POCKET_IC_LOG_DIR_LEVELS` environment variables. + +The `POCKET_IC_LOG_DIR` variable specifies the directory where the logs will be stored. It accepts any valid relative, or absolute directory path. + +The `POCKET_IC_LOG_DIR_LEVELS` variable specifies the log levels. It accepts any of the following `string` values: `trace`, `debug`, `info`, `warn`, or `error`. + +For example: + +```shell +POCKET_IC_LOG_DIR=./logs POCKET_IC_LOG_DIR_LEVELS=trace npm test +``` + +### Runtime logs + +Logs for the IC runtime can be configured when running the PocketIC server using the `showRuntimeLogs` option, for example: + +```ts +const pic = await PocketIcServer.start({ + showRuntimeLogs: true, +}); +``` diff --git a/docs/docs/guides/05-working-with-the-nns.md b/docs/docs/guides/06-working-with-the-nns.md similarity index 100% rename from docs/docs/guides/05-working-with-the-nns.md rename to docs/docs/guides/06-working-with-the-nns.md diff --git a/docs/docs/guides/06-canister-declarations.md b/docs/docs/guides/07-canister-declarations.md similarity index 100% rename from docs/docs/guides/06-canister-declarations.md rename to docs/docs/guides/07-canister-declarations.md diff --git a/examples/clock/tests/global-setup.ts b/examples/clock/tests/global-setup.ts index da244b1..78bf1f5 100644 --- a/examples/clock/tests/global-setup.ts +++ b/examples/clock/tests/global-setup.ts @@ -2,8 +2,8 @@ import { PocketIcServer } from '@hadronous/pic'; module.exports = async function (): Promise { const pic = await PocketIcServer.start({ - pipeStdout: false, - pipeStderr: true, + showCanisterLogs: true, + showRuntimeLogs: false, }); const url = pic.getUrl(); diff --git a/examples/nns_proxy/tests/src/nns-proxy.spec.ts b/examples/nns_proxy/tests/src/nns-proxy.spec.ts index 709dfc3..f92ca40 100644 --- a/examples/nns_proxy/tests/src/nns-proxy.spec.ts +++ b/examples/nns_proxy/tests/src/nns-proxy.spec.ts @@ -86,7 +86,7 @@ describe('NNS Proxy', () => { throw new Error('NNS subnet not found'); } - const rootKey = pic.getPubKey(nnsSubnet.id); + const rootKey = await pic.getPubKey(nnsSubnet.id); expect(rootKey).toBeDefined(); }); }); diff --git a/packages/pic/src/http-client.ts b/packages/pic/src/http-client.ts deleted file mode 100644 index ff29647..0000000 --- a/packages/pic/src/http-client.ts +++ /dev/null @@ -1,55 +0,0 @@ -export type HeadersInit = Record; - -export interface GetOptions { - headers?: HeadersInit; -} - -export interface PostOptions

{ - body?: P; - headers?: HeadersInit; -} - -export const JSON_HEADER: HeadersInit = { - 'Content-Type': 'application/json', -}; - -export class HttpClient { - private constructor() {} - - public static async get(url: string, options?: GetOptions): Promise { - const headers = options?.headers ?? {}; - - const response = await fetch(url, { - method: 'GET', - headers: { ...headers, ...JSON_HEADER }, - }); - - handleFetchError(response); - return (await response.json()) as R; - } - - public static async post( - url: string, - options?: PostOptions

, - ): Promise { - const body = options?.body ? JSON.stringify(options.body) : null; - const headers = options?.headers ?? {}; - - const response = await fetch(url, { - method: 'POST', - body, - headers: { ...headers, ...JSON_HEADER }, - }); - - handleFetchError(response); - return (await response.json()) as R; - } -} - -export function handleFetchError(response: Response): void { - if (!response.ok) { - console.error('Error response', response.url, response.statusText); - - throw new Error(`${response.url} ${response.statusText}`); - } -} diff --git a/packages/pic/src/http2-client.ts b/packages/pic/src/http2-client.ts new file mode 100644 index 0000000..87657af --- /dev/null +++ b/packages/pic/src/http2-client.ts @@ -0,0 +1,187 @@ +import http2, { + ClientHttp2Session, + IncomingHttpHeaders, + OutgoingHttpHeaders, +} from 'node:http2'; + +const { HTTP2_HEADER_PATH, HTTP2_HEADER_METHOD } = http2.constants; + +export interface Request { + method: Method; + path: string; + headers?: OutgoingHttpHeaders; + body?: Uint8Array; +} + +export interface JsonGetRequest { + path: string; + headers?: OutgoingHttpHeaders; +} + +export interface JsonPostRequest { + path: string; + headers?: OutgoingHttpHeaders; + body?: B; +} + +export interface Response { + status: number | undefined; + body: string; + headers: IncomingHttpHeaders; +} + +export type Method = 'GET' | 'POST' | 'PUT' | 'DELETE'; + +export const JSON_HEADER: OutgoingHttpHeaders = { + 'Content-Type': 'application/json', +}; + +export class Http2Client { + private readonly session: ClientHttp2Session; + + constructor(baseUrl: string) { + this.session = http2.connect(baseUrl); + } + + public request(init: Request): Promise { + return new Promise((resolve, reject) => { + let req = this.session.request({ + [HTTP2_HEADER_PATH]: init.path, + [HTTP2_HEADER_METHOD]: init.method, + 'content-length': init.body?.length ?? 0, + ...init.headers, + }); + + req.on('error', error => { + console.error('Erorr sending request to PocketIC server', error); + return reject(error); + }); + + req.on('response', headers => { + const status = headers[':status'] ?? -1; + + const contentLength = headers['content-length'] + ? Number(headers['content-length']) + : 0; + let buffer = Buffer.alloc(contentLength); + let bufferLength = 0; + + req.on('data', (chunk: Buffer) => { + chunk.copy(buffer, bufferLength); + bufferLength += chunk.length; + }); + + req.on('end', () => { + const body = buffer.toString('utf8'); + + return resolve({ + status, + body, + headers, + }); + }); + }); + + if (init.body) { + req.write(init.body, 'utf8'); + } + + req.end(); + }); + } + + public async jsonGet(init: JsonGetRequest): Promise { + while (true) { + const res = await this.request({ + method: 'GET', + path: init.path, + headers: { ...init.headers, ...JSON_HEADER }, + }); + + const resBody = JSON.parse(res.body) as ApiResponse; + if (!resBody) { + return resBody; + } + + // server encountered an error + if ('message' in resBody) { + console.error('PocketIC server encountered an error', resBody.message); + + throw new Error(resBody.message); + } + + // the server has started processing or is busy + if ('state_label' in resBody) { + console.error('PocketIC server is too busy to process the request'); + + if (res.status === 202) { + throw new Error('Server started processing'); + } + + if (res.status === 409) { + throw new Error('Server busy'); + } + + throw new Error('Unknown state'); + } + + return resBody; + } + } + + public async jsonPost(init: JsonPostRequest): Promise { + const reqBody = init.body + ? new TextEncoder().encode(JSON.stringify(init.body)) + : undefined; + + while (true) { + const res = await this.request({ + method: 'POST', + path: init.path, + headers: { ...init.headers, ...JSON_HEADER }, + body: reqBody, + }); + + const resBody = JSON.parse(res.body); + if (!resBody) { + return resBody; + } + + // server encountered an error + if ('message' in resBody) { + console.error('PocketIC server encountered an error', resBody.message); + + throw new Error(resBody.message); + } + + // the server has started processing or is busy + // sleep and try again + if ('state_label' in resBody) { + console.error('PocketIC server is too busy to process the request'); + + if (res.status === 202) { + throw new Error('Server started processing'); + } + + if (res.status === 409) { + throw new Error('Server busy'); + } + + throw new Error('Unknown state'); + } + + return resBody; + } + } +} + +interface StartedOrBusyApiResponse { + state_label: string; + op_id: string; +} + +interface ErrorResponse { + message: string; +} + +type ApiResponse = StartedOrBusyApiResponse | ErrorResponse | R; diff --git a/packages/pic/src/pocket-ic-client-types.ts b/packages/pic/src/pocket-ic-client-types.ts index 4f546f9..b3d87b4 100644 --- a/packages/pic/src/pocket-ic-client-types.ts +++ b/packages/pic/src/pocket-ic-client-types.ts @@ -22,6 +22,7 @@ export interface CreateInstanceRequest { bitcoin?: boolean; system?: number; application?: number; + processingTimeoutMs?: number; } export interface EncodedCreateInstanceRequest { @@ -564,15 +565,10 @@ export interface EncodedCanisterCallErrorResponse { }; } -export interface EncodedCanisterCallErrorMessageResponse { - message: string; -} - export type EncodedCanisterCallResponse = | EncodedCanisterCallSuccessResponse | EncodedCanisterCallRejectResponse - | EncodedCanisterCallErrorResponse - | EncodedCanisterCallErrorMessageResponse; + | EncodedCanisterCallErrorResponse; export function decodeCanisterCallResponse( res: EncodedCanisterCallResponse, @@ -581,10 +577,6 @@ export function decodeCanisterCallResponse( throw new Error(res.Err.description); } - if ('message' in res) { - throw new Error(res.message); - } - if ('Reject' in res.Ok) { throw new Error(res.Ok.Reject); } diff --git a/packages/pic/src/pocket-ic-client.ts b/packages/pic/src/pocket-ic-client.ts index d5bb4d4..ea61ab9 100644 --- a/packages/pic/src/pocket-ic-client.ts +++ b/packages/pic/src/pocket-ic-client.ts @@ -1,9 +1,5 @@ -import { - HeadersInit, - HttpClient, - JSON_HEADER, - handleFetchError, -} from './http-client'; +import { IncomingHttpHeaders } from 'http2'; +import { Http2Client } from './http2-client'; import { EncodedAddCyclesRequest, EncodedAddCyclesResponse, @@ -18,7 +14,6 @@ import { EncodedGetTimeResponse, EncodedSetTimeRequest, EncodedGetSubnetIdResponse, - SubnetTopology, decodeInstanceTopology, InstanceTopology, GetStableMemoryRequest, @@ -46,7 +41,6 @@ import { encodeSetTimeRequest, CanisterCallRequest, encodeCanisterCallRequest, - decodeCanisterCallResponse, CanisterCallResponse, decodeUploadBlobResponse, UploadBlobResponse, @@ -57,58 +51,65 @@ import { GetPubKeyRequest, EncodedGetPubKeyRequest, encodeGetPubKeyRequest, + EncodedSetStableMemoryRequest, + decodeCanisterCallResponse, } from './pocket-ic-client-types'; const PROCESSING_TIME_HEADER = 'processing-timeout-ms'; const PROCESSING_TIME_VALUE_MS = 300_000; -const PROCESSING_HEADER: HeadersInit = { - [PROCESSING_TIME_HEADER]: PROCESSING_TIME_VALUE_MS.toString(), -}; export class PocketIcClient { private isInstanceDeleted = false; + private readonly processingHeader: IncomingHttpHeaders; private constructor( - private readonly instanceUrl: string, - private readonly serverUrl: string, + private readonly serverClient: Http2Client, + private readonly instancePath: string, private readonly topology: InstanceTopology, - ) {} + processingTimeoutMs: number, + ) { + this.processingHeader = { + [PROCESSING_TIME_HEADER]: processingTimeoutMs.toString(), + }; + } public static async create( url: string, req?: CreateInstanceRequest, ): Promise { - const [instanceId, topology] = await PocketIcClient.createInstance( - url, - req, - ); - - return new PocketIcClient(`${url}/instances/${instanceId}`, url, topology); - } + const serverClient = new Http2Client(url); - private static async createInstance( - url: string, - req?: CreateInstanceRequest, - ): Promise<[number, Record]> { - const res = await HttpClient.post< + const res = await serverClient.jsonPost< EncodedCreateInstanceRequest, CreateInstanceResponse - >(`${url}/instances`, { body: encodeCreateInstanceRequest(req) }); + >({ + path: '/instances', + body: encodeCreateInstanceRequest(req), + }); if ('Error' in res) { + console.error('Error creating instance', res.Error.message); + throw new Error(res.Error.message); } const topology = decodeInstanceTopology(res.Created.topology); + const instanceId = res.Created.instance_id; - return [res.Created.instance_id, topology]; + return new PocketIcClient( + serverClient, + `/instances/${instanceId}`, + topology, + req?.processingTimeoutMs ?? PROCESSING_TIME_VALUE_MS, + ); } public async deleteInstance(): Promise { this.assertInstanceNotDeleted(); - await fetch(this.instanceUrl, { + await this.serverClient.request({ method: 'DELETE', + path: this.instancePath, }); this.isInstanceDeleted = true; @@ -190,30 +191,23 @@ export class PocketIcClient { public async uploadBlob(req: UploadBlobRequest): Promise { this.assertInstanceNotDeleted(); - const res = await fetch(`${this.serverUrl}/blobstore`, { + const res = await this.serverClient.request({ method: 'POST', + path: '/blobstore', body: encodeUploadBlobRequest(req), }); - return decodeUploadBlobResponse(await res.text()); + return decodeUploadBlobResponse(res.body); } public async setStableMemory(req: SetStableMemoryRequest): Promise { this.assertInstanceNotDeleted(); - // this endpoint does not return JSON encoded responses, - // so we make this request directly using fetch to avoid the automatic JSON decoding - // from HttpClient.post - const res = await fetch(`${this.instanceUrl}/update/set_stable_memory`, { - method: 'POST', - headers: { - ...JSON_HEADER, - ...PROCESSING_HEADER, - }, - body: JSON.stringify(encodeSetStableMemoryRequest(req)), + await this.serverClient.jsonPost({ + path: `${this.instancePath}/update/set_stable_memory`, + headers: this.processingHeader, + body: encodeSetStableMemoryRequest(req), }); - - handleFetchError(res); } public async getStableMemory( @@ -257,16 +251,18 @@ export class PocketIcClient { return decodeCanisterCallResponse(res); } - private async post(endpoint: string, body?: P): Promise { - return await HttpClient.post(`${this.instanceUrl}${endpoint}`, { + private async post(endpoint: string, body?: B): Promise { + return await this.serverClient.jsonPost({ + path: `${this.instancePath}${endpoint}`, + headers: this.processingHeader, body, - headers: PROCESSING_HEADER, }); } - private async get(endpoint: string): Promise { - return await HttpClient.get(`${this.instanceUrl}${endpoint}`, { - headers: PROCESSING_HEADER, + private async get(endpoint: string): Promise { + return await this.serverClient.jsonGet({ + path: `${this.instancePath}${endpoint}`, + headers: this.processingHeader, }); } diff --git a/packages/pic/src/pocket-ic-server-types.ts b/packages/pic/src/pocket-ic-server-types.ts index e294719..694819b 100644 --- a/packages/pic/src/pocket-ic-server-types.ts +++ b/packages/pic/src/pocket-ic-server-types.ts @@ -3,12 +3,12 @@ */ export interface StartServerOptions { /** - * Whether to pipe the server's stdout to the parent process's stdout. + * Whether to pipe the runtimes's logs to the parent process's stdout. */ - pipeStdout?: boolean; + showRuntimeLogs?: boolean; /** - * Whether to pipe the server's stderr to the parent process's stderr. + * Whether to pipe the canister logs to the parent process's stderr. */ - pipeStderr?: boolean; + showCanisterLogs?: boolean; } diff --git a/packages/pic/src/pocket-ic-server.ts b/packages/pic/src/pocket-ic-server.ts index ef6e168..d4979ff 100644 --- a/packages/pic/src/pocket-ic-server.ts +++ b/packages/pic/src/pocket-ic-server.ts @@ -72,11 +72,11 @@ export class PocketIcServer { const serverProcess = spawn(binPath, ['--pid', pid.toString()]); - if (options.pipeStdout) { + if (options.showRuntimeLogs) { serverProcess.stdout.pipe(process.stdout); } - if (options.pipeStderr) { + if (options.showCanisterLogs) { serverProcess.stderr.pipe(process.stderr); } diff --git a/packages/pic/src/pocket-ic-types.ts b/packages/pic/src/pocket-ic-types.ts index 3fcc534..f3d3586 100644 --- a/packages/pic/src/pocket-ic-types.ts +++ b/packages/pic/src/pocket-ic-types.ts @@ -77,6 +77,11 @@ export interface CreateInstanceOptions { * Default is `1`. */ application?: number; + + /** + * How long the PocketIC client should wait for a response from the server. + */ + processingTimeoutMs?: number; } /**