Cardano UTXORPC match filter documentation

README.md

@@ -16,7 +16,6 @@ deno install --allow-scripts && ./patch.sh
 # Build All Contracts
 deno task -r contract:compile
 
-# If running on linux set env DISABLE_YACI=true
 # Run Example Deployment Mode
 deno task -f @e2e/node quickstart
 ```

docs/site/docs/home/1200-templates/1202-wallets.md

@@ -31,7 +31,6 @@ deno install --allow-scripts && ./patch.sh
 deno task -f @e2e/evm-contracts build
 deno task -f @e2e/evm-contracts deploy:standalone
 deno task -f @e2e/midnight-contracts midnight-contract:compile
-# If running on linux set env DISABLE_YACI=true
 deno task -f @e2e/node dev
 ```
 In another terminal, run the demo:

docs/site/docs/home/200-chains/203-cardano.md

@@ -27,7 +27,7 @@ Effectstream supports syncing via **UTXO-RPC** (using Dolos/Yaci) or **Carp**.
     name: "parallelUtxoRpc",
     type: ConfigSyncProtocolType.CARDANO_UTXORPC_PARALLEL,
     rpcUrl: "http://127.0.0.1:50051", // UTXO-RPC endpoint
-    startSlot: 1,
+    startChainPoint: { slot: 1, hash: "abc123..." }, // slot and block hash to start syncing from
   })
 )
 ```

e2e/client/node/deno.json

@@ -18,7 +18,8 @@
     "clean_binaries": "cd ../../../packages/binaries/midnight-node && npm run clean && cd ../midnight-indexer && npm run clean && cd ../midnight-proof-server && npm run clean",
     "quickstart": "deno task -f @effectstream/tui clean && EFFECTSTREAM_ENV=quickstart NODE_ENV=development MIDNIGHT_STORAGE_PASSWORD=yourpasswordmypassword deno run -A --check --unstable-raw-imports scripts/start.quickstart.ts",
     "e2e": "EFFECTSTREAM_ENV=e2e deno run -A --unstable-raw-imports --check scripts/start.e2e.ts",
-    "testnet": "EFFECTSTREAM_ENV=testnet deno run -A --unstable-raw-imports --check scripts/start.testnet.ts"
+    "testnet": "EFFECTSTREAM_ENV=testnet deno run -A --unstable-raw-imports --check scripts/start.testnet.ts",
+    "cardano-midnight": "DISABLE_EVM=true DISABLE_AVAIL=true DISABLE_BITCOIN=true EFFECTSTREAM_ENV=e2e MIDNIGHT_STORAGE_PASSWORD=yourpasswordmypassword deno run -A --unstable-raw-imports --check scripts/start.cardano-midnight.ts"
   },
   "imports": {
     "pg": "npm:pg@^8.14.0",

e2e/client/node/e2e-tests/e2e.migrations.ts

@@ -6,7 +6,7 @@ const evm_enabled = !ENV.getBoolean("DISABLE_EVM");
 const midnight_enabled = !ENV.getBoolean("DISABLE_MIDNIGHT");
 const avail_enabled = !ENV.getBoolean("DISABLE_AVAIL");
 const bitcoin_enabled = !ENV.getBoolean("DISABLE_BITCOIN");
-const yaci_enabled = false; //!ENV.getBoolean("DISABLE_YACI");
+const cardano_enabled = !ENV.getBoolean("DISABLE_CARDANO");
 
 export async function testMigrations(db: Client) {
   const version = await safeQuery<{
@@ -51,7 +51,7 @@ export async function testMigrations(db: Client) {
     expectedMigrations += 5; // 5 dynamic tables
   }
 
-  if (!midnight_enabled && !avail_enabled && !bitcoin_enabled && !yaci_enabled && !evm_enabled) {
+  if (!midnight_enabled && !avail_enabled && !bitcoin_enabled && !cardano_enabled && !evm_enabled) {
     // if no other chain is enabled, then we might hit this code before blocks are generated.
     // So we know between 5 and 8 migration are expected.
     assert(

e2e/client/node/scripts/start.cardano-midnight.ts

@@ -0,0 +1,34 @@
+import {
+  OrchestratorConfig,
+  start,
+} from "@effectstream/orchestrator";
+import { ComponentNames } from "@effectstream/log";
+import { Value } from "@sinclair/typebox/value";
+import { launchCardano } from "@effectstream/orchestrator/start-cardano";
+import { launchMidnight } from "@effectstream/orchestrator/start-midnight";
+import { ENV } from "@effectstream/utils/node-env";
+
+const logs = ENV.getBoolean("EFFECTSTREAM_STDOUT") ? "stdout" : "development";
+const external_db_enabled = ENV.getBoolean("EXTERNAL_DB_ENABLED");
+
+const cardano_processes = launchCardano("@e2e/cardano-contracts");
+const midnight_processes = launchMidnight("@e2e/midnight-contracts");
+
+const config = Value.Parse(OrchestratorConfig, {
+  logs,
+  processes: {
+    [ComponentNames.TMUX]: logs === "development",
+    [ComponentNames.TUI]: logs === "development",
+    [ComponentNames.EFFECTSTREAM_PGLITE]: !external_db_enabled,
+    [ComponentNames.COLLECTOR]: true,
+  },
+
+  packageName: "@effectstream",
+
+  processesToLaunch: [
+    ...cardano_processes,
+    ...midnight_processes,
+  ],
+});
+
+await start(config);

e2e/client/node/scripts/start.e2e.ts

@@ -32,7 +32,7 @@ const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
 const external_db_enabled = ENV.getBoolean("EXTERNAL_DB_ENABLED");
 
 const evm_enabled = !ENV.getBoolean("DISABLE_EVM");
-const yaci_enabled = false; //!ENV.getBoolean("DISABLE_YACI")
+const cardano_enabled = !ENV.getBoolean("DISABLE_CARDANO");
 const midnight_enabled = !ENV.getBoolean("DISABLE_MIDNIGHT");
 const avail_enabled = !ENV.getBoolean("DISABLE_AVAIL");
 const bitcoin_enabled = !ENV.getBoolean("DISABLE_BITCOIN");
@@ -72,7 +72,7 @@ if (bitcoin_enabled) {
   lastProcess = bitcoin_processes[bitcoin_processes.length - 1].name;
 }
 
-if (yaci_enabled) {
+if (cardano_enabled) {
   cardano_processes = launchCardano("@e2e/cardano-contracts");
   // TODO Cardano processes are skipped if the dependencies are not met.
   if (cardano_processes.length > 0) {
@@ -130,7 +130,7 @@ export async function startup(): Promise<Client> {
           evm_enabled ? ComponentNames.DEPLOY_EVM_CONTRACTS : undefined,
           midnight_enabled ? ComponentNames.MIDNIGHT_CONTRACT : undefined,
           avail_enabled ? ComponentNames.AVAIL_CLIENT_WAIT : undefined,
-          yaci_enabled ? ComponentNames.DOLOS_WAIT : undefined,
+          cardano_enabled ? ComponentNames.DOLOS_WAIT : undefined,
           bitcoin_enabled ? ComponentNames.BITCOIN_WAIT_FOR_BLOCK : undefined,
         ].filter(Boolean),
       },
@@ -141,7 +141,7 @@ export async function startup(): Promise<Client> {
         dependsOn: [
           is_serial ? lastProcess : undefined,
           'batcher',
-          (yaci_enabled && cardano_processes.length > 0) ? ComponentNames.DOLOS_WAIT : undefined,
+          (cardano_enabled && cardano_processes.length > 0) ? ComponentNames.DOLOS_WAIT : undefined,
           avail_enabled ? ComponentNames.AVAIL_CLIENT_WAIT : undefined,
         ].filter(Boolean),
       },

e2e/client/node/scripts/start.quickstart.ts

@@ -14,12 +14,12 @@ import { ENV } from "@effectstream/utils/node-env";
 
 const logs = ENV.getBoolean("EFFECTSTREAM_STDOUT") ? "stdout" : "development";
 const external_db_enabled = ENV.getBoolean("EXTERNAL_DB_ENABLED");
-const yaci_enabled = false; //!ENV.getBoolean("DISABLE_YACI");
+const cardano_enabled = !ENV.getBoolean("DISABLE_CARDANO");
 const midnight_enabled = !ENV.getBoolean("DISABLE_MIDNIGHT");
 const avail_enabled = !ENV.getBoolean("DISABLE_AVAIL");
 const bitcoin_enabled = !ENV.getBoolean("DISABLE_BITCOIN");
 
-const cardano_processes = (yaci_enabled ? launchCardano("@e2e/cardano-contracts") : []);
+const cardano_processes = (cardano_enabled ? launchCardano("@e2e/cardano-contracts") : []);
 
 const config = Value.Parse(OrchestratorConfig, {
   logs,
@@ -60,7 +60,7 @@ const config = Value.Parse(OrchestratorConfig, {
       waitToExit: true,
       dependsOn: [
         'batcher',
-        (yaci_enabled && cardano_processes.length > 0) ? ComponentNames.DOLOS_WAIT : undefined,
+        (cardano_enabled && cardano_processes.length > 0) ? ComponentNames.DOLOS_WAIT : undefined,
         avail_enabled ? ComponentNames.AVAIL_CLIENT_WAIT : undefined,
       ].filter(Boolean),
     },

e2e/shared/contracts/cardano/deno.json

@@ -4,17 +4,16 @@
   "license": "MIT",
   "exports": {},
   "tasks": {
-    "dolos:exists": "test -f ./dolos",
     "devkit:start": "deno run -A --node-modules-dir npm:@bloxbean/yaci-devkit up",
     "devkit:wait": "wait-on tcp:3001",
     "dolos:fill-template": "deno run -A ./fill-template.ts",
     // rm -rf is required because of this issue: https://github.com/txpipe/dolos/issues/398
-    // TODO we need to provide newer versions of dolos binaries.
-    "dolos:start": "rm -rf ./data ./dolos.socket && ./dolos bootstrap relay && ./dolos daemon",
+    "dolos:start": "rm -rf ./data ./dolos.socket && deno run -A --node-modules-dir npm:@txpipe/dolos bootstrap relay && deno run -A --node-modules-dir npm:@txpipe/dolos daemon",
     "dolos:wait": "wait-on tcp:50051" // utxorpc port
   },
   "imports": {
-    "@txpipe/dolos": "npm:@txpipe/dolos@0.19.1",
+    "@bloxbean/yaci-devkit": "@bloxbean/yaci-devkit@0.11.0-beta1",
+    "@txpipe/dolos": "npm:@txpipe/dolos@1.0.0-rc.10",
     "toml": "jsr:@std/toml"
   }
 }

e2e/shared/data-types/src/config.ts

@@ -43,7 +43,7 @@ const isEnvTrue = (key: string, defaultValue?: boolean) => {
 // TODO: This is a workaround to disable yaci-devkit in linux for testing.
 //       There is a unknown error when launching this process.
 //       error: Text file busy (os error 26)
-const yaci_enabled = false; //!isEnvTrue("DISABLE_YACI");
+const cardano_enabled = !isEnvTrue("DISABLE_CARDANO");
 
 // NOTE: This disable midnight sync, allowing for faster testing.
 const midnight_enabled = !isEnvTrue("DISABLE_MIDNIGHT");
@@ -76,8 +76,8 @@ if (typeof Deno !== 'undefined' && Deno) {
   const dbConn = getConnection();
   try {
     const result = await dbConn.query(`
-      SELECT * FROM effectstream.sync_protocol_pagination 
-      WHERE protocol_name = '${mainSyncProtocolName}' 
+      SELECT * FROM effectstream.sync_protocol_pagination
+      WHERE protocol_name = '${mainSyncProtocolName}'
       ORDER BY page_number ASC
       LIMIT 1
     `);
@@ -92,11 +92,13 @@ if (typeof Deno !== 'undefined' && Deno) {
   }
 
     // We fetch the latest block from the dolos mini blockfrost endpoint
-    if (yaci_enabled) {
-      const response = await fetch("http://localhost:3000/blocks/latest");
-      yaciDevKitStartTime = (await response.json()).time * 1000;
+    if (cardano_enabled) {
+      const latestResponse = await fetch("http://localhost:3000/blocks/latest");
+      const latestBlock = await latestResponse.json();
+      yaciDevKitStartTime = latestBlock.time * 1000;
       yaciDevKitStartTime = new Date().getTime() - yaciDevKitStartTime;
       console.log("yaciDevKitStartTime", yaciDevKitStartTime);
+
     }
 }
 
@@ -154,7 +156,7 @@ export const config = new ConfigBuilder()
         });
     }
 
-    if (yaci_enabled) {
+    if (cardano_enabled) {
       b = b
         .addNetwork({
           name: "yaci",
@@ -256,7 +258,7 @@ export const config = new ConfigBuilder()
         );
     }
 
-    if (yaci_enabled) {
+    if (cardano_enabled) {
 
       result = result
         .addParallel(
@@ -265,7 +267,7 @@ export const config = new ConfigBuilder()
             name: "parallelUtxoRpc",
             type: ConfigSyncProtocolType.CARDANO_UTXORPC_PARALLEL,
             rpcUrl: "http://127.0.0.1:50051", // dolos utxorpc address
-            startSlot: 1,
+            startChainPoint: "origin",
             // TODO: The exact delay is not correct, but it's close.
             // byron-genesis.json startTime
             // 633 skipped slots
@@ -456,7 +458,7 @@ export const config = new ConfigBuilder()
         }),
       );
     }
-    if (yaci_enabled) {
+    if (cardano_enabled) {
       b = b.addPrimitive(
         (syncProtocols) => (syncProtocols as any).parallelUtxoRpc,
         (network, deployments, syncProtocol) => ({

packages/build-tools/orchestrator/scripts/launch-cardano.ts

@@ -1,7 +1,4 @@
 import { ComponentNames } from "@effectstream/log";
-import { statSync } from "node:fs";
-import { spawnSync } from "node:child_process";
-import { getEnv } from "@effectstream/utils/runtime";
 
 // Start Cardano Node and Indexer.
 //
@@ -31,38 +28,12 @@ export const launchCardano = (packageName: string): {
   cwd?: string;
   command?: string;
 }[] => {
-  const home = getEnv("HOME");
-  const yaciDir = `${home}/.yaci-cli`;
-  const yaciCliPath = `${yaciDir}/yaci-cli`;
 
-  // TODO $HOME/.yaci-cli/yaci-cli must be installed to allow this to work.
-  // At the time the npm packages is incompatible with deno.
-  try {
-    statSync(yaciCliPath);
-  } catch (_error) {
-    throw new Error(
-      `Cardano launcher skipped: missing ${yaciCliPath}. Run yaci-cli setup.`,
-    );
-  }
-  // TODO We require the latest dolos binary built from source.
-  // At the time there is not npm package for the latest dolos binary.
-  try {
-    const dolosExists = spawnSync("deno", ["task", "-f", packageName, "dolos:exists"], { encoding: "utf-8" });
-    if (dolosExists.status !== 0) throw new Error();
-  } catch (_error) {
-    throw new Error(
-      "Cardano launcher skipped: dolos binary is missing.",
-    );
-  }
-
-
  return [
     {
       stopProcessAtPort: [8090, 10000, 50051, 3001],
-      cwd: yaciDir,
-      command: "./yaci-cli",
-      args: ["up"],
       name: ComponentNames.YACI_DEVKIT,
+      args: ["task", "-f", packageName, "devkit:start"],
       waitToExit: false,
       type: "system-dependency",
       logsStartDisabled: true,
@@ -72,6 +43,12 @@ export const launchCardano = (packageName: string): {
       args: ["task", "-f", packageName, "devkit:wait"],
       // dependsOn: [ComponentNames.YACI_DEVKIT],
     },
+    {
+      name: ComponentNames.DOLOS_FILL_TEMPLATE,
+      args: ["task", "-f", packageName, "dolos:fill-template"],
+      waitToExit: true,
+      dependsOn: [ComponentNames.YACI_DEVKIT_WAIT],
+    },
     {
       name: ComponentNames.DOLOS,
       args: ["task", "-f", packageName, "dolos:start"],
@@ -80,7 +57,7 @@ export const launchCardano = (packageName: string): {
       logsStartDisabled: true,
       disableStderr: true,
       type: "system-dependency",
-      dependsOn: [ComponentNames.YACI_DEVKIT_WAIT],
+      dependsOn: [ComponentNames.DOLOS_FILL_TEMPLATE],
     },
     {
       name: ComponentNames.DOLOS_WAIT,

packages/effectstream-sdk/config/src/schema/common.ts

@@ -38,6 +38,28 @@ export const StartStopSlot = new ConfigSchema({
   }),
 });
 
+export const CardanoChainPoint = Type.Object({
+  slot: TypeboxHelpers.AbsoluteSlotNumber(),
+  hash: TypeboxHelpers.Cardano.BlockHash,
+});
+
+export const StartChainPointValue = Type.Union([
+  Type.Literal("origin"),
+  Type.Literal("tip"),
+  CardanoChainPoint,
+]);
+
+export const StartStopChainPoint = new ConfigSchema({
+  required: Type.Object({
+    startChainPoint: StartChainPointValue,
+  }),
+  optional: Type.Object({
+    stopChainPoint: TypeboxHelpers.Nullable(CardanoChainPoint, {
+      default: null,
+    }),
+  }),
+});
+
 export const StartStopTimestamp = new ConfigSchema({
   required: Type.Object({
     startTimestamp: TypeboxHelpers.TimestampMs(),