Don't dump hash on HS, allow cache minification via schema, fix psql for local dev

DZakh · DZakh · commit 68be8887b000 · 2025-07-18T17:27:25.000+03:00
diff --git a/codegenerator/cli/npm/envio/src/Envio.res b/codegenerator/cli/npm/envio/src/Envio.res
@@ -50,6 +50,7 @@ let experimental_createEffect = (
       >
     ),
     callsCount: 0,
+    input: options.input->(Utils.magic: S.t<'input> => S.t<Internal.effectInput>),
     output: options.output->(Utils.magic: S.t<'output> => S.t<Internal.effectOutput>),
     cache: options.cache->Belt.Option.getWithDefault(false),
   }->(Utils.magic: Internal.effect => effect<'input, 'output>)
diff --git a/codegenerator/cli/npm/envio/src/Internal.res b/codegenerator/cli/npm/envio/src/Internal.res
@@ -199,6 +199,7 @@ type effect = {
   handler: effectArgs => promise<effectOutput>,
   cache: bool,
   output: S.t<effectOutput>,
+  input: S.t<effectInput>,
   mutable callsCount: int,
 }
 
diff --git a/codegenerator/cli/npm/envio/src/PgStorage.res b/codegenerator/cli/npm/envio/src/PgStorage.res
@@ -387,7 +387,7 @@ let setOrThrow = async (sql, ~items, ~table: Table.table, ~itemSchema, ~pgSchema
       raise(
         Persistence.StorageError({
           message: `Failed to insert items into table "${table.tableName}"`,
-          reason: exn,
+          reason: exn->Internal.prettifyExn,
         }),
       )
     }
@@ -455,22 +455,27 @@ let makeSchemaCacheTableInfoQuery = (~pgSchema) => {
 type psqlExecState =
   Unknown | Pending(promise<result<string, string>>) | Resolved(result<string, string>)
 
-let getPsqlExec = {
+let getConnectedPsqlExec = {
+  let pgDockerServiceName = "envio-postgres"
+  // Should use the default port, since we're executing the command
+  // from the postgres container's network.
+  let pgDockerServicePort = 5432
+
   // For development: We run the indexer process locally,
   //   and there might not be psql installed on the user's machine.
   //   So we use docker-compose to run psql existing in the postgres container.
   // For production: We expect indexer to be running in a container,
   //   with psql installed. So we can call it directly.
   let psqlExecState = ref(Unknown)
-  async (~pgUser, ~pgHost) => {
+  async (~pgUser, ~pgHost, ~pgDatabase, ~pgPort) => {
     switch psqlExecState.contents {
     | Unknown => {
         let promise = Promise.make((resolve, _reject) => {
           let binary = "psql"
           NodeJs.ChildProcess.exec(`${binary} --version`, (~error, ~stdout as _, ~stderr as _) => {
             switch error {
             | Value(_) => {
-                let binary = `docker-compose exec -T -u ${pgUser} ${pgHost} psql`
+                let binary = `docker-compose exec -T -u ${pgUser} ${pgDockerServiceName} psql`
                 NodeJs.ChildProcess.exec(
                   `${binary} --version`,
                   (~error, ~stdout as _, ~stderr as _) => {
@@ -479,12 +484,22 @@ let getPsqlExec = {
                       resolve(
                         Error(`Please check if "psql" binary is installed or docker-compose is running for the local indexer.`),
                       )
-                    | Null => resolve(Ok(binary))
+                    | Null =>
+                      resolve(
+                        Ok(
+                          `${binary} -h ${pgHost} -p ${pgDockerServicePort->Js.Int.toString} -U ${pgUser} -d ${pgDatabase}`,
+                        ),
+                      )
                     }
                   },
                 )
               }
-            | Null => resolve(Ok(binary))
+            | Null =>
+              resolve(
+                Ok(
+                  `${binary} -h ${pgHost} -p ${pgPort->Js.Int.toString} -U ${pgUser} -d ${pgDatabase}`,
+                ),
+              )
             }
           })
         })
@@ -676,7 +691,7 @@ let make = (
       cacheTablePrefix ++ effectName,
       ~fields=[
         Table.mkField("id", Text, ~fieldSchema=S.string, ~isPrimaryKey=true),
-        Table.mkField("output", JsonB, ~fieldSchema=outputSchema),
+        Table.mkField("output", JsonB, ~fieldSchema=S.json(~validate=false)),
       ],
       ~compositeIndices=[],
     )
@@ -731,7 +746,7 @@ let make = (
         // Command for testing. Run from generated
         // docker-compose exec -T -u postgres envio-postgres psql -d envio-dev -c 'COPY "public"."envio_effect_getTokenMetadata" TO STDOUT (FORMAT text, HEADER);' > ../.envio/cache/getTokenMetadata.tsv
 
-        switch await getPsqlExec(~pgUser, ~pgHost) {
+        switch await getConnectedPsqlExec(~pgUser, ~pgHost, ~pgDatabase, ~pgPort) {
         | Ok(psqlExec) => {
             Logging.info(
               `Dumping cache: ${cacheTableInfo
@@ -746,7 +761,7 @@ let make = (
               let outputFile =
                 NodeJs.Path.join(cacheDirPath, cacheName ++ ".tsv")->NodeJs.Path.toString
 
-              let command = `${psqlExec} -h ${pgHost} -p ${pgPort->Js.Int.toString} -U ${pgUser} -d ${pgDatabase} -c 'COPY "${pgSchema}"."${tableName}" TO STDOUT WITH (FORMAT text, HEADER);' > ${outputFile}`
+              let command = `${psqlExec} -c 'COPY "${pgSchema}"."${tableName}" TO STDOUT WITH (FORMAT text, HEADER);' > ${outputFile}`
 
               Promise.make((resolve, reject) => {
                 NodeJs.ChildProcess.execWithOptions(
@@ -782,7 +797,7 @@ let make = (
         NodeJs.Fs.Promises.readdir(cacheDirPath)
         ->Promise.thenResolve(e => Ok(e))
         ->Promise.catch(_ => Promise.resolve(Error(nothingToUploadErrorMessage))),
-        getPsqlExec(~pgUser, ~pgHost),
+        getConnectedPsqlExec(~pgUser, ~pgHost, ~pgDatabase, ~pgPort),
       )) {
       | (Ok(entries), Ok(psqlExec)) => {
           let cacheFiles = entries->Js.Array2.filter(entry => {
@@ -808,7 +823,7 @@ let make = (
               ->Promise.then(() => {
                 let inputFile = NodeJs.Path.join(cacheDirPath, entry)->NodeJs.Path.toString
 
-                let command = `${psqlExec} -h ${pgHost} -p ${pgPort->Js.Int.toString} -U ${pgUser} -d ${pgDatabase} -c 'COPY "${pgSchema}"."${tableName}" FROM STDIN WITH (FORMAT text, HEADER);' < ${inputFile}`
+                let command = `${psqlExec} -c 'COPY "${pgSchema}"."${tableName}" FROM STDIN WITH (FORMAT text, HEADER);' < ${inputFile}`
 
                 Promise.make(
                   (resolve, reject) => {
@@ -827,6 +842,8 @@ let make = (
               })
             })
             ->Promise.all
+
+          Logging.info("Successfully uploaded cache.")
         }
       | (Error(message), _)
       | (_, Error(message)) =>
diff --git a/codegenerator/cli/templates/static/codegen/src/Config.res b/codegenerator/cli/templates/static/codegen/src/Config.res
@@ -91,46 +91,54 @@ let codegenPersistence = Persistence.make(
     ~pgSchema=storagePgSchema,
     ~pgHost=Env.Db.host,
     ~pgUser=Env.Db.user,
+    ~pgPort=Env.Db.port,
     ~pgDatabase=Env.Db.database,
     ~pgPassword=Env.Db.password,
-    ~pgPort=Env.Db.port,
-    ~onInitialize=() => {
+    ~onInitialize=?{
       if Env.Hasura.enabled {
-        Hasura.trackDatabase(
-          ~endpoint=Env.Hasura.graphqlEndpoint,
-          ~auth={
-            role: Env.Hasura.role,
-            secret: Env.Hasura.secret,
+        Some(
+          () => {
+            Hasura.trackDatabase(
+              ~endpoint=Env.Hasura.graphqlEndpoint,
+              ~auth={
+                role: Env.Hasura.role,
+                secret: Env.Hasura.secret,
+              },
+              ~pgSchema=storagePgSchema,
+              ~allStaticTables=Db.allStaticTables,
+              ~allEntityTables=Db.allEntityTables,
+              ~responseLimit=Env.Hasura.responseLimit,
+              ~schema=Db.schema,
+              ~aggregateEntities=Env.Hasura.aggregateEntities,
+            )->Promise.catch(err => {
+              Logging.errorWithExn(
+                err->Internal.prettifyExn,
+                `EE803: Error tracking tables`,
+              )->Promise.resolve
+            })
           },
-          ~pgSchema=storagePgSchema,
-          ~allStaticTables=Db.allStaticTables,
-          ~allEntityTables=Db.allEntityTables,
-          ~responseLimit=Env.Hasura.responseLimit,
-          ~schema=Db.schema,
-          ~aggregateEntities=Env.Hasura.aggregateEntities,
-        )->Promise.catch(err => {
-          Logging.errorWithExn(
-            err->Internal.prettifyExn,
-            `EE803: Error tracking tables`,
-          )->Promise.resolve
-        })
+        )
       } else {
-        Promise.resolve()
+        None
       }
     },
-    ~onNewTables=(~tableNames) => {
+    ~onNewTables=?{
       if Env.Hasura.enabled {
-        Hasura.trackTables(
-          ~endpoint=Env.Hasura.graphqlEndpoint,
-          ~auth={
-            role: Env.Hasura.role,
-            secret: Env.Hasura.secret,
+        Some(
+          (~tableNames) => {
+            Hasura.trackTables(
+              ~endpoint=Env.Hasura.graphqlEndpoint,
+              ~auth={
+                role: Env.Hasura.role,
+                secret: Env.Hasura.secret,
+              },
+              ~pgSchema=storagePgSchema,
+              ~tableNames,
+            )
           },
-          ~pgSchema=storagePgSchema,
-          ~tableNames,
         )
       } else {
-        Promise.resolve()
+        None
       }
     },
   ),
diff --git a/codegenerator/cli/templates/static/codegen/src/LoadLayer.res b/codegenerator/cli/templates/static/codegen/src/LoadLayer.res
@@ -83,7 +83,7 @@ let loadEffect = (
             `envio_effect_${effect.name}`,
             ~fields=[
               Table.mkField("id", Text, ~fieldSchema=S.string, ~isPrimaryKey=true),
-              Table.mkField("output", JsonB, ~fieldSchema=effect.output),
+              Table.mkField("output", JsonB, ~fieldSchema=S.json(~validate=false)),
             ],
             ~compositeIndices=[],
           ),
diff --git a/codegenerator/cli/templates/static/codegen/src/UserContext.res b/codegenerator/cli/templates/static/codegen/src/UserContext.res
@@ -34,7 +34,7 @@ let rec initEffect = (params: contextParams) => (
     ~effectArgs={
       input,
       context: params->Utils.Proxy.make(effectTraps)->Utils.magic,
-      cacheKey: input->Utils.Hash.makeOrThrow,
+      cacheKey: input->S.reverseConvertOrThrow(effect.input)->Utils.Hash.makeOrThrow,
     },
     ~inMemoryStore=params.inMemoryStore,
     ~shouldGroup=params.isPreload,
diff --git a/codegenerator/cli/templates/static/codegen/src/globalState/GlobalState.res b/codegenerator/cli/templates/static/codegen/src/globalState/GlobalState.res
@@ -862,8 +862,6 @@ let injectedTaskReducer = (
     switch shouldExit {
     | ExitWithSuccess =>
       updateChainMetadataTable(chainManager, ~throttler=writeThrottlers.chainMetaData)
-      // Dump cache to disk when we reached the end block with tui off
-      ->Promise.then(_ => state.config.persistence.storage.dumpEffectCache())
       ->Promise.thenResolve(_ => dispatchAction(SuccessExit))
       ->ignore
     | NoExit | NoExitWithCacheDump =>

Original file line number	Diff line number	Diff line change
`@@ -50,6 +50,7 @@ let experimental_createEffect = (`
`50`	`50`	`>`
`51`	`51`	`),`
`52`	`52`	`callsCount: 0,`
	`53`	`+ input: options.input->(Utils.magic: S.t<'input> => S.t<Internal.effectInput>),`
`53`	`54`	`output: options.output->(Utils.magic: S.t<'output> => S.t<Internal.effectOutput>),`
`54`	`55`	`cache: options.cache->Belt.Option.getWithDefault(false),`
`55`	`56`	`}->(Utils.magic: Internal.effect => effect<'input, 'output>)`
Original file line number	Diff line number	Diff line change
`@@ -199,6 +199,7 @@ type effect = {`
`199`	`199`	`handler: effectArgs => promise<effectOutput>,`
`200`	`200`	`cache: bool,`
`201`	`201`	`output: S.t<effectOutput>,`
	`202`	`+ input: S.t<effectInput>,`
`202`	`203`	`mutable callsCount: int,`
`203`	`204`	`}`
`204`	`205`