Allow remote shared cache testing from a network isolated sandbox (#1498

)

Creates the basic infrastructure needed to run postgres in an isolated test harness, runs the postgres tests in CI

.github/workflows/build.yaml

@@ -19,7 +19,7 @@ jobs:
             fetch-depth: 0
 
         - name: Install Deps
-          run: sudo apt-get update && sudo apt-get install -y build-essential fuse libfuse-dev libsqlite3-dev libgmp-dev libncurses5-dev pkg-config git g++ gcc libre2-dev python3-sphinx clang-format-12
+          run: sudo apt-get update && sudo apt-get install -y build-essential fuse libfuse-dev libsqlite3-dev libgmp-dev libncurses5-dev pkg-config git g++ gcc libre2-dev python3-sphinx clang-format-12 postgresql
 
         - name: Check C/C++ Formatting
           run: clang-format-12 --style=file --Werror -n $(./scripts/which_clang_files all)
@@ -33,6 +33,27 @@ jobs:
         - name: Check Wake Formatting
           run: ./bin/wake-format.native-cpp14-release --auto --dry-run
 
+        - name: Download rust
+          run: pushd /var/tmp && wget https://static.rust-lang.org/dist/rust-1.72.0-x86_64-unknown-linux-gnu.tar.gz && popd
+
+        - name: Unpack rust
+          run: pushd /var/tmp && tar -xf rust-1.72.0-x86_64-unknown-linux-gnu.tar.gz && popd
+
+        - name: Install rust
+          run: pushd /var/tmp && mkdir rust-install && ./rust-1.72.0-x86_64-unknown-linux-gnu/install.sh --prefix=/var/tmp/rust-install --verbose && popd
+
+        - name: Checkout postgres
+          run: pushd /var/tmp/ && git clone --depth 1 https://github.com/postgres/postgres.git && cd postgres && git fetch origin c372fbbd8e911f2412b80a8c39d7079366565d67 && git checkout c372fbbd8e911f2412b80a8c39d7079366565d67 && popd
+
+        - name: Build postgres
+          run: pushd /var/tmp/postgres && ./configure --prefix /var/tmp/pg-server && make install && popd
+
+        - name: Vendor remote cache deps
+          run: cd rust/rsc && cargo vendor --locked && cd ..
+
+        - name: Run Remote Cache Tests
+          run: POSTGRES_DIR=/var/tmp/pg-server CARGO_HOME=/var/tmp/rust-install make remoteCacheTests
+
         - name: Generate Docs
           run: mkdir www && ./bin/wake --no-workspace --html > www/index.html
 

.gitignore

@@ -51,3 +51,4 @@ compile_commands.json
 /rust/entity/target/*
 /rust/log_viewer/target/*
 /rust/rsc/target/*
+/rust/rsc/vendor/*

.wakemanifest

@@ -14,6 +14,7 @@ build.wake
 extensions/vscode/vscode.wake
 tests/tests.wake
 rust/log_viewer/build.wake
+rust/rsc/postgres-tests.wake
 share/wake/lib/gcc_wake/gcc.wake
 share/wake/lib/gcc_wake/pkgconfig.wake
 share/wake/lib/rust_wake/cargo.wake

Makefile

@@ -80,6 +80,9 @@ test:		wake.db
 unittest:	all
 	$(WAKE_ENV) ./bin/wake --in test_wake runUnitTests
 
+remoteCacheTests:	all
+	$(WAKE_ENV) ./bin/wake -d -x 'testPostgres Unit'
+
 tarball:	wake.db
 	$(WAKE_ENV) ./bin/wake build tarball
 

rust/rsc/postgres-tests.wake

@@ -0,0 +1,144 @@
+# Copyright 2024 SiFive, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You should have received a copy of LICENSE.Apache2 along with
+# this software. If not, you may obtain a copy at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package build_wake
+
+from wake import _
+from rust import findCargoPath
+
+def createFileOp dst =
+    JObject (
+        "type" :-> JString "create-file",
+        "destination" :-> JString dst,
+    )
+
+def createDirOp dst =
+    JObject (
+        "type" :-> JString "create-dir",
+        "destination" :-> JString dst,
+    )
+
+def tmpMountOp dst =
+    JObject (
+        "type" :-> JString "tmpfs",
+        "destination" :-> JString dst,
+    )
+
+def bindMountOp src dst =
+    JObject (
+        "type" :-> JString "bind",
+        "source" :-> JString src,
+        "destination" :-> JString dst,
+    )
+
+def postgresSpecJson stdinFile cargoPath =
+    JObject (
+        "label" :-> JString "postgres cargo test",
+        "environment" :-> JArray (JString "PATH=/usr/lib64/ccache:/usr/bin:/usr/sbin:/bin:{dirname cargoPath}",),
+        "command" :-> JArray (JString "bash", Nil),
+        "user-id" :-> JInteger 0,
+        "group-id" :-> JInteger 0,
+        "visible" :-> JArray Nil,
+        "directory" :-> JString ".",
+        "stdin" :-> JString "{stdinFile}",
+        "resources" :-> JArray Nil,
+        "isolate-pids" :-> JBoolean True,
+        "isolate-network" :-> JBoolean True,
+        "usage" :-> JObject (
+            "status" :-> JInteger 0,
+            "runtime" :-> JInteger 0,
+            "cputime" :-> JInteger 0,
+            "membytes" :-> JInteger 0,
+            "inbytes" :-> JInteger 0,
+            "outputs" :-> JInteger 0,
+        ),
+        "mount-ops" :-> JArray (
+            tmpMountOp "/tmp",
+            tmpMountOp "/root",
+            createDirOp "rust/rsc/.cargo",
+            createFileOp "rust/rsc/.cargo/config.toml",
+            bindMountOp "rust/rsc/vendor-config.toml" "rust/rsc/.cargo/config.toml",
+        ),
+    )
+
+export target testPostgres Unit =
+    # First we define a few directories, using various
+    # environment variables
+    def dataDir = "/tmp/pg_data"
+
+    require Pass postgresDir = match (getenv "POSTGRES_DIR")
+        Some dir -> Pass "{dir}/bin"
+        None ->
+            require Some postgresPath = whichInEnvPath "postgres"
+            else failWithError "could not find postgres on path"
+
+            Pass (dirname postgresPath)
+
+    require Pass cargoPath = findCargoPath Unit
+
+    # This is normally unsafe but we're just using it for
+    # a warning, you should never depend on the result here
+    require _, _ = files "{@here}/vendor" `.*`
+    else failWithError "You forgot to run `cargo vendor --locked` inside {@here}"
+
+    # This script is piped in via stdin so that
+    # as shells are opened one after the other,
+    # the successive commands are run as intended
+    def script =
+        """
+        set -ex
+        trap 'exit $(cat /tmp/status)' 0
+        unshare -U
+        export HOME=/root
+        mkdir %{dataDir}
+        echo $? > /tmp/status
+        %{postgresDir}/initdb -D %{dataDir}
+        echo $? > /tmp/status
+        %{postgresDir}/postgres -D %{dataDir} &
+        trap "kill $!" 0
+        sleep 2s
+        %{postgresDir}/createuser -h localhost root
+        echo $? > /tmp/status
+        %{postgresDir}/createdb -h localhost shim
+        echo $? > /tmp/status
+        cd %{@here}
+        %{cargoPath} test
+        echo $? > /tmp/status
+        exit $(cat /tmp/status) # exit the unshare shell
+        """
+
+    # Jobs require that we write stdin out to a file in order to
+    # pipe it in
+
+    require Pass scriptPath = write ".build/postgres-cargo.sh" script
+
+    def _ = println "Cargo Path: {cargoPath}"
+    def pgJson = postgresSpecJson scriptPath.getPathName cargoPath
+
+    require Pass pgJsonPath = write ".build/postgres-cmd-spec.json" (prettyJSON pgJson)
+
+    def emitVendorWarning status =
+        require (Exited 0) = status
+        else failWithError "cargo tests failed: Consider running `cargo vendor --locked` in {@here}"
+
+        Pass status
+
+    makeExecPlan ("{wakePath}/wakebox", "-p", pgJsonPath.getPathName, Nil) Nil
+    | editPlanEnvironment (addEnvironmentPath "/usr/sbin")
+    | setPlanPersistence ReRun
+    | setPlanEcho logReport
+    | runJobWith localRunner
+    | getJobStatus
+    | emitVendorWarning

rust/rsc/src/rsc/main.rs

@@ -4,14 +4,16 @@ use axum::{
     Router,
 };
 use clap::Parser;
+use data_encoding::HEXLOWER;
 use migration::{Migrator, MigratorTrait};
+use rand_core::{OsRng, RngCore};
 use std::io::{Error, ErrorKind};
 use std::sync::Arc;
 use tracing;
 
 use sea_orm::{
-    ActiveModelTrait, ActiveValue::*, ColumnTrait, Database, DatabaseConnection, DeleteResult,
-    EntityTrait, QueryFilter,
+    ActiveModelTrait, ActiveValue::*, ColumnTrait, ConnectionTrait, Database, DatabaseConnection,
+    DeleteResult, EntityTrait, QueryFilter,
 };
 
 use chrono::{Duration, Utc};
@@ -102,7 +104,16 @@ fn create_router(conn: Arc<DatabaseConnection>, config: Arc<config::RSCConfig>)
 }
 
 async fn create_standalone_db() -> Result<DatabaseConnection, sea_orm::DbErr> {
-    let db = Database::connect("sqlite::memory:").await?;
+    let shim_db = Database::connect("postgres://127.0.0.1/shim").await?;
+    let mut buf = [0u8; 24];
+    OsRng.fill_bytes(&mut buf);
+    let rng_str = HEXLOWER.encode(&buf);
+    let db = format!("db_{}", rng_str);
+    shim_db
+        .execute_unprepared(&format!("CREATE DATABASE {}", db))
+        .await?;
+    drop(shim_db);
+    let db = Database::connect(format!("postgres://127.0.0.1/{}", db)).await?;
     Migrator::up(&db, None).await?;
     Ok(db)
 }

rust/rsc/vendor-config.toml

@@ -0,0 +1,5 @@
+[source.crates-io]
+replace-with = "vendored-sources"
+
+[source.vendored-sources]
+directory = "vendor"

scripts/which_wake_files

@@ -16,8 +16,8 @@ if [[ ${1} == "all" ]]; then
 fi
 
 if [[ ${1} == "changed" ]]; then
-  CHANGED_FILES=$(git --git-dir ${WAKE_ROOT}.git diff --name-only | grep '\.wake' | grep -v 'tests/' | grep -v "manifest.wake")
-  STAGED_FILES=$(git --git-dir ${WAKE_ROOT}.git diff --name-only --cached | grep '\.wake' | grep -v 'tests/' | grep -v "manifest.wake")
+  CHANGED_FILES=$(git --git-dir ${WAKE_ROOT}.git diff --name-only | grep '\.wake$' | grep -v 'tests/' | grep -v "manifest.wake")
+  STAGED_FILES=$(git --git-dir ${WAKE_ROOT}.git diff --name-only --cached | grep '\.wake$' | grep -v 'tests/' | grep -v "manifest.wake")
   MERGED="${CHANGED_FILES} ${STAGED_FILES}"
 
   echo ${MERGED}

share/wake/lib/rust_wake/cargo.wake

@@ -15,7 +15,7 @@
 
 package rust
 
-target findCargoPath Unit =
+export target findCargoPath Unit =
     require Some cargo_home = getenv "CARGO_HOME"
     else
         require Some path = getenv "PATH"