Merge pull request #254 from blockfrost/chore/offline-compilation

Allow offline compilation and `--config <PATH>`

Author: sandtreader

Cargo.lock

@@ -37,6 +37,7 @@ caryatid_module_clock = "0.12"
 caryatid_module_spy = "0.12"
 anyhow = "1.0"
 chrono = "0.4"
+clap = { version = "4.5", features = ["derive"] }
 config = "0.15.11"
 dashmap = "6.1.0"
 hex = "0.4"

Cargo.toml

@@ -26,6 +26,7 @@ tracing = { workspace = true }
 [build-dependencies]
 anyhow = { workspace = true }
 reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
+serde_json = { workspace = true }
 tokio = { workspace = true }
 
 [lib]

modules/genesis_bootstrapper/Cargo.toml

@@ -1,18 +1,36 @@
 // Build-time script to download generics
+use std::collections::HashMap;
+use std::env;
 use std::fs;
+use std::fs::File;
 use std::io::Write;
 use std::path::Path;
 
 use anyhow::{Context, Result};
+use serde_json::from_reader;
 
 const OUTPUT_DIR: &str = "downloads";
 
+async fn fetch_bytes(client: &reqwest::Client, url: &str) -> Result<Vec<u8>> {
+    if let Ok(path) = env::var("ACROPOLIS_OFFLINE_MIRROR") {
+        if let Ok(file) = File::open(&path) {
+            if let Ok(map) = from_reader::<_, HashMap<String, String>>(file) {
+                if let Some(path) = map.get(url) {
+                    if let Ok(bytes) = fs::read(&Path::new(path).to_path_buf()) {
+                        return Ok(bytes);
+                    }
+                }
+            }
+        }
+    }
+    let req = client.get(url).build().with_context(|| format!("Failed to request {url}"))?;
+    let resp = client.execute(req).await.with_context(|| format!("Failed to fetch {url}"))?;
+    Ok(resp.bytes().await.context("Failed to read response")?.to_vec())
+}
+
 /// Download a URL to a file in OUTPUT_DIR
 async fn download(client: &reqwest::Client, url: &str, filename: &str) -> Result<()> {
-    let request = client.get(url).build().with_context(|| format!("Failed to request {url}"))?;
-    let response =
-        client.execute(request).await.with_context(|| format!("Failed to fetch {url}"))?;
-    let data = response.bytes().await.context("Failed to read response")?;
+    let data = fetch_bytes(client, url).await?;
 
     let output_path = Path::new(OUTPUT_DIR);
     if !output_path.exists() {

modules/genesis_bootstrapper/build.rs

@@ -29,6 +29,7 @@ tracing = { workspace = true }
 
 [build-dependencies]
 reqwest = { version = "0.11", features = ["blocking"] }
+serde_json = { workspace = true }
 
 [lib]
 path = "src/parameters_state.rs"

modules/parameters_state/Cargo.toml

@@ -1,16 +1,36 @@
 // Build-time script to download generics
 use reqwest::blocking::get;
+use serde_json::from_reader;
+use std::collections::HashMap;
+use std::env;
 use std::fs;
+use std::fs::File;
 use std::io::Write;
 use std::path::Path;
 
 const OUTPUT_DIR: &str = "downloads";
 
+fn fetch_text(url: &str) -> Result<String, Box<dyn std::error::Error>> {
+    if let Ok(path) = env::var("ACROPOLIS_OFFLINE_MIRROR") {
+        if !path.is_empty() {
+            if let Ok(file) = File::open(path) {
+                if let Ok(map) = from_reader::<_, HashMap<String, String>>(file) {
+                    if let Some(path_str) = map.get(url) {
+                        if let Ok(s) = fs::read_to_string(&Path::new(path_str).to_path_buf()) {
+                            return Ok(s);
+                        }
+                    }
+                }
+            }
+        }
+    }
+    Ok(get(url)?.error_for_status()?.text()?)
+}
+
 /// Download a URL to a file in OUTPUT_DIR
 fn download(url_base: &str, epoch: &str, filename: &str, rename: &Vec<(&str, &str)>) {
     let url = format!("{}/{}-genesis.json", url_base, epoch);
-    let response = get(url).expect("Failed to fetch {url}");
-    let mut data = response.text().expect("Failed to read response");
+    let mut data = fetch_text(&url).expect("Failed to fetch {url}");
 
     for (what, with) in rename.iter() {
         data = data.replace(&format!("\"{what}\""), &format!("\"{with}\""));

modules/parameters_state/build.rs

@@ -35,6 +35,7 @@ caryatid_module_rest_server = { workspace = true }
 caryatid_module_spy = { workspace = true }
 
 anyhow = { workspace = true }
+clap = { workspace = true }
 config = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { version = "0.3.20", features = ["registry", "env-filter"] }

processes/omnibus/Cargo.toml

@@ -45,9 +45,18 @@ use tikv_jemallocator::Jemalloc;
 #[global_allocator]
 static GLOBAL: Jemalloc = Jemalloc;
 
+#[derive(Debug, clap::Parser)]
+#[command(name = "acropolis_process_omnibus")]
+struct Args {
+    #[arg(long, value_name = "PATH", default_value_t = option_env!("ACROPOLIS_OMNIBUS_DEFAULT_CONFIG").unwrap_or("omnibus.toml").to_string())]
+    config: String,
+}
+
 /// Standard main
 #[tokio::main]
 pub async fn main() -> Result<()> {
+    let args = <self::Args as clap::Parser>::parse();
+
     // Standard logging using RUST_LOG for log levels default to INFO for events only
     let fmt_layer = fmt::layer()
         .with_filter(EnvFilter::from_default_env().add_directive(filter::LevelFilter::INFO.into()))
@@ -77,7 +86,7 @@ pub async fn main() -> Result<()> {
     // Read the config
     let config = Arc::new(
         Config::builder()
-            .add_source(File::with_name("omnibus"))
+            .add_source(File::with_name(&args.config))
             .add_source(Environment::with_prefix("ACROPOLIS"))
             .build()
             .unwrap(),

processes/omnibus/src/main.rs

@@ -30,6 +30,7 @@ caryatid_module_rest_server = { workspace = true }
 caryatid_module_spy = { workspace = true }
 
 anyhow = { workspace = true }
+clap = { workspace = true }
 config = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { version = "0.3.20", features = ["registry", "env-filter"] }