fix(hydroflow_plus): adjust default features to allow compilation to …

…musl targets

Previously, the default `deploy` feature would pull in Hydro Deploy and its transitive native dependencies.

Also sets up `examples/paxos.rs` with CLI flags to deploy to GCP.

.github/workflows/ci.yml

@@ -181,10 +181,10 @@ jobs:
         run: curl -LsSf https://get.nexte.st/latest/windows-tar | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
 
       - name: Run cargo nextest on all targets
-        run: cargo nextest run --no-fail-fast --features python --all-targets
+        run: cargo nextest run --no-fail-fast --features python --features deploy --all-targets
 
       - name: Run doctests
-        run: cargo test --no-fail-fast --features python --doc
+        run: cargo test --no-fail-fast --features python --features deploy --doc
 
       - name: Install Python
         uses: actions/setup-python@v4

.vscode/settings.json

@@ -16,7 +16,4 @@
     "files.watcherExclude": {
         "**/target": true
     },
-    "rust-analyzer.cargo.features": [
-        "python"
-    ]
 }

hydroflow_plus/Cargo.toml

@@ -14,10 +14,11 @@ workspace = true
 path = "src/lib.rs"
 
 [features]
-default = ["deploy"]
+default = ["deploy_runtime"]
 diagnostics = [ "hydroflow_lang/diagnostics" ]
 stageleft_devel = []
-deploy = [ "hydroflow/deploy_integration", "dep:hydro_deploy", "dep:trybuild-internals-api", "dep:toml", "dep:prettyplease" ]
+deploy_runtime = [ "hydroflow/deploy_integration" ]
+deploy = [ "deploy_runtime", "dep:hydro_deploy", "dep:trybuild-internals-api", "dep:toml", "dep:prettyplease" ]
 
 [dependencies]
 quote = "1.0.35"

hydroflow_plus/src/deploy/deploy_graph.rs

@@ -17,7 +17,7 @@ use hydroflow::futures::StreamExt;
 use hydroflow::util::deploy::ConnectedSource;
 use nameof::name_of;
 use serde::de::DeserializeOwned;
-use serde::{Deserialize, Serialize};
+use serde::Serialize;
 use sha2::{Digest, Sha256};
 use stageleft::{Quoted, RuntimeData};
 use syn::visit_mut::VisitMut;
@@ -31,13 +31,6 @@ use crate::futures::SinkExt;
 use crate::lang::graph::HydroflowGraph;
 use crate::util::deploy::ConnectedSink;
 
-#[derive(Default, Serialize, Deserialize)]
-pub struct HydroflowPlusMeta {
-    pub clusters: HashMap<usize, Vec<u32>>,
-    pub cluster_id: Option<u32>,
-    pub subgraph_id: usize,
-}
-
 pub struct HydroDeploy {}
 
 impl<'a> Deploy<'a> for HydroDeploy {

hydroflow_plus/src/deploy/deploy_runtime.rs

@@ -1,10 +1,19 @@
+use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
 use stageleft::{q, Quoted, RuntimeData};
 
-use super::HydroflowPlusMeta;
 use crate::util::deploy::{
     ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource, ConnectedTagged, DeployPorts,
 };
 
+#[derive(Default, Serialize, Deserialize)]
+pub struct HydroflowPlusMeta {
+    pub clusters: HashMap<usize, Vec<u32>>,
+    pub cluster_id: Option<u32>,
+    pub subgraph_id: usize,
+}
+
 pub fn cluster_members(
     cli: RuntimeData<&DeployPorts<HydroflowPlusMeta>>,
     of_cluster: usize,
@@ -123,7 +132,7 @@ pub fn deploy_m2m(
 
 pub fn deploy_e2o(
     env: RuntimeData<&DeployPorts<HydroflowPlusMeta>>,
-    e1_port: &str,
+    _e1_port: &str,
     p2_port: &str,
 ) -> syn::Expr {
     q!({
@@ -137,7 +146,7 @@ pub fn deploy_e2o(
 pub fn deploy_o2e(
     env: RuntimeData<&DeployPorts<HydroflowPlusMeta>>,
     p1_port: &str,
-    e2_port: &str,
+    _e2_port: &str,
 ) -> syn::Expr {
     q!({
         env.port(p1_port)

hydroflow_plus/src/deploy/mod.rs

@@ -9,6 +9,7 @@ use serde::de::DeserializeOwned;
 use serde::Serialize;
 use stageleft::Quoted;
 
+#[cfg(feature = "deploy_runtime")]
 pub mod macro_runtime;
 
 #[cfg(feature = "deploy")]
@@ -18,8 +19,10 @@ pub use macro_runtime::*;
 #[cfg(feature = "deploy")]
 pub use trybuild::init_test;
 
-#[allow(clippy::allow_attributes, unused, reason = "stageleft")]
-pub(crate) mod deploy_runtime;
+#[cfg(feature = "deploy_runtime")]
+pub mod deploy_runtime;
+#[cfg(feature = "deploy_runtime")]
+pub use deploy_runtime::HydroflowPlusMeta;
 
 #[cfg(feature = "deploy")]
 pub mod deploy_graph;

hydroflow_plus_test/Cargo.toml

@@ -26,3 +26,4 @@ insta = "1.39"
 hydro_deploy = { path = "../hydro_deploy/core", version = "^0.9.0" }
 hydroflow_plus = { path = "../hydroflow_plus", version = "^0.9.0", features = [ "deploy" ] }
 futures = "0.3.0"
+async-ssh2-lite = { version = "0.5.0", features = ["vendored-openssl"] }

hydroflow_plus_test/examples/paxos.rs

@@ -1,15 +1,40 @@
-use hydro_deploy::Deployment;
+use std::sync::Arc;
+
+use hydro_deploy::gcp::GcpNetwork;
+use hydro_deploy::{Deployment, Host};
 use hydroflow_plus::deploy::TrybuildHost;
+use tokio::sync::RwLock;
+
+type HostCreator = Box<dyn Fn(&mut Deployment) -> Arc<dyn Host>>;
 
 #[tokio::main]
 async fn main() {
     let mut deployment = Deployment::new();
-    let localhost = deployment.Localhost();
+    let host_arg = std::env::args().nth(1).unwrap_or_default();
+
+    let create_host: HostCreator = if host_arg == *"gcp" {
+        let project = std::env::args().nth(2).unwrap();
+        let network = Arc::new(RwLock::new(GcpNetwork::new(&project, None)));
+
+        Box::new(move |deployment| -> Arc<dyn Host> {
+            deployment
+                .GcpComputeEngineHost()
+                .project(&project)
+                .machine_type("n2-highcpu-2")
+                .image("debian-cloud/debian-11")
+                .region("us-west1-a")
+                .network(network.clone())
+                .add()
+        })
+    } else {
+        let localhost = deployment.Localhost();
+        Box::new(move |_| -> Arc<dyn Host> { localhost.clone() })
+    };
 
     let builder = hydroflow_plus::FlowBuilder::new();
     let f = 1;
     let num_clients = 1;
-    let num_clients_per_node = 1; // Change based on experiment between 1, 50, 100.
+    let num_clients_per_node = 100; // Change based on experiment between 1, 50, 100.
     let median_latency_window_size = 1000;
     let checkpoint_frequency = 1000; // Num log entries
     let i_am_leader_send_timeout = 5; // Sec
@@ -35,25 +60,25 @@ async fn main() {
         .with_cluster(
             &proposers,
             (0..f + 1)
-                .map(|_| TrybuildHost::new(localhost.clone()).rustflags(rustflags))
+                .map(|_| TrybuildHost::new(create_host(&mut deployment)).rustflags(rustflags))
                 .collect::<Vec<_>>(),
         )
         .with_cluster(
             &acceptors,
             (0..2 * f + 1)
-                .map(|_| TrybuildHost::new(localhost.clone()).rustflags(rustflags))
+                .map(|_| TrybuildHost::new(create_host(&mut deployment)).rustflags(rustflags))
                 .collect::<Vec<_>>(),
         )
         .with_cluster(
             &clients,
             (0..num_clients)
-                .map(|_| TrybuildHost::new(localhost.clone()).rustflags(rustflags))
+                .map(|_| TrybuildHost::new(create_host(&mut deployment)).rustflags(rustflags))
                 .collect::<Vec<_>>(),
         )
         .with_cluster(
             &replicas,
             (0..f + 1)
-                .map(|_| TrybuildHost::new(localhost.clone()).rustflags(rustflags))
+                .map(|_| TrybuildHost::new(create_host(&mut deployment)).rustflags(rustflags))
                 .collect::<Vec<_>>(),
         )
         .deploy(&mut deployment);