Embed static files in binary for release builds (#603)

Author: mwylde

.github/workflows/ci.yml

@@ -87,6 +87,10 @@ jobs:
           sudo service mosquitto start
       - name: Check Formatting
         run: cargo fmt -- --check
+      - name: Build console
+        run: |
+          cd webui
+          pnpm build        
       - name: Build
         run: cargo build --all-features
       - name: Validate API

Cargo.lock

@@ -75,6 +75,8 @@ proc-macro2 = "1"
 postgres-types = { version = "*", features = ["derive"] }
 tokio-postgres = { version = "*", features = ["with-serde_json-1", "with-time-0_3", "with-uuid-1"] }
 deadpool-postgres = { version = "0.10" }
+
+
 futures = "0.3"
 futures-util = "0.3.28"
 time = "0.3"
@@ -84,6 +86,8 @@ uuid = "1.3.3"
 regress = "0.6.0"
 apache-avro = "0.16.0"
 toml = "0.8"
+rust-embed = { version = "6.8.1", features = ["axum"] }
+mime_guess = "2.0.4"
 
 [build-dependencies]
 cornucopia = { version = "0.9" }

crates/arroyo-api/Cargo.toml

@@ -1,19 +1,15 @@
-use axum::body::Body;
-use axum::response::IntoResponse;
+use axum::response::{Html, IntoResponse, Response};
 use axum::{
     routing::{delete, get, patch, post},
     Json, Router,
 };
 use deadpool_postgres::Pool;
 
-use once_cell::sync::Lazy;
+use http::{header, StatusCode, Uri};
+use rust_embed::RustEmbed;
 use std::env;
-use std::path::PathBuf;
-use std::str::FromStr;
-use tower::service_fn;
 use tower_http::cors;
 use tower_http::cors::CorsLayer;
-use tower_http::services::ServeDir;
 use utoipa::OpenApi;
 use utoipa_swagger_ui::SwaggerUi;
 
@@ -37,7 +33,11 @@ use crate::pipelines::{
 use crate::rest_utils::not_found;
 use crate::udfs::{create_udf, delete_udf, get_udfs, validate_udf};
 use crate::ApiDoc;
-use arroyo_types::{telemetry_enabled, API_ENDPOINT_ENV, ASSET_DIR_ENV};
+use arroyo_types::{telemetry_enabled, API_ENDPOINT_ENV};
+
+#[derive(RustEmbed)]
+#[folder = "../../webui/dist"]
+struct Assets;
 
 #[derive(Clone)]
 pub struct AppState {
@@ -62,30 +62,54 @@ pub async fn api_fallback() -> impl IntoResponse {
     not_found("Route")
 }
 
-pub fn create_rest_app(pool: Pool, controller_addr: &str) -> Router {
-    let asset_dir = env::var(ASSET_DIR_ENV).unwrap_or_else(|_| "webui/dist".to_string());
+async fn not_found_static() -> Response {
+    (StatusCode::NOT_FOUND, "404").into_response()
+}
 
-    static INDEX_HTML: Lazy<String> = Lazy::new(|| {
-        let asset_dir = env::var(ASSET_DIR_ENV).unwrap_or_else(|_| "webui/dist".to_string());
+async fn static_handler(uri: Uri) -> impl IntoResponse {
+    let path = uri.path().trim_start_matches('/');
 
-        let endpoint = env::var(API_ENDPOINT_ENV).unwrap_or_else(|_| String::new());
+    if path.is_empty() || path == "index.html" {
+        return index_html().await;
+    }
 
-        std::fs::read_to_string(PathBuf::from_str(&asset_dir).unwrap()
-            .join("index.html"))
-            .expect("Could not find index.html in asset dir (you may need to build the console sources)")
-            .replace("{{API_ENDPOINT}}", &endpoint)
-            .replace("{{CLUSTER_ID}}", &arroyo_server_common::get_cluster_id())
-            .replace("{{DISABLE_TELEMETRY}}", if telemetry_enabled() { "false" } else { "true" })
-    });
+    match Assets::get(path) {
+        Some(content) => {
+            let mime = mime_guess::from_path(path).first_or_octet_stream();
 
-    let fallback = service_fn(|_: http::Request<_>| async move {
-        let body = Body::from(INDEX_HTML.as_str());
-        let res = http::Response::new(body);
-        Ok::<_, _>(res)
-    });
+            ([(header::CONTENT_TYPE, mime.as_ref())], content.data).into_response()
+        }
+        None => {
+            if path.contains('.') {
+                return not_found_static().await;
+            }
+
+            index_html().await
+        }
+    }
+}
 
-    let serve_dir = ServeDir::new(asset_dir).not_found_service(fallback);
+async fn index_html() -> Response {
+    match Assets::get("index.html") {
+        Some(content) => {
+            let endpoint = env::var(API_ENDPOINT_ENV).unwrap_or_else(|_| String::new());
 
+            let replaced = String::from_utf8(content.data.to_vec())
+                .expect("index.html is invalid UTF-8")
+                .replace("{{API_ENDPOINT}}", &endpoint)
+                .replace("{{CLUSTER_ID}}", &arroyo_server_common::get_cluster_id())
+                .replace(
+                    "{{DISABLE_TELEMETRY}}",
+                    if telemetry_enabled() { "false" } else { "true" },
+                );
+
+            Html(replaced).into_response()
+        }
+        None => not_found_static().await,
+    }
+}
+
+pub fn create_rest_app(pool: Pool, controller_addr: &str) -> Router {
     // TODO: enable in development only!!!
     let cors = CorsLayer::new()
         .allow_methods(cors::Any)
@@ -146,8 +170,7 @@ pub fn create_rest_app(pool: Pool, controller_addr: &str) -> Router {
                 .url("/api/v1/api-docs/openapi.json", ApiDoc::openapi()),
         )
         .nest("/api/v1", api_routes)
-        .route_service("/", fallback)
-        .fallback_service(serve_dir)
+        .fallback(static_handler)
         .with_state(AppState {
             controller_addr: controller_addr.to_string(),
             pool,

crates/arroyo-api/src/rest.rs

@@ -68,7 +68,7 @@ use datafusion_execution::FunctionRegistry;
 use std::time::{Duration, SystemTime};
 use std::{collections::HashMap, sync::Arc};
 use syn::Item;
-use tracing::{info, warn};
+use tracing::{debug, info, warn};
 use unicase::UniCase;
 
 const DEFAULT_IDLE_TIME: Option<Duration> = Some(Duration::from_secs(5 * 60));
@@ -520,7 +520,7 @@ pub async fn parse_and_get_arrow_program(
         plan_rewrite.visit(&mut metadata)?;
         used_connections.extend(metadata.connection_ids.iter());
 
-        info!("Logical plan: {}", plan_rewrite.display_graphviz());
+        debug!("Logical plan: {}", plan_rewrite.display_graphviz());
 
         let sink = match sink_name {
             Some(sink_name) => {

crates/arroyo-df/src/lib.rs

@@ -30,7 +30,6 @@ use datafusion_expr::{
 use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
 use std::time::Duration;
-use tracing::info;
 
 /// Rewrites a logical plan to move projections out of table scans
 /// and into a separate projection node which may include virtual fields,
@@ -149,20 +148,11 @@ impl<'a> SourceRewriter<'a> {
             )),
         });
 
-        info!(
-            "table source schema: {:?}",
-            table_source_extension.schema().fields()
-        );
-
         let (projection_input, projection) = if table.is_updating() {
             let mut projection_offsets = table_scan.projection.clone();
             if let Some(offsets) = projection_offsets.as_mut() {
                 offsets.push(table.fields.len())
             }
-            info!(
-                "table source schema: {:?}",
-                table_source_extension.schema().fields()
-            );
             (
                 LogicalPlan::Extension(Extension {
                     node: Arc::new(DebeziumUnrollingExtension::try_new(table_source_extension)?),
@@ -188,8 +178,6 @@ impl<'a> SourceRewriter<'a> {
     ) -> DFResult<LogicalPlan> {
         let input = self.projection(table_scan, table)?;
 
-        info!("table scan plan:\n{:?}", input);
-
         let schema = input.schema().clone();
         let remote = LogicalPlan::Extension(Extension {
             node: Arc::new(RemoteTableExtension {

crates/arroyo-df/src/rewriters.rs

@@ -55,7 +55,6 @@ COPY docker/install_deps.sh /install_deps.sh
 RUN sh /install_deps.sh run
 
 COPY --from=builder /arroyo-bin ./
-COPY --from=builder /app/webui/dist ./dist
 
 COPY docker/supervisord.conf /supervisord.conf
 COPY docker/entrypoint.sh /entrypoint.sh
@@ -78,7 +77,6 @@ RUN apt-get update && \
     apt-get -y install libsasl2-2 ca-certificates curl
 
 COPY --from=builder /arroyo-bin ./
-COPY --from=builder /app/webui/dist ./dist
 
 ENV PRODUCTION=true \
     ASSET_DIR="/app/dist" \
@@ -94,7 +92,6 @@ RUN apt-get update && \
     apt-get -y install libsasl2-2 ca-certificates curl
 
 COPY --from=builder /arroyo-bin ./
-COPY --from=builder /app/webui/dist ./dist
 
 ENV PRODUCTION=true \
     ASSET_DIR="/app/dist"