From 103f1cff5be3788586238b70710d7bd0e973058e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81ngel=20M?= Date: Wed, 5 Jul 2023 19:12:47 +0200 Subject: [PATCH] feat: add the HTTP bindings to perform HTTP requests from workers (#168) * feat: add the HTTP bindings to perform HTTP requests from workers * fix: exclude wit-bindgen-backport packages from cargo deny * fix: remove unnecesary channel when calling reqwest * fix: remove unnecesary intermediate variable * improve: implement From to build the final error for the binding --- Cargo.lock | 96 +++++++++++++++++++++-- Cargo.toml | 2 + crates/project/Cargo.toml | 3 +- crates/worker/Cargo.toml | 5 ++ crates/worker/src/bindings/http.rs | 119 +++++++++++++++++++++++++++++ crates/worker/src/bindings/mod.rs | 4 + crates/worker/src/lib.rs | 18 ++++- deny.toml | 26 ++++++- wit/core/http-types.wit | 60 +++++++++++++++ wit/core/http.wit | 4 + 10 files changed, 326 insertions(+), 11 deletions(-) create mode 100644 crates/worker/src/bindings/http.rs create mode 100644 crates/worker/src/bindings/mod.rs create mode 100644 wit/core/http-types.wit create mode 100644 wit/core/http.wit diff --git a/Cargo.lock b/Cargo.lock index 7dd6f38a..da43de96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -355,9 +355,9 @@ checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341" [[package]] name = "async-trait" -version = "0.1.68" +version = "0.1.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +checksum = "7b2d0f03b3640e3a630367e40c468cb7f309529c708ed1d88597047b0e7c6ef7" dependencies = [ "proc-macro2", "quote", @@ -654,7 +654,7 @@ version = "4.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "syn 2.0.18", @@ -1345,6 +1345,15 @@ dependencies = [ "ahash 0.8.3", ] +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "heck" version = "0.4.1" @@ -2903,6 +2912,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + [[package]] name = "unicode-width" version = "0.1.10" @@ -3255,7 +3270,7 @@ dependencies = [ "syn 1.0.109", "wasmtime-component-util", "wasmtime-wit-bindgen", - "wit-parser", + "wit-parser 0.8.0", ] [[package]] @@ -3476,8 +3491,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3334b0466a4d340de345cda83474d1d2c429770c3d667877971407672bc618a" dependencies = [ "anyhow", - "heck", - "wit-parser", + "heck 0.4.1", + "wit-parser 0.8.0", ] [[package]] @@ -3568,7 +3583,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df96ee6bea595fabf0346c08c553f684b08e88fad6fdb125e6efde047024f7b" dependencies = [ "anyhow", - "heck", + "heck 0.4.1", "proc-macro2", "quote", "shellexpand", @@ -3787,6 +3802,69 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "wit-bindgen-gen-core" +version = "0.2.0" +source = "git+https://github.com/fermyon/wit-bindgen-backport?rev=b89d5079ba5b07b319631a1b191d2139f126c976#b89d5079ba5b07b319631a1b191d2139f126c976" +dependencies = [ + "anyhow", + "wit-parser 0.2.0", +] + +[[package]] +name = "wit-bindgen-gen-rust" +version = "0.2.0" +source = "git+https://github.com/fermyon/wit-bindgen-backport?rev=b89d5079ba5b07b319631a1b191d2139f126c976#b89d5079ba5b07b319631a1b191d2139f126c976" +dependencies = [ + "heck 0.3.3", + "wit-bindgen-gen-core", +] + +[[package]] +name = "wit-bindgen-gen-wasmtime" +version = "0.2.0" +source = "git+https://github.com/fermyon/wit-bindgen-backport?rev=b89d5079ba5b07b319631a1b191d2139f126c976#b89d5079ba5b07b319631a1b191d2139f126c976" +dependencies = [ + "heck 0.3.3", + "wit-bindgen-gen-core", + "wit-bindgen-gen-rust", +] + +[[package]] +name = "wit-bindgen-wasmtime" +version = "0.2.0" +source = "git+https://github.com/fermyon/wit-bindgen-backport?rev=b89d5079ba5b07b319631a1b191d2139f126c976#b89d5079ba5b07b319631a1b191d2139f126c976" +dependencies = [ + "anyhow", + "bitflags 1.3.2", + "thiserror", + "wasmtime", + "wit-bindgen-wasmtime-impl", +] + +[[package]] +name = "wit-bindgen-wasmtime-impl" +version = "0.2.0" +source = "git+https://github.com/fermyon/wit-bindgen-backport?rev=b89d5079ba5b07b319631a1b191d2139f126c976#b89d5079ba5b07b319631a1b191d2139f126c976" +dependencies = [ + "proc-macro2", + "syn 1.0.109", + "wit-bindgen-gen-core", + "wit-bindgen-gen-wasmtime", +] + +[[package]] +name = "wit-parser" +version = "0.2.0" +source = "git+https://github.com/fermyon/wit-bindgen-backport?rev=b89d5079ba5b07b319631a1b191d2139f126c976#b89d5079ba5b07b319631a1b191d2139f126c976" +dependencies = [ + "anyhow", + "id-arena", + "pulldown-cmark", + "unicode-normalization", + "unicode-xid", +] + [[package]] name = "wit-parser" version = "0.8.0" @@ -3889,6 +3967,7 @@ dependencies = [ "serde", "serde_json", "sha256", + "tokio", "toml 0.7.4", "url", "wws-store", @@ -3950,13 +4029,16 @@ dependencies = [ "actix-web", "anyhow", "base64", + "reqwest", "serde", "serde_json", "sha256", + "tokio", "toml 0.7.4", "wasi-common", "wasmtime", "wasmtime-wasi", + "wit-bindgen-wasmtime", "wws-config", "wws-data-kv", "wws-runtimes", diff --git a/Cargo.toml b/Cargo.toml index d3bd8afe..d509d10b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,8 +68,10 @@ exclude = [ actix-web = "4" anyhow = "1.0.66" lazy_static = "1.4.0" +reqwest = "0.11" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.85" +tokio = "1.28" toml = "0.7.0" wws-config = { path = "./crates/config" } wws-runtimes = { path = "./crates/runtimes" } diff --git a/crates/project/Cargo.toml b/crates/project/Cargo.toml index 6312b030..aff5ca11 100644 --- a/crates/project/Cargo.toml +++ b/crates/project/Cargo.toml @@ -8,13 +8,14 @@ repository = { workspace = true } [dependencies] anyhow = { workspace = true } +reqwest = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +tokio = { workspace = true } toml = { workspace = true } wws-store = { workspace = true } url = "2.3.1" sha256 = "1.1.1" -reqwest = "0.11" git2 = "0.17.2" # Not all platforms require OpenSSL openssl = { workspace = true, optional = true } diff --git a/crates/worker/Cargo.toml b/crates/worker/Cargo.toml index 36431f44..5f7495eb 100644 --- a/crates/worker/Cargo.toml +++ b/crates/worker/Cargo.toml @@ -12,8 +12,10 @@ doctest = false [dependencies] actix-web = { workspace = true } anyhow = { workspace = true } +reqwest = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +tokio = { workspace = true } toml = { workspace = true } wasmtime = { workspace = true } wasmtime-wasi = { workspace = true } @@ -21,5 +23,8 @@ wasi-common = { workspace = true } wws-config = { workspace = true } wws-data-kv = { workspace = true } wws-runtimes = { workspace = true } +# We didn't integrate components yet. For an initial binding implementation, +# we will use the wit-bindgen-wasmtime crate maintained by the Fermyon team. +wit-bindgen-wasmtime = { git = "https://github.com/fermyon/wit-bindgen-backport", rev = "b89d5079ba5b07b319631a1b191d2139f126c976" } base64 = "0.21.0" sha256 = "1.1.1" diff --git a/crates/worker/src/bindings/http.rs b/crates/worker/src/bindings/http.rs new file mode 100644 index 00000000..bc8461b9 --- /dev/null +++ b/crates/worker/src/bindings/http.rs @@ -0,0 +1,119 @@ +// Copyright 2023 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use reqwest::Method; +use tokio::runtime::Builder; + +// Implement the HTTP bindings for the workers. +wit_bindgen_wasmtime::export!({paths: ["../../wit/core/http.wit"]}); +use http::{Http, HttpError, HttpMethod, HttpRequest, HttpRequestError, HttpResponse}; + +pub use http::add_to_linker; + +pub struct HttpBindings {} + +/// Map the reqwest error to a known http-error +/// HttpError comes from the HTTP bindings +impl From for HttpError { + fn from(value: reqwest::Error) -> Self { + if value.is_timeout() { + HttpError::Timeout + } else if value.is_redirect() { + HttpError::RedirectLoop + } else if value.is_request() { + HttpError::InvalidRequest + } else if value.is_body() { + HttpError::InvalidRequestBody + } else if value.is_decode() { + HttpError::InvalidResponseBody + } else { + HttpError::InternalError + } + } +} + +impl Http for HttpBindings { + fn send_http_request( + &mut self, + req: HttpRequest<'_>, + ) -> Result { + // Create local variables from the request + let mut headers = Vec::new(); + let uri = req.uri.to_string(); + let body = req.body.unwrap_or(&[]).to_vec(); + + for (key, value) in req.headers { + headers.push((key.to_string(), value.to_string())); + } + + // Run the request in an async thread + let thread_result = std::thread::spawn(move || { + Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { + let client = reqwest::Client::new(); + + let method = match req.method { + HttpMethod::Get => Method::GET, + HttpMethod::Post => Method::POST, + HttpMethod::Put => Method::PUT, + HttpMethod::Patch => Method::PATCH, + HttpMethod::Delete => Method::DELETE, + HttpMethod::Options => Method::OPTIONS, + HttpMethod::Head => Method::HEAD, + }; + + let mut builder = client.request(method, uri); + + for (key, value) in headers { + builder = builder.header(key, value); + } + + builder = builder.body(body); + + match builder.send().await { + Ok(res) => { + let mut headers = Vec::new(); + let status = res.status().as_u16(); + + for (name, value) in res.headers().iter() { + headers + .push((name.to_string(), value.to_str().unwrap().to_string())); + } + + let body = res.bytes().await; + + Ok(HttpResponse { + headers, + status, + body: Some(body.unwrap().to_vec()), + }) + } + Err(e) => { + let message = e.to_string(); + + // Manage the different possible errors from Reqwest + Err(HttpRequestError { + error: e.into(), + message, + }) + } + } + }) + }) + .join(); + + match thread_result { + Ok(res) => match res { + Ok(res) => Ok(res), + Err(err) => Err(err), + }, + Err(_) => Err(HttpRequestError { + error: HttpError::InternalError, + message: "There was an error processing the request on the host side.".to_string(), + }), + } + } +} diff --git a/crates/worker/src/bindings/mod.rs b/crates/worker/src/bindings/mod.rs new file mode 100644 index 00000000..54281d22 --- /dev/null +++ b/crates/worker/src/bindings/mod.rs @@ -0,0 +1,4 @@ +// Copyright 2023 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +pub mod http; diff --git a/crates/worker/src/lib.rs b/crates/worker/src/lib.rs index 42635342..232a6808 100644 --- a/crates/worker/src/lib.rs +++ b/crates/worker/src/lib.rs @@ -1,12 +1,14 @@ // Copyright 2022 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 +mod bindings; pub mod config; pub mod io; mod stdio; use actix_web::HttpRequest; use anyhow::{anyhow, Result}; +use bindings::http::{add_to_linker as http_add_to_linker, HttpBindings}; use config::Config; use io::{WasmInput, WasmOutput}; use sha256::digest as sha256_digest; @@ -14,6 +16,7 @@ use std::fs::{self, File}; use std::path::PathBuf; use std::{collections::HashMap, path::Path}; use stdio::Stdio; +use wasi_common::WasiCtx; use wasmtime::{Engine, Linker, Module, Store}; use wasmtime_wasi::{ambient_authority, Dir, WasiCtxBuilder}; use wws_config::Config as ProjectConfig; @@ -37,6 +40,11 @@ pub struct Worker { path: PathBuf, } +struct WorkerState { + pub wasi: WasiCtx, + pub http: HttpBindings, +} + impl Worker { /// Creates a new Worker pub fn new(project_root: &Path, path: &Path, project_config: &ProjectConfig) -> Result { @@ -97,7 +105,9 @@ impl Worker { let stdio = Stdio::new(&input, stderr_file); let mut linker = Linker::new(&self.engine); - wasmtime_wasi::add_to_linker(&mut linker, |s| s)?; + + http_add_to_linker(&mut linker, |s: &mut WorkerState| &mut s.http)?; + wasmtime_wasi::add_to_linker(&mut linker, |s: &mut WorkerState| &mut s.wasi)?; // I have to use `String` as it's required by WasiCtxBuilder let tuple_vars: Vec<(String, String)> = @@ -126,7 +136,11 @@ impl Worker { wasi_builder = self.runtime.prepare_wasi_ctx(wasi_builder)?; let wasi = wasi_builder.build(); - let mut store = Store::new(&self.engine, wasi); + let state = WorkerState { + wasi, + http: HttpBindings {}, + }; + let mut store = Store::new(&self.engine, state); linker.module(&mut store, "", &self.module)?; linker diff --git a/deny.toml b/deny.toml index bd0bed8b..d96e57ef 100644 --- a/deny.toml +++ b/deny.toml @@ -12,6 +12,30 @@ targets = [ { triple = "aarch64-pc-windows-msvc" }, ] +# I'm excluding these packages as cargo deny is failing due to a missing license +# in the package definition [1]. The project license is Apache-2.0 as stated in the +# LICENSE file [2]. +# +# I tried to use a license.clarify option to better document this [3]. However, this +# option requires to specify the LICENSE file you're pointing to. Since the crate doesn't +# include the LICENSE file (it's part of the cargo workspace), it cannot find it [4]. +# +# For these reasons I added them as excluded packages for now. I introduced this +# exception on https://github.com/vmware-labs/wasm-workers-server/pull/168. +# +# - [1] https://github.com/fermyon/wit-bindgen-backport/blob/b89d5079ba5b07b319631a1b191d2139f126c976/crates/wasmtime-impl/Cargo.toml +# - [2] https://github.com/fermyon/wit-bindgen-backport/blob/b89d5079ba5b07b319631a1b191d2139f126c976/LICENSE +# - [3] https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html#the-clarify-field-optional +# - [4] https://github.com/EmbarkStudios/cargo-deny/issues/373 +exclude = [ + "wit-parser", + "wit-bindgen-wasmtime-impl", + "wit-bindgen-wasmtime", + "wit-bindgen-gen-wasmtime", + "wit-bindgen-gen-rust", + "wit-bindgen-gen-core" +] + # More documentation for the advisories section can be found here: # https://embarkstudios.github.io/cargo-deny/checks/advisories/cfg.html [advisories] @@ -63,4 +87,4 @@ deny = [ # Certain crates/versions that will be skipped when doing duplicate detection. skip = [ #{ name = "ansi_term", version = "=0.11.0" }, -] \ No newline at end of file +] diff --git a/wit/core/http-types.wit b/wit/core/http-types.wit new file mode 100644 index 00000000..05d40413 --- /dev/null +++ b/wit/core/http-types.wit @@ -0,0 +1,60 @@ +// URI +type uri = string + +// HTTP Status +type http-status = u16 + +// Header +type http-header = tuple +type http-headers = list + +// Methods +enum http-method { + get, + post, + put, + patch, + delete, + options, + head +} + +// URL params +type http-param = tuple +type http-params = list + +// The body content +type http-body = list + +// A complete HTTP request +record http-request { + body: option, + headers: http-headers, + method: http-method, + params: http-params, + uri: uri, +} + +// Return information about a failed request +record http-request-error { + error: http-error, + message: string +} + +// A complete HTTP response +record http-response { + body: option, + headers: http-headers, + status: http-status, +} + +// The list of errors +enum http-error { + invalid-request, + invalid-request-body, + invalid-response-body, + not-allowed, + internal-error, + timeout, + redirect-loop, +} diff --git a/wit/core/http.wit b/wit/core/http.wit new file mode 100644 index 00000000..038a3ea3 --- /dev/null +++ b/wit/core/http.wit @@ -0,0 +1,4 @@ +use * from http-types + +// Send a HTTP request from inside the worker. +send-http-request: func(request: http-request) -> expected