From 7255abc934ebc1021c5abe2dc63dda91bd24ac24 Mon Sep 17 00:00:00 2001 From: Leonard Vincent Simon Pahlke <58054077+leonardpahlke@users.noreply.github.com> Date: Wed, 3 Jan 2024 23:39:55 +0100 Subject: [PATCH] add sqs spin trigger (#192) Signed-off-by: leonardpahlke --- containerd-shim-spin/Cargo.lock | 526 ++++++++++++++++++++++++++++- containerd-shim-spin/Cargo.toml | 1 + containerd-shim-spin/src/engine.rs | 16 +- 3 files changed, 536 insertions(+), 7 deletions(-) diff --git a/containerd-shim-spin/Cargo.lock b/containerd-shim-spin/Cargo.lock index c8bd849..d45bd62 100644 --- a/containerd-shim-spin/Cargo.lock +++ b/containerd-shim-spin/Cargo.lock @@ -372,6 +372,296 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-config" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7688e1dfbb9f7804fab0a830820d7e827b8d973906763cf1a855ce4719292f5" +dependencies = [ + "aws-http", + "aws-sdk-sso", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-types", + "bytes", + "hex", + "http", + "hyper 0.14.27", + "ring 0.16.20", + "time", + "tokio", + "tower", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-endpoint" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "253d7cd480bfa59a5323390e9e91885a8f06a275e0517d81eeb1070b6aa7d271" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "aws-types", + "http", + "regex", + "tracing", +] + +[[package]] +name = "aws-http" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd1b83859383e46ea8fda633378f9f3f02e6e3a446fd89f0240b5c3662716c9" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "http-body 0.4.5", + "lazy_static", + "percent-encoding", + "pin-project-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sqs" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b489461c4309514f1019a924687fbdafeb88eeeac0800e8aca6fef6a66877e73" +dependencies = [ + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-query", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "http", + "tokio-stream", + "tower", +] + +[[package]] +name = "aws-sdk-sso" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf03342c2b3f52b180f484e60586500765474f2bfc7dcd4ffe893a7a1929db1d" +dependencies = [ + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "tokio-stream", + "tower", +] + +[[package]] +name = "aws-sdk-sts" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa1de4e07ea87a30a317c7b563b3a40fd18a843ad794216dda81672b6e174bce" +dependencies = [ + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-query", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "http", + "tower", + "tracing", +] + +[[package]] +name = "aws-sig-auth" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6126c4ff918e35fb9ae1bf2de71157fad36f0cc6a2b1d0f7197ee711713700fc" +dependencies = [ + "aws-sigv4", + "aws-smithy-http", + "aws-types", + "http", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "0.52.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1ee2d9e6e268a77bdf3a0dc2cc8767401627e60abaa32883aaa3d8b47428e9" +dependencies = [ + "aws-smithy-http", + "form_urlencoded", + "hex", + "hmac", + "http", + "once_cell", + "percent-encoding", + "regex", + "sha2", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e6a895d68852dd1564328e63ef1583e5eb307dd2a5ebf35d862a5c402957d5e" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", + "tokio-stream", +] + +[[package]] +name = "aws-smithy-client" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f505bf793eb3e6d7c166ef1275c27b4b2cd5361173fe950ac8e2cfc08c29a7ef" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-types", + "bytes", + "fastrand 1.9.0", + "http", + "http-body 0.4.5", + "hyper 0.14.27", + "hyper-rustls 0.23.2", + "lazy_static", + "pin-project-lite", + "tokio", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-http" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e4b4304b7ea4af1af3e08535100eb7b6459d5a6264b92078bf85176d04ab85" +dependencies = [ + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body 0.4.5", + "hyper 0.14.27", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tokio", + "tokio-util 0.7.10", + "tracing", +] + +[[package]] +name = "aws-smithy-http-tower" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e86072ecc4dc4faf3e2071144285cfd539263fe7102b701d54fb991eafb04af8" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "http", + "http-body 0.4.5", + "pin-project-lite", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e3ddd9275b167bc59e9446469eca56177ec0b51225632f90aaa2cd5f41c940e" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b19d2e0b3ce20e460bad0d0d974238673100edebba6978c2c1aadd925602f7" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-types" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "987b1e37febb9bd409ca0846e82d35299e572ad8279bc404778caeb5fc05ad56" +dependencies = [ + "base64-simd", + "itoa", + "num-integer", + "ryu", + "time", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ce3791e14eec75ffac851a5a559f1ce6b31843297f42cc8bfba82714a6a5d8" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c05adca3e2bcf686dd2c47836f216ab52ed7845c177d180c84b08522c1166a3" +dependencies = [ + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-types", + "http", + "rustc_version", + "tracing", + "zeroize", +] + [[package]] name = "axum" version = "0.6.20" @@ -500,6 +790,15 @@ version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +[[package]] +name = "base64-simd" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "781dd20c3aff0bd194fe7d2a977dd92f21c173891f3a03b677359e5fa457e5d5" +dependencies = [ + "simd-abstraction", +] + [[package]] name = "base64ct" version = "1.6.0" @@ -608,6 +907,16 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "bytesize" version = "1.3.0" @@ -994,6 +1303,7 @@ dependencies = [ "spin-trigger", "spin-trigger-http", "tokio", + "trigger-sqs", "url", "wasmtime 14.0.4 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2372,6 +2682,22 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" +dependencies = [ + "http", + "hyper 0.14.27", + "log", + "rustls 0.20.9", + "rustls-native-certs", + "tokio", + "tokio-rustls 0.23.4", + "webpki-roots 0.22.6", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -2547,6 +2873,17 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +[[package]] +name = "is-terminal" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +dependencies = [ + "hermit-abi 0.3.3", + "rustix 0.38.21", + "windows-sys 0.48.0", +] + [[package]] name = "itertools" version = "0.8.2" @@ -3069,6 +3406,15 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d" +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + [[package]] name = "matchit" version = "0.7.3" @@ -3334,6 +3680,16 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-bigint" version = "0.4.4" @@ -3649,6 +4005,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "outref" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f222829ae9293e33a9f5e9f440c6760a3d450a64affe1846486b140db81c1f4" + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "page_size" version = "0.6.0" @@ -4417,10 +4785,19 @@ checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ "aho-corasick 1.1.2", "memchr", - "regex-automata", + "regex-automata 0.4.3", "regex-syntax 0.8.2", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", +] + [[package]] name = "regex-automata" version = "0.4.3" @@ -4432,6 +4809,12 @@ dependencies = [ "regex-syntax 0.8.2", ] +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.7.5" @@ -4460,7 +4843,7 @@ dependencies = [ "http", "http-body 0.4.5", "hyper 0.14.27", - "hyper-rustls", + "hyper-rustls 0.24.2", "hyper-tls", "ipnet", "js-sys", @@ -4486,7 +4869,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots", + "webpki-roots 0.25.2", "winreg", ] @@ -4673,6 +5056,18 @@ dependencies = [ "sct", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile 1.0.3", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "0.3.0" @@ -4919,6 +5314,15 @@ dependencies = [ "digest", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shellexpand" version = "2.1.2" @@ -4962,6 +5366,15 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-abstraction" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cadb29c57caadc51ff8346233b5cec1d240b68ce55cf1afc764818791876987" +dependencies = [ + "outref", +] + [[package]] name = "siphasher" version = "0.3.11" @@ -5755,6 +6168,16 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "thread_local" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +dependencies = [ + "cfg-if 1.0.0", + "once_cell", +] + [[package]] name = "time" version = "0.3.30" @@ -5946,6 +6369,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-scoped" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4beb8ba13bc53ac53ce1d52b42f02e5d8060f0f42138862869beb769722b256" +dependencies = [ + "tokio", + "tokio-stream", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -6133,6 +6566,58 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "trigger-sqs" +version = "0.4.0" +source = "git+https://github.com/0xE282B0/spin-trigger-sqs?branch=sql-trigger-library#388501385735ccba16875f37c177191d189c6bcc" +dependencies = [ + "anyhow", + "async-trait", + "aws-config", + "aws-sdk-sqs", + "clap", + "futures", + "is-terminal", + "serde", + "spin-core", + "spin-trigger", + "tokio", + "tokio-scoped", + "tracing", + "tracing-subscriber", + "wasmtime 14.0.4 (git+https://github.com/fermyon/wasmtime?rev=a2fa8fe7de1e918eae06d78de53451832ba380b6)", ] [[package]] @@ -6298,6 +6783,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "uuid" version = "1.5.0" @@ -6307,6 +6798,12 @@ dependencies = [ "getrandom 0.2.10", ] +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "value-bag" version = "1.4.2" @@ -7158,7 +7655,7 @@ dependencies = [ "tracing", "wasmtime 14.0.4 (git+https://github.com/fermyon/wasmtime?rev=a2fa8fe7de1e918eae06d78de53451832ba380b6)", "wasmtime-wasi", - "webpki-roots", + "webpki-roots 0.25.2", ] [[package]] @@ -7261,6 +7758,15 @@ dependencies = [ "untrusted 0.9.0", ] +[[package]] +name = "webpki-roots" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +dependencies = [ + "webpki", +] + [[package]] name = "webpki-roots" version = "0.25.2" @@ -7618,6 +8124,12 @@ dependencies = [ "libc", ] +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "zerocopy" version = "0.7.25" @@ -7638,6 +8150,12 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "zeroize" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" + [[package]] name = "zip" version = "0.6.6" diff --git a/containerd-shim-spin/Cargo.toml b/containerd-shim-spin/Cargo.toml index ab1454a..f21952f 100644 --- a/containerd-shim-spin/Cargo.toml +++ b/containerd-shim-spin/Cargo.toml @@ -18,6 +18,7 @@ spin-core = { git = "https://github.com/fermyon/spin", tag = "v2.0.1" } spin-trigger = { git = "https://github.com/fermyon/spin", tag = "v2.0.1" } spin-trigger-http = { git = "https://github.com/fermyon/spin", tag = "v2.0.1" } spin-redis-engine = { git = "https://github.com/fermyon/spin", tag = "v2.0.1" } +trigger-sqs = { git = "https://github.com/fermyon/spin-trigger-sqs", rev = "34ef90888a7374ca53d32654dcf02b3fbfd82119" } spin-manifest = { git = "https://github.com/fermyon/spin", tag = "v2.0.1" } spin-loader = { git = "https://github.com/fermyon/spin", tag = "v2.0.1" } spin-oci = { git = "https://github.com/fermyon/spin", tag = "v2.0.1" } diff --git a/containerd-shim-spin/src/engine.rs b/containerd-shim-spin/src/engine.rs index faea270..18d6e05 100644 --- a/containerd-shim-spin/src/engine.rs +++ b/containerd-shim-spin/src/engine.rs @@ -7,6 +7,7 @@ use spin_loader::cache::Cache; use spin_loader::FilesMountStrategy; use spin_manifest::schema::v2::AppManifest; use spin_redis_engine::RedisTrigger; +use trigger_sqs::SqsTrigger; use spin_trigger::TriggerHooks; use spin_trigger::{loader, RuntimeConfig, TriggerExecutor, TriggerExecutorBuilder}; use spin_trigger_http::HttpTrigger; @@ -172,8 +173,17 @@ impl SpinEngine { info!(" >>> running spin trigger"); redis_trigger.run(spin_trigger::cli::NoArgs) } + SqsTrigger::TRIGGER_TYPE => { + let sqs_trigger: SqsTrigger = self + .build_spin_trigger(working_dir, app) + .await + .context("failed to build spin trigger")?; + + info!(" >>> running spin trigger"); + sqs_trigger.run(spin_trigger::cli::NoArgs) + } _ => { - todo!("Only Http and Redis triggers are currently supported.") + todo!("Only Http, Redis and SQS triggers are currently supported.") } }; info!(" >>> notifying main thread we are about to start"); @@ -297,9 +307,9 @@ fn trigger_command_for_resolved_app_source(resolved: &ResolvedAppSource) -> Resu let trigger_type = resolved.trigger_type()?; match trigger_type { - RedisTrigger::TRIGGER_TYPE | HttpTrigger::TRIGGER_TYPE => Ok(trigger_type.to_owned()), + RedisTrigger::TRIGGER_TYPE | HttpTrigger::TRIGGER_TYPE | SqsTrigger::TRIGGER_TYPE => Ok(trigger_type.to_owned()), _ => { - todo!("Only Http and Redis triggers are currently supported.") + todo!("Only Http, Redis and SQS triggers are currently supported.") } } }