diff --git a/Cargo.lock b/Cargo.lock index 3e99b431..1c62c15a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -184,8 +184,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d" dependencies = [ "proc-macro2", - "quote 1.0.8", - "syn 1.0.58", + "quote", + "syn", ] [[package]] @@ -262,17 +262,6 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" -[[package]] -name = "bincode" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e103c8b299b28a9c6990458b7013dc4a8356a9b854c51b9883241f5866fac36e" -dependencies = [ - "byteorder", - "num-traits 0.1.43", - "serde", -] - [[package]] name = "bincode" version = "1.3.1" @@ -401,12 +390,6 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" -[[package]] -name = "case" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88b166b48e29667f5443df64df3c61dc07dc2b1a0b0d231800e07f09a33ecc1" - [[package]] name = "cc" version = "1.0.66" @@ -436,7 +419,7 @@ checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" dependencies = [ "libc", "num-integer", - "num-traits 0.2.14", + "num-traits", "time", "winapi 0.3.9", ] @@ -601,15 +584,6 @@ dependencies = [ "scopeguard", ] -[[package]] -name = "crossbeam-queue" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b" -dependencies = [ - "crossbeam-utils 0.6.6", -] - [[package]] name = "crossbeam-queue" version = "0.2.3" @@ -621,16 +595,6 @@ dependencies = [ "maybe-uninit", ] -[[package]] -name = "crossbeam-utils" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" -dependencies = [ - "cfg-if 0.1.10", - "lazy_static", -] - [[package]] name = "crossbeam-utils" version = "0.7.2" @@ -683,17 +647,6 @@ dependencies = [ "libc", ] -[[package]] -name = "derive-error" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629f1bb3abce791912ca85a24676fff54464f7deb122906adabc90fb96e876d3" -dependencies = [ - "case", - "quote 0.3.15", - "syn 0.11.11", -] - [[package]] name = "difference" version = "2.0.0" @@ -812,8 +765,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ "proc-macro2", - "quote 1.0.8", - "syn 1.0.58", + "quote", + "syn", "synstructure", ] @@ -856,7 +809,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1267f4ac4f343772758f7b1bdcbe767c218bbab93bb432acbf5162bbf85a6c4" dependencies = [ - "num-traits 0.2.14", + "num-traits", ] [[package]] @@ -968,7 +921,6 @@ dependencies = [ "futures-core", "futures-task", "futures-util", - "num_cpus", ] [[package]] @@ -992,6 +944,16 @@ dependencies = [ "waker-fn", ] +[[package]] +name = "futures-locks" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c4e684ddb2d8a4db5ca8a02b35156da129674ba4412b6f528698d58c594954" +dependencies = [ + "futures 0.3.9", + "tokio 0.2.24", +] + [[package]] name = "futures-macro" version = "0.3.9" @@ -1000,8 +962,8 @@ checksum = "0f8719ca0e1f3c5e34f3efe4570ef2c0610ca6da85ae7990d472e9cbfba13664" dependencies = [ "proc-macro-hack", "proc-macro2", - "quote 1.0.8", - "syn 1.0.58", + "quote", + "syn", ] [[package]] @@ -1025,7 +987,6 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "036a2107cdeb57f6d7322f1b6c363dad67cd63ca3b7d1b925bdf75bd5d96cda9" dependencies = [ - "futures 0.1.30", "futures-channel", "futures-core", "futures-io", @@ -1105,7 +1066,7 @@ dependencies = [ "bytes 0.4.12", "fnv", "futures 0.1.30", - "http", + "http 0.1.21", "indexmap", "log 0.4.11", "slab", @@ -1113,6 +1074,26 @@ dependencies = [ "tokio-io", ] +[[package]] +name = "h2" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" +dependencies = [ + "bytes 0.5.6", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.4", + "indexmap", + "slab", + "tokio 0.2.24", + "tokio-util", + "tracing", + "tracing-futures", +] + [[package]] name = "hashbrown" version = "0.9.1" @@ -1149,6 +1130,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11" +dependencies = [ + "bytes 1.0.0", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.1.0" @@ -1157,16 +1149,32 @@ checksum = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d" dependencies = [ "bytes 0.4.12", "futures 0.1.30", - "http", + "http 0.1.21", "tokio-buf", ] +[[package]] +name = "http-body" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +dependencies = [ + "bytes 0.5.6", + "http 0.2.4", +] + [[package]] name = "httparse" version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" +[[package]] +name = "httpdate" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + [[package]] name = "humantime" version = "2.0.1" @@ -1182,9 +1190,9 @@ dependencies = [ "bytes 0.4.12", "futures 0.1.30", "futures-cpupool", - "h2", - "http", - "http-body", + "h2 0.1.26", + "http 0.1.21", + "http-body 0.1.0", "httparse", "iovec", "itoa", @@ -1200,7 +1208,31 @@ dependencies = [ "tokio-tcp", "tokio-threadpool", "tokio-timer", - "want", + "want 0.2.0", +] + +[[package]] +name = "hyper" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a6f157065790a3ed2f88679250419b5cdd96e714a0d65f7797fd337186e96bb" +dependencies = [ + "bytes 0.5.6", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.2.7", + "http 0.2.4", + "http-body 0.3.1", + "httparse", + "httpdate", + "itoa", + "pin-project 1.0.6", + "socket2", + "tokio 0.2.24", + "tower-service", + "tracing", + "want 0.3.0", ] [[package]] @@ -1211,20 +1243,33 @@ checksum = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f" dependencies = [ "bytes 0.4.12", "futures 0.1.30", - "hyper", + "hyper 0.12.35", "native-tls", "tokio-io", ] +[[package]] +name = "hyper-tls" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" +dependencies = [ + "bytes 0.5.6", + "hyper 0.13.10", + "native-tls", + "tokio 0.2.24", + "tokio-tls", +] + [[package]] name = "hyperx" -version = "0.12.0" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78e2d2253d7a17929560fc3adf48c48fc924c94fa4507e037a60e6bc55c0eda6" +checksum = "e4a94cbc2c6f63028e5736ca4e811ae36d3990059c384cbe68298c66728a9776" dependencies = [ - "base64 0.9.3", + "base64 0.10.1", "bytes 0.4.12", - "http", + "http 0.1.21", "httparse", "language-tags", "log 0.4.11", @@ -1284,6 +1329,12 @@ dependencies = [ "libc", ] +[[package]] +name = "ipnet" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" + [[package]] name = "itertools" version = "0.9.0" @@ -1698,7 +1749,7 @@ checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" dependencies = [ "autocfg 1.0.1", "num-integer", - "num-traits 0.2.14", + "num-traits", ] [[package]] @@ -1708,16 +1759,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" dependencies = [ "autocfg 1.0.1", - "num-traits 0.2.14", -] - -[[package]] -name = "num-traits" -version = "0.1.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" -dependencies = [ - "num-traits 0.2.14", + "num-traits", ] [[package]] @@ -1796,6 +1838,22 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "parity-tokio-ipc" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd7f6c69d7687501b2205fe51ade1d7b8797bb3aa141fe5bf13dd78c0483bc89" +dependencies = [ + "futures 0.3.9", + "libc", + "log 0.4.11", + "mio-named-pipes", + "miow 0.3.6", + "rand 0.7.3", + "tokio 0.2.24", + "winapi 0.3.9", +] + [[package]] name = "parking" version = "2.0.0" @@ -1890,6 +1948,46 @@ dependencies = [ "unicase 1.4.2", ] +[[package]] +name = "pin-project" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918192b5c59119d51e0cd221f4d49dde9112824ba717369e903c97d076083d0f" +dependencies = [ + "pin-project-internal 0.4.28", +] + +[[package]] +name = "pin-project" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc174859768806e91ae575187ada95c91a29e96a98dc5d2cd9a1fed039501ba6" +dependencies = [ + "pin-project-internal 1.0.6", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be26700300be6d9d23264c73211d8190e755b6b5ca7a1b28230025511b52a5e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a490329918e856ed1b083f244e3bfe2d8c4f336407e4ea9e1a9f479ff09049e5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.1.11" @@ -1980,7 +2078,7 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" dependencies = [ - "unicode-xid 0.2.1", + "unicode-xid", ] [[package]] @@ -2011,12 +2109,6 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" -[[package]] -name = "quote" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" - [[package]] name = "quote" version = "1.0.8" @@ -2286,16 +2378,16 @@ dependencies = [ "encoding_rs", "flate2", "futures 0.1.30", - "http", - "hyper", - "hyper-tls", + "http 0.1.21", + "hyper 0.12.35", + "hyper-tls 0.3.2", "log 0.4.11", "mime 0.3.16", "mime_guess 2.0.3", "native-tls", "serde", "serde_json", - "serde_urlencoded", + "serde_urlencoded 0.5.5", "time", "tokio 0.1.22", "tokio-executor", @@ -2304,7 +2396,43 @@ dependencies = [ "tokio-timer", "url 1.7.2", "uuid 0.7.4", - "winreg", + "winreg 0.6.2", +] + +[[package]] +name = "reqwest" +version = "0.10.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0718f81a8e14c4dbb3b34cf23dc6aaf9ab8a0dfec160c534b3dbca1aaa21f47c" +dependencies = [ + "base64 0.13.0", + "bytes 0.5.6", + "encoding_rs", + "futures-core", + "futures-util", + "http 0.2.4", + "http-body 0.3.1", + "hyper 0.13.10", + "hyper-tls 0.4.3", + "ipnet", + "js-sys", + "lazy_static", + "log 0.4.11", + "mime 0.3.16", + "mime_guess 2.0.3", + "native-tls", + "percent-encoding 2.1.0", + "pin-project-lite 0.2.1", + "serde", + "serde_json", + "serde_urlencoded 0.7.0", + "tokio 0.2.24", + "tokio-tls", + "url 2.2.0", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg 0.7.0", ] [[package]] @@ -2409,11 +2537,13 @@ dependencies = [ "anyhow", "ar", "assert_cmd", + "async-trait", "atty", "base64 0.13.0", - "bincode 1.3.1", + "bincode", "blake3", "byteorder", + "bytes 0.5.6", "cc", "chrono", "clap", @@ -2424,11 +2554,11 @@ dependencies = [ "env_logger", "filetime", "flate2", - "futures 0.1.30", "futures 0.3.9", + "futures-locks", "hmac", - "http", - "hyper", + "http 0.2.4", + "hyper 0.13.10", "hyperx", "itertools 0.10.0", "jobserver", @@ -2445,12 +2575,13 @@ dependencies = [ "num_cpus", "number_prefix", "openssl", + "parity-tokio-ipc", "percent-encoding 2.1.0", "predicates", "rand 0.7.3", "redis", "regex", - "reqwest", + "reqwest 0.10.10", "retry", "ring", "rouille", @@ -2464,15 +2595,9 @@ dependencies = [ "syslog", "tar", "tempfile", - "tokio-compat", - "tokio-io", - "tokio-named-pipes", - "tokio-process", - "tokio-reactor", - "tokio-serde-bincode", - "tokio-tcp", - "tokio-timer", - "tokio-uds", + "tokio 0.2.24", + "tokio-serde", + "tokio-util", "toml", "tower", "untrusted", @@ -2532,7 +2657,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01173ad274e14fafa534a5e660d950ca1939ccebd3955df987b7df7e4e301108" dependencies = [ - "reqwest", + "reqwest 0.9.24", "serde", "serde_derive", "serde_json", @@ -2570,8 +2695,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df" dependencies = [ "proc-macro2", - "quote 1.0.8", - "syn 1.0.58", + "quote", + "syn", ] [[package]] @@ -2597,6 +2722,18 @@ dependencies = [ "url 1.7.2", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha-1" version = "0.9.2" @@ -2646,7 +2783,7 @@ checksum = "692ca13de57ce0613a363c8c2f1de925adebc81b04c923ac60c5488bb44abe4b" dependencies = [ "chrono", "num-bigint", - "num-traits 0.2.14", + "num-traits", ] [[package]] @@ -2727,17 +2864,6 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" -[[package]] -name = "syn" -version = "0.11.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" -dependencies = [ - "quote 0.3.15", - "synom", - "unicode-xid 0.0.4", -] - [[package]] name = "syn" version = "1.0.58" @@ -2745,17 +2871,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc60a3d73ea6594cd712d830cc1f0390fd71542d8c8cd24e70cc54cdfd5e05d5" dependencies = [ "proc-macro2", - "quote 1.0.8", - "unicode-xid 0.2.1", -] - -[[package]] -name = "synom" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" -dependencies = [ - "unicode-xid 0.0.4", + "quote", + "unicode-xid", ] [[package]] @@ -2765,9 +2882,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" dependencies = [ "proc-macro2", - "quote 1.0.8", - "syn 1.0.58", - "unicode-xid 0.2.1", + "quote", + "syn", + "unicode-xid", ] [[package]] @@ -2863,8 +2980,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9be73a2caec27583d0046ef3796c3794f868a5bc813db689eed00c7631275cd1" dependencies = [ "proc-macro2", - "quote 1.0.8", - "syn 1.0.58", + "quote", + "syn", ] [[package]] @@ -2935,18 +3052,13 @@ dependencies = [ "futures 0.1.30", "mio", "num_cpus", - "tokio-codec", "tokio-current-thread", "tokio-executor", - "tokio-fs", "tokio-io", "tokio-reactor", - "tokio-sync", "tokio-tcp", "tokio-threadpool", "tokio-timer", - "tokio-udp", - "tokio-uds", ] [[package]] @@ -2963,10 +3075,14 @@ dependencies = [ "libc", "memchr", "mio", + "mio-named-pipes", "mio-uds", "num_cpus", "pin-project-lite 0.1.11", + "signal-hook-registry", "slab", + "tokio-macros", + "winapi 0.3.9", ] [[package]] @@ -2980,34 +3096,6 @@ dependencies = [ "futures 0.1.30", ] -[[package]] -name = "tokio-codec" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25b2998660ba0e70d18684de5d06b70b70a3a747469af9dea7618cc59e75976b" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.30", - "tokio-io", -] - -[[package]] -name = "tokio-compat" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "107b625135aa7b9297dd2d99ccd6ca6ab124a5d1230778e159b9095adca4c722" -dependencies = [ - "futures 0.1.30", - "futures-core", - "futures-util", - "pin-project-lite 0.1.11", - "tokio 0.2.24", - "tokio-current-thread", - "tokio-executor", - "tokio-reactor", - "tokio-timer", -] - [[package]] name = "tokio-current-thread" version = "0.1.7" @@ -3028,17 +3116,6 @@ dependencies = [ "futures 0.1.30", ] -[[package]] -name = "tokio-fs" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297a1206e0ca6302a0eed35b700d292b275256f596e2f3fea7729d5e629b6ff4" -dependencies = [ - "futures 0.1.30", - "tokio-io", - "tokio-threadpool", -] - [[package]] name = "tokio-io" version = "0.1.13" @@ -3051,35 +3128,14 @@ dependencies = [ ] [[package]] -name = "tokio-named-pipes" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d282d483052288b2308ba5ee795f5673b159c9bdf63c385a05609da782a5eae" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.30", - "mio", - "mio-named-pipes", - "tokio 0.1.22", -] - -[[package]] -name = "tokio-process" -version = "0.2.5" +name = "tokio-macros" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "382d90f43fa31caebe5d3bc6cfd854963394fff3b8cb59d5146607aaae7e7e43" +checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" dependencies = [ - "crossbeam-queue 0.1.2", - "futures 0.1.30", - "lazy_static", - "libc", - "log 0.4.11", - "mio", - "mio-named-pipes", - "tokio-io", - "tokio-reactor", - "tokio-signal", - "winapi 0.3.9", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -3103,43 +3159,13 @@ dependencies = [ [[package]] name = "tokio-serde" -version = "0.1.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "894168193c4f80862a2244ff953b69145a9961a9efba39500e0970b083d0649c" +checksum = "ebdd897b01021779294eb09bb3b52b6e11b0747f9f7e333a84bef532b656de99" dependencies = [ - "bytes 0.4.12", - "futures 0.1.30", -] - -[[package]] -name = "tokio-serde-bincode" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e35c8d60a5e87cfb30dd562a309e56f8a6d36617b0a76c87f04d5466607ca8" -dependencies = [ - "bincode 0.8.0", - "bytes 0.4.12", - "derive-error", - "futures 0.1.30", - "serde", - "tokio-serde", -] - -[[package]] -name = "tokio-signal" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c34c6e548f101053321cba3da7cbb87a610b85555884c41b07da2eb91aff12" -dependencies = [ - "futures 0.1.30", - "libc", - "mio", - "mio-uds", - "signal-hook-registry", - "tokio-executor", - "tokio-io", - "tokio-reactor", - "winapi 0.3.9", + "bytes 0.5.6", + "futures 0.3.9", + "pin-project 0.4.28", ] [[package]] @@ -3173,7 +3199,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" dependencies = [ "crossbeam-deque", - "crossbeam-queue 0.2.3", + "crossbeam-queue", "crossbeam-utils 0.7.2", "futures 0.1.30", "lazy_static", @@ -3196,36 +3222,13 @@ dependencies = [ ] [[package]] -name = "tokio-udp" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2a0b10e610b39c38b031a2fcab08e4b82f16ece36504988dcbd81dbba650d82" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.30", - "log 0.4.11", - "mio", - "tokio-codec", - "tokio-io", - "tokio-reactor", -] - -[[package]] -name = "tokio-uds" -version = "0.2.7" +name = "tokio-tls" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab57a4ac4111c8c9dbcf70779f6fc8bc35ae4b2454809febac840ad19bd7e4e0" +checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343" dependencies = [ - "bytes 0.4.12", - "futures 0.1.30", - "iovec", - "libc", - "log 0.4.11", - "mio", - "mio-uds", - "tokio-codec", - "tokio-io", - "tokio-reactor", + "native-tls", + "tokio 0.2.24", ] [[package]] @@ -3253,11 +3256,11 @@ dependencies = [ [[package]] name = "tower" -version = "0.1.1" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc72f33b6a72c75c9df0037afce313018bae845f0ec7fdb9201b8768427a917f" +checksum = "fd3169017c090b7a28fce80abaad0ab4f5566423677c9331bb320af7e49cfe62" dependencies = [ - "futures 0.1.30", + "futures-core", "tower-buffer", "tower-discover", "tower-layer", @@ -3271,13 +3274,13 @@ dependencies = [ [[package]] name = "tower-buffer" -version = "0.1.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7b83e1ccf5b23dd109dd6ae2c07b8e2beec7a51a21f29da2dba576317370e0" +checksum = "c4887dc2a65d464c8b9b66e0e4d51c2fd6cf5b3373afc72805b0a60bce00446a" dependencies = [ - "futures 0.1.30", - "tokio-executor", - "tokio-sync", + "futures-core", + "pin-project 0.4.28", + "tokio 0.2.24", "tower-layer", "tower-service", "tracing", @@ -3285,91 +3288,101 @@ dependencies = [ [[package]] name = "tower-discover" -version = "0.1.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73a7632286f78164d65d18fd0e570307acde9362489aa5c8c53e6315cc2bde47" +checksum = "0f6b5000c3c54d269cc695dff28136bb33d08cbf1df2c48129e143ab65bf3c2a" dependencies = [ - "futures 0.1.30", + "futures-core", + "pin-project 0.4.28", "tower-service", ] [[package]] name = "tower-layer" -version = "0.1.0" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" + +[[package]] +name = "tower-limit" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ddf07e10c07dcc8f41da6de036dc66def1a85b70eb8a385159e3908bb258328" +checksum = "92c3040c5dbed68abffaa0d4517ac1a454cd741044f33ab0eefab6b8d1361404" dependencies = [ - "futures 0.1.30", + "futures-core", + "pin-project 0.4.28", + "tokio 0.2.24", + "tower-layer", + "tower-load", "tower-service", ] [[package]] -name = "tower-limit" -version = "0.1.3" +name = "tower-load" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c21ba835a08fd54b63cd91ae0548a7b6e2a91075147dfa3dc8e1a940c1b6f18f" +checksum = "8cc79fc3afd07492b7966d7efa7c6c50f8ed58d768a6075dd7ae6591c5d2017b" dependencies = [ - "futures 0.1.30", - "tokio-sync", - "tokio-timer", - "tower-layer", + "futures-core", + "log 0.4.11", + "pin-project 0.4.28", + "tokio 0.2.24", + "tower-discover", "tower-service", - "tracing", ] [[package]] name = "tower-load-shed" -version = "0.1.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04fbaf5bfb63d84204db87b9b2aeec61549613f2bbb8706dcc36f5f3ea8cd769" +checksum = "9f021e23900173dc315feb4b6922510dae3e79c689b74c089112066c11f0ae4e" dependencies = [ - "futures 0.1.30", + "futures-core", + "pin-project 0.4.28", "tower-layer", "tower-service", ] [[package]] name = "tower-retry" -version = "0.1.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09e80588125061f276ed2a7b0939988b411e570a2dbb2965b1382ef4f71036f7" +checksum = "e6727956aaa2f8957d4d9232b308fe8e4e65d99db30f42b225646e86c9b6a952" dependencies = [ - "futures 0.1.30", - "tokio-timer", + "futures-core", + "pin-project 0.4.28", + "tokio 0.2.24", "tower-layer", "tower-service", ] [[package]] name = "tower-service" -version = "0.2.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc0c98637d23732f8de6dfd16494c9f1559c3b9e20b4a46462c8f9b9e827bfa" -dependencies = [ - "futures 0.1.30", -] +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tower-timeout" -version = "0.1.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c06bbc2fbd056f810940a8c6f0cc194557d36da3c22999a755a7a6612447da9" +checksum = "127b8924b357be938823eaaec0608c482d40add25609481027b96198b2e4b31e" dependencies = [ - "futures 0.1.30", - "tokio-timer", + "pin-project 0.4.28", + "tokio 0.2.24", "tower-layer", "tower-service", ] [[package]] name = "tower-util" -version = "0.1.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4792342fac093db5d2558655055a89a04ca909663467a4310c7739d9f8b64698" +checksum = "d1093c19826d33807c72511e68f73b4a0469a3f22c2bd5f7d5212178b4b89674" dependencies = [ - "futures 0.1.30", - "tokio-io", - "tower-layer", + "futures-core", + "futures-util", + "pin-project 0.4.28", "tower-service", ] @@ -3393,8 +3406,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80e0ccfc3378da0cce270c946b676a376943f5cd16aeba64568e7939806f4ada" dependencies = [ "proc-macro2", - "quote 1.0.8", - "syn 1.0.58", + "quote", + "syn", ] [[package]] @@ -3406,6 +3419,16 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project 1.0.6", + "tracing", +] + [[package]] name = "treeline" version = "0.1.0" @@ -3484,12 +3507,6 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" -[[package]] -name = "unicode-xid" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" - [[package]] name = "unicode-xid" version = "0.2.1" @@ -3647,6 +3664,16 @@ dependencies = [ "try-lock", ] +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log 0.4.11", + "try-lock", +] + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -3666,6 +3693,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3cd364751395ca0f68cafb17666eee36b63077fb5ecd972bbcd74c90c4bf736e" dependencies = [ "cfg-if 1.0.0", + "serde", + "serde_json", "wasm-bindgen-macro", ] @@ -3679,8 +3708,8 @@ dependencies = [ "lazy_static", "log 0.4.11", "proc-macro2", - "quote 1.0.8", - "syn 1.0.58", + "quote", + "syn", "wasm-bindgen-shared", ] @@ -3702,7 +3731,7 @@ version = "0.2.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a6ac8995ead1f084a8dea1e65f194d0973800c7f571f6edd70adf06ecf77084" dependencies = [ - "quote 1.0.8", + "quote", "wasm-bindgen-macro-support", ] @@ -3713,8 +3742,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" dependencies = [ "proc-macro2", - "quote 1.0.8", - "syn 1.0.58", + "quote", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3806,6 +3835,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "ws2_32-sys" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 13cbf696..504ee4ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,11 +24,13 @@ required-features = ["dist-server"] [dependencies] anyhow = "1.0" ar = { version = "0.8", optional = true } +async-trait = "0.1" atty = "0.2.6" base64 = "0.13" bincode = "1" blake3 = "0.3" byteorder = "1.0" +bytes = "0.5" chrono = { version = "0.4", optional = true } clap = "2.23.0" counted-array = "0.1" @@ -36,12 +38,12 @@ directories = "3" env_logger = "0.8" filetime = "0.2" flate2 = { version = "1.0", optional = true, default-features = false, features = ["rust_backend"] } -futures = "0.1.11" -futures_03 = { package = "futures", version = "0.3", features = ["compat", "thread-pool"] } +futures = "0.3" +futures-locks = "0.6" hmac = { version = "0.10", optional = true } -http = "0.1" -hyper = { version = "0.12", optional = true } -hyperx = { version = "0.12", optional = true } +http = "0.2" +hyper = { version = "0.13", optional = true } +hyperx = { version = "0.13", optional = true } jobserver = "0.1" jsonwebtoken = { version = "7", optional = true } lazy_static = "1.0.0" @@ -58,7 +60,7 @@ percent-encoding = { version = "2", optional = true } rand = "0.7" redis = { version = "0.17", optional = true } regex = "1" -reqwest = { version = "0.9.11", optional = true } +reqwest = { version = "0.10", features = ["json", "blocking"], optional = true } retry = "1" ring = { version = "0.16", optional = true, features = ["std"] } sha-1 = { version = "0.9", optional = true } @@ -69,13 +71,10 @@ serde_json = "1.0" strip-ansi-escapes = "0.1" tar = "0.4" tempfile = "3" -tokio-compat = "0.1" -tokio-io = "0.1" -tokio-process = "0.2" -tokio-serde-bincode = "0.1" -tower = "0.1" -tokio-tcp = "0.1" -tokio-timer = "0.2" +tokio = { version = "0.2", features = ["rt-threaded", "blocking", "io-util", "time", "uds", "tcp", "process", "macros"] } +tokio-serde = "0.6" +tokio-util = { version = "0.3", features = ["codec"] } +tower = "0.3" toml = "0.5" untrusted = { version = "0.7", optional = true } url = { version = "2", optional = true } @@ -105,11 +104,9 @@ selenium-rs = "0.1" [target.'cfg(unix)'.dependencies] daemonize = "0.4" -tokio-uds = "0.2" [target.'cfg(windows)'.dependencies] -tokio-named-pipes = "0.1" -tokio-reactor = "0.1" +parity-tokio-ipc = "0.8" [target.'cfg(windows)'.dependencies.winapi] version = "0.3" diff --git a/src/azure/blobstore.rs b/src/azure/blobstore.rs index b85897f9..48f227a6 100644 --- a/src/azure/blobstore.rs +++ b/src/azure/blobstore.rs @@ -14,14 +14,11 @@ // limitations under the License. use crate::azure::credentials::*; -use futures::{Future, Stream}; use hmac::{Hmac, Mac, NewMac}; -use hyper::header::HeaderValue; -use hyper::Method; use hyperx::header; use md5::{Digest, Md5}; -use reqwest::r#async::{Client, Request}; use reqwest::Url; +use reqwest::{header::HeaderValue, Client, Method, Request}; use sha2::Sha256; use std::fmt; use std::str::FromStr; @@ -72,7 +69,7 @@ impl BlobContainer { }) } - pub fn get(&self, key: &str, creds: &AzureCredentials) -> SFuture> { + pub async fn get(&self, key: &str, creds: &AzureCredentials) -> Result> { let url_string = format!("{}{}", self.url, key); let uri = Url::from_str(&url_string).unwrap(); let dt = chrono::Utc::now(); @@ -90,10 +87,7 @@ impl BlobContainer { creds, ); - let uri_copy = uri.clone(); - let uri_second_copy = uri.clone(); - - let mut request = Request::new(Method::GET, uri); + let mut request = Request::new(Method::GET, uri.clone()); request.headers_mut().insert( "x-ms-date", HeaderValue::from_str(&date).expect("Date is an invalid header value"), @@ -108,46 +102,34 @@ impl BlobContainer { ); } - Box::new( - self.client - .execute(request) - .fwith_context(move || format!("failed GET: {}", uri_copy)) - .and_then(|res| { - if res.status().is_success() { - let content_length = res - .headers() - .get_hyperx::() - .map(|header::ContentLength(len)| len); - Ok((res.into_body(), content_length)) - } else { - Err(BadHttpStatusError(res.status()).into()) - } - }) - .and_then(|(body, content_length)| { - body.fold(Vec::new(), |mut body, chunk| { - body.extend_from_slice(&chunk); - Ok::<_, reqwest::Error>(body) - }) - .fcontext("failed to read HTTP body") - .and_then(move |bytes| { - if let Some(len) = content_length { - if len != bytes.len() as u64 { - bail!(format!( - "Bad HTTP body size read: {}, expected {}", - bytes.len(), - len - )); - } else { - info!("Read {} bytes from {}", bytes.len(), uri_second_copy); - } - } - Ok(bytes) - }) - }), - ) + let res = self + .client + .execute(request) + .await + .with_context(|| format!("failed GET: {}", &uri))?; + + let (bytes, content_length) = if res.status().is_success() { + let content_length = res.content_length(); + (res.bytes().await?, content_length) + } else { + return Err(BadHttpStatusError(res.status()).into()); + }; + + if let Some(len) = content_length { + if len != bytes.len() as u64 { + bail!(format!( + "Bad HTTP body size read: {}, expected {}", + bytes.len(), + len + )); + } else { + info!("Read {} bytes from {}", bytes.len(), &uri); + } + } + Ok(bytes.into_iter().collect()) } - pub fn put(&self, key: &str, content: Vec, creds: &AzureCredentials) -> SFuture<()> { + pub async fn put(&self, key: &str, content: Vec, creds: &AzureCredentials) -> Result<()> { let url_string = format!("{}{}", self.url, key); let uri = Url::from_str(&url_string).unwrap(); let dt = chrono::Utc::now(); @@ -206,7 +188,7 @@ impl BlobContainer { *request.body_mut() = Some(content.into()); - Box::new(self.client.execute(request).then(|result| match result { + match self.client.execute(request).await { Ok(res) => { if res.status().is_success() { trace!("PUT succeeded"); @@ -220,7 +202,7 @@ impl BlobContainer { trace!("PUT failed with error: {:?}", e); Err(e.into()) } - })) + } } } @@ -285,7 +267,7 @@ fn canonicalize_resource(uri: &Url, account_name: &str) -> String { #[cfg(test)] mod test { use super::*; - use tokio_compat::runtime::current_thread::Runtime; + use tokio::runtime::Runtime; #[test] fn test_signing() { diff --git a/src/bin/sccache-dist/token_check.rs b/src/bin/sccache-dist/token_check.rs index 2f405418..bc721427 100644 --- a/src/bin/sccache-dist/token_check.rs +++ b/src/bin/sccache-dist/token_check.rs @@ -77,7 +77,7 @@ const MOZ_USERINFO_ENDPOINT: &str = "https://auth.mozilla.auth0.com/userinfo"; // Mozilla-specific check by forwarding the token onto the auth0 userinfo endpoint pub struct MozillaCheck { auth_cache: Mutex>, // token, token_expiry - client: reqwest::Client, + client: reqwest::blocking::Client, required_groups: Vec, } @@ -96,7 +96,7 @@ impl MozillaCheck { pub fn new(required_groups: Vec) -> Self { Self { auth_cache: Mutex::new(HashMap::new()), - client: reqwest::Client::new(), + client: reqwest::blocking::Client::new(), required_groups, } } @@ -145,22 +145,18 @@ impl MozillaCheck { let header = hyperx::header::Authorization(hyperx::header::Bearer { token: token.to_owned(), }); - let mut res = self + let res = self .client .get(url.clone()) .set_header(header) .send() .context("Failed to make request to mozilla userinfo")?; + let status = res.status(); let res_text = res .text() .context("Failed to interpret response from mozilla userinfo as string")?; - if !res.status().is_success() { - bail!( - "JWT forwarded to {} returned {}: {}", - url, - res.status(), - res_text - ) + if !status.is_success() { + bail!("JWT forwarded to {} returned {}: {}", url, status, res_text) } // The API didn't return a HTTP error code, let's check the response @@ -241,7 +237,7 @@ fn test_auth_verify_check_mozilla_profile() { // Don't check a token is valid (it may not even be a JWT) just forward it to // an API and check for success pub struct ProxyTokenCheck { - client: reqwest::Client, + client: reqwest::blocking::Client, maybe_auth_cache: Option, Duration)>>, url: String, } @@ -265,7 +261,7 @@ impl ProxyTokenCheck { let maybe_auth_cache: Option, Duration)>> = cache_secs.map(|secs| Mutex::new((HashMap::new(), Duration::from_secs(secs)))); Self { - client: reqwest::Client::new(), + client: reqwest::blocking::Client::new(), maybe_auth_cache, url, } @@ -330,7 +326,7 @@ impl ClientAuthCheck for ValidJWTCheck { impl ValidJWTCheck { pub fn new(audience: String, issuer: String, jwks_url: &str) -> Result { - let mut res = reqwest::get(jwks_url).context("Failed to make request to JWKs url")?; + let res = reqwest::blocking::get(jwks_url).context("Failed to make request to JWKs url")?; if !res.status().is_success() { bail!("Could not retrieve JWKs, HTTP error: {}", res.status()) } diff --git a/src/cache/azure.rs b/src/cache/azure.rs index 63bae033..757eedc9 100644 --- a/src/cache/azure.rs +++ b/src/cache/azure.rs @@ -16,15 +16,14 @@ use crate::azure::BlobContainer; use crate::azure::*; use crate::cache::{Cache, CacheRead, CacheWrite, Storage}; -use futures::future::Future; use std::io; -use std::rc::Rc; +use std::sync::Arc; use std::time::{Duration, Instant}; use crate::errors::*; pub struct AzureBlobCache { - container: Rc, + container: Arc, credentials: AzureCredentials, } @@ -44,53 +43,48 @@ impl AzureBlobCache { }; Ok(AzureBlobCache { - container: Rc::new(container), + container: Arc::new(container), credentials, }) } } +#[async_trait] impl Storage for AzureBlobCache { - fn get(&self, key: &str) -> SFuture { - Box::new( - self.container - .get(key, &self.credentials) - .then(|result| match result { - Ok(data) => { - let hit = CacheRead::from(io::Cursor::new(data))?; - Ok(Cache::Hit(hit)) - } - Err(e) => { - warn!("Got Azure error: {:?}", e); - Ok(Cache::Miss) - } - }), - ) + async fn get(&self, key: &str) -> Result { + match self.container.get(&key, &self.credentials).await { + Ok(data) => { + let hit = CacheRead::from(io::Cursor::new(data))?; + Ok(Cache::Hit(hit)) + } + Err(e) => { + warn!("Got Azure error: {:?}", e); + Ok(Cache::Miss) + } + } } - fn put(&self, key: &str, entry: CacheWrite) -> SFuture { + async fn put(&self, key: &str, entry: CacheWrite) -> Result { let start = Instant::now(); - let data = match entry.finish() { - Ok(data) => data, - Err(e) => return f_err(e), - }; + let data = entry.finish()?; - let response = self + let _ = self .container .put(key, data, &self.credentials) - .fcontext("Failed to put cache entry in Azure"); + .await + .context("Failed to put cache entry in Azure")?; - Box::new(response.map(move |_| start.elapsed())) + Ok(start.elapsed()) } fn location(&self) -> String { format!("Azure, container: {}", self.container) } - fn current_size(&self) -> SFuture> { - f_ok(None) + async fn current_size(&self) -> Result> { + Ok(None) } - fn max_size(&self) -> SFuture> { - f_ok(None) + async fn max_size(&self) -> Result> { + Ok(None) } } diff --git a/src/cache/cache.rs b/src/cache/cache.rs index faf20516..ae6ee794 100644 --- a/src/cache/cache.rs +++ b/src/cache/cache.rs @@ -24,8 +24,6 @@ use crate::cache::redis::RedisCache; #[cfg(feature = "s3")] use crate::cache::s3::S3Cache; use crate::config::{self, CacheType, Config}; -use crate::util::SpawnExt; -use futures_03::executor::ThreadPool; use std::fmt; use std::fs; #[cfg(feature = "gcs")] @@ -151,11 +149,15 @@ impl CacheRead { bytes } - pub fn extract_objects(mut self, objects: T, pool: &ThreadPool) -> SFuture<()> + pub async fn extract_objects( + mut self, + objects: T, + pool: &tokio::runtime::Handle, + ) -> Result<()> where T: IntoIterator + Send + Sync + 'static, { - Box::new(pool.spawn_fn(move || { + pool.spawn_blocking(move || { for (key, path) in objects { let dir = match path.parent() { Some(d) => d, @@ -172,7 +174,8 @@ impl CacheRead { } } Ok(()) - })) + }) + .await? } } @@ -190,11 +193,11 @@ impl CacheWrite { } /// Create a new cache entry populated with the contents of `objects`. - pub fn from_objects(objects: T, pool: &ThreadPool) -> SFuture + pub async fn from_objects(objects: T, pool: &tokio::runtime::Handle) -> Result where T: IntoIterator + Send + Sync + 'static, { - Box::new(pool.spawn_fn(move || -> Result<_> { + pool.spawn_blocking(move || { let mut entry = CacheWrite::new(); for (key, path) in objects { let mut f = fs::File::open(&path)?; @@ -204,7 +207,8 @@ impl CacheWrite { .with_context(|| format!("failed to put object `{:?}` in cache entry", path))?; } Ok(entry) - })) + }) + .await? } /// Add an object containing the contents of `from` to this cache entry at `name`. @@ -259,7 +263,8 @@ impl Default for CacheWrite { } /// An interface to cache storage. -pub trait Storage { +#[async_trait] +pub trait Storage: Send + Sync { /// Get a cache entry by `key`. /// /// If an error occurs, this method should return a `Cache::Error`. @@ -267,27 +272,27 @@ pub trait Storage { /// it should return a `Cache::Miss`. /// If the entry is successfully found in the cache, it should /// return a `Cache::Hit`. - fn get(&self, key: &str) -> SFuture; + async fn get(&self, key: &str) -> Result; /// Put `entry` in the cache under `key`. /// /// Returns a `Future` that will provide the result or error when the put is /// finished. - fn put(&self, key: &str, entry: CacheWrite) -> SFuture; + async fn put(&self, key: &str, entry: CacheWrite) -> Result; /// Get the storage location. fn location(&self) -> String; /// Get the current storage usage, if applicable. - fn current_size(&self) -> SFuture>; + async fn current_size(&self) -> Result>; /// Get the maximum storage size, if applicable. - fn max_size(&self) -> SFuture>; + async fn max_size(&self) -> Result>; } /// Get a suitable `Storage` implementation from configuration. #[allow(clippy::cognitive_complexity)] // TODO simplify! -pub fn storage_from_config(config: &Config, pool: &ThreadPool) -> Arc { +pub fn storage_from_config(config: &Config, pool: &tokio::runtime::Handle) -> Arc { for cache_type in config.caches.iter() { match *cache_type { CacheType::Azure(config::AzureCacheConfig) => { diff --git a/src/cache/disk.rs b/src/cache/disk.rs index 6eb7148f..49766a64 100644 --- a/src/cache/disk.rs +++ b/src/cache/disk.rs @@ -15,8 +15,6 @@ use crate::cache::{Cache, CacheRead, CacheWrite, Storage}; use crate::lru_disk_cache::Error as LruError; use crate::lru_disk_cache::LruDiskCache; -use crate::util::SpawnExt; -use futures_03::executor::ThreadPool; use std::ffi::OsStr; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; @@ -30,12 +28,16 @@ pub struct DiskCache { /// `LruDiskCache` does all the real work here. lru: Arc>, /// Thread pool to execute disk I/O - pool: ThreadPool, + pool: tokio::runtime::Handle, } impl DiskCache { /// Create a new `DiskCache` rooted at `root`, with `max_size` as the maximum cache size on-disk, in bytes. - pub fn new>(root: &T, max_size: u64, pool: &ThreadPool) -> DiskCache { + pub fn new>( + root: &T, + max_size: u64, + pool: &tokio::runtime::Handle, + ) -> DiskCache { DiskCache { //TODO: change this function to return a Result lru: Arc::new(Mutex::new( @@ -51,53 +53,60 @@ fn make_key_path(key: &str) -> PathBuf { Path::new(&key[0..1]).join(&key[1..2]).join(key) } +#[async_trait] impl Storage for DiskCache { - fn get(&self, key: &str) -> SFuture { + async fn get(&self, key: &str) -> Result { trace!("DiskCache::get({})", key); let path = make_key_path(key); let lru = self.lru.clone(); let key = key.to_owned(); - Box::new(self.pool.spawn_fn(move || { - let mut lru = lru.lock().unwrap(); - let f = match lru.get(&path) { - Ok(f) => f, - Err(LruError::FileNotInCache) => { - trace!("DiskCache::get({}): FileNotInCache", key); - return Ok(Cache::Miss); - } - Err(LruError::Io(e)) => { - trace!("DiskCache::get({}): IoError: {:?}", key, e); - return Err(e.into()); - } - Err(_) => unreachable!(), - }; - let hit = CacheRead::from(f)?; - Ok(Cache::Hit(hit)) - })) + + self.pool + .spawn_blocking(move || { + let mut lru = lru.lock().unwrap(); + let io = match lru.get(&path) { + Ok(f) => f, + Err(LruError::FileNotInCache) => { + trace!("DiskCache::get({}): FileNotInCache", key); + return Ok(Cache::Miss); + } + Err(LruError::Io(e)) => { + trace!("DiskCache::get({}): IoError: {:?}", key, e); + return Err(e.into()); + } + Err(_) => unreachable!(), + }; + let hit = CacheRead::from(io)?; + Ok(Cache::Hit(hit)) + }) + .await? } - fn put(&self, key: &str, entry: CacheWrite) -> SFuture { + async fn put(&self, key: &str, entry: CacheWrite) -> Result { // We should probably do this on a background thread if we're going to buffer // everything in memory... trace!("DiskCache::finish_put({})", key); let lru = self.lru.clone(); let key = make_key_path(key); - Box::new(self.pool.spawn_fn(move || { - let start = Instant::now(); - let v = entry.finish()?; - lru.lock().unwrap().insert_bytes(key, &v)?; - Ok(start.elapsed()) - })) + + self.pool + .spawn_blocking(move || { + let start = Instant::now(); + let v = entry.finish()?; + lru.lock().unwrap().insert_bytes(key, &v)?; + Ok(start.elapsed()) + }) + .await? } fn location(&self) -> String { format!("Local disk: {:?}", self.lru.lock().unwrap().path()) } - fn current_size(&self) -> SFuture> { - f_ok(Some(self.lru.lock().unwrap().size())) + async fn current_size(&self) -> Result> { + Ok(Some(self.lru.lock().unwrap().size())) } - fn max_size(&self) -> SFuture> { - f_ok(Some(self.lru.lock().unwrap().capacity())) + async fn max_size(&self) -> Result> { + Ok(Some(self.lru.lock().unwrap().capacity())) } } diff --git a/src/cache/gcs.rs b/src/cache/gcs.rs index b99c01a4..13877ef5 100644 --- a/src/cache/gcs.rs +++ b/src/cache/gcs.rs @@ -13,21 +13,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::{cell::RefCell, fmt, io, rc::Rc, time}; +use std::{fmt, io, sync::Arc, time}; use crate::{ cache::{Cache, CacheRead, CacheWrite, Storage}, errors::*, util::HeadersExt, }; -use futures::{ - future::{self, Shared}, - Async, Future, Stream, -}; use hyper::Method; use hyperx::header::{Authorization, Bearer, ContentLength, ContentType}; use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS}; -use reqwest::r#async::{Client, Request}; +use reqwest::{Client, Request}; use serde::de; use url::form_urlencoded; @@ -62,7 +58,11 @@ impl Bucket { Ok(Bucket { name, client }) } - fn get(&self, key: &str, cred_provider: &Option) -> SFuture> { + async fn get( + &self, + key: &str, + cred_provider: &Option, + ) -> Result> { let url = format!( "https://www.googleapis.com/download/storage/v1/b/{}/o/{}?alt=media", utf8_percent_encode(&self.name, PATH_SEGMENT), @@ -71,95 +71,85 @@ impl Bucket { let client = self.client.clone(); - let creds_opt_future = if let Some(ref cred_provider) = *cred_provider { - future::Either::A( - cred_provider - .credentials(&self.client) - .map_err(|err| { - warn!("Error getting credentials: {:?}", err); - err - }) - .map(Some), - ) + let creds_opt = if let Some(ref cred_provider) = *cred_provider { + cred_provider + .credentials(&self.client) + .await + .map_err(|err| { + warn!("Error getting credentials: {:?}", err); + err + }) + .map(Some)? } else { - future::Either::B(future::ok(None)) + None }; - Box::new(creds_opt_future.and_then(move |creds_opt| { - let mut request = Request::new(Method::GET, url.parse().unwrap()); - if let Some(creds) = creds_opt { - request - .headers_mut() - .set(Authorization(Bearer { token: creds.token })); - } - client - .execute(request) - .fwith_context(move || format!("failed GET: {}", url)) - .and_then(|res| { - if res.status().is_success() { - Ok(res.into_body()) - } else { - Err(BadHttpStatusError(res.status()).into()) - } - }) - .and_then(|body| { - body.fold(Vec::new(), |mut body, chunk| { - body.extend_from_slice(&chunk); - Ok::<_, reqwest::Error>(body) - }) - .fcontext("failed to read HTTP body") - }) - })) + let mut request = Request::new(Method::GET, url.parse().unwrap()); + if let Some(creds) = creds_opt { + request + .headers_mut() + .set(Authorization(Bearer { token: creds.token })); + } + let res = client + .execute(request) + .await + .with_context(|| format!("failed GET: {}", url))?; + if res.status().is_success() { + let bytes = res.bytes().await.context("failed to read HTTP body")?; + Ok(bytes.into_iter().collect()) + } else { + Err(BadHttpStatusError(res.status()).into()) + } } - fn put( + async fn put( &self, key: &str, content: Vec, cred_provider: &Option, - ) -> SFuture<()> { + ) -> Result<()> { let url = format!( "https://www.googleapis.com/upload/storage/v1/b/{}/o?name={}&uploadType=media", utf8_percent_encode(&self.name, PATH_SEGMENT), utf8_percent_encode(key, QUERY) ); + let url = url.parse().unwrap(); let client = self.client.clone(); - let creds_opt_future = if let Some(ref cred_provider) = cred_provider { - future::Either::A(cred_provider.credentials(&self.client).map(Some)) + let creds_opt = if let Some(ref cred_provider) = cred_provider { + let val = cred_provider.credentials(&self.client).await?; + Some(val) } else { - future::Either::B(future::ok(None)) + None }; - Box::new(creds_opt_future.and_then(move |creds_opt| { - let mut request = Request::new(Method::POST, url.parse().unwrap()); - { - let headers = request.headers_mut(); - if let Some(creds) = creds_opt { - headers.set(Authorization(Bearer { token: creds.token })); - } - headers.set(ContentType::octet_stream()); - headers.set(ContentLength(content.len() as u64)); + let mut request = Request::new(Method::POST, url); + { + let headers = request.headers_mut(); + if let Some(creds) = creds_opt { + headers.set(Authorization(Bearer { token: creds.token })); } - *request.body_mut() = Some(content.into()); - - client.execute(request).then(|result| match result { - Ok(res) => { - if res.status().is_success() { - trace!("PUT succeeded"); - Ok(()) - } else { - trace!("PUT failed with HTTP status: {}", res.status()); - Err(BadHttpStatusError(res.status()).into()) - } - } - Err(e) => { - trace!("PUT failed with error: {:?}", e); - Err(e.into()) + headers.set(ContentType::octet_stream()); + headers.set(ContentLength(content.len() as u64)); + } + *request.body_mut() = Some(content.into()); + + match client.execute(request).await { + Ok(res) => { + if res.status().is_success() { + trace!("PUT succeeded"); + Ok(()) + } else { + trace!("PUT failed with HTTP status: {}", res.status()); + Err(BadHttpStatusError(res.status()).into()) } - }) - })) + } + Err(e) => { + trace!("PUT failed with error: {:?}", e); + Err(e.into()) + } + } } } @@ -170,7 +160,7 @@ impl Bucket { pub struct GCSCredentialProvider { rw_mode: RWMode, sa_info: ServiceAccountInfo, - cached_credentials: RefCell>>>, + cached_credentials: futures_locks::Mutex>, } /// ServiceAccountInfo either contains a URL to fetch the oauth token @@ -334,7 +324,7 @@ impl GCSCredentialProvider { GCSCredentialProvider { rw_mode, sa_info, - cached_credentials: RefCell::new(None), + cached_credentials: futures_locks::Mutex::new(None), } } @@ -364,129 +354,92 @@ impl GCSCredentialProvider { ) } - fn request_new_token( + async fn request_new_token( &self, sa_key: &ServiceAccountKey, client: &Client, - ) -> SFuture { - let client = client.clone(); + ) -> Result { let expires_at = chrono::offset::Utc::now() + chrono::Duration::minutes(59); - let auth_jwt = self.auth_request_jwt(sa_key, &expires_at); + let auth_jwt = self.auth_request_jwt(sa_key, &expires_at)?; let url = sa_key.token_uri.clone(); - // Request credentials - Box::new( - future::result(auth_jwt) - .and_then(move |auth_jwt| { - let params = form_urlencoded::Serializer::new(String::new()) - .append_pair("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer") - .append_pair("assertion", &auth_jwt) - .finish(); - - let mut request = Request::new(Method::POST, url.parse().unwrap()); - { - let headers = request.headers_mut(); - headers.set(ContentType::form_url_encoded()); - headers.set(ContentLength(params.len() as u64)); - } - *request.body_mut() = Some(params.into()); + let params = form_urlencoded::Serializer::new(String::new()) + .append_pair("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer") + .append_pair("assertion", &auth_jwt) + .finish(); - client.execute(request).map_err(Into::into) - }) - .and_then(move |res| { - if res.status().is_success() { - Ok(res.into_body()) - } else { - Err(BadHttpStatusError(res.status()).into()) - } - }) - .and_then(move |body| { - // Concatenate body chunks into a single Vec - body.fold(Vec::new(), |mut body, chunk| { - body.extend_from_slice(&chunk); - Ok::<_, reqwest::Error>(body) - }) - .fcontext("failed to read HTTP body") - }) - .and_then(move |body| { - // Convert body to string and parse the token out of the response - let body_str = String::from_utf8(body)?; - let token_msg: TokenMsg = serde_json::from_str(&body_str)?; - - Ok(GCSCredential { - token: token_msg.access_token, - expiration_time: expires_at, - }) - }), - ) + let mut request = Request::new(Method::POST, url.parse().unwrap()); + { + let headers = request.headers_mut(); + headers.set(ContentType::form_url_encoded()); + headers.set(ContentLength(params.len() as u64)); + } + *request.body_mut() = Some(params.into()); + + let res = client.execute(request).await?; + + let token_msg = if res.status().is_success() { + let token_msg = res.json::().await?; + Ok(token_msg) + } else { + Err(BadHttpStatusError(res.status())) + }?; + + Ok(GCSCredential { + token: token_msg.access_token, + expiration_time: expires_at, + }) } - fn request_new_token_from_tcauth(&self, url: &str, client: &Client) -> SFuture { - Box::new( - client - .get(url) - .send() - .map_err(Into::into) - .and_then(move |res| { - if res.status().is_success() { - Ok(res.into_body()) - } else { - Err(BadHttpStatusError(res.status()).into()) - } - }) - .and_then(move |body| { - body.fold(Vec::new(), |mut body, chunk| { - body.extend_from_slice(&chunk); - Ok::<_, reqwest::Error>(body) - }) - .fcontext("failed to read HTTP body") - }) - .and_then(move |body| { - let body_str = String::from_utf8(body)?; - let resp: AuthResponse = serde_json::from_str(&body_str)?; - Ok(GCSCredential { - token: resp.access_token, - expiration_time: resp.expire_time.parse()?, - }) - }), - ) + async fn request_new_token_from_tcauth( + &self, + url: &str, + client: &Client, + ) -> Result { + let res = client.get(url).send().await?; + + if res.status().is_success() { + let resp = res.json::().await?; + Ok(GCSCredential { + token: resp.access_token, + expiration_time: resp.expire_time.parse()?, + }) + } else { + Err(BadHttpStatusError(res.status()).into()) + } } - pub fn credentials(&self, client: &Client) -> SFuture { - let mut future_opt = self.cached_credentials.borrow_mut(); + pub async fn credentials(&self, client: &Client) -> Result { + // NOTE: Only this function is responsible for managing credentials and + // its cache; make sure we hold the lock across the yield points + let mut cache = self.cached_credentials.lock().await; - let needs_refresh = match Option::as_mut(&mut future_opt).map(|f| f.poll()) { - None => true, - Some(Ok(Async::Ready(ref creds))) => creds.expiration_time < chrono::offset::Utc::now(), - _ => false, - }; + match *cache { + Some(ref creds) if creds.expiration_time >= chrono::offset::Utc::now() => { + Ok(creds.clone()) + } + _ => { + let new_creds = match self.sa_info { + ServiceAccountInfo::AccountKey(ref sa_key) => { + self.request_new_token(sa_key, client).await + } + ServiceAccountInfo::URL(ref url) => { + self.request_new_token_from_tcauth(url, client).await + } + }?; - if needs_refresh { - let credentials = match self.sa_info { - ServiceAccountInfo::AccountKey(ref sa_key) => { - self.request_new_token(sa_key, client) - } - ServiceAccountInfo::URL(ref url) => self.request_new_token_from_tcauth(url, client), - }; - *future_opt = Some(credentials.shared()); - }; + *cache = Some(new_creds.clone()); - Box::new( - Option::as_mut(&mut future_opt) - .unwrap() - .clone() - .then(|result| match result { - Ok(e) => Ok((*e).clone()), - Err(e) => Err(anyhow!(e.to_string())), - }), - ) + Ok(new_creds) + } + } } } /// A cache that stores entries in Google Cloud Storage pub struct GCSCache { /// The GCS bucket - bucket: Rc, + bucket: Arc, /// Credential provider for GCS credential_provider: Option, /// Read-only or not @@ -501,74 +454,74 @@ impl GCSCache { rw_mode: RWMode, ) -> Result { Ok(GCSCache { - bucket: Rc::new(Bucket::new(bucket)?), + bucket: Arc::new(Bucket::new(bucket)?), rw_mode, credential_provider, }) } } +#[async_trait] impl Storage for GCSCache { - fn get(&self, key: &str) -> SFuture { - Box::new( - self.bucket - .get(&key, &self.credential_provider) - .then(|result| match result { - Ok(data) => { - let hit = CacheRead::from(io::Cursor::new(data))?; - Ok(Cache::Hit(hit)) - } - Err(e) => { - warn!("Got GCS error: {:?}", e); - Ok(Cache::Miss) - } - }), - ) + async fn get(&self, key: &str) -> Result { + match self.bucket.get(&key, &self.credential_provider).await { + Ok(data) => { + let hit = CacheRead::from(io::Cursor::new(data))?; + Ok(Cache::Hit(hit)) + } + Err(e) => { + warn!("Got GCS error: {:?}", e); + Ok(Cache::Miss) + } + } } - fn put(&self, key: &str, entry: CacheWrite) -> SFuture { + async fn put(&self, key: &str, entry: CacheWrite) -> Result { if let RWMode::ReadOnly = self.rw_mode { - return Box::new(future::ok(time::Duration::new(0, 0))); + return Ok(time::Duration::new(0, 0)); } let start = time::Instant::now(); - let data = match entry.finish() { - Ok(data) => data, - Err(e) => return Box::new(future::err(e)), - }; + let data = entry.finish()?; + let bucket = self.bucket.clone(); - let response = bucket + let _ = bucket .put(&key, data, &self.credential_provider) - .fcontext("failed to put cache entry in GCS"); + .await + .context("failed to put cache entry in GCS")?; - Box::new(response.map(move |_| start.elapsed())) + Ok(start.elapsed()) } fn location(&self) -> String { format!("GCS, bucket: {}", self.bucket) } - fn current_size(&self) -> SFuture> { - Box::new(future::ok(None)) + async fn current_size(&self) -> Result> { + Ok(None) } - fn max_size(&self) -> SFuture> { - Box::new(future::ok(None)) + + async fn max_size(&self) -> Result> { + Ok(None) } } -#[test] -fn test_gcs_credential_provider() { +#[tokio::test] +async fn test_gcs_credential_provider() { + use futures::{FutureExt, TryFutureExt}; + use std::convert::Infallible; + const EXPIRE_TIME: &str = "3000-01-01T00:00:00.0Z"; let addr = ([127, 0, 0, 1], 3000).into(); - let make_service = || { - hyper::service::service_fn_ok(|_| { + let make_service = hyper::service::make_service_fn(|_socket| async { + Ok::<_, Infallible>(hyper::service::service_fn(|_request| async { let token = serde_json::json!({ "accessToken": "1234567890", "expireTime": EXPIRE_TIME, }); - hyper::Response::new(hyper::Body::from(token.to_string())) - }) - }; + Ok::<_, Infallible>(hyper::Response::new(hyper::Body::from(token.to_string()))) + })) + }); let server = hyper::Server::bind(&addr).serve(make_service); @@ -580,7 +533,7 @@ fn test_gcs_credential_provider() { let client = Client::new(); let cred_fut = credential_provider .credentials(&client) - .map(move |credential| { + .map_ok(move |credential| { assert_eq!(credential.token, "1234567890"); assert_eq!( credential.expiration_time.timestamp(), @@ -592,5 +545,5 @@ fn test_gcs_credential_provider() { }) .map_err(move |err| panic!(err.to_string())); - server.with_graceful_shutdown(cred_fut); + let _ = server.with_graceful_shutdown(cred_fut.map(drop)).await; } diff --git a/src/cache/memcached.rs b/src/cache/memcached.rs index 7ab0ad45..a3c96e63 100644 --- a/src/cache/memcached.rs +++ b/src/cache/memcached.rs @@ -15,8 +15,6 @@ use crate::cache::{Cache, CacheRead, CacheWrite, Storage}; use crate::errors::*; -use crate::util::SpawnExt; -use futures_03::executor::ThreadPool; use memcached::client::Client; use memcached::proto::NoReplyOperation; use memcached::proto::Operation; @@ -32,11 +30,11 @@ thread_local! { #[derive(Clone)] pub struct MemcachedCache { url: String, - pool: ThreadPool, + pool: tokio::runtime::Handle, } impl MemcachedCache { - pub fn new(url: &str, pool: &ThreadPool) -> Result { + pub fn new(url: &str, pool: &tokio::runtime::Handle) -> Result { Ok(MemcachedCache { url: url.to_owned(), pool: pool.clone(), @@ -66,36 +64,43 @@ impl MemcachedCache { } } +#[async_trait] impl Storage for MemcachedCache { - fn get(&self, key: &str) -> SFuture { + async fn get(&self, key: &str) -> Result { let key = key.to_owned(); let me = self.clone(); - Box::new(self.pool.spawn_fn(move || { - me.exec(|c| c.get(&key.as_bytes())) - .map(|(d, _)| CacheRead::from(Cursor::new(d)).map(Cache::Hit)) - .unwrap_or(Ok(Cache::Miss)) - })) + + self.pool + .spawn_blocking(move || { + me.exec(|c| c.get(&key.as_bytes())) + .map(|(d, _)| CacheRead::from(Cursor::new(d)).map(Cache::Hit)) + .unwrap_or(Ok(Cache::Miss)) + }) + .await? } - fn put(&self, key: &str, entry: CacheWrite) -> SFuture { + async fn put(&self, key: &str, entry: CacheWrite) -> Result { let key = key.to_owned(); let me = self.clone(); - Box::new(self.pool.spawn_fn(move || { - let start = Instant::now(); - let d = entry.finish()?; - me.exec(|c| c.set_noreply(&key.as_bytes(), &d, 0, 0))?; - Ok(start.elapsed()) - })) + + self.pool + .spawn_blocking(move || { + let start = Instant::now(); + let d = entry.finish()?; + me.exec(|c| c.set_noreply(&key.as_bytes(), &d, 0, 0))?; + Ok(start.elapsed()) + }) + .await? } fn location(&self) -> String { format!("Memcached: {}", self.url) } - fn current_size(&self) -> SFuture> { - f_ok(None) + async fn current_size(&self) -> Result> { + Ok(None) } - fn max_size(&self) -> SFuture> { - f_ok(None) + async fn max_size(&self) -> Result> { + Ok(None) } } diff --git a/src/cache/redis.rs b/src/cache/redis.rs index 02a25b90..d657bb5e 100644 --- a/src/cache/redis.rs +++ b/src/cache/redis.rs @@ -15,7 +15,6 @@ use crate::cache::{Cache, CacheRead, CacheWrite, Storage}; use crate::errors::*; -use futures_03::prelude::*; use redis::aio::Connection; use redis::{cmd, Client, InfoDict}; use std::collections::HashMap; @@ -39,44 +38,31 @@ impl RedisCache { } /// Returns a connection with configured read and write timeouts. - async fn connect(self) -> Result { + async fn connect(&self) -> Result { Ok(self.client.get_async_connection().await?) } } +#[async_trait] impl Storage for RedisCache { /// Open a connection and query for a key. - fn get(&self, key: &str) -> SFuture { - let key = key.to_owned(); - let me = self.clone(); - Box::new( - Box::pin(async move { - let mut c = me.connect().await?; - let d: Vec = cmd("GET").arg(key).query_async(&mut c).await?; - if d.is_empty() { - Ok(Cache::Miss) - } else { - CacheRead::from(Cursor::new(d)).map(Cache::Hit) - } - }) - .compat(), - ) + async fn get(&self, key: &str) -> Result { + let mut c = self.connect().await?; + let d: Vec = cmd("GET").arg(key).query_async(&mut c).await?; + if d.is_empty() { + Ok(Cache::Miss) + } else { + CacheRead::from(Cursor::new(d)).map(Cache::Hit) + } } /// Open a connection and store a object in the cache. - fn put(&self, key: &str, entry: CacheWrite) -> SFuture { - let key = key.to_owned(); - let me = self.clone(); + async fn put(&self, key: &str, entry: CacheWrite) -> Result { let start = Instant::now(); - Box::new( - Box::pin(async move { - let mut c = me.connect().await?; - let d = entry.finish()?; - cmd("SET").arg(key).arg(d).query_async(&mut c).await?; - Ok(start.elapsed()) - }) - .compat(), - ) + let mut c = self.connect().await?; + let d = entry.finish()?; + cmd("SET").arg(key).arg(d).query_async(&mut c).await?; + Ok(start.elapsed()) } /// Returns the cache location. @@ -86,40 +72,27 @@ impl Storage for RedisCache { /// Returns the current cache size. This value is aquired via /// the Redis INFO command (used_memory). - fn current_size(&self) -> SFuture> { - let me = self.clone(); // TODO Remove clone - Box::new( - Box::pin(async move { - let mut c = me.connect().await?; - let v: InfoDict = cmd("INFO").query_async(&mut c).await?; - Ok(v.get("used_memory")) - }) - .compat(), - ) + async fn current_size(&self) -> Result> { + let mut c = self.connect().await?; + let v: InfoDict = cmd("INFO").query_async(&mut c).await?; + Ok(v.get("used_memory")) } /// Returns the maximum cache size. This value is read via /// the Redis CONFIG command (maxmemory). If the server has no /// configured limit, the result is None. - fn max_size(&self) -> SFuture> { - let me = self.clone(); // TODO Remove clone - Box::new( - Box::pin(async move { - let mut c = me.connect().await?; - let result: redis::RedisResult> = cmd("CONFIG") - .arg("GET") - .arg("maxmemory") - .query_async(&mut c) - .await; - match result { - Ok(h) => { - Ok(h.get("maxmemory") - .and_then(|&s| if s != 0 { Some(s as u64) } else { None })) - } - Err(_) => Ok(None), - } - }) - .compat(), - ) + async fn max_size(&self) -> Result> { + let mut c = self.connect().await?; + let result: redis::RedisResult> = cmd("CONFIG") + .arg("GET") + .arg("maxmemory") + .query_async(&mut c) + .await; + match result { + Ok(h) => Ok(h + .get("maxmemory") + .and_then(|&s| if s != 0 { Some(s as u64) } else { None })), + Err(_) => Ok(None), + } } } diff --git a/src/cache/s3.rs b/src/cache/s3.rs index b08fcee9..d11b3ecf 100644 --- a/src/cache/s3.rs +++ b/src/cache/s3.rs @@ -17,10 +17,8 @@ use crate::simples3::{ AutoRefreshingProvider, Bucket, ChainProvider, ProfileProvider, ProvideAwsCredentials, Ssl, }; use directories::UserDirs; -use futures::future; -use futures::future::Future; use std::io; -use std::rc::Rc; +use std::sync::Arc; use std::time::{Duration, Instant}; use crate::errors::*; @@ -28,7 +26,7 @@ use crate::errors::*; /// A cache that stores entries in Amazon S3. pub struct S3Cache { /// The S3 bucket. - bucket: Rc, + bucket: Arc, /// Credentials provider. provider: AutoRefreshingProvider, /// Prefix to be used for bucket keys. @@ -51,7 +49,7 @@ impl S3Cache { let provider = AutoRefreshingProvider::new(ChainProvider::with_profile_providers(profile_providers)); let ssl_mode = if use_ssl { Ssl::Yes } else { Ssl::No }; - let bucket = Rc::new(Bucket::new(bucket, endpoint, ssl_mode)?); + let bucket = Arc::new(Bucket::new(bucket, endpoint, ssl_mode)?); Ok(S3Cache { bucket, provider, @@ -71,11 +69,21 @@ impl S3Cache { } } +#[async_trait] impl Storage for S3Cache { - fn get(&self, key: &str) -> SFuture { + async fn get(&self, key: &str) -> Result { let key = self.normalize_key(key); - let result_cb = |result| match result { + let credentials = self.provider.credentials().await; + let result = match credentials { + Ok(creds) => self.bucket.get(&key, Some(&creds)).await, + Err(e) => { + debug!("Could not load AWS creds: {}", e); + self.bucket.get(&key, None).await + } + }; + + match result { Ok(data) => { let hit = CacheRead::from(io::Cursor::new(data))?; Ok(Cache::Hit(hit)) @@ -84,53 +92,37 @@ impl Storage for S3Cache { warn!("Got AWS error: {:?}", e); Ok(Cache::Miss) } - }; - - let bucket = self.bucket.clone(); - let response = self - .provider - .credentials() - .then(move |credentials| match credentials { - Ok(creds) => bucket.get(&key, Some(&creds)), - Err(e) => { - debug!("Could not load AWS creds: {}", e); - bucket.get(&key, None) - } - }) - .then(result_cb); - Box::new(response) + } } - fn put(&self, key: &str, entry: CacheWrite) -> SFuture { + async fn put(&self, key: &str, entry: CacheWrite) -> Result { let key = self.normalize_key(&key); let start = Instant::now(); - let data = match entry.finish() { - Ok(data) => data, - Err(e) => return f_err(e), - }; + let data = entry.finish()?; + let credentials = self .provider .credentials() - .fcontext("failed to get AWS credentials"); + .await + .context("failed to get AWS credentials")?; let bucket = self.bucket.clone(); - let response = credentials.and_then(move |credentials| { - bucket - .put(&key, data, &credentials) - .fcontext("failed to put cache entry in s3") - }); + let _ = bucket + .put(&key, data, &credentials) + .await + .context("failed to put cache entry in s3")?; - Box::new(response.map(move |_| start.elapsed())) + Ok(start.elapsed()) } fn location(&self) -> String { format!("S3, bucket: {}", self.bucket) } - fn current_size(&self) -> SFuture> { - Box::new(future::ok(None)) + async fn current_size(&self) -> Result> { + Ok(None) } - fn max_size(&self) -> SFuture> { - Box::new(future::ok(None)) + async fn max_size(&self) -> Result> { + Ok(None) } } diff --git a/src/commands.rs b/src/commands.rs index 19babb27..df845c2f 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -23,7 +23,6 @@ use crate::server::{self, DistInfo, ServerInfo, ServerStartup}; use crate::util::daemonize; use atty::Stream; use byteorder::{BigEndian, ByteOrder}; -use futures::Future; use log::Level::Trace; use std::env; use std::ffi::{OsStr, OsString}; @@ -34,10 +33,8 @@ use std::os::unix::process::ExitStatusExt; use std::path::Path; use std::process; use strip_ansi_escapes::Writer; -use tokio_compat::runtime::current_thread::Runtime; -use tokio_io::io::read_exact; -use tokio_io::AsyncRead; -use tokio_timer::Timeout; +use tokio::io::AsyncReadExt; +use tokio::runtime::Runtime; use which::which_in; use crate::errors::*; @@ -56,33 +53,31 @@ fn get_port() -> u16 { .unwrap_or(DEFAULT_PORT) } -fn read_server_startup_status( - server: R, -) -> impl Future { +async fn read_server_startup_status( + mut server: R, +) -> Result { // This is an async equivalent of ServerConnection::read_one_response - read_exact(server, [0u8; 4]) - .map_err(Error::from) - .and_then(|(server, bytes)| { - let len = BigEndian::read_u32(&bytes); - let data = vec![0; len as usize]; - read_exact(server, data) - .map_err(Error::from) - .and_then(|(_server, data)| Ok(bincode::deserialize(&data)?)) - }) + let mut bytes = [0u8; 4]; + server.read_exact(&mut bytes[..]).await?; + + let len = BigEndian::read_u32(&bytes); + let mut data = vec![0; len as usize]; + server.read_exact(data.as_mut_slice()).await?; + + Ok(bincode::deserialize(&data)?) } /// Re-execute the current executable as a background server, and wait /// for it to start up. #[cfg(not(windows))] fn run_server_process() -> Result { - use futures::Stream; + use futures::StreamExt; use std::time::Duration; trace!("run_server_process"); let tempdir = tempfile::Builder::new().prefix("sccache").tempdir()?; let socket_path = tempdir.path().join("sock"); let mut runtime = Runtime::new()?; - let listener = tokio_uds::UnixListener::bind(&socket_path)?; let exe_path = env::current_exe()?; let workdir = exe_path.parent().expect("executable path has no parent?!"); let _child = process::Command::new(&exe_path) @@ -92,23 +87,21 @@ fn run_server_process() -> Result { .env("RUST_BACKTRACE", "1") .spawn()?; - let startup = listener.incoming().into_future().map_err(|e| e.0); - let startup = startup.map_err(Error::from).and_then(|(socket, _rest)| { + let startup = async move { + let mut listener = tokio::net::UnixListener::bind(&socket_path)?; + let socket = listener.incoming().next().await; let socket = socket.unwrap(); // incoming() never returns None - read_server_startup_status(socket) - }); + + read_server_startup_status(socket?).await + }; let timeout = Duration::from_millis(SERVER_STARTUP_TIMEOUT_MS.into()); - let timeout = Timeout::new(startup, timeout).or_else(|err| { - if err.is_elapsed() { - Ok(ServerStartup::TimedOut) - } else if err.is_inner() { - Err(err.into_inner().unwrap()) - } else { - Err(err.into_timer().unwrap().into()) + runtime.block_on(async move { + match tokio::time::timeout(timeout, startup).await { + Ok(result) => result, + Err(_elapsed) => Ok(ServerStartup::TimedOut), } - }); - runtime.block_on(timeout) + }) } #[cfg(not(windows))] @@ -146,13 +139,11 @@ fn redirect_error_log() -> Result<()> { /// Re-execute the current executable as a background server. #[cfg(windows)] fn run_server_process() -> Result { - use futures::future; + use futures::StreamExt; use std::mem; use std::os::windows::ffi::OsStrExt; use std::ptr; use std::time::Duration; - use tokio_named_pipes::NamedPipe; - use tokio_reactor::Handle; use uuid::Uuid; use winapi::shared::minwindef::{DWORD, FALSE, LPVOID, TRUE}; use winapi::um::handleapi::CloseHandle; @@ -166,22 +157,6 @@ fn run_server_process() -> Result { // Create a mini event loop and register our named pipe server let mut runtime = Runtime::new()?; let pipe_name = format!(r"\\.\pipe\{}", Uuid::new_v4().to_simple_ref()); - let server = runtime.block_on(future::lazy(|| { - NamedPipe::new( - &pipe_name, - #[allow(deprecated)] - &Handle::current(), - ) - }))?; - - // Connect a client to our server, and we'll wait below if it's still in - // progress. - - match server.connect() { - Ok(()) => {} - Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {} - Err(e) => return Err(e.into()), - } // Spawn a server which should come back and connect to us let exe_path = env::current_exe()?; @@ -250,19 +225,21 @@ fn run_server_process() -> Result { return Err(io::Error::last_os_error().into()); } - let result = read_server_startup_status(server); + let startup = async move { + let listener = parity_tokio_ipc::Endpoint::new(pipe_name); + let socket = listener.incoming()?.next().await; + let socket = socket.unwrap(); // incoming() never returns None + + read_server_startup_status(socket?).await + }; let timeout = Duration::from_millis(SERVER_STARTUP_TIMEOUT_MS.into()); - let timeout = Timeout::new(result, timeout).or_else(|err| { - if err.is_elapsed() { - Ok(ServerStartup::TimedOut) - } else if err.is_inner() { - Err(err.into_inner().unwrap()) - } else { - Err(err.into_timer().unwrap().into()) + runtime.block_on(async move { + match tokio::time::timeout(timeout, startup).await { + Ok(result) => result, + Err(_elapsed) => Ok(ServerStartup::TimedOut), } - }); - runtime.block_on(timeout) + }) } /// Attempt to connect to an sccache server listening on `port`, or start one if no server is running. @@ -518,10 +495,14 @@ where if log_enabled!(Trace) { trace!("running command: {:?}", cmd); } - let status = runtime.block_on( - cmd.spawn() - .and_then(|c| c.wait().fcontext("failed to wait for child")), - )?; + + let status = runtime.block_on(async move { + let child = cmd.spawn().await?; + child + .wait() + .await + .with_context(|| "failed to wait for a child") + })?; Ok(status.code().unwrap_or_else(|| { if let Some(sig) = status_signal(status) { @@ -678,22 +659,22 @@ pub fn run_command(cmd: Command) -> Result { #[cfg(feature = "dist-client")] Command::PackageToolchain(executable, out) => { use crate::compiler; - use futures_03::executor::ThreadPool; trace!("Command::PackageToolchain({})", executable.display()); let mut runtime = Runtime::new()?; let jobserver = unsafe { Client::new() }; let creator = ProcessCommandCreator::new(&jobserver); let env: Vec<_> = env::vars_os().collect(); - let pool = ThreadPool::builder().pool_size(1).create()?; let out_file = File::create(out)?; let cwd = env::current_dir().expect("A current working dir should exist"); - let compiler = - compiler::get_compiler_info(creator, &executable, &cwd, &env, &pool, None); - let packager = compiler.map(|c| c.0.get_toolchain_packager()); - let res = packager.and_then(|p| p.write_pkg(out_file)); - runtime.block_on(res)? + let pool = runtime.handle().clone(); + runtime.block_on(async move { + compiler::get_compiler_info(creator, &executable, &cwd, &env, &pool, None) + .await + .map(|compiler| compiler.0.get_toolchain_packager()) + .and_then(|packager| packager.write_pkg(out_file)) + })? } #[cfg(not(feature = "dist-client"))] Command::PackageToolchain(_executable, _out) => bail!( diff --git a/src/compiler/c.rs b/src/compiler/c.rs index 7ac6dab6..a82fabcf 100644 --- a/src/compiler/c.rs +++ b/src/compiler/c.rs @@ -23,8 +23,6 @@ use crate::dist; use crate::dist::pkg; use crate::mock_command::CommandCreatorSync; use crate::util::{hash_all, Digest, HashToDigest}; -use futures::Future; -use futures_03::executor::ThreadPool; use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::ffi::{OsStr, OsString}; @@ -163,7 +161,8 @@ pub enum CCompilerKind { } /// An interface to a specific C compiler. -pub trait CCompilerImpl: Clone + fmt::Debug + Send + 'static { +#[async_trait] +pub trait CCompilerImpl: Clone + fmt::Debug + Send + Sync + 'static { /// Return the kind of compiler. fn kind(&self) -> CCompilerKind; /// Return true iff this is g++ or clang++. @@ -176,7 +175,7 @@ pub trait CCompilerImpl: Clone + fmt::Debug + Send + 'static { ) -> CompilerArguments; /// Run the C preprocessor with the specified set of arguments. #[allow(clippy::too_many_arguments)] - fn preprocess( + async fn preprocess( &self, creator: &T, executable: &Path, @@ -185,7 +184,7 @@ pub trait CCompilerImpl: Clone + fmt::Debug + Send + 'static { env_vars: &[(OsString, OsString)], may_dist: bool, rewrite_includes_only: bool, - ) -> SFuture + ) -> Result where T: CommandCreatorSync; /// Generate a command that can be used to invoke the C compiler to perform @@ -205,28 +204,28 @@ impl CCompiler where I: CCompilerImpl, { - pub fn new( + pub async fn new( compiler: I, executable: PathBuf, version: Option, - pool: &ThreadPool, - ) -> SFuture> { - Box::new( - Digest::file(executable.clone(), &pool).map(move |digest| CCompiler { - executable, - executable_digest: { - if let Some(version) = version { - let mut m = Digest::new(); - m.update(digest.as_bytes()); - m.update(version.as_bytes()); - m.finish() - } else { - digest - } - }, - compiler, - }), - ) + pool: &tokio::runtime::Handle, + ) -> Result> { + let digest = Digest::file(executable.clone(), pool).await?; + + Ok(CCompiler { + executable, + executable_digest: { + if let Some(version) = version { + let mut m = Digest::new(); + m.update(digest.as_bytes()); + m.update(version.as_bytes()); + m.finish() + } else { + digest + } + }, + compiler, + }) } } @@ -265,125 +264,121 @@ impl Compiler for CCompiler { } } +#[async_trait] impl CompilerHasher for CCompilerHasher where T: CommandCreatorSync, I: CCompilerImpl, { - fn generate_hash_key( + async fn generate_hash_key( self: Box, creator: &T, cwd: PathBuf, env_vars: Vec<(OsString, OsString)>, may_dist: bool, - pool: &ThreadPool, + pool: &tokio::runtime::Handle, rewrite_includes_only: bool, - ) -> SFuture { - let me = *self; + ) -> Result { let CCompilerHasher { parsed_args, executable, executable_digest, compiler, - } = me; - let result = compiler.preprocess( - creator, - &executable, - &parsed_args, - &cwd, - &env_vars, - may_dist, - rewrite_includes_only, - ); + } = *self; + + let result = compiler + .preprocess( + creator, + &executable, + &parsed_args, + &cwd, + &env_vars, + may_dist, + rewrite_includes_only, + ) + .await; let out_pretty = parsed_args.output_pretty().into_owned(); - let result = result.map_err(move |e| { + let result = result.map_err(|e| { debug!("[{}]: preprocessor failed: {:?}", out_pretty, e); e }); - let out_pretty = parsed_args.output_pretty().into_owned(); - let extra_hashes = hash_all(&parsed_args.extra_hash_files, &pool.clone()); + + let extra_hashes = hash_all(&parsed_args.extra_hash_files, &pool.clone()).await?; let outputs = parsed_args.outputs.clone(); let args_cwd = cwd.clone(); - Box::new( - result - .or_else(move |err| { - // Errors remove all traces of potential output. - debug!("removing files {:?}", &outputs); - - let v: std::result::Result<(), std::io::Error> = - outputs.values().fold(Ok(()), |r, f| { - r.and_then(|_| { - let mut path = (&args_cwd).clone(); - path.push(&f); - match fs::metadata(&path) { - // File exists, remove it. - Ok(_) => fs::remove_file(&path), - _ => Ok(()), - } - }) - }); - if v.is_err() { - warn!("Could not remove files after preprocessing failed!\n"); - } - - match err.downcast::() { - Ok(ProcessError(output)) => { - debug!( - "[{}]: preprocessor returned error status {:?}", - out_pretty, - output.status.code() - ); - // Drop the stdout since it's the preprocessor output, - // just hand back stderr and the exit status. - bail!(ProcessError(process::Output { - stdout: vec!(), - ..output - })) + let preprocessor_result = result.or_else(move |err| { + // Errors remove all traces of potential output. + debug!("removing files {:?}", &outputs); + + let v: std::result::Result<(), std::io::Error> = + outputs.values().fold(Ok(()), |r, f| { + r.and_then(|_| { + let mut path = (&args_cwd).clone(); + path.push(&f); + match fs::metadata(&path) { + // File exists, remove it. + Ok(_) => fs::remove_file(&path), + _ => Ok(()), } - Err(err) => Err(err), - } - }) - .and_then(move |preprocessor_result| { - trace!( - "[{}]: Preprocessor output is {} bytes", - parsed_args.output_pretty(), - preprocessor_result.stdout.len() - ); + }) + }); + if v.is_err() { + warn!("Could not remove files after preprocessing failed!"); + } - Box::new(extra_hashes.and_then(move |extra_hashes| { - let key = { - hash_key( - &executable_digest, - parsed_args.language, - &parsed_args.common_args, - &extra_hashes, - &env_vars, - &preprocessor_result.stdout, - compiler.plusplus(), - ) - }; - // A compiler binary may be a symlink to another and so has the same digest, but that means - // the toolchain will not contain the correct path to invoke the compiler! Add the compiler - // executable path to try and prevent this - let weak_toolchain_key = - format!("{}-{}", executable.to_string_lossy(), executable_digest); - Ok(HashResult { - key, - compilation: Box::new(CCompilation { - parsed_args, - #[cfg(feature = "dist-client")] - preprocessed_input: preprocessor_result.stdout, - executable, - compiler, - cwd, - env_vars, - }), - weak_toolchain_key, - }) + match err.downcast::() { + Ok(ProcessError(output)) => { + debug!( + "[{}]: preprocessor returned error status {:?}", + out_pretty, + output.status.code() + ); + // Drop the stdout since it's the preprocessor output, + // just hand back stderr and the exit status. + bail!(ProcessError(process::Output { + stdout: vec!(), + ..output })) - }), - ) + } + Err(err) => Err(err), + } + })?; + + trace!( + "[{}]: Preprocessor output is {} bytes", + parsed_args.output_pretty(), + preprocessor_result.stdout.len() + ); + + let key = { + hash_key( + &executable_digest, + parsed_args.language, + &parsed_args.common_args, + &extra_hashes, + &env_vars, + &preprocessor_result.stdout, + compiler.plusplus(), + ) + }; + // A compiler binary may be a symlink to another and so has the same digest, but that means + // the toolchain will not contain the correct path to invoke the compiler! Add the compiler + // executable path to try and prevent this + let weak_toolchain_key = format!("{}-{}", executable.to_string_lossy(), executable_digest); + Ok(HashResult { + key, + compilation: Box::new(CCompilation { + parsed_args, + #[cfg(feature = "dist-client")] + preprocessed_input: preprocessor_result.stdout, + executable, + compiler, + cwd, + env_vars, + }), + weak_toolchain_key, + }) } fn color_mode(&self) -> ColorMode { diff --git a/src/compiler/clang.rs b/src/compiler/clang.rs index fa65a5d5..39c6af1f 100644 --- a/src/compiler/clang.rs +++ b/src/compiler/clang.rs @@ -21,9 +21,9 @@ use crate::compiler::{gcc, write_temp_file, Cacheable, CompileCommand, CompilerA use crate::dist; use crate::mock_command::{CommandCreator, CommandCreatorSync, RunCommand}; use crate::util::{run_input_output, OsStrExt}; -use futures::future::{self, Future}; use std::ffi::OsString; use std::fs::File; +use std::future::Future; use std::io::{self, Write}; use std::path::{Path, PathBuf}; use std::process; @@ -37,6 +37,7 @@ pub struct Clang { pub clangplusplus: bool, } +#[async_trait] impl CCompilerImpl for Clang { fn kind(&self) -> CCompilerKind { CCompilerKind::Clang @@ -57,7 +58,8 @@ impl CCompilerImpl for Clang { ) } - fn preprocess( + #[allow(clippy::too_many_arguments)] + async fn preprocess( &self, creator: &T, executable: &Path, @@ -66,7 +68,7 @@ impl CCompilerImpl for Clang { env_vars: &[(OsString, OsString)], may_dist: bool, rewrite_includes_only: bool, - ) -> SFuture + ) -> Result where T: CommandCreatorSync, { @@ -80,6 +82,7 @@ impl CCompilerImpl for Clang { self.kind(), rewrite_includes_only, ) + .await } fn generate_compile_commands( @@ -136,8 +139,8 @@ mod test { use crate::compiler::*; use crate::mock_command::*; use crate::test::utils::*; - use futures::Future; use std::collections::HashMap; + use std::future::Future; use std::path::PathBuf; fn parse_arguments_(arguments: Vec) -> CompilerArguments { diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 4e3be3e0..145088e6 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -26,10 +26,8 @@ use crate::dist; use crate::dist::pkg; use crate::lru_disk_cache; use crate::mock_command::{exit_status, CommandChild, CommandCreatorSync, RunCommand}; -use crate::util::{fmt_duration_as_secs, ref_env, run_input_output, SpawnExt}; +use crate::util::{fmt_duration_as_secs, ref_env, run_input_output}; use filetime::FileTime; -use futures::Future; -use futures_03::executor::ThreadPool; use std::borrow::Cow; use std::collections::HashMap; use std::ffi::OsString; @@ -37,14 +35,15 @@ use std::fmt; #[cfg(feature = "dist-client")] use std::fs; use std::fs::File; +use std::future::Future; use std::io::prelude::*; use std::path::{Path, PathBuf}; +use std::pin::Pin; use std::process::{self, Stdio}; use std::str; use std::sync::Arc; use std::time::{Duration, Instant}; use tempfile::TempDir; -use tokio_timer::Timeout; use crate::errors::*; @@ -66,7 +65,7 @@ pub struct CompileCommand { } impl CompileCommand { - pub fn execute(self, creator: &T) -> SFuture + pub async fn execute(self, creator: &T) -> Result where T: CommandCreatorSync, { @@ -75,7 +74,7 @@ impl CompileCommand { .env_clear() .envs(self.env_vars) .current_dir(self.cwd); - Box::new(run_input_output(cmd, None)) + run_input_output(cmd, None).await } } @@ -150,14 +149,21 @@ where creator: T, cwd: PathBuf, env_vars: &[(OsString, OsString)], - ) -> SFuture<(PathBuf, FileTime)>; + ) -> Pin> + Send + 'static>>; /// Create a clone of `Self` and puts it in a `Box` fn box_clone(&self) -> Box>; } +impl Clone for Box> { + fn clone(&self) -> Box> { + self.box_clone() + } +} + /// An interface to a compiler for hash key generation, the result of /// argument parsing. +#[async_trait] pub trait CompilerHasher: fmt::Debug + Send + 'static where T: CommandCreatorSync, @@ -165,15 +171,15 @@ where /// Given information about a compiler command, generate a hash key /// that can be used for cache lookups, as well as any additional /// information that can be reused for compilation if necessary. - fn generate_hash_key( + async fn generate_hash_key( self: Box, creator: &T, cwd: PathBuf, env_vars: Vec<(OsString, OsString)>, may_dist: bool, - pool: &ThreadPool, + pool: &tokio::runtime::Handle, rewrite_includes_only: bool, - ) -> SFuture; + ) -> Result; /// Return the state of any `--color` option passed to the compiler. fn color_mode(&self) -> ColorMode; @@ -181,228 +187,197 @@ where /// Look up a cached compile result in `storage`. If not found, run the /// compile and store the result. #[allow(clippy::too_many_arguments)] - fn get_cached_or_compile( + async fn get_cached_or_compile( self: Box, - dist_client: Result>>, + dist_client: Option>, creator: T, storage: Arc, arguments: Vec, cwd: PathBuf, env_vars: Vec<(OsString, OsString)>, cache_control: CacheControl, - pool: ThreadPool, - ) -> SFuture<(CompileResult, process::Output)> { + pool: tokio::runtime::Handle, + ) -> Result<(CompileResult, process::Output)> { let out_pretty = self.output_pretty().into_owned(); debug!("[{}]: get_cached_or_compile: {:?}", out_pretty, arguments); let start = Instant::now(); - let may_dist = matches!(dist_client, Ok(Some(_))); + let may_dist = dist_client.is_some(); let rewrite_includes_only = match dist_client { - Ok(Some(ref client)) => client.rewrite_includes_only(), + Some(ref client) => client.rewrite_includes_only(), _ => false, }; - let result = self.generate_hash_key( - &creator, - cwd.clone(), - env_vars, - may_dist, - &pool, - rewrite_includes_only, + let result = self + .generate_hash_key( + &creator, + cwd.clone(), + env_vars, + may_dist, + &pool, + rewrite_includes_only, + ) + .await; + debug!( + "[{}]: generate_hash_key took {}", + out_pretty, + fmt_duration_as_secs(&start.elapsed()) ); - Box::new(result.then(move |res| -> SFuture<_> { - debug!( - "[{}]: generate_hash_key took {}", - out_pretty, - fmt_duration_as_secs(&start.elapsed()) - ); - let (key, compilation, weak_toolchain_key) = match res { - Err(e) => { - return match e.downcast::() { - Ok(ProcessError(output)) => f_ok((CompileResult::Error, output)), - Err(e) => f_err(e), - }; + let (key, compilation, weak_toolchain_key) = match result { + Err(e) => { + return match e.downcast::() { + Ok(ProcessError(output)) => Ok((CompileResult::Error, output)), + Err(e) => Err(e), + }; + } + Ok(HashResult { + key, + compilation, + weak_toolchain_key, + }) => (key, compilation, weak_toolchain_key), + }; + trace!("[{}]: Hash key: {}", out_pretty, key); + // If `ForceRecache` is enabled, we won't check the cache. + let start = Instant::now(); + let cache_status = async { + if cache_control == CacheControl::ForceRecache { + Ok(Cache::Recache) + } else { + storage.get(&key).await + } + }; + + // Set a maximum time limit for the cache to respond before we forge + // ahead ourselves with a compilation. + let timeout = Duration::new(60, 0); + let cache_status = tokio::time::timeout(timeout, cache_status); + + // Check the result of the cache lookup. + let duration = start.elapsed(); + let outputs = compilation + .outputs() + .map(|(key, path)| (key.to_string(), cwd.join(path))) + .collect::>(); + + let lookup = match cache_status.await { + Ok(Ok(Cache::Hit(mut entry))) => { + debug!( + "[{}]: Cache hit in {}", + out_pretty, + fmt_duration_as_secs(&duration) + ); + let stdout = entry.get_stdout(); + let stderr = entry.get_stderr(); + let output = process::Output { + status: exit_status(0), + stdout, + stderr, + }; + let hit = CompileResult::CacheHit(duration); + match entry.extract_objects(outputs.clone(), &pool).await { + Ok(()) => Ok(CacheLookupResult::Success(hit, output)), + Err(e) => { + if e.downcast_ref::().is_some() { + debug!("[{}]: Failed to decompress object", out_pretty); + Ok(CacheLookupResult::Miss(MissType::CacheReadError)) + } else { + Err(e) + } + } } - Ok(HashResult { - key, + } + Ok(Ok(Cache::Miss)) => { + debug!( + "[{}]: Cache miss in {}", + out_pretty, + fmt_duration_as_secs(&duration) + ); + Ok(CacheLookupResult::Miss(MissType::Normal)) + } + Ok(Ok(Cache::Recache)) => { + debug!( + "[{}]: Cache recache in {}", + out_pretty, + fmt_duration_as_secs(&duration) + ); + Ok(CacheLookupResult::Miss(MissType::ForcedRecache)) + } + Ok(Err(err)) => { + error!("[{}]: Cache read error: {:?}", out_pretty, err); + Ok(CacheLookupResult::Miss(MissType::CacheReadError)) + } + Err(_elapsed) => { + debug!( + "[{}]: Cache timed out {}", + out_pretty, + fmt_duration_as_secs(&duration) + ); + Ok(CacheLookupResult::Miss(MissType::TimedOut)) + } + }?; + + match lookup { + CacheLookupResult::Success(compile_result, output) => { + Ok::<_, Error>((compile_result, output)) + } + CacheLookupResult::Miss(miss_type) => { + // Cache miss, so compile it. + let start = Instant::now(); + + let (cacheable, dist_type, compiler_result) = dist_or_local_compile( + dist_client, + creator, + cwd, compilation, weak_toolchain_key, - }) => (key, compilation, weak_toolchain_key), - }; - trace!("[{}]: Hash key: {}", out_pretty, key); - // If `ForceRecache` is enabled, we won't check the cache. - let start = Instant::now(); - let cache_status = if cache_control == CacheControl::ForceRecache { - f_ok(Cache::Recache) - } else { - storage.get(&key) - }; + out_pretty.clone(), + ) + .await?; + let duration = start.elapsed(); + if !compiler_result.status.success() { + debug!( + "[{}]: Compiled but failed, not storing in cache", + out_pretty + ); + return Ok((CompileResult::CompileFailed, compiler_result)); + } + if cacheable != Cacheable::Yes { + // Not cacheable + debug!("[{}]: Compiled but not cacheable", out_pretty); + return Ok((CompileResult::NotCacheable, compiler_result)); + } + debug!( + "[{}]: Compiled in {}, storing in cache", + out_pretty, + fmt_duration_as_secs(&duration) + ); + let mut entry = CacheWrite::from_objects(outputs, &pool) + .await + .context("failed to zip up compiler outputs")?; - // Set a maximum time limit for the cache to respond before we forge - // ahead ourselves with a compilation. - let timeout = Duration::new(60, 0); - let cache_status = Timeout::new(cache_status, timeout); + entry.put_stdout(&compiler_result.stdout)?; + entry.put_stderr(&compiler_result.stderr)?; - // Check the result of the cache lookup. - Box::new(cache_status.then(move |result| { let out_pretty2 = out_pretty.clone(); - let duration = start.elapsed(); - let outputs = compilation - .outputs() - .map(|(key, path)| (key.to_string(), cwd.join(path))) - .collect::>(); - - let miss_type = Box::new(match result { - Ok(Cache::Hit(mut entry)) => { - debug!( - "[{}]: Cache hit in {}", - out_pretty, - fmt_duration_as_secs(&duration) - ); - let stdout = entry.get_stdout(); - let stderr = entry.get_stderr(); - let write = entry.extract_objects(outputs.clone(), &pool); - let output = process::Output { - status: exit_status(0), - stdout, - stderr, - }; - let hit = CompileResult::CacheHit(duration); - Box::new(write.then(move |result| match result { - Ok(()) => f_ok(CacheLookupResult::Success(hit, output)), - Err(e) => { - if e.downcast_ref::().is_some() { - debug!("[{}]: Failed to decompress object", out_pretty); - f_ok(CacheLookupResult::Miss(MissType::CacheReadError)) - } else { - f_err(e) - } - } - })) - } - Ok(Cache::Miss) => { - debug!( - "[{}]: Cache miss in {}", - out_pretty, - fmt_duration_as_secs(&duration) - ); - f_ok(CacheLookupResult::Miss(MissType::Normal)) - } - Ok(Cache::Recache) => { - debug!( - "[{}]: Cache recache in {}", - out_pretty, - fmt_duration_as_secs(&duration) - ); - f_ok(CacheLookupResult::Miss(MissType::ForcedRecache)) - } - Err(err) => { - if err.is_elapsed() { - debug!( - "[{}]: Cache timed out {}", - out_pretty, - fmt_duration_as_secs(&duration) - ); - f_ok(CacheLookupResult::Miss(MissType::TimedOut)) - } else { - error!("[{}]: Cache read error: {}", out_pretty, err); - if err.is_inner() { - let err = err.into_inner().unwrap(); - for e in err.chain().skip(1) { - error!("[{}] \t{}", out_pretty, e); - } - } - f_ok(CacheLookupResult::Miss(MissType::CacheReadError)) - } + // Try to finish storing the newly-written cache + // entry. We'll get the result back elsewhere. + let future = async move { + match storage.put(&key, entry).await { + Ok(_) => debug!("[{}]: Stored in cache successfully!", out_pretty2), + Err(ref e) => debug!("[{}]: Cache write error: {:?}", out_pretty2, e), } - }); - Box::new(miss_type.and_then(move |result| { - match result { - CacheLookupResult::Success(compile_result, output) => { - f_ok((compile_result, output)) - } - CacheLookupResult::Miss(miss_type) => { - // Cache miss, so compile it. - let start = Instant::now(); - let compile = dist_or_local_compile( - dist_client, - creator, - cwd, - compilation, - weak_toolchain_key, - out_pretty2.clone(), - ); - - Box::new(compile.and_then( - move |(cacheable, dist_type, compiler_result)| { - let duration = start.elapsed(); - if !compiler_result.status.success() { - debug!( - "[{}]: Compiled but failed, not storing in cache", - out_pretty2 - ); - return f_ok((CompileResult::CompileFailed, compiler_result)) - as SFuture<_>; - } - if cacheable != Cacheable::Yes { - // Not cacheable - debug!("[{}]: Compiled but not cacheable", out_pretty2); - return f_ok(( - CompileResult::NotCacheable, - compiler_result, - )); - } - debug!( - "[{}]: Compiled in {}, storing in cache", - out_pretty2, - fmt_duration_as_secs(&duration) - ); - let write = CacheWrite::from_objects(outputs, &pool); - let write = write.fcontext("failed to zip up compiler outputs"); - let o = out_pretty2.clone(); - Box::new( - write - .and_then(move |mut entry| { - entry.put_stdout(&compiler_result.stdout)?; - entry.put_stderr(&compiler_result.stderr)?; - - // Try to finish storing the newly-written cache - // entry. We'll get the result back elsewhere. - let future = - storage.put(&key, entry).then(move |res| { - match res { - Ok(_) => debug!( - "[{}]: Stored in cache successfully!", - out_pretty2 - ), - Err(ref e) => debug!( - "[{}]: Cache write error: {:?}", - out_pretty2, e - ), - } - res.map(|duration| CacheWriteInfo { - object_file_pretty: out_pretty2, - duration, - }) - }); - let future = Box::new(future); - Ok(( - CompileResult::CacheMiss( - miss_type, dist_type, duration, future, - ), - compiler_result, - )) - }) - .fwith_context(move || { - format!("failed to store `{}` to cache", o) - }), - ) - }, - )) - } - } - })) - })) - })) + Ok(CacheWriteInfo { + object_file_pretty: out_pretty2, + duration, + }) + }; + let future = Box::pin(future); + Ok(( + CompileResult::CacheMiss(miss_type, dist_type, duration, future), + compiler_result, + )) + } + } + .with_context(|| format!("failed to store `{}` to cache", out_pretty)) } /// A descriptive string about the file that we're going to be producing. @@ -415,210 +390,235 @@ where } #[cfg(not(feature = "dist-client"))] -fn dist_or_local_compile( - _dist_client: Result>>, +async fn dist_or_local_compile( + _dist_client: Option>, creator: T, _cwd: PathBuf, compilation: Box, _weak_toolchain_key: String, out_pretty: String, -) -> SFuture<(Cacheable, DistType, process::Output)> +) -> Result<(Cacheable, DistType, process::Output)> where T: CommandCreatorSync, { let mut path_transformer = dist::PathTransformer::default(); - let compile_commands = compilation + let (compile_cmd, _dist_compile_cmd, cacheable) = compilation .generate_compile_commands(&mut path_transformer, true) - .context("Failed to generate compile commands"); - let (compile_cmd, _dist_compile_cmd, cacheable) = match compile_commands { - Ok(cmds) => cmds, - Err(e) => return f_err(e), - }; + .context("Failed to generate compile commands")?; debug!("[{}]: Compiling locally", out_pretty); - Box::new( - compile_cmd - .execute(&creator) - .map(move |o| (cacheable, DistType::NoDist, o)), - ) + compile_cmd + .execute(&creator) + .await + .map(move |o| (cacheable, DistType::NoDist, o)) } #[cfg(feature = "dist-client")] -fn dist_or_local_compile( - dist_client: Result>>, +async fn dist_or_local_compile( + dist_client: Option>, creator: T, cwd: PathBuf, compilation: Box, weak_toolchain_key: String, out_pretty: String, -) -> SFuture<(Cacheable, DistType, process::Output)> +) -> Result<(Cacheable, DistType, process::Output)> where T: CommandCreatorSync, { - use futures::future; use std::io; let rewrite_includes_only = match dist_client { - Ok(Some(ref client)) => client.rewrite_includes_only(), + Some(ref client) => client.rewrite_includes_only(), _ => false, }; let mut path_transformer = dist::PathTransformer::default(); - let compile_commands = compilation + let (compile_cmd, dist_compile_cmd, cacheable) = compilation .generate_compile_commands(&mut path_transformer, rewrite_includes_only) - .context("Failed to generate compile commands"); - let (compile_cmd, dist_compile_cmd, cacheable) = match compile_commands { - Ok(cmds) => cmds, - Err(e) => return f_err(e), - }; + .context("Failed to generate compile commands")?; let dist_client = match dist_client { - Ok(Some(dc)) => dc, - Ok(None) => { + Some(dc) => dc, + None => { debug!("[{}]: Compiling locally", out_pretty); - return Box::new( - compile_cmd - .execute(&creator) - .map(move |o| (cacheable, DistType::NoDist, o)), - ); - } - Err(e) => { - return f_err(e); + return compile_cmd + .execute(&creator) + .await + .map(move |o| (cacheable, DistType::NoDist, o)); } }; debug!("[{}]: Attempting distributed compilation", out_pretty); - let compile_out_pretty = out_pretty.clone(); - let compile_out_pretty2 = out_pretty.clone(); - let compile_out_pretty3 = out_pretty.clone(); - let compile_out_pretty4 = out_pretty; + let out_pretty2 = out_pretty.clone(); + let local_executable = compile_cmd.executable.clone(); - let local_executable2 = local_executable.clone(); - // TODO: the number of map_errs is subideal, but there's no futures-based carrier trait AFAIK - Box::new(future::result(dist_compile_cmd.context("Could not create distributed compile command")) - .and_then(move |dist_compile_cmd| { - debug!("[{}]: Creating distributed compile request", compile_out_pretty); - let dist_output_paths = compilation.outputs() - .map(|(_key, path)| path_transformer.as_dist_abs(&cwd.join(path))) - .collect::>() - .context("Failed to adapt an output path for distributed compile")?; - compilation.into_dist_packagers(path_transformer) - .map(|packagers| (dist_compile_cmd, packagers, dist_output_paths)) - }) - .and_then(move |(mut dist_compile_cmd, (inputs_packager, toolchain_packager, outputs_rewriter), dist_output_paths)| { - debug!("[{}]: Identifying dist toolchain for {:?}", compile_out_pretty2, local_executable); - dist_client.put_toolchain(&local_executable, &weak_toolchain_key, toolchain_packager) - .and_then(|(dist_toolchain, maybe_dist_compile_executable)| { - let mut tc_archive = None; - if let Some((dist_compile_executable, archive_path)) = maybe_dist_compile_executable { - dist_compile_cmd.executable = dist_compile_executable; - tc_archive = Some(archive_path); - } - Ok((dist_client, dist_compile_cmd, dist_toolchain, inputs_packager, outputs_rewriter, dist_output_paths, tc_archive)) - }) - }) - .and_then(move |(dist_client, dist_compile_cmd, dist_toolchain, inputs_packager, outputs_rewriter, dist_output_paths, tc_archive)| { - debug!("[{}]: Requesting allocation", compile_out_pretty3); - dist_client.do_alloc_job(dist_toolchain.clone()) - .and_then(move |jares| { - let alloc = match jares { - dist::AllocJobResult::Success { job_alloc, need_toolchain: true } => { - debug!("[{}]: Sending toolchain {} for job {}", - compile_out_pretty3, dist_toolchain.archive_id, job_alloc.job_id); - Box::new(dist_client.do_submit_toolchain(job_alloc.clone(), dist_toolchain) - .and_then(move |res| { - match res { - dist::SubmitToolchainResult::Success => Ok(job_alloc), - dist::SubmitToolchainResult::JobNotFound => - bail!("Job {} not found on server", job_alloc.job_id), - dist::SubmitToolchainResult::CannotCache => - bail!("Toolchain for job {} could not be cached by server", job_alloc.job_id), - } - }) - .fcontext("Could not submit toolchain")) - }, - dist::AllocJobResult::Success { job_alloc, need_toolchain: false } => - f_ok(job_alloc), - dist::AllocJobResult::Fail { msg } => - f_err(anyhow!("Failed to allocate job").context(msg)), - }; - alloc - .and_then(move |job_alloc| { - let job_id = job_alloc.job_id; - let server_id = job_alloc.server_id; - debug!("[{}]: Running job", compile_out_pretty3); - dist_client.do_run_job(job_alloc, dist_compile_cmd, dist_output_paths, inputs_packager) - .map(move |res| ((job_id, server_id), res)) - .fwith_context(move || format!("could not run distributed compilation job on {:?}", server_id)) - }) - }) - .and_then(move |((job_id, server_id), (jres, path_transformer))| { - let jc = match jres { - dist::RunJobResult::Complete(jc) => jc, - dist::RunJobResult::JobNotFound => bail!("Job {} not found on server", job_id), - }; - info!("fetched {:?}", jc.outputs.iter().map(|&(ref p, ref bs)| (p, bs.lens().to_string())).collect::>()); - let mut output_paths: Vec = vec![]; - macro_rules! try_or_cleanup { - ($v:expr) => {{ - match $v { - Ok(v) => v, - Err(e) => { - // Do our best to clear up. We may end up deleting a file that we just wrote over - // the top of, but it's better to clear up too much than too little - for local_path in output_paths.iter() { - if let Err(e) = fs::remove_file(local_path) { - if e.kind() != io::ErrorKind::NotFound { - warn!("{} while attempting to clear up {}", e, local_path.display()) - } - } - } - return Err(e) - }, - } - }}; + let local_executable2 = compile_cmd.executable.clone(); + + let do_dist_compile = async move { + let mut dist_compile_cmd = + dist_compile_cmd.context("Could not create distributed compile command")?; + debug!("[{}]: Creating distributed compile request", out_pretty); + let dist_output_paths = compilation + .outputs() + .map(|(_key, path)| path_transformer.as_dist_abs(&cwd.join(path))) + .collect::>() + .context("Failed to adapt an output path for distributed compile")?; + let (inputs_packager, toolchain_packager, outputs_rewriter) = + compilation.into_dist_packagers(path_transformer)?; + + debug!( + "[{}]: Identifying dist toolchain for {:?}", + out_pretty, local_executable + ); + let (dist_toolchain, maybe_dist_compile_executable) = dist_client + .put_toolchain(local_executable, weak_toolchain_key, toolchain_packager) + .await?; + let mut tc_archive = None; + if let Some((dist_compile_executable, archive_path)) = maybe_dist_compile_executable { + dist_compile_cmd.executable = dist_compile_executable; + tc_archive = Some(archive_path); + } + + debug!("[{}]: Requesting allocation", out_pretty); + let jares = dist_client.do_alloc_job(dist_toolchain.clone()).await?; + let job_alloc = match jares { + dist::AllocJobResult::Success { + job_alloc, + need_toolchain: true, + } => { + debug!( + "[{}]: Sending toolchain {} for job {}", + out_pretty, dist_toolchain.archive_id, job_alloc.job_id + ); + + match dist_client + .do_submit_toolchain(job_alloc.clone(), dist_toolchain) + .await + .map_err(|e| e.context("Could not submit toolchain"))? + { + dist::SubmitToolchainResult::Success => Ok(job_alloc), + dist::SubmitToolchainResult::JobNotFound => { + bail!("Job {} not found on server", job_alloc.job_id) } + dist::SubmitToolchainResult::CannotCache => bail!( + "Toolchain for job {} could not be cached by server", + job_alloc.job_id + ), + } + } + dist::AllocJobResult::Success { + job_alloc, + need_toolchain: false, + } => Ok(job_alloc), + dist::AllocJobResult::Fail { msg } => { + Err(anyhow!("Failed to allocate job").context(msg)) + } + }?; + let job_id = job_alloc.job_id; + let server_id = job_alloc.server_id; + debug!("[{}]: Running job", out_pretty); + let ((job_id, server_id), (jres, path_transformer)) = dist_client + .do_run_job( + job_alloc, + dist_compile_cmd, + dist_output_paths, + inputs_packager, + ) + .await + .map(move |res| ((job_id, server_id), res)) + .with_context(|| { + format!( + "could not run distributed compilation job on {:?}", + server_id + ) + })?; - for (path, output_data) in jc.outputs { - let len = output_data.lens().actual; - let local_path = try_or_cleanup!(path_transformer.to_local(&path) - .with_context(|| format!("unable to transform output path {}", path))); - output_paths.push(local_path); - // Do this first so cleanup works correctly - let local_path = output_paths.last().expect("nothing in vec after push"); + let jc = match jres { + dist::RunJobResult::Complete(jc) => jc, + dist::RunJobResult::JobNotFound => bail!("Job {} not found on server", job_id), + }; + info!( + "fetched {:?}", + jc.outputs + .iter() + .map(|&(ref p, ref bs)| (p, bs.lens().to_string())) + .collect::>() + ); + let mut output_paths: Vec = vec![]; + macro_rules! try_or_cleanup { + ($v:expr) => {{ + match $v { + Ok(v) => v, + Err(e) => { + // Do our best to clear up. We may end up deleting a file that we just wrote over + // the top of, but it's better to clear up too much than too little + for local_path in output_paths.iter() { + if let Err(e) = fs::remove_file(local_path) { + if e.kind() != io::ErrorKind::NotFound { + warn!("{} while attempting to clear up {}", e, local_path.display()) + } + } + } + return Err(e) + }, + } + }}; + } - let mut file = try_or_cleanup!(File::create(&local_path) - .with_context(|| format!("Failed to create output file {}", local_path.display()))); - let count = try_or_cleanup!(io::copy(&mut output_data.into_reader(), &mut file) - .with_context(|| format!("Failed to write output to {}", local_path.display()))); + for (path, output_data) in jc.outputs { + let len = output_data.lens().actual; + let local_path = try_or_cleanup!(path_transformer + .to_local(&path) + .with_context(|| format!("unable to transform output path {}", path))); + output_paths.push(local_path); + // Do this first so cleanup works correctly + let local_path = output_paths.last().expect("nothing in vec after push"); + + let mut file = try_or_cleanup!(File::create(&local_path) + .with_context(|| format!("Failed to create output file {}", local_path.display()))); + let count = try_or_cleanup!(io::copy(&mut output_data.into_reader(), &mut file) + .with_context(|| format!("Failed to write output to {}", local_path.display()))); + + assert!(count == len); + } + let extra_inputs = match tc_archive { + Some(p) => vec![p], + None => vec![], + }; + try_or_cleanup!(outputs_rewriter + .handle_outputs(&path_transformer, &output_paths, &extra_inputs) + .with_context(|| "failed to rewrite outputs from compile")); + Ok((DistType::Ok(server_id), jc.output.into())) + }; - assert!(count == len); - } - let extra_inputs = match tc_archive { - Some(p) => vec![p], - None => vec![], - }; - try_or_cleanup!(outputs_rewriter.handle_outputs(&path_transformer, &output_paths, &extra_inputs) - .with_context(|| "failed to rewrite outputs from compile")); - Ok((DistType::Ok(server_id), jc.output.into())) - }) - }) - .or_else(move |e| { + use futures::TryFutureExt; + do_dist_compile + .or_else(move |e| async move { if let Some(HttpClientError(_)) = e.downcast_ref::() { - f_err(e) - } else if let Some(lru_disk_cache::Error::FileTooLarge) = e.downcast_ref::() { - f_err(anyhow!( + Err(e) + } else if let Some(lru_disk_cache::Error::FileTooLarge) = + e.downcast_ref::() + { + Err(anyhow!( "Could not cache dist toolchain for {:?} locally. - Increase `toolchain_cache_size` or decrease the toolchain archive size.", - local_executable2)) + Increase `toolchain_cache_size` or decrease the toolchain archive size.", + local_executable2 + )) } else { // `{:#}` prints the error and the causes in a single line. let errmsg = format!("{:#}", e); - warn!("[{}]: Could not perform distributed compile, falling back to local: {}", compile_out_pretty4, errmsg); - Box::new(compile_cmd.execute(&creator).map(|o| (DistType::Error, o))) + warn!( + "[{}]: Could not perform distributed compile, falling back to local: {}", + out_pretty2, errmsg + ); + + compile_cmd + .execute(&creator) + .await + .map(|o| (DistType::Error, o)) } }) - .map(move |(dt, o)| (cacheable, dt, o)) - ) + .map_ok(move |(dt, o)| (cacheable, dt, o)) + .await } impl Clone for Box> { @@ -628,7 +628,7 @@ impl Clone for Box> { } /// An interface to a compiler for actually invoking compilation. -pub trait Compilation { +pub trait Compilation: Send { /// Given information about a compiler command, generate a command that can /// execute the compiler. fn generate_compile_commands( @@ -652,7 +652,7 @@ pub trait Compilation { } #[cfg(feature = "dist-client")] -pub trait OutputsRewriter { +pub trait OutputsRewriter: Send { /// Perform any post-compilation handling of outputs, given a Vec of the dist_path and local_path fn handle_outputs( self: Box, @@ -755,7 +755,12 @@ pub enum CompileResult { /// /// The `CacheWriteFuture` will resolve when the result is finished /// being stored in the cache. - CacheMiss(MissType, DistType, Duration, SFuture), + CacheMiss( + MissType, + DistType, + Duration, + Pin> + Send>>, + ), /// Not in cache, but the compilation result was determined to be not cacheable. NotCacheable, /// Not in cache, but compilation failed. @@ -833,31 +838,32 @@ pub enum CacheControl { /// /// Note that when the `TempDir` is dropped it will delete all of its contents /// including the path returned. -pub fn write_temp_file( - pool: &ThreadPool, +pub async fn write_temp_file( + pool: &tokio::runtime::Handle, path: &Path, contents: Vec, -) -> SFuture<(TempDir, PathBuf)> { +) -> Result<(TempDir, PathBuf)> { let path = path.to_owned(); - pool.spawn_fn(move || -> Result<_> { + pool.spawn_blocking(move || { let dir = tempfile::Builder::new().prefix("sccache").tempdir()?; let src = dir.path().join(path); let mut file = File::create(&src)?; file.write_all(&contents)?; - Ok((dir, src)) + Ok::<_, anyhow::Error>((dir, src)) }) - .fcontext("failed to write temporary file") + .await? + .context("failed to write temporary file") } /// If `executable` is a known compiler, return `Some(Box)`. -fn detect_compiler( +async fn detect_compiler( creator: T, executable: &Path, cwd: &Path, env: &[(OsString, OsString)], - pool: &ThreadPool, + pool: &tokio::runtime::Handle, dist_archive: Option, -) -> SFuture<(Box>, Option>>)> +) -> Result<(Box>, Option>>)> where T: CommandCreatorSync, { @@ -865,7 +871,7 @@ where // First, see if this looks like rustc. let filename = match executable.file_stem() { - None => return f_err(anyhow!("could not determine compiler kind")), + None => bail!("could not determine compiler kind"), Some(f) => f, }; let filename = filename.to_string_lossy().to_lowercase(); @@ -876,116 +882,106 @@ where let mut child = creator.clone().new_command_sync(executable); child.env_clear().envs(ref_env(env)).args(&["-vV"]); - Box::new(run_input_output(child, None).map(|output| { + run_input_output(child, None).await.map(|output| { if let Ok(stdout) = String::from_utf8(output.stdout.clone()) { if stdout.starts_with("rustc ") { return Some(Ok(stdout)); } } Some(Err(ProcessError(output))) - })) + })? } else { - f_ok(None) + None }; let creator1 = creator.clone(); - let creator2 = creator.clone(); let executable = executable.to_owned(); let executable2 = executable.clone(); - let env1 = env.to_owned(); - let env2 = env.to_owned(); - let env3 = env.to_owned(); let pool = pool.clone(); let cwd = cwd.to_owned(); - Box::new( - rustc_vv - .and_then(move |rustc_vv| match rustc_vv { - Some(Ok(rustc_verbose_version)) => { - debug!("Found rustc"); - - Box::new( - RustupProxy::find_proxy_executable::(&executable2,"rustup", creator, &env1) - .and_then(move |proxy : Result>| -> SFuture<(Option, PathBuf)> { - match proxy { - Ok(Some(proxy)) => { - trace!("Found rustup proxy executable"); - let fut = - proxy - .resolve_proxied_executable(creator1, cwd, &env2) - .then(move |res| { - // take the pathbuf for rustc as resolved by the proxy - match res { - Ok((resolved_path, _time)) => { - trace!("Resolved path with rustup proxy {:?}", &resolved_path); - f_ok((Some(proxy), resolved_path)) - }, - Err(e) => { - trace!("Could not resolve compiler with rustup proxy: {}", e); - f_ok((None, executable)) - }, - } - }); - Box::new(fut) - }, - Ok(None) => { - trace!("Did not find rustup"); - f_ok((None, executable)) - }, - Err(e) => { - trace!("Did not find rustup due to {}", e); - f_ok((None, executable)) - }, + match rustc_vv { + Some(Ok(rustc_verbose_version)) => { + debug!("Found rustc"); + + let proxy = RustupProxy::find_proxy_executable::( + &executable2, + "rustup", + creator.clone(), + &env, + ); + use futures::TryFutureExt; + let res = proxy.and_then(move |proxy| async move { + match proxy { + Ok(Some(proxy)) => { + trace!("Found rustup proxy executable"); + // take the pathbuf for rustc as resolved by the proxy + match proxy.resolve_proxied_executable(creator1, cwd, &env).await { + Ok((resolved_path, _time)) => { + trace!("Resolved path with rustup proxy {:?}", &resolved_path); + Ok((Some(proxy), resolved_path)) } - }) - .then(move |res: Result<(Option, PathBuf)>| { - let (proxy, resolved_rustc) : (_, PathBuf) - = res - .map(|(proxy,resolved_compiler_executable)| { - ( - proxy.map(Box::new).map(|x : Box| { - x as Box> - }), - resolved_compiler_executable - ) - }) - .unwrap_or_else(|_e| { - trace!("Compiling rust without proxy"); - (None, executable2) - }); - - Rust::new( - creator2, - resolved_rustc, - &env3, - &rustc_verbose_version, - dist_archive, - pool, - ) - .map(|c| { - ( - Box::new(c) as Box >, - proxy as Option>> - ) - }) + Err(e) => { + trace!("Could not resolve compiler with rustup proxy: {}", e); + Ok((None, executable)) + } + } } - ) + Ok(None) => { + trace!("Did not find rustup"); + Ok((None, executable)) + } + Err(e) => { + trace!("Did not find rustup due to {}, compiling without proxy", e); + Ok((None, executable)) + } + } + }); + + let (proxy, resolved_rustc) = res + .await + .map(|(proxy, resolved_compiler_executable)| { + ( + proxy + .map(Box::new) + .map(|x: Box| x as Box>), + resolved_compiler_executable, + ) + }) + .unwrap_or_else(|_e| { + trace!("Compiling rust without proxy"); + (None, executable2) + }); + + Rust::new( + creator, + resolved_rustc, + &env, + &rustc_verbose_version, + dist_archive, + pool, ) - } - Some(Err(e)) => f_err(e), - None => { - let cc = detect_c_compiler(creator, executable, env1.to_vec(), pool); - Box::new(cc.map(|c : Box>| { (c, None ) })) - }, - }) - ) + .await + .map(|c| { + ( + Box::new(c) as Box>, + proxy as Option>>, + ) + }) + } + Some(Err(e)) => Err(e).context("Failed to launch subprocess for compiler determination"), + None => { + let cc = detect_c_compiler(creator, executable, env.to_vec(), pool).await; + cc.map(|c| (c, None)) + } + } } -fn detect_c_compiler( +async fn detect_c_compiler( creator: T, executable: PathBuf, env: Vec<(OsString, OsString)>, - pool: ThreadPool, -) -> SFuture>> + pool: tokio::runtime::Handle, +) -> Result>> where T: CommandCreatorSync, { @@ -1018,164 +1014,153 @@ unknown __VERSION__ " .to_vec(); - let write = write_temp_file(&pool, "testfile.c".as_ref(), test); + let (tempdir, src) = write_temp_file(&pool, "testfile.c".as_ref(), test).await?; let mut cmd = creator.clone().new_command_sync(&executable); cmd.stdout(Stdio::piped()) .stderr(Stdio::piped()) .envs(env.iter().map(|s| (&s.0, &s.1))); - let output = write.and_then(move |(tempdir, src)| { - cmd.arg("-E").arg(src); - trace!("compiler {:?}", cmd); - cmd.spawn() - .and_then(|child| { - child - .wait_with_output() - .fcontext("failed to read child output") - }) - .map(|e| { - drop(tempdir); - e - }) - }); - Box::new(output.and_then(move |output| -> SFuture<_> { - let stdout = match str::from_utf8(&output.stdout) { - Ok(s) => s, - Err(_) => return f_err(anyhow!("Failed to parse output")), - }; - let mut lines = stdout.lines().filter_map(|line| { - let line = line.trim(); - if line.is_empty() || line.starts_with('#') { - None - } else { - Some(line) + cmd.arg("-E").arg(src); + trace!("compiler {:?}", cmd); + let child = cmd.spawn().await?; + let output = child + .wait_with_output() + .await + .context("failed to read child output")?; + + drop(tempdir); + + let stdout = match str::from_utf8(&output.stdout) { + Ok(s) => s, + Err(_) => bail!("Failed to parse output"), + }; + let mut lines = stdout.lines().filter_map(|line| { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + None + } else { + Some(line) + } + }); + if let Some(kind) = lines.next() { + let version = lines + .next() + // In case the compiler didn't expand the macro. + .filter(|&line| line != "__VERSION__") + .map(str::to_owned); + match kind { + "clang" | "clang++" => { + debug!("Found {}", kind); + return CCompiler::new( + Clang { + clangplusplus: kind == "clang++", + }, + executable, + version, + &pool, + ) + .await + .map(|c| Box::new(c) as Box>); } - }); - if let Some(kind) = lines.next() { - let version = lines - .next() - // In case the compiler didn't expand the macro. - .filter(|&line| line != "__VERSION__") - .map(str::to_owned); - match kind { - "clang" | "clang++" => { - debug!("Found {}", kind); - return Box::new( - CCompiler::new( - Clang { - clangplusplus: kind == "clang++", - }, - executable, - version, - &pool, - ) - .map(|c| Box::new(c) as Box>), - ); - } - "diab" => { - debug!("Found diab"); - return Box::new( - CCompiler::new(Diab, executable, version, &pool) - .map(|c| Box::new(c) as Box>), - ); - } - "gcc" | "g++" => { - debug!("Found {}", kind); - return Box::new( - CCompiler::new( - GCC { - gplusplus: kind == "g++", - }, - executable, - version, - &pool, - ) - .map(|c| Box::new(c) as Box>), - ); - } - "msvc" | "msvc-clang" => { - let is_clang = kind == "msvc-clang"; - debug!("Found MSVC (is clang: {})", is_clang); - let prefix = msvc::detect_showincludes_prefix( - &creator, - executable.as_ref(), + "diab" => { + debug!("Found diab"); + return CCompiler::new(Diab, executable, version, &pool) + .await + .map(|c| Box::new(c) as Box>); + } + "gcc" | "g++" => { + debug!("Found {}", kind); + return CCompiler::new( + GCC { + gplusplus: kind == "g++", + }, + executable, + version, + &pool, + ) + .await + .map(|c| Box::new(c) as Box>); + } + "msvc" | "msvc-clang" => { + let is_clang = kind == "msvc-clang"; + debug!("Found MSVC (is clang: {})", is_clang); + let prefix = msvc::detect_showincludes_prefix( + &creator, + executable.as_ref(), + is_clang, + env, + &pool, + ) + .await?; + trace!("showIncludes prefix: '{}'", prefix); + + return CCompiler::new( + MSVC { + includes_prefix: prefix, is_clang, - env, - &pool, - ); - return Box::new(prefix.and_then(move |prefix| { - trace!("showIncludes prefix: '{}'", prefix); - CCompiler::new( - MSVC { - includes_prefix: prefix, - is_clang, - }, - executable, - version, - &pool, - ) - .map(|c| Box::new(c) as Box>) - })); - } - "nvcc" => { - debug!("Found NVCC"); - return Box::new( - CCompiler::new(NVCC, executable, version, &pool) - .map(|c| Box::new(c) as Box>), - ); - } - _ => (), + }, + executable, + version, + &pool, + ) + .await + .map(|c| Box::new(c) as Box>); + } + "nvcc" => { + debug!("Found NVCC"); + return CCompiler::new(NVCC, executable, version, &pool) + .await + .map(|c| Box::new(c) as Box>); } + _ => (), } + } - let stderr = String::from_utf8_lossy(&output.stderr); - debug!("nothing useful in detection output {:?}", stdout); - debug!("compiler status: {}", output.status); - debug!("compiler stderr:\n{}", stderr); + let stderr = String::from_utf8_lossy(&output.stderr); + debug!("nothing useful in detection output {:?}", stdout); + debug!("compiler status: {}", output.status); + debug!("compiler stderr:\n{}", stderr); - f_err(anyhow!(stderr.into_owned())) - })) + bail!(stderr.into_owned()) } /// If `executable` is a known compiler, return a `Box` containing information about it. -pub fn get_compiler_info( +pub async fn get_compiler_info( creator: T, executable: &Path, cwd: &Path, env: &[(OsString, OsString)], - pool: &ThreadPool, + pool: &tokio::runtime::Handle, dist_archive: Option, -) -> SFuture<(Box>, Option>>)> +) -> Result<(Box>, Option>>)> where T: CommandCreatorSync, { let pool = pool.clone(); - detect_compiler(creator, executable, cwd, env, &pool, dist_archive) + detect_compiler(creator, executable, cwd, env, &pool, dist_archive).await } #[cfg(test)] mod test { use super::*; use crate::cache::disk::DiskCache; - use crate::cache::Storage; use crate::mock_command::*; use crate::test::mock_storage::MockStorage; use crate::test::utils::*; - use futures::{future, Future}; - use futures_03::executor::ThreadPool; use std::fs::{self, File}; use std::io::Write; use std::sync::Arc; use std::time::Duration; use std::u64; - use tokio_compat::runtime::current_thread::Runtime; + use tokio::runtime::Runtime; #[test] fn test_detect_compiler_kind_gcc() { let f = TestFixture::new(); let creator = new_creator(); - let pool = ThreadPool::sized(1); + let runtime = single_threaded_runtime(); + let pool = runtime.handle(); next_command(&creator, Ok(MockChild::new(exit_status(0), "\n\ngcc", ""))); let c = detect_compiler(creator, &f.bins[0], f.tempdir.path(), &[], &pool, None) .wait() @@ -1188,7 +1173,8 @@ mod test { fn test_detect_compiler_kind_clang() { let f = TestFixture::new(); let creator = new_creator(); - let pool = ThreadPool::sized(1); + let runtime = single_threaded_runtime(); + let pool = runtime.handle(); next_command(&creator, Ok(MockChild::new(exit_status(0), "clang\n", ""))); let c = detect_compiler(creator, &f.bins[0], f.tempdir.path(), &[], &pool, None) .wait() @@ -1201,7 +1187,8 @@ mod test { fn test_detect_compiler_kind_msvc() { drop(env_logger::try_init()); let creator = new_creator(); - let pool = ThreadPool::sized(1); + let runtime = single_threaded_runtime(); + let pool = runtime.handle(); let f = TestFixture::new(); let srcfile = f.touch("test.h").unwrap(); let mut s = srcfile.to_str().unwrap(); @@ -1228,7 +1215,8 @@ mod test { fn test_detect_compiler_kind_nvcc() { let f = TestFixture::new(); let creator = new_creator(); - let pool = ThreadPool::sized(1); + let runtime = single_threaded_runtime(); + let pool = runtime.handle(); next_command(&creator, Ok(MockChild::new(exit_status(0), "nvcc\n", ""))); let c = detect_compiler(creator, &f.bins[0], f.tempdir.path(), &[], &pool, None) .wait() @@ -1245,7 +1233,8 @@ mod test { fs::create_dir(f.tempdir.path().join("bin")).unwrap(); let rustc = f.mk_bin("rustc").unwrap(); let creator = new_creator(); - let pool = ThreadPool::sized(1); + let runtime = single_threaded_runtime(); + let pool = runtime.handle(); // rustc --vV next_command( &creator, @@ -1278,7 +1267,8 @@ LLVM version: 6.0", fn test_detect_compiler_kind_diab() { let f = TestFixture::new(); let creator = new_creator(); - let pool = ThreadPool::sized(1); + let runtime = single_threaded_runtime(); + let pool = runtime.handle(); next_command(&creator, Ok(MockChild::new(exit_status(0), "\ndiab\n", ""))); let c = detect_compiler(creator, &f.bins[0], f.tempdir.path(), &[], &pool, None) .wait() @@ -1291,7 +1281,8 @@ LLVM version: 6.0", fn test_detect_compiler_kind_unknown() { let f = TestFixture::new(); let creator = new_creator(); - let pool = ThreadPool::sized(1); + let runtime = single_threaded_runtime(); + let pool = runtime.handle(); next_command( &creator, Ok(MockChild::new(exit_status(0), "something", "")), @@ -1312,7 +1303,8 @@ LLVM version: 6.0", fn test_detect_compiler_kind_process_fail() { let f = TestFixture::new(); let creator = new_creator(); - let pool = ThreadPool::sized(1); + let runtime = single_threaded_runtime(); + let pool = runtime.handle(); next_command(&creator, Ok(MockChild::new(exit_status(1), "", ""))); assert!(detect_compiler( creator, @@ -1330,7 +1322,8 @@ LLVM version: 6.0", fn test_compiler_version_affects_hash() { let f = TestFixture::new(); let creator = new_creator(); - let pool = ThreadPool::sized(1); + let runtime = single_threaded_runtime(); + let pool = runtime.handle(); let arguments = ovec!["-c", "foo.c", "-o", "foo.o"]; let cwd = f.tempdir.path(); @@ -1371,7 +1364,8 @@ LLVM version: 6.0", #[test] fn test_get_compiler_info() { let creator = new_creator(); - let pool = ThreadPool::sized(1); + let runtime = single_threaded_runtime(); + let pool = runtime.handle(); let f = TestFixture::new(); // Pretend to be GCC. next_command(&creator, Ok(MockChild::new(exit_status(0), "gcc", ""))); @@ -1388,10 +1382,10 @@ LLVM version: 6.0", drop(env_logger::try_init()); let creator = new_creator(); let f = TestFixture::new(); - let pool = ThreadPool::sized(1); let mut runtime = Runtime::new().unwrap(); + let pool = runtime.handle().clone(); let storage = DiskCache::new(&f.tempdir.path().join("cache"), u64::MAX, &pool); - let storage: Arc = Arc::new(storage); + let storage = Arc::new(storage); // Pretend to be GCC. next_command(&creator, Ok(MockChild::new(exit_status(0), "gcc", ""))); let c = get_compiler_info( @@ -1433,18 +1427,20 @@ LLVM version: 6.0", }; let hasher2 = hasher.clone(); let (cached, res) = runtime - .block_on(future::lazy(|| { - hasher.get_cached_or_compile( - Ok(None), - creator.clone(), - storage.clone(), - arguments.clone(), - cwd.to_path_buf(), - vec![], - CacheControl::Default, - pool.clone(), - ) - })) + .block_on(async { + hasher + .get_cached_or_compile( + None, + creator.clone(), + storage.clone(), + arguments.clone(), + cwd.to_path_buf(), + vec![], + CacheControl::Default, + pool.clone(), + ) + .await + }) .unwrap(); // Ensure that the object file was created. assert!(fs::metadata(&obj).map(|m| m.len() > 0).unwrap()); @@ -1467,18 +1463,20 @@ LLVM version: 6.0", ); // There should be no actual compiler invocation. let (cached, res) = runtime - .block_on(future::lazy(|| { - hasher2.get_cached_or_compile( - Ok(None), - creator, - storage, - arguments, - cwd.to_path_buf(), - vec![], - CacheControl::Default, - pool, - ) - })) + .block_on(async { + hasher2 + .get_cached_or_compile( + None, + creator, + storage, + arguments, + cwd.to_path_buf(), + vec![], + CacheControl::Default, + pool, + ) + .await + }) .unwrap(); // Ensure that the object file was created. assert!(fs::metadata(&obj).map(|m| m.len() > 0).unwrap()); @@ -1494,10 +1492,10 @@ LLVM version: 6.0", drop(env_logger::try_init()); let creator = new_creator(); let f = TestFixture::new(); - let pool = ThreadPool::sized(1); let mut runtime = Runtime::new().unwrap(); + let pool = runtime.handle().clone(); let storage = DiskCache::new(&f.tempdir.path().join("cache"), u64::MAX, &pool); - let storage: Arc = Arc::new(storage); + let storage = Arc::new(storage); // Pretend to be GCC. next_command(&creator, Ok(MockChild::new(exit_status(0), "gcc", ""))); let c = get_compiler_info( @@ -1534,18 +1532,20 @@ LLVM version: 6.0", }; let hasher2 = hasher.clone(); let (cached, res) = runtime - .block_on(future::lazy(|| { - hasher.get_cached_or_compile( - Ok(dist_client.clone()), - creator.clone(), - storage.clone(), - arguments.clone(), - cwd.to_path_buf(), - vec![], - CacheControl::Default, - pool.clone(), - ) - })) + .block_on(async { + hasher + .get_cached_or_compile( + dist_client.clone(), + creator.clone(), + storage.clone(), + arguments.clone(), + cwd.to_path_buf(), + vec![], + CacheControl::Default, + pool.clone(), + ) + .await + }) .unwrap(); // Ensure that the object file was created. assert!(fs::metadata(&obj).map(|m| m.len() > 0).unwrap()); @@ -1568,18 +1568,20 @@ LLVM version: 6.0", ); // There should be no actual compiler invocation. let (cached, res) = runtime - .block_on(future::lazy(|| { - hasher2.get_cached_or_compile( - Ok(dist_client.clone()), - creator, - storage, - arguments, - cwd.to_path_buf(), - vec![], - CacheControl::Default, - pool, - ) - })) + .block_on(async { + hasher2 + .get_cached_or_compile( + dist_client.clone(), + creator, + storage, + arguments, + cwd.to_path_buf(), + vec![], + CacheControl::Default, + pool, + ) + .await + }) .unwrap(); // Ensure that the object file was created. assert!(fs::metadata(&obj).map(|m| m.len() > 0).unwrap()); @@ -1596,8 +1598,8 @@ LLVM version: 6.0", drop(env_logger::try_init()); let creator = new_creator(); let f = TestFixture::new(); - let pool = ThreadPool::sized(1); let mut runtime = Runtime::new().unwrap(); + let pool = runtime.handle().clone(); let storage = MockStorage::new(); let storage: Arc = Arc::new(storage); // Pretend to be GCC. @@ -1640,27 +1642,25 @@ LLVM version: 6.0", o => panic!("Bad result from parse_arguments: {:?}", o), }; // The cache will return an error. - storage.next_get(f_err(anyhow!("Some Error"))); + storage.next_get(Err(anyhow!("Some Error"))); let (cached, res) = runtime - .block_on(future::lazy(|| { - hasher.get_cached_or_compile( - Ok(None), - creator, - storage, - arguments.clone(), - cwd.to_path_buf(), - vec![], - CacheControl::Default, - pool, - ) - })) + .block_on(hasher.get_cached_or_compile( + None, + creator, + storage, + arguments.clone(), + cwd.to_path_buf(), + vec![], + CacheControl::Default, + pool, + )) .unwrap(); // Ensure that the object file was created. assert!(fs::metadata(&obj).map(|m| m.len() > 0).unwrap()); match cached { CompileResult::CacheMiss(MissType::CacheReadError, DistType::NoDist, _, f) => { // wait on cache write future so we don't race with it! - f.wait().unwrap(); + let _ = f.wait(); } _ => panic!("Unexpected compile result: {:?}", cached), } @@ -1675,10 +1675,10 @@ LLVM version: 6.0", drop(env_logger::try_init()); let creator = new_creator(); let f = TestFixture::new(); - let pool = ThreadPool::sized(1); - let mut runtime = Runtime::new().unwrap(); + let mut runtime = single_threaded_runtime(); + let pool = runtime.handle().clone(); let storage = DiskCache::new(&f.tempdir.path().join("cache"), u64::MAX, &pool); - let storage: Arc = Arc::new(storage); + let storage = Arc::new(storage); // Pretend to be GCC. next_command(&creator, Ok(MockChild::new(exit_status(0), "gcc", ""))); let c = get_compiler_info( @@ -1724,18 +1724,20 @@ LLVM version: 6.0", }; let hasher2 = hasher.clone(); let (cached, res) = runtime - .block_on(future::lazy(|| { - hasher.get_cached_or_compile( - Ok(None), - creator.clone(), - storage.clone(), - arguments.clone(), - cwd.to_path_buf(), - vec![], - CacheControl::Default, - pool.clone(), - ) - })) + .block_on(async { + hasher + .get_cached_or_compile( + None, + creator.clone(), + storage.clone(), + arguments.clone(), + cwd.to_path_buf(), + vec![], + CacheControl::Default, + pool.clone(), + ) + .await + }) .unwrap(); // Ensure that the object file was created. assert!(fs::metadata(&obj).map(|m| m.len() > 0).unwrap()); @@ -1753,7 +1755,7 @@ LLVM version: 6.0", fs::remove_file(&obj).unwrap(); let (cached, res) = hasher2 .get_cached_or_compile( - Ok(None), + None, creator, storage, arguments, @@ -1783,10 +1785,10 @@ LLVM version: 6.0", drop(env_logger::try_init()); let creator = new_creator(); let f = TestFixture::new(); - let pool = ThreadPool::sized(1); - let mut runtime = Runtime::new().unwrap(); + let mut runtime = single_threaded_runtime(); + let pool = runtime.handle().clone(); let storage = DiskCache::new(&f.tempdir.path().join("cache"), u64::MAX, &pool); - let storage: Arc = Arc::new(storage); + let storage = Arc::new(storage); // Pretend to be GCC. Also inject a fake object file that the subsequent // preprocessor failure should remove. let obj = f.tempdir.path().join("foo.o"); @@ -1826,18 +1828,20 @@ LLVM version: 6.0", o => panic!("Bad result from parse_arguments: {:?}", o), }; let (cached, res) = runtime - .block_on(future::lazy(|| { - hasher.get_cached_or_compile( - Ok(None), - creator, - storage, - arguments, - cwd.to_path_buf(), - vec![], - CacheControl::Default, - pool, - ) - })) + .block_on(async { + hasher + .get_cached_or_compile( + None, + creator, + storage, + arguments, + cwd.to_path_buf(), + vec![], + CacheControl::Default, + pool, + ) + .await + }) .unwrap(); assert_eq!(cached, CompileResult::Error); assert_eq!(exit_status(1), res.status); @@ -1854,7 +1858,8 @@ LLVM version: 6.0", drop(env_logger::try_init()); let creator = new_creator(); let f = TestFixture::new(); - let pool = ThreadPool::sized(1); + let runtime = Runtime::new().unwrap(); + let pool = runtime.handle().clone(); let dist_clients = vec![ test_dist::ErrorPutToolchainClient::new(), test_dist::ErrorAllocJobClient::new(), @@ -1862,7 +1867,7 @@ LLVM version: 6.0", test_dist::ErrorRunJobClient::new(), ]; let storage = DiskCache::new(&f.tempdir.path().join("cache"), u64::MAX, &pool); - let storage: Arc = Arc::new(storage); + let storage = Arc::new(storage); // Pretend to be GCC. next_command(&creator, Ok(MockChild::new(exit_status(0), "gcc", ""))); let c = get_compiler_info( @@ -1914,7 +1919,7 @@ LLVM version: 6.0", let hasher = hasher.clone(); let (cached, res) = hasher .get_cached_or_compile( - Ok(Some(dist_client.clone())), + Some(dist_client.clone()), creator.clone(), storage.clone(), arguments.clone(), @@ -1924,7 +1929,7 @@ LLVM version: 6.0", pool.clone(), ) .wait() - .unwrap(); + .expect("Does not error if storage put fails. qed"); // Ensure that the object file was created. assert!(fs::metadata(&obj).map(|m| m.len() > 0).unwrap()); match cached { @@ -1950,9 +1955,8 @@ mod test_dist { PathTransformer, ProcessOutput, RunJobResult, SchedulerStatusResult, ServerId, SubmitToolchainResult, Toolchain, }; - use std::cell::Cell; - use std::path::{Path, PathBuf}; - use std::sync::Arc; + use std::path::PathBuf; + use std::sync::{atomic::AtomicBool, Arc}; use crate::errors::*; @@ -1963,32 +1967,37 @@ mod test_dist { Arc::new(ErrorPutToolchainClient) } } + #[async_trait] impl dist::Client for ErrorPutToolchainClient { - fn do_alloc_job(&self, _: Toolchain) -> SFuture { + async fn do_alloc_job(&self, _: Toolchain) -> Result { unreachable!() } - fn do_get_status(&self) -> SFuture { + async fn do_get_status(&self) -> Result { unreachable!() } - fn do_submit_toolchain(&self, _: JobAlloc, _: Toolchain) -> SFuture { + async fn do_submit_toolchain( + &self, + _: JobAlloc, + _: Toolchain, + ) -> Result { unreachable!() } - fn do_run_job( + async fn do_run_job( &self, _: JobAlloc, _: CompileCommand, _: Vec, _: Box, - ) -> SFuture<(RunJobResult, PathTransformer)> { + ) -> Result<(RunJobResult, PathTransformer)> { unreachable!() } - fn put_toolchain( + async fn put_toolchain( &self, - _: &Path, - _: &str, + _: PathBuf, + _: String, _: Box, - ) -> SFuture<(Toolchain, Option<(String, PathBuf)>)> { - f_err(anyhow!("put toolchain failure")) + ) -> Result<(Toolchain, Option<(String, PathBuf)>)> { + Err(anyhow!("MOCK: put toolchain failure")) } fn rewrite_includes_only(&self) -> bool { false @@ -2011,33 +2020,38 @@ mod test_dist { }) } } + #[async_trait] impl dist::Client for ErrorAllocJobClient { - fn do_alloc_job(&self, tc: Toolchain) -> SFuture { + async fn do_alloc_job(&self, tc: Toolchain) -> Result { assert_eq!(self.tc, tc); - f_err(anyhow!("alloc job failure")) + Err(anyhow!("MOCK: alloc job failure")) } - fn do_get_status(&self) -> SFuture { + async fn do_get_status(&self) -> Result { unreachable!() } - fn do_submit_toolchain(&self, _: JobAlloc, _: Toolchain) -> SFuture { + async fn do_submit_toolchain( + &self, + _: JobAlloc, + _: Toolchain, + ) -> Result { unreachable!() } - fn do_run_job( + async fn do_run_job( &self, _: JobAlloc, _: CompileCommand, _: Vec, _: Box, - ) -> SFuture<(RunJobResult, PathTransformer)> { + ) -> Result<(RunJobResult, PathTransformer)> { unreachable!() } - fn put_toolchain( + async fn put_toolchain( &self, - _: &Path, - _: &str, + _: PathBuf, + _: String, _: Box, - ) -> SFuture<(Toolchain, Option<(String, PathBuf)>)> { - f_ok((self.tc.clone(), None)) + ) -> Result<(Toolchain, Option<(String, PathBuf)>)> { + Ok((self.tc.clone(), None)) } fn rewrite_includes_only(&self) -> bool { false @@ -2048,25 +2062,29 @@ mod test_dist { } pub struct ErrorSubmitToolchainClient { - has_started: Cell, + has_started: AtomicBool, tc: Toolchain, } impl ErrorSubmitToolchainClient { #[allow(clippy::new_ret_no_self)] pub fn new() -> Arc { Arc::new(Self { - has_started: Cell::new(false), + has_started: AtomicBool::default(), tc: Toolchain { archive_id: "somearchiveid".to_owned(), }, }) } } + + #[async_trait] impl dist::Client for ErrorSubmitToolchainClient { - fn do_alloc_job(&self, tc: Toolchain) -> SFuture { - assert!(!self.has_started.replace(true)); + async fn do_alloc_job(&self, tc: Toolchain) -> Result { + assert!(!self + .has_started + .swap(true, std::sync::atomic::Ordering::AcqRel)); assert_eq!(self.tc, tc); - f_ok(AllocJobResult::Success { + Ok(AllocJobResult::Success { job_alloc: JobAlloc { auth: "abcd".to_owned(), job_id: JobId(0), @@ -2075,34 +2093,34 @@ mod test_dist { need_toolchain: true, }) } - fn do_get_status(&self) -> SFuture { - unreachable!() + async fn do_get_status(&self) -> Result { + unreachable!("fn do_get_status is not used for this test. qed") } - fn do_submit_toolchain( + async fn do_submit_toolchain( &self, job_alloc: JobAlloc, tc: Toolchain, - ) -> SFuture { + ) -> Result { assert_eq!(job_alloc.job_id, JobId(0)); assert_eq!(self.tc, tc); - f_err(anyhow!("submit toolchain failure")) + Err(anyhow!("MOCK: submit toolchain failure")) } - fn do_run_job( + async fn do_run_job( &self, _: JobAlloc, _: CompileCommand, _: Vec, _: Box, - ) -> SFuture<(RunJobResult, PathTransformer)> { - unreachable!() + ) -> Result<(RunJobResult, PathTransformer)> { + unreachable!("fn do_run_job is not used for this test. qed") } - fn put_toolchain( + async fn put_toolchain( &self, - _: &Path, - _: &str, + _: PathBuf, + _: String, _: Box, - ) -> SFuture<(Toolchain, Option<(String, PathBuf)>)> { - f_ok((self.tc.clone(), None)) + ) -> Result<(Toolchain, Option<(String, PathBuf)>)> { + Ok((self.tc.clone(), None)) } fn rewrite_includes_only(&self) -> bool { false @@ -2113,25 +2131,29 @@ mod test_dist { } pub struct ErrorRunJobClient { - has_started: Cell, + has_started: AtomicBool, tc: Toolchain, } impl ErrorRunJobClient { #[allow(clippy::new_ret_no_self)] pub fn new() -> Arc { Arc::new(Self { - has_started: Cell::new(false), + has_started: AtomicBool::default(), tc: Toolchain { archive_id: "somearchiveid".to_owned(), }, }) } } + + #[async_trait] impl dist::Client for ErrorRunJobClient { - fn do_alloc_job(&self, tc: Toolchain) -> SFuture { - assert!(!self.has_started.replace(true)); + async fn do_alloc_job(&self, tc: Toolchain) -> Result { + assert!(!self + .has_started + .swap(true, std::sync::atomic::Ordering::AcqRel)); assert_eq!(self.tc, tc); - f_ok(AllocJobResult::Success { + Ok(AllocJobResult::Success { job_alloc: JobAlloc { auth: "abcd".to_owned(), job_id: JobId(0), @@ -2140,36 +2162,36 @@ mod test_dist { need_toolchain: true, }) } - fn do_get_status(&self) -> SFuture { + async fn do_get_status(&self) -> Result { unreachable!() } - fn do_submit_toolchain( + async fn do_submit_toolchain( &self, job_alloc: JobAlloc, tc: Toolchain, - ) -> SFuture { + ) -> Result { assert_eq!(job_alloc.job_id, JobId(0)); assert_eq!(self.tc, tc); - f_ok(SubmitToolchainResult::Success) + Ok(SubmitToolchainResult::Success) } - fn do_run_job( + async fn do_run_job( &self, job_alloc: JobAlloc, command: CompileCommand, _: Vec, _: Box, - ) -> SFuture<(RunJobResult, PathTransformer)> { + ) -> Result<(RunJobResult, PathTransformer)> { assert_eq!(job_alloc.job_id, JobId(0)); assert_eq!(command.executable, "/overridden/compiler"); - f_err(anyhow!("run job failure")) + Err(anyhow!("MOCK: run job failure")) } - fn put_toolchain( + async fn put_toolchain( &self, - _: &Path, - _: &str, + _: PathBuf, + _: String, _: Box, - ) -> SFuture<(Toolchain, Option<(String, PathBuf)>)> { - f_ok(( + ) -> Result<(Toolchain, Option<(String, PathBuf)>)> { + Ok(( self.tc.clone(), Some(( "/overridden/compiler".to_owned(), @@ -2186,7 +2208,7 @@ mod test_dist { } pub struct OneshotClient { - has_started: Cell, + has_started: AtomicBool, tc: Toolchain, output: ProcessOutput, } @@ -2195,7 +2217,7 @@ mod test_dist { #[allow(clippy::new_ret_no_self)] pub fn new(code: i32, stdout: Vec, stderr: Vec) -> Arc { Arc::new(Self { - has_started: Cell::new(false), + has_started: AtomicBool::default(), tc: Toolchain { archive_id: "somearchiveid".to_owned(), }, @@ -2204,12 +2226,15 @@ mod test_dist { } } + #[async_trait] impl dist::Client for OneshotClient { - fn do_alloc_job(&self, tc: Toolchain) -> SFuture { - assert!(!self.has_started.replace(true)); + async fn do_alloc_job(&self, tc: Toolchain) -> Result { + assert!(!self + .has_started + .swap(true, std::sync::atomic::Ordering::AcqRel)); assert_eq!(self.tc, tc); - f_ok(AllocJobResult::Success { + Ok(AllocJobResult::Success { job_alloc: JobAlloc { auth: "abcd".to_owned(), job_id: JobId(0), @@ -2218,26 +2243,26 @@ mod test_dist { need_toolchain: true, }) } - fn do_get_status(&self) -> SFuture { - unreachable!() + async fn do_get_status(&self) -> Result { + unreachable!("fn do_get_status is not used for this test. qed") } - fn do_submit_toolchain( + async fn do_submit_toolchain( &self, job_alloc: JobAlloc, tc: Toolchain, - ) -> SFuture { + ) -> Result { assert_eq!(job_alloc.job_id, JobId(0)); assert_eq!(self.tc, tc); - f_ok(SubmitToolchainResult::Success) + Ok(SubmitToolchainResult::Success) } - fn do_run_job( + async fn do_run_job( &self, job_alloc: JobAlloc, command: CompileCommand, outputs: Vec, inputs_packager: Box, - ) -> SFuture<(RunJobResult, PathTransformer)> { + ) -> Result<(RunJobResult, PathTransformer)> { assert_eq!(job_alloc.job_id, JobId(0)); assert_eq!(command.executable, "/overridden/compiler"); @@ -2255,15 +2280,15 @@ mod test_dist { output: self.output.clone(), outputs, }); - f_ok((result, path_transformer)) + Ok((result, path_transformer)) } - fn put_toolchain( + async fn put_toolchain( &self, - _: &Path, - _: &str, + _: PathBuf, + _: String, _: Box, - ) -> SFuture<(Toolchain, Option<(String, PathBuf)>)> { - f_ok(( + ) -> Result<(Toolchain, Option<(String, PathBuf)>)> { + Ok(( self.tc.clone(), Some(( "/overridden/compiler".to_owned(), diff --git a/src/compiler/diab.rs b/src/compiler/diab.rs index ae02bd52..7864c540 100644 --- a/src/compiler/diab.rs +++ b/src/compiler/diab.rs @@ -34,6 +34,7 @@ use std::process; #[derive(Clone, Debug)] pub struct Diab; +#[async_trait] impl CCompilerImpl for Diab { fn kind(&self) -> CCompilerKind { CCompilerKind::Diab @@ -49,7 +50,8 @@ impl CCompilerImpl for Diab { parse_arguments(arguments, cwd, &ARGS[..]) } - fn preprocess( + #[allow(clippy::too_many_arguments)] + async fn preprocess( &self, creator: &T, executable: &Path, @@ -58,11 +60,11 @@ impl CCompilerImpl for Diab { env_vars: &[(OsString, OsString)], may_dist: bool, _rewrite_includes_only: bool, - ) -> SFuture + ) -> Result where T: CommandCreatorSync, { - preprocess(creator, executable, parsed_args, cwd, env_vars, may_dist) + preprocess(creator, executable, parsed_args, cwd, env_vars, may_dist).await } fn generate_compile_commands( @@ -284,14 +286,14 @@ where }) } -pub fn preprocess( +pub async fn preprocess( creator: &T, executable: &Path, parsed_args: &ParsedArguments, cwd: &Path, env_vars: &[(OsString, OsString)], _may_dist: bool, -) -> SFuture +) -> Result where T: CommandCreatorSync, { @@ -308,7 +310,7 @@ where if log_enabled!(Trace) { trace!("preprocess: {:?}", cmd); } - Box::new(run_input_output(cmd, None)) + run_input_output(cmd, None).await } pub fn generate_compile_commands( @@ -422,7 +424,6 @@ mod test { use crate::compiler::*; use crate::mock_command::*; use crate::test::utils::*; - use futures::Future; use std::fs::File; use std::io::Write; diff --git a/src/compiler/gcc.rs b/src/compiler/gcc.rs index 03d6c481..ce6dad0a 100644 --- a/src/compiler/gcc.rs +++ b/src/compiler/gcc.rs @@ -34,6 +34,7 @@ pub struct GCC { pub gplusplus: bool, } +#[async_trait] impl CCompilerImpl for GCC { fn kind(&self) -> CCompilerKind { CCompilerKind::GCC @@ -49,7 +50,8 @@ impl CCompilerImpl for GCC { parse_arguments(arguments, cwd, &ARGS[..], self.gplusplus) } - fn preprocess( + #[allow(clippy::too_many_arguments)] + async fn preprocess( &self, creator: &T, executable: &Path, @@ -58,7 +60,7 @@ impl CCompilerImpl for GCC { env_vars: &[(OsString, OsString)], may_dist: bool, rewrite_includes_only: bool, - ) -> SFuture + ) -> Result where T: CommandCreatorSync, { @@ -72,6 +74,7 @@ impl CCompilerImpl for GCC { self.kind(), rewrite_includes_only, ) + .await } fn generate_compile_commands( @@ -495,7 +498,7 @@ where } #[allow(clippy::too_many_arguments)] -pub fn preprocess( +pub async fn preprocess( creator: &T, executable: &Path, parsed_args: &ParsedArguments, @@ -504,7 +507,7 @@ pub fn preprocess( may_dist: bool, kind: CCompilerKind, rewrite_includes_only: bool, -) -> SFuture +) -> Result where T: CommandCreatorSync, { @@ -547,7 +550,7 @@ where if log_enabled!(Trace) { trace!("preprocess: {:?}", cmd); } - Box::new(run_input_output(cmd, None)) + run_input_output(cmd, None).await } pub fn generate_compile_commands( @@ -736,7 +739,6 @@ mod test { use crate::compiler::*; use crate::mock_command::*; use crate::test::utils::*; - use futures::Future; fn parse_arguments_( arguments: Vec, diff --git a/src/compiler/msvc.rs b/src/compiler/msvc.rs index fd2ec9b6..cb0f086d 100644 --- a/src/compiler/msvc.rs +++ b/src/compiler/msvc.rs @@ -19,9 +19,7 @@ use crate::compiler::{ }; use crate::dist; use crate::mock_command::{CommandCreatorSync, RunCommand}; -use crate::util::{run_input_output, SpawnExt}; -use futures::future::Future; -use futures_03::executor::ThreadPool; +use crate::util::run_input_output; use local_encoding::{Encoder, Encoding}; use log::Level::Debug; use std::collections::{HashMap, HashSet}; @@ -43,6 +41,7 @@ pub struct MSVC { pub is_clang: bool, } +#[async_trait] impl CCompilerImpl for MSVC { fn kind(&self) -> CCompilerKind { CCompilerKind::MSVC @@ -58,7 +57,8 @@ impl CCompilerImpl for MSVC { parse_arguments(arguments, cwd, self.is_clang) } - fn preprocess( + #[allow(clippy::too_many_arguments)] + async fn preprocess( &self, creator: &T, executable: &Path, @@ -67,7 +67,7 @@ impl CCompilerImpl for MSVC { env_vars: &[(OsString, OsString)], may_dist: bool, rewrite_includes_only: bool, - ) -> SFuture + ) -> Result where T: CommandCreatorSync, { @@ -82,6 +82,7 @@ impl CCompilerImpl for MSVC { rewrite_includes_only, self.is_clang, ) + .await } fn generate_compile_commands( @@ -102,94 +103,89 @@ fn from_local_codepage(bytes: &[u8]) -> io::Result { } /// Detect the prefix included in the output of MSVC's -showIncludes output. -pub fn detect_showincludes_prefix( +pub async fn detect_showincludes_prefix( creator: &T, exe: &OsStr, is_clang: bool, env: Vec<(OsString, OsString)>, - pool: &ThreadPool, -) -> SFuture + pool: &tokio::runtime::Handle, +) -> Result where T: CommandCreatorSync, { - let write = write_temp_file(pool, "test.c".as_ref(), b"#include \"test.h\"\n".to_vec()); + let (tempdir, input) = + write_temp_file(pool, "test.c".as_ref(), b"#include \"test.h\"\n".to_vec()).await?; let exe = exe.to_os_string(); let mut creator = creator.clone(); let pool = pool.clone(); - let write2 = write.and_then(move |(tempdir, input)| { - let header = tempdir.path().join("test.h"); - pool.spawn_fn(move || -> Result<_> { + + let header = tempdir.path().join("test.h"); + let tempdir = pool + .spawn_blocking(move || { let mut file = File::create(&header)?; file.write_all(b"/* empty */\n")?; - Ok((tempdir, input)) + Ok::<_, std::io::Error>(tempdir) }) - .fcontext("failed to write temporary file") - }); - let output = write2.and_then(move |(tempdir, input)| { - let mut cmd = creator.new_command_sync(&exe); - // clang.exe on Windows reports the same set of built-in preprocessor defines as clang-cl, - // but it doesn't accept MSVC commandline arguments unless you pass --driver-mode=cl. - // clang-cl.exe will accept this argument as well, so always add it in this case. - if is_clang { - cmd.arg("--driver-mode=cl"); - } - cmd.args(&["-nologo", "-showIncludes", "-c", "-Fonul", "-I."]) - .arg(&input) - .current_dir(&tempdir.path()) - // The MSDN docs say the -showIncludes output goes to stderr, - // but that's not true unless running with -E. - .stdout(Stdio::piped()) - .stderr(Stdio::null()); - for (k, v) in env { - cmd.env(k, v); - } - trace!("detect_showincludes_prefix: {:?}", cmd); + .await? + .context("Failed to write temporary file")?; + + let mut cmd = creator.new_command_sync(&exe); + // clang.exe on Windows reports the same set of built-in preprocessor defines as clang-cl, + // but it doesn't accept MSVC commandline arguments unless you pass --driver-mode=cl. + // clang-cl.exe will accept this argument as well, so always add it in this case. + if is_clang { + cmd.arg("--driver-mode=cl"); + } + cmd.args(&["-nologo", "-showIncludes", "-c", "-Fonul", "-I."]) + .arg(&input) + .current_dir(&tempdir.path()) + // The MSDN docs say the -showIncludes output goes to stderr, + // but that's not true unless running with -E. + .stdout(Stdio::piped()) + .stderr(Stdio::null()); + for (k, v) in env { + cmd.env(k, v); + } + trace!("detect_showincludes_prefix: {:?}", cmd); - run_input_output(cmd, None).map(|e| { - // Keep the tempdir around so test.h still exists for the - // checks below. - (e, tempdir) - }) - }); + let output = run_input_output(cmd, None).await?; - Box::new(output.and_then(|(output, tempdir)| { - if !output.status.success() { - bail!("Failed to detect showIncludes prefix") - } + if !output.status.success() { + bail!("Failed to detect showIncludes prefix") + } - let process::Output { - stdout: stdout_bytes, - .. - } = output; - let stdout = from_local_codepage(&stdout_bytes) - .context("Failed to convert compiler stdout while detecting showIncludes prefix")?; - for line in stdout.lines() { - if !line.ends_with("test.h") { + let process::Output { + stdout: stdout_bytes, + .. + } = output; + let stdout = from_local_codepage(&stdout_bytes) + .context("Failed to convert compiler stdout while detecting showIncludes prefix")?; + for line in stdout.lines() { + if !line.ends_with("test.h") { + continue; + } + for (i, c) in line.char_indices().rev() { + if c != ' ' { continue; } - for (i, c) in line.char_indices().rev() { - if c != ' ' { - continue; - } - let path = tempdir.path().join(&line[i + 1..]); - // See if the rest of this line is a full pathname. - if path.exists() { - // Everything from the beginning of the line - // to this index is the prefix. - return Ok(line[..=i].to_owned()); - } + let path = tempdir.path().join(&line[i + 1..]); + // See if the rest of this line is a full pathname. + if path.exists() { + // Everything from the beginning of the line + // to this index is the prefix. + return Ok(line[..=i].to_owned()); } } - drop(tempdir); + } + drop(tempdir); - debug!( - "failed to detect showIncludes prefix with output: {}", - stdout - ); + debug!( + "failed to detect showIncludes prefix with output: {}", + stdout + ); - bail!("Failed to detect showIncludes prefix") - })) + bail!("Failed to detect showIncludes prefix") } #[cfg(unix)] @@ -718,7 +714,7 @@ fn normpath(path: &str) -> String { } #[allow(clippy::too_many_arguments)] -pub fn preprocess( +pub async fn preprocess( creator: &T, executable: &Path, parsed_args: &ParsedArguments, @@ -728,7 +724,7 @@ pub fn preprocess( includes_prefix: &str, rewrite_includes_only: bool, is_clang: bool, -) -> SFuture +) -> Result where T: CommandCreatorSync, { @@ -769,65 +765,65 @@ where let includes_prefix = includes_prefix.to_string(); let cwd = cwd.to_owned(); - Box::new(run_input_output(cmd, None).and_then(move |output| { - let parsed_args = &parsed_args; - if let (Some(ref objfile), &Some(ref depfile)) = - (parsed_args.outputs.get("obj"), &parsed_args.depfile) - { - let f = File::create(cwd.join(depfile))?; - let mut f = BufWriter::new(f); - - encode_path(&mut f, &objfile) - .with_context(|| format!("Couldn't encode objfile filename: '{:?}'", objfile))?; - write!(f, ": ")?; - encode_path(&mut f, &parsed_args.input) - .with_context(|| format!("Couldn't encode input filename: '{:?}'", objfile))?; - write!(f, " ")?; - let process::Output { - status, - stdout, - stderr: stderr_bytes, - } = output; - let stderr = from_local_codepage(&stderr_bytes) - .context("Failed to convert preprocessor stderr")?; - let mut deps = HashSet::new(); - let mut stderr_bytes = vec![]; - for line in stderr.lines() { - if line.starts_with(&includes_prefix) { - let dep = normpath(line[includes_prefix.len()..].trim()); - trace!("included: {}", dep); - if deps.insert(dep.clone()) && !dep.contains(' ') { - write!(f, "{} ", dep)?; - } - if !parsed_args.msvc_show_includes { - continue; - } + let output = run_input_output(cmd, None).await?; + + let parsed_args = &parsed_args; + if let (Some(ref objfile), &Some(ref depfile)) = + (parsed_args.outputs.get("obj"), &parsed_args.depfile) + { + let f = File::create(cwd.join(depfile))?; + let mut f = BufWriter::new(f); + + encode_path(&mut f, &objfile) + .with_context(|| format!("Couldn't encode objfile filename: '{:?}'", objfile))?; + write!(f, ": ")?; + encode_path(&mut f, &parsed_args.input) + .with_context(|| format!("Couldn't encode input filename: '{:?}'", objfile))?; + write!(f, " ")?; + let process::Output { + status, + stdout, + stderr: stderr_bytes, + } = output; + let stderr = + from_local_codepage(&stderr_bytes).context("Failed to convert preprocessor stderr")?; + let mut deps = HashSet::new(); + let mut stderr_bytes = vec![]; + for line in stderr.lines() { + if line.starts_with(&includes_prefix) { + let dep = normpath(line[includes_prefix.len()..].trim()); + trace!("included: {}", dep); + if deps.insert(dep.clone()) && !dep.contains(' ') { + write!(f, "{} ", dep)?; } - stderr_bytes.extend_from_slice(line.as_bytes()); - stderr_bytes.push(b'\n'); - } - writeln!(f)?; - // Write extra rules for each dependency to handle - // removed files. - encode_path(&mut f, &parsed_args.input) - .with_context(|| format!("Couldn't encode filename: '{:?}'", parsed_args.input))?; - writeln!(f, ":")?; - let mut sorted = deps.into_iter().collect::>(); - sorted.sort(); - for dep in sorted { - if !dep.contains(' ') { - writeln!(f, "{}:", dep)?; + if !parsed_args.msvc_show_includes { + continue; } } - Ok(process::Output { - status, - stdout, - stderr: stderr_bytes, - }) - } else { - Ok(output) + stderr_bytes.extend_from_slice(line.as_bytes()); + stderr_bytes.push(b'\n'); + } + writeln!(f)?; + // Write extra rules for each dependency to handle + // removed files. + encode_path(&mut f, &parsed_args.input) + .with_context(|| format!("Couldn't encode filename: '{:?}'", parsed_args.input))?; + writeln!(f, ":")?; + let mut sorted = deps.into_iter().collect::>(); + sorted.sort(); + for dep in sorted { + if !dep.contains(' ') { + writeln!(f, "{}:", dep)?; + } } - })) + Ok(process::Output { + status, + stdout, + stderr: stderr_bytes, + }) + } else { + Ok(output) + } } fn generate_compile_commands( @@ -916,8 +912,6 @@ mod test { use crate::compiler::*; use crate::mock_command::*; use crate::test::utils::*; - use futures::Future; - use futures_03::executor::ThreadPool; fn parse_arguments(arguments: Vec) -> CompilerArguments { super::parse_arguments(&arguments, &std::env::current_dir().unwrap(), false) @@ -927,7 +921,8 @@ mod test { fn test_detect_showincludes_prefix() { drop(env_logger::try_init()); let creator = new_creator(); - let pool = ThreadPool::sized(1); + let runtime = single_threaded_runtime(); + let pool = runtime.handle().clone(); let f = TestFixture::new(); let srcfile = f.touch("test.h").unwrap(); let mut s = srcfile.to_str().unwrap(); diff --git a/src/compiler/nvcc.rs b/src/compiler/nvcc.rs index c715a8a2..e1aaaefd 100644 --- a/src/compiler/nvcc.rs +++ b/src/compiler/nvcc.rs @@ -21,10 +21,10 @@ use crate::compiler::{gcc, write_temp_file, Cacheable, CompileCommand, CompilerA use crate::dist; use crate::mock_command::{CommandCreator, CommandCreatorSync, RunCommand}; use crate::util::{run_input_output, OsStrExt}; -use futures::future::{self, Future}; use log::Level::Trace; use std::ffi::OsString; use std::fs::File; +use std::future::Future; use std::io::{self, Write}; use std::path::{Path, PathBuf}; use std::process; @@ -35,6 +35,7 @@ use crate::errors::*; #[derive(Clone, Debug)] pub struct NVCC; +#[async_trait] impl CCompilerImpl for NVCC { fn kind(&self) -> CCompilerKind { CCompilerKind::NVCC @@ -50,7 +51,8 @@ impl CCompilerImpl for NVCC { gcc::parse_arguments(arguments, cwd, (&gcc::ARGS[..], &ARGS[..]), false) } - fn preprocess( + #[allow(clippy::too_many_arguments)] + async fn preprocess( &self, creator: &T, executable: &Path, @@ -59,7 +61,7 @@ impl CCompilerImpl for NVCC { env_vars: &[(OsString, OsString)], may_dist: bool, rewrite_includes_only: bool, - ) -> SFuture + ) -> Result where T: CommandCreatorSync, { @@ -131,9 +133,13 @@ impl CCompilerImpl for NVCC { if !parsed_args.dependency_args.is_empty() { let first = run_input_output(dep_before_preprocessor(), None); let second = run_input_output(cmd, None); - Box::new(first.join(second).map(|(f, s)| s)) + // TODO: If we need to chain these to emulate a frontend, shouldn't + // we explicitly wait on the first one before starting the second one? + // (rather than via which drives these concurrently) + let (_f, s) = futures::future::try_join(first, second).await?; + Ok(s) } else { - Box::new(run_input_output(cmd, None)) + run_input_output(cmd, None).await } } @@ -205,7 +211,6 @@ mod test { use crate::compiler::*; use crate::mock_command::*; use crate::test::utils::*; - use futures::Future; use std::collections::HashMap; use std::path::PathBuf; diff --git a/src/compiler/rust.rs b/src/compiler/rust.rs index 6ac684ff..1f45bbca 100644 --- a/src/compiler/rust.rs +++ b/src/compiler/rust.rs @@ -26,10 +26,8 @@ use crate::dist::pkg; use crate::lru_disk_cache::{LruCache, Meter}; use crate::mock_command::{CommandCreatorSync, RunCommand}; use crate::util::{fmt_duration_as_secs, hash_all, run_input_output, Digest}; -use crate::util::{ref_env, HashToDigest, OsStrExt, SpawnExt}; +use crate::util::{ref_env, HashToDigest, OsStrExt}; use filetime::FileTime; -use futures::Future; -use futures_03::executor::ThreadPool; use log::Level::Trace; #[cfg(feature = "dist-client")] #[cfg(feature = "dist-client")] @@ -44,12 +42,14 @@ use std::env::consts::{DLL_PREFIX, EXE_EXTENSION}; use std::ffi::OsString; use std::fmt; use std::fs; +use std::future::Future; use std::hash::Hash; #[cfg(feature = "dist-client")] use std::io; use std::io::Read; use std::iter; use std::path::{Path, PathBuf}; +use std::pin::Pin; use std::process; #[cfg(feature = "dist-client")] use std::sync::{Arc, Mutex}; @@ -197,24 +197,24 @@ lazy_static! { const CACHE_VERSION: &[u8] = b"6"; /// Get absolute paths for all source files listed in rustc's dep-info output. -fn get_source_files( +async fn get_source_files( creator: &T, crate_name: &str, executable: &Path, arguments: &[OsString], cwd: &Path, env_vars: &[(OsString, OsString)], - pool: &ThreadPool, -) -> SFuture> + pool: &tokio::runtime::Handle, +) -> Result> where T: CommandCreatorSync, { let start = time::Instant::now(); // Get the full list of source files from rustc's dep-info. - let temp_dir = ftry!(tempfile::Builder::new() + let temp_dir = tempfile::Builder::new() .prefix("sccache") .tempdir() - .context("Failed to create temp dir")); + .context("Failed to create temp dir")?; let dep_file = temp_dir.path().join("deps.d"); let mut cmd = creator.clone().new_command_sync(executable); cmd.args(&arguments) @@ -225,29 +225,29 @@ where .envs(ref_env(env_vars)) .current_dir(cwd); trace!("[{}]: get dep-info: {:?}", crate_name, cmd); - let dep_info = run_input_output(cmd, None); + // Output of command is in file under dep_file, so we ignore stdout&stderr + let _dep_info = run_input_output(cmd, None).await?; // Parse the dep-info file, then hash the contents of those files. - let pool = pool.clone(); let cwd = cwd.to_owned(); - let crate_name = crate_name.to_owned(); - Box::new(dep_info.and_then(move |_| -> SFuture<_> { - let name2 = crate_name.clone(); - let parsed = pool.spawn_fn(move || { + let name2 = crate_name.to_owned(); + let parsed = pool + .spawn_blocking(move || { parse_dep_file(&dep_file, &cwd) .with_context(|| format!("Failed to parse dep info for {}", name2)) - }); - Box::new(parsed.map(move |files| { - trace!( - "[{}]: got {} source files from dep-info in {}", - crate_name, - files.len(), - fmt_duration_as_secs(&start.elapsed()) - ); - // Just to make sure we capture temp_dir. - drop(temp_dir); - files - })) - })) + }) + .await?; + + parsed.map(move |files| { + trace!( + "[{}]: got {} source files from dep-info in {}", + crate_name, + files.len(), + fmt_duration_as_secs(&start.elapsed()) + ); + // Just to make sure we capture temp_dir. + drop(temp_dir); + files + }) } /// Parse dependency info from `file` and return a Vec of files mentioned. @@ -315,15 +315,15 @@ where } /// Run `rustc --print file-names` to get the outputs of compilation. -fn get_compiler_outputs( +async fn get_compiler_outputs( creator: &T, executable: &Path, arguments: Vec, cwd: &Path, env_vars: &[(OsString, OsString)], -) -> SFuture> +) -> Result> where - T: CommandCreatorSync, + T: Clone + CommandCreatorSync, { let mut cmd = creator.clone().new_command_sync(executable); cmd.args(&arguments) @@ -334,37 +334,36 @@ where if log_enabled!(Trace) { trace!("get_compiler_outputs: {:?}", cmd); } - let outputs = run_input_output(cmd, None); - Box::new(outputs.and_then(move |output| -> Result<_> { - let outstr = String::from_utf8(output.stdout).context("Error parsing rustc output")?; - if log_enabled!(Trace) { - trace!("get_compiler_outputs: {:?}", outstr); - } - Ok(outstr.lines().map(|l| l.to_owned()).collect()) - })) + let outputs = run_input_output(cmd, None).await?; + + let outstr = String::from_utf8(outputs.stdout).context("Error parsing rustc output")?; + if log_enabled!(Trace) { + trace!("get_compiler_outputs: {:?}", outstr); + } + Ok(outstr.lines().map(|l| l.to_owned()).collect()) } impl Rust { /// Create a new Rust compiler instance, calculating the hashes of /// all the shared libraries in its sysroot. - pub fn new( + pub async fn new( mut creator: T, executable: PathBuf, env_vars: &[(OsString, OsString)], rustc_verbose_version: &str, dist_archive: Option, - pool: ThreadPool, - ) -> SFuture + pool: tokio::runtime::Handle, + ) -> Result where T: CommandCreatorSync, { // Taken from Cargo - let host = ftry!(rustc_verbose_version + let host = rustc_verbose_version .lines() .find(|l| l.starts_with("host: ")) .map(|l| &l[6..]) - .context("rustc verbose version didn't have a line for `host:`")) - .to_string(); + .context("rustc verbose version didn't have a line for `host:`")? + .to_string(); // it's fine to use the `executable` directly no matter if proxied or not let mut cmd = creator.new_command_sync(&executable); @@ -373,8 +372,8 @@ impl Rust { .arg("--print=sysroot") .env_clear() .envs(ref_env(env_vars)); - let output = run_input_output(cmd, None); - let sysroot_and_libs = output.and_then(move |output| -> Result<_> { + let sysroot_and_libs = async move { + let output = run_input_output(cmd, None).await?; //debug!("output.and_then: {}", output); let outstr = String::from_utf8(output.stdout).context("Error parsing sysroot")?; let sysroot = PathBuf::from(outstr.trim_end()); @@ -401,45 +400,48 @@ impl Rust { libs.push(path); }; libs.sort(); - Ok((sysroot, libs)) - }); - - #[cfg(feature = "dist-client")] - let rlib_dep_reader = { - let executable = executable.clone(); - let env_vars = env_vars.to_owned(); - pool.spawn_fn(move || Ok(RlibDepReader::new_with_check(executable, &env_vars))) + Result::Ok((sysroot, libs)) }; #[cfg(feature = "dist-client")] - return Box::new(sysroot_and_libs.join(rlib_dep_reader).and_then(move |((sysroot, libs), rlib_dep_reader)| { + { + use futures::TryFutureExt; + let rlib_dep_reader = { + let executable = executable.clone(); + let env_vars = env_vars.to_owned(); + pool.spawn_blocking(move || RlibDepReader::new_with_check(executable, &env_vars)) + .map_err(anyhow::Error::from) + }; + + let ((sysroot, libs), rlib_dep_reader) = + futures::future::try_join(sysroot_and_libs, rlib_dep_reader).await?; + let rlib_dep_reader = match rlib_dep_reader { Ok(r) => Some(Arc::new(r)), Err(e) => { warn!("Failed to initialise RlibDepDecoder, distributed compiles will be inefficient: {}", e); None - }, - }; - hash_all(&libs, &pool).map(move |digests| { - Rust { - executable, - host, - sysroot, - compiler_shlibs_digests: digests, - rlib_dep_reader, } + }; + hash_all(&libs, &pool).await.map(move |digests| Rust { + executable, + host, + sysroot, + compiler_shlibs_digests: digests, + rlib_dep_reader, }) - })); + } #[cfg(not(feature = "dist-client"))] - return Box::new(sysroot_and_libs.and_then(move |(sysroot, libs)| { - hash_all(&libs, &pool).map(move |digests| Rust { + { + let (sysroot, libs) = sysroot_and_libs.await?; + hash_all(&libs, &pool).await.map(move |digests| Rust { executable, host, sysroot, compiler_shlibs_digests: digests, }) - })); + } } } @@ -503,49 +505,40 @@ where mut creator: T, cwd: PathBuf, env: &[(OsString, OsString)], - ) -> SFuture<(PathBuf, FileTime)> { - let proxy_executable = self.proxy_executable.clone(); - - let mut child = creator.new_command_sync(&proxy_executable); + ) -> Pin> + Send>> { + let mut child = creator.new_command_sync(&self.proxy_executable); child .current_dir(&cwd) .env_clear() .envs(ref_env(&env)) .args(&["which", "rustc"]); - let lookup = run_input_output(child, None) - .map_err(|e| anyhow!("Failed to execute rustup which rustc: {}", e)) - .and_then(move |output| { - String::from_utf8(output.stdout) - .map_err(|e| anyhow!("Failed to parse output of rustup which rustc: {}", e)) - .and_then(|stdout| { - let proxied_compiler = PathBuf::from(stdout.trim()); - trace!( - "proxy: rustup which rustc produced: {:?}", - &proxied_compiler - ); - let res = fs::metadata(proxied_compiler.as_path()) - .map_err(|e| { - anyhow!( - "Failed to obtain metadata of the resolved, true rustc: {}", - e - ) - }) - .and_then(|attr| { - if attr.is_file() { - Ok(FileTime::from_last_modification_time(&attr)) - } else { - Err(anyhow!( - "proxy: rustup resolved compiler is not of type file" - )) - } - }) - .map(|filetime| (proxied_compiler, filetime)); - res - }) - }); + Box::pin(async move { + let output = run_input_output(child, None) + .await + .context("Failed to execute rustup which rustc")?; + + let stdout = String::from_utf8(output.stdout) + .context("Failed to parse output of rustup which rustc")?; + + let proxied_compiler = PathBuf::from(stdout.trim()); + trace!( + "proxy: rustup which rustc produced: {:?}", + &proxied_compiler + ); + // TODO: Delegate FS access to a thread pool if possible + let attr = fs::metadata(proxied_compiler.as_path()) + .context("Failed to obtain metadata of the resolved, true rustc")?; - Box::new(lookup) + if attr.is_file() { + Ok(FileTime::from_last_modification_time(&attr)) + } else { + Err(anyhow!( + "proxy: rustup resolved compiler is not of type file" + )) + } + .map(move |filetime| (proxied_compiler, filetime)) + }) } fn box_clone(&self) -> Box> { @@ -567,23 +560,15 @@ impl RustupProxy { }) } - pub fn find_proxy_executable( + pub async fn find_proxy_executable( compiler_executable: &Path, proxy_name: &str, mut creator: T, env: &[(OsString, OsString)], - ) -> SFuture>> + ) -> Result>> where T: CommandCreatorSync, { - let compiler_executable1 = compiler_executable.to_owned(); - let compiler_executable2 = compiler_executable.to_owned(); - let proxy_name1 = proxy_name.to_owned(); - let proxy_name2 = proxy_name.to_owned(); - - let env1 = env.to_owned(); - let env2 = env.to_owned(); - enum ProxyPath { Candidate(PathBuf), ToBeDiscovered, @@ -613,93 +598,82 @@ impl RustupProxy { // verify rustc is proxy let mut child = creator.new_command_sync(compiler_executable.to_owned()); - child.env_clear().envs(ref_env(&env1)).args(&["+stable"]); - let find_candidate = run_input_output(child, None) - .map(move |output| { - if output.status.success() { - trace!("proxy: Found a compiler proxy managed by rustup"); - ProxyPath::ToBeDiscovered - } else { - trace!("proxy: Found a regular compiler"); - ProxyPath::None - } - }) - .and_then(move |state| { - let state = match state { - ProxyPath::Candidate(_) => { unreachable!("qed") } - ProxyPath::ToBeDiscovered => { - // simple check: is there a rustup in the same parent dir as rustc? - // that would be the prefered one - Ok(match compiler_executable1.parent().map(|parent| { parent.to_owned() }) { - Some(mut parent) => { - parent.push(proxy_name1); - let proxy_candidate = parent; - if proxy_candidate.exists() { - trace!("proxy: Found a compiler proxy at {}", proxy_candidate.display()); - ProxyPath::Candidate(proxy_candidate) - } else { - ProxyPath::ToBeDiscovered - } - }, - None => { - ProxyPath::ToBeDiscovered - }, - }) - }, - x => Ok(x), - }; - f_ok(state) - }).and_then(move |state| { - let state = match state { - Ok(ProxyPath::ToBeDiscovered) => { - // still no rustup found, use which crate to find one - match which::which(&proxy_name2) { - Ok(proxy_candidate) => { - warn!("proxy: rustup found, but not where it was expected (next to rustc {})", compiler_executable2.display()); - Ok(ProxyPath::Candidate(proxy_candidate)) - }, - Err(e) => { - trace!("proxy: rustup is not present: {}", e); - Ok(ProxyPath::ToBeDiscovered) - }, + child.env_clear().envs(ref_env(&env)).args(&["+stable"]); + let state = run_input_output(child, None).await.map(move |output| { + if output.status.success() { + trace!("proxy: Found a compiler proxy managed by rustup"); + ProxyPath::ToBeDiscovered + } else { + trace!("proxy: Found a regular compiler"); + ProxyPath::None + } + }); + + let state = match state { + Ok(ProxyPath::Candidate(_)) => unreachable!("Q.E.D."), + Ok(ProxyPath::ToBeDiscovered) => { + // simple check: is there a rustup in the same parent dir as rustc? + // that would be the prefered one + Ok(match compiler_executable.parent().map(Path::to_owned) { + Some(parent) => { + let proxy_candidate = parent.join(proxy_name); + if proxy_candidate.exists() { + trace!( + "proxy: Found a compiler proxy at {}", + proxy_candidate.display() + ); + ProxyPath::Candidate(proxy_candidate) + } else { + ProxyPath::ToBeDiscovered } } - x => x, - }; - f_ok(state) - }); - - let f = find_candidate.and_then(move |state| { - match state { - Err(e) => f_ok(Err(e)), - Ok(ProxyPath::ToBeDiscovered) => f_ok(Err(anyhow!( - "Failed to discover a rustup executable, but rustc behaves like a proxy" - ))), - Ok(ProxyPath::None) => f_ok(Ok(None)), - Ok(ProxyPath::Candidate(proxy_executable)) => { - // verify the candidate is a rustup - let mut child = creator.new_command_sync(proxy_executable.to_owned()); - child.env_clear().envs(ref_env(&env2)).args(&["--version"]); - let rustup_candidate_check = run_input_output(child, None).map(move |output| { - String::from_utf8(output.stdout) - .map_err(|_e| { - anyhow!("Response of `rustup --version` is not valid UTF-8") - }) - .and_then(|stdout| { - if stdout.trim().starts_with("rustup ") { - trace!("PROXY rustup --version produced: {}", &stdout); - Self::new(&proxy_executable).map(Some) - } else { - Err(anyhow!("Unexpected output or `rustup --version`")) - } - }) - }); - Box::new(rustup_candidate_check) + None => ProxyPath::ToBeDiscovered, + }) + } + x => x, + }; + let state = match state { + Ok(ProxyPath::ToBeDiscovered) => { + // still no rustup found, use which crate to find one + match which::which(&proxy_name) { + Ok(proxy_candidate) => { + warn!( + "proxy: rustup found, but not where it was expected (next to rustc {})", + compiler_executable.display() + ); + Ok(ProxyPath::Candidate(proxy_candidate)) + } + Err(e) => { + trace!("proxy: rustup is not present: {}", e); + Ok(ProxyPath::ToBeDiscovered) + } } } - }); + x => x, + }; - Box::new(f) + match state { + Err(e) => Err(e), + Ok(ProxyPath::ToBeDiscovered) => Ok(Err(anyhow!( + "Failed to discover a rustup executable, but rustc behaves like a proxy" + ))), + Ok(ProxyPath::None) => Ok(Ok(None)), + Ok(ProxyPath::Candidate(proxy_executable)) => { + // verify the candidate is a rustup + let mut child = creator.new_command_sync(proxy_executable.to_owned()); + child.env_clear().envs(ref_env(&env)).args(&["--version"]); + let rustup_candidate_check = run_input_output(child, None).await?; + + let stdout = String::from_utf8(rustup_candidate_check.stdout) + .map_err(|_e| anyhow!("Response of `rustup --version` is not valid UTF-8"))?; + Ok(if stdout.trim().starts_with("rustup ") { + trace!("PROXY rustup --version produced: {}", &stdout); + Self::new(&proxy_executable).map(Some) + } else { + Err(anyhow!("Unexpected output or `rustup --version`")) + }) + } + } } } @@ -1211,19 +1185,21 @@ fn parse_arguments(arguments: &[OsString], cwd: &Path) -> CompilerArguments CompilerHasher for RustHasher where T: CommandCreatorSync, { - fn generate_hash_key( + async fn generate_hash_key( self: Box, creator: &T, cwd: PathBuf, env_vars: Vec<(OsString, OsString)>, _may_dist: bool, - pool: &ThreadPool, + pool: &tokio::runtime::Handle, _rewrite_includes_only: bool, - ) -> SFuture { + ) -> Result { let RustHasher { executable, host, @@ -1274,19 +1250,21 @@ where .collect::>(); // Find all the source files and hash them let source_hashes_pool = pool.clone(); - let source_files = get_source_files( - creator, - &crate_name, - &executable, - &filtered_arguments, - &cwd, - &env_vars, - pool, - ); - let source_files_and_hashes = source_files.and_then(move |source_files| { - hash_all(&source_files, &source_hashes_pool) - .map(|source_hashes| (source_files, source_hashes)) - }); + let source_files_and_hashes = async { + let source_files = get_source_files( + creator, + &crate_name, + &executable, + &filtered_arguments, + &cwd, + &env_vars, + pool, + ) + .await?; + let source_hashes = hash_all(&source_files, &source_hashes_pool).await?; + Ok((source_files, source_hashes)) + }; + // Hash the contents of the externs listed on the commandline. trace!("[{}]: hashing {} externs", crate_name, externs.len()); let abs_externs = externs.iter().map(|e| cwd.join(e)).collect::>(); @@ -1295,196 +1273,190 @@ where trace!("[{}]: hashing {} staticlibs", crate_name, staticlibs.len()); let abs_staticlibs = staticlibs.iter().map(|s| cwd.join(s)).collect::>(); let staticlib_hashes = hash_all(&abs_staticlibs, pool); - let creator = creator.clone(); - let hashes = source_files_and_hashes.join3(extern_hashes, staticlib_hashes); - Box::new(hashes.and_then( - move |((source_files, source_hashes), extern_hashes, staticlib_hashes)| -> SFuture<_> { - // If you change any of the inputs to the hash, you should change `CACHE_VERSION`. - let mut m = Digest::new(); - // Hash inputs: - // 1. A version - m.update(CACHE_VERSION); - // 2. compiler_shlibs_digests - for d in compiler_shlibs_digests { - m.update(d.as_bytes()); - } - let weak_toolchain_key = m.clone().finish(); - // 3. The full commandline (self.arguments) - // TODO: there will be full paths here, it would be nice to - // normalize them so we can get cross-machine cache hits. - // A few argument types are not passed in a deterministic order - // by cargo: --extern, -L, --cfg. We'll filter those out, sort them, - // and append them to the rest of the arguments. - let args = { - let (mut sortables, rest): (Vec<_>, Vec<_>) = os_string_arguments - .iter() - // We exclude a few arguments from the hash: - // -L, --extern, --out-dir - // These contain paths which aren't relevant to the output, and the compiler inputs - // in those paths (rlibs and static libs used in the compilation) are used as hash - // inputs below. - .filter(|&&(ref arg, _)| { - !(arg == "--extern" || arg == "-L" || arg == "--out-dir") - }) - // A few argument types were not passed in a deterministic order - // by older versions of cargo: --extern, -L, --cfg. We'll filter the rest of those - // out, sort them, and append them to the rest of the arguments. - .partition(|&&(ref arg, _)| arg == "--cfg"); - sortables.sort(); - rest.into_iter() - .chain(sortables) - .flat_map(|&(ref arg, ref val)| iter::once(arg).chain(val.as_ref())) - .fold(OsString::new(), |mut a, b| { - a.push(b); - a - }) - }; - args.hash(&mut HashToDigest { digest: &mut m }); - // 4. The digest of all source files (this includes src file from cmdline). - // 5. The digest of all files listed on the commandline (self.externs). - // 6. The digest of all static libraries listed on the commandline (self.staticlibs). - for h in source_hashes - .into_iter() - .chain(extern_hashes) - .chain(staticlib_hashes) - { - m.update(h.as_bytes()); + + let ((source_files, source_hashes), extern_hashes, staticlib_hashes) = + futures::try_join!(source_files_and_hashes, extern_hashes, staticlib_hashes)?; + // If you change any of the inputs to the hash, you should change `CACHE_VERSION`. + let mut m = Digest::new(); + // Hash inputs: + // 1. A version + m.update(CACHE_VERSION); + // 2. compiler_shlibs_digests + for d in compiler_shlibs_digests { + m.update(d.as_bytes()); + } + let weak_toolchain_key = m.clone().finish(); + // 3. The full commandline (self.arguments) + // TODO: there will be full paths here, it would be nice to + // normalize them so we can get cross-machine cache hits. + // A few argument types are not passed in a deterministic order + // by cargo: --extern, -L, --cfg. We'll filter those out, sort them, + // and append them to the rest of the arguments. + let args = { + let (mut sortables, rest): (Vec<_>, Vec<_>) = os_string_arguments + .iter() + // We exclude a few arguments from the hash: + // -L, --extern, --out-dir + // These contain paths which aren't relevant to the output, and the compiler inputs + // in those paths (rlibs and static libs used in the compilation) are used as hash + // inputs below. + .filter(|&&(ref arg, _)| !(arg == "--extern" || arg == "-L" || arg == "--out-dir")) + // A few argument types were not passed in a deterministic order + // by older versions of cargo: --extern, -L, --cfg. We'll filter the rest of those + // out, sort them, and append them to the rest of the arguments. + .partition(|&&(ref arg, _)| arg == "--cfg"); + sortables.sort(); + rest.into_iter() + .chain(sortables) + .flat_map(|&(ref arg, ref val)| iter::once(arg).chain(val.as_ref())) + .fold(OsString::new(), |mut a, b| { + a.push(b); + a + }) + }; + args.hash(&mut HashToDigest { digest: &mut m }); + // 4. The digest of all source files (this includes src file from cmdline). + // 5. The digest of all files listed on the commandline (self.externs). + // 6. The digest of all static libraries listed on the commandline (self.staticlibs). + for h in source_hashes + .into_iter() + .chain(extern_hashes) + .chain(staticlib_hashes) + { + m.update(h.as_bytes()); + } + // 7. Environment variables. Ideally we'd use anything referenced + // via env! in the program, but we don't have a way to determine that + // currently, and hashing all environment variables is too much, so + // we'll just hash the CARGO_ env vars and hope that's sufficient. + // Upstream Rust issue tracking getting information about env! usage: + // https://github.com/rust-lang/rust/issues/40364 + let mut env_vars: Vec<_> = env_vars + .iter() + // Filter out RUSTC_COLOR since we control color usage with command line flags. + // rustc reports an error when both are present. + .filter(|(ref k, _)| k != "RUSTC_COLOR") + .cloned() + .collect(); + env_vars.sort(); + for &(ref var, ref val) in env_vars.iter() { + // CARGO_MAKEFLAGS will have jobserver info which is extremely non-cacheable. + if var.starts_with("CARGO_") && var != "CARGO_MAKEFLAGS" { + var.hash(&mut HashToDigest { digest: &mut m }); + m.update(b"="); + val.hash(&mut HashToDigest { digest: &mut m }); + } + } + // 8. The cwd of the compile. This will wind up in the rlib. + cwd.hash(&mut HashToDigest { digest: &mut m }); + // Turn arguments into a simple Vec to calculate outputs. + let flat_os_string_arguments: Vec = os_string_arguments + .into_iter() + .flat_map(|(arg, val)| iter::once(arg).chain(val)) + .collect(); + + let mut outputs = get_compiler_outputs( + creator, + &executable, + flat_os_string_arguments, + &cwd, + &env_vars, + ) + .await?; + + // metadata / dep-info don't ever generate binaries, but + // rustc still makes them appear in the --print + // file-names output (see + // https://github.com/rust-lang/rust/pull/68799). + // + // So if we see a binary in the rustc output and figure + // out that we're not _actually_ generating it, then we + // can avoid generating everything that isn't an rlib / + // rmeta. + // + // This can go away once the above rustc PR makes it in. + let emit_generates_only_metadata = + !emit.is_empty() && emit.iter().all(|e| e == "metadata" || e == "dep-info"); + + if emit_generates_only_metadata { + outputs.retain(|o| o.ends_with(".rlib") || o.ends_with(".rmeta")); + } + + if emit.contains("metadata") { + // rustc currently does not report rmeta outputs with --print file-names + // --emit metadata the rlib is printed, and with --emit metadata,link + // only the rlib is printed. + let rlibs: HashSet<_> = outputs + .iter() + .cloned() + .filter(|p| p.ends_with(".rlib")) + .collect(); + for lib in rlibs { + let rmeta = lib.replacen(".rlib", ".rmeta", 1); + // Do this defensively for future versions of rustc that may + // be fixed. + if !outputs.contains(&rmeta) { + outputs.push(rmeta); } - // 7. Environment variables. Ideally we'd use anything referenced - // via env! in the program, but we don't have a way to determine that - // currently, and hashing all environment variables is too much, so - // we'll just hash the CARGO_ env vars and hope that's sufficient. - // Upstream Rust issue tracking getting information about env! usage: - // https://github.com/rust-lang/rust/issues/40364 - let mut env_vars: Vec<_> = env_vars - .iter() - // Filter out RUSTC_COLOR since we control color usage with command line flags. - // rustc reports an error when both are present. - .filter(|(ref k, _)| k != "RUSTC_COLOR") - .cloned() - .collect(); - env_vars.sort(); - for &(ref var, ref val) in env_vars.iter() { - // CARGO_MAKEFLAGS will have jobserver info which is extremely non-cacheable. - if var.eq("CARGO") || (var.starts_with("CARGO_") && var != "CARGO_MAKEFLAGS") { - var.hash(&mut HashToDigest { digest: &mut m }); - m.update(b"="); - val.hash(&mut HashToDigest { digest: &mut m }); - } + if !emit.contains("link") { + outputs.retain(|p| *p != lib); } - // 8. The cwd of the compile. This will wind up in the rlib. - cwd.hash(&mut HashToDigest { digest: &mut m }); - // Turn arguments into a simple Vec to calculate outputs. - let flat_os_string_arguments: Vec = os_string_arguments - .into_iter() - .flat_map(|(arg, val)| iter::once(arg).chain(val)) - .collect(); - Box::new( - get_compiler_outputs( - &creator, - &executable, - flat_os_string_arguments, - &cwd, - &env_vars, - ) - .map(move |mut outputs| { - // metadata / dep-info don't ever generate binaries, but - // rustc still makes them appear in the --print - // file-names output (see - // https://github.com/rust-lang/rust/pull/68799). - // - // So if we see a binary in the rustc output and figure - // out that we're not _actually_ generating it, then we - // can avoid generating everything that isn't an rlib / - // rmeta. - // - // This can go away once the above rustc PR makes it in. - let emit_generates_only_metadata = !emit.is_empty() - && emit.iter().all(|e| e == "metadata" || e == "dep-info"); - - if emit_generates_only_metadata { - outputs.retain(|o| o.ends_with(".rlib") || o.ends_with(".rmeta")); - } + } + } - if emit.contains("metadata") { - // rustc currently does not report rmeta outputs with --print file-names - // --emit metadata the rlib is printed, and with --emit metadata,link - // only the rlib is printed. - let rlibs: HashSet<_> = outputs - .iter() - .cloned() - .filter(|p| p.ends_with(".rlib")) - .collect(); - for lib in rlibs { - let rmeta = lib.replacen(".rlib", ".rmeta", 1); - // Do this defensively for future versions of rustc that may - // be fixed. - if !outputs.contains(&rmeta) { - outputs.push(rmeta); - } - if !emit.contains("link") { - outputs.retain(|p| *p != lib); - } - } - } + // Convert output files into a map of basename -> full + // path, and remove some unneeded / non-existing ones, + // see https://github.com/rust-lang/rust/pull/68799. + let mut outputs = outputs + .into_iter() + .map(|o| { + let p = output_dir.join(&o); + (o, p) + }) + .collect::>(); + let dep_info = if let Some(dep_info) = dep_info { + let p = output_dir.join(&dep_info); + outputs.insert(dep_info.to_string_lossy().into_owned(), p.clone()); + Some(p) + } else { + None + }; + let mut arguments = arguments; + // Request color output unless json was requested. The client will strip colors if needed. + if !has_json { + arguments.push(Argument::WithValue( + "--color", + ArgData::Color("always".into()), + ArgDisposition::Separated, + )); + } - // Convert output files into a map of basename -> full - // path, and remove some unneeded / non-existing ones, - // see https://github.com/rust-lang/rust/pull/68799. - let mut outputs = outputs - .into_iter() - .map(|o| { - let p = output_dir.join(&o); - (o, p) - }) - .collect::>(); - let dep_info = if let Some(dep_info) = dep_info { - let p = output_dir.join(&dep_info); - outputs.insert(dep_info.to_string_lossy().into_owned(), p.clone()); - Some(p) - } else { - None - }; - let mut arguments = arguments; - // Request color output unless json was requested. The client will strip colors if needed. - if !has_json { - arguments.push(Argument::WithValue( - "--color", - ArgData::Color("always".into()), - ArgDisposition::Separated, - )); - } + let inputs = source_files + .into_iter() + .chain(abs_externs) + .chain(abs_staticlibs) + .collect(); - let inputs = source_files - .into_iter() - .chain(abs_externs) - .chain(abs_staticlibs) - .collect(); - - HashResult { - key: m.finish(), - compilation: Box::new(RustCompilation { - executable, - host, - sysroot, - arguments, - inputs, - outputs, - crate_link_paths, - crate_name, - crate_types, - dep_info, - cwd, - env_vars, - #[cfg(feature = "dist-client")] - rlib_dep_reader, - }), - weak_toolchain_key, - } - }), - ) - }, - )) + Ok(HashResult { + key: m.finish(), + compilation: Box::new(RustCompilation { + executable, + host, + sysroot, + arguments, + inputs, + outputs, + crate_link_paths, + crate_name, + crate_types, + dep_info, + cwd, + env_vars, + #[cfg(feature = "dist-client")] + rlib_dep_reader, + }), + weak_toolchain_key, + }) } fn color_mode(&self) -> ColorMode { @@ -2956,7 +2928,8 @@ c:/foo/bar.rs: let creator = new_creator(); mock_dep_info(&creator, &["foo.rs", "bar.rs"]); mock_file_names(&creator, &["foo.rlib", "foo.a"]); - let pool = ThreadPool::sized(1); + let runtime = single_threaded_runtime(); + let pool = runtime.handle().clone(); let res = hasher .generate_hash_key( &creator, @@ -3046,7 +3019,9 @@ c:/foo/bar.rs: }); let creator = new_creator(); - let pool = ThreadPool::sized(1); + let runtime = single_threaded_runtime(); + let pool = runtime.handle().clone(); + mock_dep_info(&creator, &["foo.rs"]); mock_file_names(&creator, &["foo.rlib"]); hasher diff --git a/src/dist/client_auth.rs b/src/dist/client_auth.rs index 2a050ff8..b43689cd 100644 --- a/src/dist/client_auth.rs +++ b/src/dist/client_auth.rs @@ -1,21 +1,16 @@ -use futures::future; -use futures::prelude::*; -use futures::sync::oneshot; +use futures::channel::oneshot; use http::StatusCode; -use hyper::body::Payload; use hyper::server::conn::AddrIncoming; -use hyper::service::Service; -use hyper::{Body, Request, Response, Server}; +use hyper::{Body, Response, Server}; use hyperx::header::{ContentLength, ContentType}; use serde::Serialize; use std::collections::HashMap; use std::error::Error as StdError; use std::io; -use std::marker::PhantomData; use std::net::{TcpStream, ToSocketAddrs}; use std::sync::mpsc; use std::time::Duration; -use tokio_compat::runtime::current_thread::Runtime; +use tokio::runtime::Runtime; use url::Url; use uuid::Uuid; @@ -29,43 +24,6 @@ pub const VALID_PORTS: &[u16] = &[12731, 32492, 56909]; const MIN_TOKEN_VALIDITY: Duration = Duration::from_secs(2 * 24 * 60 * 60); const MIN_TOKEN_VALIDITY_WARNING: &str = "two days"; -trait ServeFn: - Fn(Request) -> Box, Error = hyper::Error> + Send> - + Copy - + Send - + 'static -{ -} -impl ServeFn for T where - T: Fn(Request) -> Box, Error = hyper::Error> + Send> - + Copy - + Send - + 'static -{ -} - -fn serve_sfuture(serve: fn(Request) -> SFutureSend>) -> impl ServeFn { - move |req: Request| { - let uri = req.uri().to_owned(); - Box::new(serve(req).or_else(move |e| { - // `{:?}` prints the full cause chain and backtrace. - let body = format!("{:?}", e); - eprintln!( - "sccache: Error during a request to {} on the client auth web server\n{}", - uri, body - ); - let len = body.len(); - let mut builder = Response::builder(); - builder.status(StatusCode::INTERNAL_SERVER_ERROR); - Ok(builder - .set_header(ContentType::text()) - .set_header(ContentLength(len as u64)) - .body(body.into()) - .unwrap()) - })) as Box + Send> - } -} - fn query_pairs(url: &str) -> Result> { // Url::parse operates on absolute URLs, so ensure there's a prefix let url = Url::parse("http://unused_base") @@ -128,8 +86,7 @@ mod code_grant_pkce { html_response, json_response, query_pairs, MIN_TOKEN_VALIDITY, MIN_TOKEN_VALIDITY_WARNING, REDIRECT_WITH_AUTH_JSON, }; - use futures::future; - use futures::sync::oneshot; + use futures::channel::oneshot; use hyper::{Body, Method, Request, Response, StatusCode}; use rand::{rngs::OsRng, RngCore}; use sha2::{Digest, Sha256}; @@ -241,19 +198,19 @@ mod code_grant_pkce { "##; - pub fn serve(req: Request) -> SFutureSend> { + pub fn serve(req: Request) -> Result> { let mut state = STATE.lock().unwrap(); let state = state.as_mut().unwrap(); debug!("Handling {} {}", req.method(), req.uri()); let response = match (req.method(), req.uri().path()) { (&Method::GET, "/") => html_response(REDIRECT_WITH_AUTH_JSON), - (&Method::GET, "/auth_detail.json") => ftry_send!(json_response(&state.auth_url)), + (&Method::GET, "/auth_detail.json") => json_response(&state.auth_url)?, (&Method::GET, "/redirect") => { - let query_pairs = ftry_send!(query_pairs(&req.uri().to_string())); - let (code, auth_state) = ftry_send!(handle_code_response(query_pairs) - .context("Failed to handle response from redirect")); + let query_pairs = query_pairs(&req.uri().to_string())?; + let (code, auth_state) = handle_code_response(query_pairs) + .context("Failed to handle response from redirect")?; if auth_state != state.auth_state_value { - return ftry_send!(Err(anyhow!("Mismatched auth states after redirect"))); + return Err(anyhow!("Mismatched auth states after redirect")); } // Deliberately in reverse order for a 'happens-before' relationship state.code_tx.send(code).unwrap(); @@ -264,12 +221,11 @@ mod code_grant_pkce { warn!("Route not found"); Response::builder() .status(StatusCode::NOT_FOUND) - .body("".into()) - .unwrap() + .body("".into())? } }; - Box::new(future::ok(response)) + Ok(response) } pub fn code_to_token( @@ -286,8 +242,8 @@ mod code_grant_pkce { grant_type: GRANT_TYPE_PARAM_VALUE, redirect_uri, }; - let client = reqwest::Client::new(); - let mut res = client.post(token_url).json(&token_request).send()?; + let client = reqwest::blocking::Client::new(); + let res = client.post(token_url).json(&token_request).send()?; if !res.status().is_success() { bail!( "Sending code to {} failed, HTTP error: {}", @@ -319,8 +275,7 @@ mod implicit { html_response, json_response, query_pairs, MIN_TOKEN_VALIDITY, MIN_TOKEN_VALIDITY_WARNING, REDIRECT_WITH_AUTH_JSON, }; - use futures::future; - use futures::sync::oneshot; + use futures::channel::oneshot; use hyper::{Body, Method, Request, Response, StatusCode}; use std::collections::HashMap; use std::sync::mpsc; @@ -420,21 +375,20 @@ mod implicit { "##; - pub fn serve(req: Request) -> SFutureSend> { + pub fn serve(req: Request) -> Result> { let mut state = STATE.lock().unwrap(); let state = state.as_mut().unwrap(); debug!("Handling {} {}", req.method(), req.uri()); let response = match (req.method(), req.uri().path()) { (&Method::GET, "/") => html_response(REDIRECT_WITH_AUTH_JSON), - (&Method::GET, "/auth_detail.json") => ftry_send!(json_response(&state.auth_url)), + (&Method::GET, "/auth_detail.json") => json_response(&state.auth_url)?, (&Method::GET, "/redirect") => html_response(SAVE_AUTH_AFTER_REDIRECT), (&Method::POST, "/save_auth") => { - let query_pairs = ftry_send!(query_pairs(&req.uri().to_string())); - let (token, expires_at, auth_state) = ftry_send!( - handle_response(query_pairs).context("Failed to save auth after redirect") - ); + let query_pairs = query_pairs(&req.uri().to_string())?; + let (token, expires_at, auth_state) = + handle_response(query_pairs).context("Failed to save auth after redirect")?; if auth_state != state.auth_state_value { - return ftry_send!(Err(anyhow!("Mismatched auth states after redirect"))); + return Err(anyhow!("Mismatched auth states after redirect")); } if expires_at - Instant::now() < MIN_TOKEN_VALIDITY { warn!( @@ -449,7 +403,7 @@ mod implicit { // Deliberately in reverse order for a 'happens-before' relationship state.token_tx.send(token).unwrap(); state.shutdown_tx.take().unwrap().send(()).unwrap(); - ftry_send!(json_response(&"")) + json_response(&"")? } _ => { warn!("Route not found"); @@ -460,58 +414,51 @@ mod implicit { } }; - Box::new(future::ok(response)) + Ok(response) } } -fn service_fn(f: F) -> ServiceFn -where - F: Fn(Request) -> S, - S: IntoFuture, -{ - ServiceFn { - f, - _req: PhantomData, - } -} - -struct ServiceFn { - f: F, - _req: PhantomData, +// Typing out a hyper service is a major pain, so let's focus on our simple +// `fn(Request) -> Response` handler functions; to reduce repetition +// we create a relevant service using hyper's own helper factory functions. +macro_rules! make_service { + ($serve_fn: expr) => {{ + use core::convert::Infallible; + use hyper::server::conn::AddrStream; + use hyper::service::{make_service_fn, service_fn}; + use hyper::{Body, Request}; + + make_service_fn(|_socket: &AddrStream| async { + Ok::<_, Infallible>(service_fn(|req: Request| async move { + let uri = req.uri().clone(); + $serve_fn(req).or_else(|e| error_code_response(uri, e)) + })) + }) + }}; } -impl Service for ServiceFn +#[allow(clippy::unnecessary_wraps)] +fn error_code_response(uri: hyper::Uri, e: E) -> hyper::Result> where - F: Fn(Request) -> Ret, - ReqBody: Payload, - Ret: IntoFuture>, - Ret::Error: Into>, - ResBody: Payload, + E: std::fmt::Debug, { - type ReqBody = ReqBody; - type ResBody = ResBody; - type Error = Ret::Error; - type Future = Ret::Future; - - fn call(&mut self, req: Request) -> Self::Future { - (self.f)(req).into_future() - } -} - -impl IntoFuture for ServiceFn { - type Future = future::FutureResult; - type Item = Self; - type Error = hyper::Error; - - fn into_future(self) -> Self::Future { - future::ok(self) - } + let body = format!("{:?}", e); + eprintln!( + "sccache: Error during a request to {} on the client auth web server\n{}", + uri, body + ); + let len = body.len(); + let builder = Response::builder().status(StatusCode::INTERNAL_SERVER_ERROR); + let res = builder + .set_header(ContentType::text()) + .set_header(ContentLength(len as u64)) + .body(body.into()) + .unwrap(); + Ok::, hyper::Error>(res) } -fn try_serve(serve: T) -> Result ServiceFn>> -where - T: ServeFn, -{ +/// Try to bind a TCP stream to any of the available port out of [`VALID_PORTS`]. +fn try_bind() -> Result> { // Try all the valid ports for &port in VALID_PORTS { let mut addrs = ("localhost", port) @@ -533,9 +480,8 @@ where } } - let new_service = move || service_fn(serve); match Server::try_bind(&addr) { - Ok(s) => return Ok(s.serve(new_service)), + Ok(s) => return Ok(s), Err(ref err) if err .source() @@ -557,7 +503,7 @@ pub fn get_token_oauth2_code_grant_pkce( mut auth_url: Url, token_url: &str, ) -> Result { - let server = try_serve(serve_sfuture(code_grant_pkce::serve))?; + let server = try_bind()?.serve(make_service!(code_grant_pkce::serve)); let port = server.local_addr().port(); let redirect_uri = format!("http://localhost:{}/redirect", port); @@ -585,15 +531,16 @@ pub fn get_token_oauth2_code_grant_pkce( shutdown_tx: Some(shutdown_tx), }; *code_grant_pkce::STATE.lock().unwrap() = Some(state); - let shutdown_signal = shutdown_rx.map_err(|e| { - warn!( - "Something went wrong while waiting for auth server shutdown: {}", - e - ) - }); let mut runtime = Runtime::new()?; - runtime.block_on(server.with_graceful_shutdown(shutdown_signal))?; + runtime.block_on(server.with_graceful_shutdown(async move { + if let Err(e) = shutdown_rx.await { + warn!( + "Something went wrong while waiting for auth server shutdown: {}", + e + ) + } + }))?; info!("Server finished, using code to request token"); let code = code_rx @@ -605,7 +552,8 @@ pub fn get_token_oauth2_code_grant_pkce( // https://auth0.com/docs/api-auth/tutorials/implicit-grant pub fn get_token_oauth2_implicit(client_id: &str, mut auth_url: Url) -> Result { - let server = try_serve(serve_sfuture(implicit::serve))?; + let server = try_bind()?.serve(make_service!(implicit::serve)); + let port = server.local_addr().port(); let redirect_uri = format!("http://localhost:{}/redirect", port); @@ -626,15 +574,16 @@ pub fn get_token_oauth2_implicit(client_id: &str, mut auth_url: Url) -> Result) -> Self; fn bearer_auth(self, token: String) -> Self; } - impl ReqwestRequestBuilderExt for reqwest::RequestBuilder { + impl ReqwestRequestBuilderExt for reqwest::blocking::RequestBuilder { fn bincode(self, bincode: &T) -> Result { let bytes = bincode::serialize(bincode).context("Failed to serialize body to bincode")?; @@ -54,7 +53,7 @@ mod common { self.set_header(header::Authorization(header::Bearer { token })) } } - impl ReqwestRequestBuilderExt for reqwest::r#async::RequestBuilder { + impl ReqwestRequestBuilderExt for reqwest::RequestBuilder { fn bincode(self, bincode: &T) -> Result { let bytes = bincode::serialize(bincode).context("Failed to serialize body to bincode")?; @@ -71,7 +70,7 @@ mod common { } pub fn bincode_req( - req: reqwest::RequestBuilder, + req: reqwest::blocking::RequestBuilder, ) -> Result { // Work around tiny_http issue #151 by disabling HTTP pipeline with // `Connection: close`. @@ -92,41 +91,29 @@ mod common { } } #[cfg(feature = "dist-client")] - pub fn bincode_req_fut( - req: reqwest::r#async::RequestBuilder, - ) -> SFuture { - Box::new( - // Work around tiny_http issue #151 by disabling HTTP pipeline with - // `Connection: close`. - req.set_header(header::Connection::close()) - .send() - .map_err(Into::into) - .and_then(|res| { - let status = res.status(); - res.into_body() - .concat2() - .map(move |b| (status, b)) - .map_err(Into::into) - }) - .and_then(|(status, body)| { - if !status.is_success() { - let errmsg = format!( - "Error {}: {}", - status.as_u16(), - String::from_utf8_lossy(&body) - ); - if status.is_client_error() { - return f_err(HttpClientError(errmsg)); - } else { - return f_err(anyhow!(errmsg)); - } - } - match bincode::deserialize(&body) { - Ok(r) => f_ok(r), - Err(e) => f_err(e), - } - }), - ) + pub async fn bincode_req_fut( + req: reqwest::RequestBuilder, + ) -> Result { + // Work around tiny_http issue #151 by disabling HTTP pipeline with + // `Connection: close`. + let res = req.set_header(header::Connection::close()).send().await?; + + let status = res.status(); + let bytes = res.bytes().await?; + if !status.is_success() { + let errmsg = format!( + "Error {}: {}", + status.as_u16(), + String::from_utf8_lossy(&bytes) + ); + if status.is_client_error() { + anyhow::bail!(HttpClientError(errmsg)); + } else { + anyhow::bail!(errmsg); + } + } else { + Ok(bincode::deserialize(&*bytes)?) + } } #[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq)] @@ -691,7 +678,7 @@ mod server { check_server_auth, } = self; let requester = SchedulerRequester { - client: Mutex::new(reqwest::Client::new()), + client: Mutex::new(reqwest::blocking::Client::new()), }; macro_rules! check_server_auth_or_err { @@ -727,7 +714,7 @@ mod server { } fn maybe_update_certs( - client: &mut reqwest::Client, + client: &mut reqwest::blocking::Client, certs: &mut HashMap, Vec)>, server_id: ServerId, cert_digest: Vec, @@ -742,7 +729,7 @@ mod server { "Adding new certificate for {} to scheduler", server_id.addr() ); - let mut client_builder = reqwest::ClientBuilder::new(); + let mut client_builder = reqwest::blocking::ClientBuilder::new(); // Add all the certificates we know about client_builder = client_builder.add_root_certificate( reqwest::Certificate::from_pem(&cert_pem) @@ -856,7 +843,7 @@ mod server { } struct SchedulerRequester { - client: Mutex, + client: Mutex, } impl dist::SchedulerOutgoing for SchedulerRequester { @@ -938,14 +925,14 @@ mod server { let job_authorizer = JWTJobAuthorizer::new(jwt_key); let heartbeat_url = urls::scheduler_heartbeat_server(&scheduler_url); let requester = ServerRequester { - client: reqwest::Client::new(), + client: reqwest::blocking::Client::new(), scheduler_url, scheduler_auth: scheduler_auth.clone(), }; // TODO: detect if this panics thread::spawn(move || { - let client = reqwest::Client::new(); + let client = reqwest::blocking::Client::new(); loop { trace!("Performing heartbeat"); match bincode_req( @@ -1031,7 +1018,7 @@ mod server { } struct ServerRequester { - client: reqwest::Client, + client: reqwest::blocking::Client, scheduler_url: reqwest::Url, scheduler_auth: String, } @@ -1063,12 +1050,10 @@ mod client { self, AllocJobResult, CompileCommand, JobAlloc, PathTransformer, RunJobResult, SchedulerStatusResult, SubmitToolchainResult, Toolchain, }; - use crate::util::SpawnExt; + use byteorder::{BigEndian, WriteBytesExt}; use flate2::write::ZlibEncoder as ZlibWriteEncoder; use flate2::Compression; - use futures::Future; - use futures_03::executor::ThreadPool; use std::collections::HashMap; use std::io::Write; use std::path::{Path, PathBuf}; @@ -1092,16 +1077,16 @@ mod client { server_certs: Arc, Vec>>>, // TODO: this should really only use the async client, but reqwest async bodies are extremely limited // and only support owned bytes, which means the whole toolchain would end up in memory - client: Arc>, - client_async: Arc>, - pool: ThreadPool, + client: Arc>, + client_async: Arc>, + pool: tokio::runtime::Handle, tc_cache: Arc, rewrite_includes_only: bool, } impl Client { pub fn new( - pool: &ThreadPool, + pool: &tokio::runtime::Handle, scheduler_url: reqwest::Url, cache_dir: &Path, cache_size: u64, @@ -1111,12 +1096,12 @@ mod client { ) -> Result { let timeout = Duration::new(REQUEST_TIMEOUT_SECS, 0); let connect_timeout = Duration::new(CONNECT_TIMEOUT_SECS, 0); - let client = reqwest::ClientBuilder::new() + let client = reqwest::blocking::ClientBuilder::new() .timeout(timeout) .connect_timeout(connect_timeout) .build() .context("failed to create a HTTP client")?; - let client_async = reqwest::r#async::ClientBuilder::new() + let client_async = reqwest::ClientBuilder::new() .timeout(timeout) .connect_timeout(connect_timeout) .build() @@ -1137,14 +1122,14 @@ mod client { } fn update_certs( - client: &mut reqwest::Client, - client_async: &mut reqwest::r#async::Client, + client: &mut reqwest::blocking::Client, + client_async: &mut reqwest::Client, certs: &mut HashMap, Vec>, cert_digest: Vec, cert_pem: Vec, ) -> Result<()> { - let mut client_builder = reqwest::ClientBuilder::new(); - let mut client_async_builder = reqwest::r#async::ClientBuilder::new(); + let mut client_builder = reqwest::blocking::ClientBuilder::new(); + let mut client_async_builder = reqwest::ClientBuilder::new(); // Add all the certificates we know about client_builder = client_builder.add_root_certificate( reqwest::Certificate::from_pem(&cert_pem) @@ -1180,24 +1165,26 @@ mod client { } } + #[async_trait] impl dist::Client for Client { - fn do_alloc_job(&self, tc: Toolchain) -> SFuture { + async fn do_alloc_job(&self, tc: Toolchain) -> Result { let scheduler_url = self.scheduler_url.clone(); let url = urls::scheduler_alloc_job(&scheduler_url); let mut req = self.client_async.lock().unwrap().post(url); - req = ftry!(req.bearer_auth(self.auth_token.clone()).bincode(&tc)); + req = req.bearer_auth(self.auth_token.clone()).bincode(&tc)?; let client = self.client.clone(); let client_async = self.client_async.clone(); let server_certs = self.server_certs.clone(); - Box::new(bincode_req_fut(req).and_then(move |res| match res { + + match bincode_req_fut(req).await? { AllocJobHttpResponse::Success { job_alloc, need_toolchain, cert_digest, } => { let server_id = job_alloc.server_id; - let alloc_job_res = f_ok(AllocJobResult::Success { + let alloc_job_res = Ok(AllocJobResult::Success { job_alloc, need_toolchain, }); @@ -1210,101 +1197,127 @@ mod client { ); let url = urls::scheduler_server_certificate(&scheduler_url, server_id); let req = client_async.lock().unwrap().get(url); - Box::new( - bincode_req_fut(req) - .map_err(|e| e.context("GET to scheduler server_certificate failed")) - .and_then(move |res: ServerCertificateHttpResponse| { - ftry!(Self::update_certs( - &mut client.lock().unwrap(), - &mut client_async.lock().unwrap(), - &mut server_certs.lock().unwrap(), - res.cert_digest, - res.cert_pem, - )); - alloc_job_res - }), - ) + let res: ServerCertificateHttpResponse = bincode_req_fut(req) + .await + .context("GET to scheduler server_certificate failed")?; + + // TODO: Move to asynchronous reqwest client only. + // This function internally builds a blocking reqwest client; + // However, it does so by utilizing a runtime which it drops, + // triggering (rightfully) a sanity check that prevents from + // dropping a runtime in asynchronous context. + // For the time being, we work around this by off-loading it + // to a dedicated blocking-friendly thread pool. + let _ = self + .pool + .spawn_blocking(move || { + Self::update_certs( + &mut client.lock().unwrap(), + &mut client_async.lock().unwrap(), + &mut server_certs.lock().unwrap(), + res.cert_digest, + res.cert_pem, + ) + .context("Failed to update certificate") + .unwrap_or_else(|e| warn!("Failed to update certificate: {:?}", e)); + }) + .await; + + alloc_job_res } - AllocJobHttpResponse::Fail { msg } => f_ok(AllocJobResult::Fail { msg }), - })) + AllocJobHttpResponse::Fail { msg } => Ok(AllocJobResult::Fail { msg }), + } } - fn do_get_status(&self) -> SFuture { + + async fn do_get_status(&self) -> Result { let scheduler_url = self.scheduler_url.clone(); let url = urls::scheduler_status(&scheduler_url); let req = self.client.lock().unwrap().get(url); - Box::new(self.pool.spawn_fn(move || bincode_req(req))) + + self.pool.spawn_blocking(move || bincode_req(req)).await? } - fn do_submit_toolchain( + + async fn do_submit_toolchain( &self, job_alloc: JobAlloc, tc: Toolchain, - ) -> SFuture { + ) -> Result { match self.tc_cache.get_toolchain(&tc) { Ok(Some(toolchain_file)) => { let url = urls::server_submit_toolchain(job_alloc.server_id, job_alloc.job_id); let req = self.client.lock().unwrap().post(url); - - Box::new(self.pool.spawn_fn(move || { - let req = req.bearer_auth(job_alloc.auth.clone()).body(toolchain_file); - bincode_req(req) - })) + self.pool + .spawn_blocking(move || { + let toolchain_file_size = toolchain_file.metadata()?.len(); + let body = + reqwest::blocking::Body::sized(toolchain_file, toolchain_file_size); + let req = req.bearer_auth(job_alloc.auth).body(body); + bincode_req(req) + }) + .await? } - Ok(None) => f_err(anyhow!("couldn't find toolchain locally")), - Err(e) => f_err(e), + Ok(None) => Err(anyhow!("couldn't find toolchain locally")), + Err(e) => Err(e), } } - fn do_run_job( + + async fn do_run_job( &self, job_alloc: JobAlloc, command: CompileCommand, outputs: Vec, inputs_packager: Box, - ) -> SFuture<(RunJobResult, PathTransformer)> { + ) -> Result<(RunJobResult, PathTransformer)> { let url = urls::server_run_job(job_alloc.server_id, job_alloc.job_id); let mut req = self.client.lock().unwrap().post(url); - Box::new(self.pool.spawn_fn(move || { - let bincode = bincode::serialize(&RunJobHttpRequest { command, outputs }) - .context("failed to serialize run job request")?; - let bincode_length = bincode.len(); - - let mut body = vec![]; - body.write_u32::(bincode_length as u32) - .expect("Infallible write of bincode length to vec failed"); - body.write_all(&bincode) - .expect("Infallible write of bincode body to vec failed"); - let path_transformer; - { - let mut compressor = ZlibWriteEncoder::new(&mut body, Compression::fast()); - path_transformer = inputs_packager - .write_inputs(&mut compressor) - .context("Could not write inputs for compilation")?; - compressor.flush().context("failed to flush compressor")?; - trace!( - "Compressed inputs from {} -> {}", - compressor.total_in(), - compressor.total_out() - ); - compressor.finish().context("failed to finish compressor")?; - } + self.pool + .spawn_blocking(move || { + let bincode = bincode::serialize(&RunJobHttpRequest { command, outputs }) + .context("failed to serialize run job request")?; + let bincode_length = bincode.len(); + + let mut body = vec![]; + body.write_u32::(bincode_length as u32) + .expect("Infallible write of bincode length to vec failed"); + body.write_all(&bincode) + .expect("Infallible write of bincode body to vec failed"); + let path_transformer; + { + let mut compressor = ZlibWriteEncoder::new(&mut body, Compression::fast()); + path_transformer = inputs_packager + .write_inputs(&mut compressor) + .context("Could not write inputs for compilation")?; + compressor.flush().context("failed to flush compressor")?; + trace!( + "Compressed inputs from {} -> {}", + compressor.total_in(), + compressor.total_out() + ); + compressor.finish().context("failed to finish compressor")?; + } - req = req.bearer_auth(job_alloc.auth.clone()).bytes(body); - bincode_req(req).map(|res| (res, path_transformer)) - })) + req = req.bearer_auth(job_alloc.auth.clone()).bytes(body); + bincode_req(req).map(|res| (res, path_transformer)) + }) + .await? } - fn put_toolchain( + async fn put_toolchain( &self, - compiler_path: &Path, - weak_key: &str, + compiler_path: PathBuf, + weak_key: String, toolchain_packager: Box, - ) -> SFuture<(Toolchain, Option<(String, PathBuf)>)> { + ) -> Result<(Toolchain, Option<(String, PathBuf)>)> { let compiler_path = compiler_path.to_owned(); let weak_key = weak_key.to_owned(); let tc_cache = self.tc_cache.clone(); - Box::new(self.pool.spawn_fn(move || { - tc_cache.put_toolchain(&compiler_path, &weak_key, toolchain_packager) - })) + + self.pool + .spawn_blocking(move || { + tc_cache.put_toolchain(&compiler_path, &weak_key, toolchain_packager) + }) + .await? } fn rewrite_includes_only(&self) -> bool { diff --git a/src/dist/mod.rs b/src/dist/mod.rs index d91c96ea..cbf96f81 100644 --- a/src/dist/mod.rs +++ b/src/dist/mod.rs @@ -18,7 +18,7 @@ use std::ffi::OsString; use std::fmt; use std::io::{self, Read}; use std::net::SocketAddr; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::process; use std::str::FromStr; #[cfg(feature = "dist-server")] @@ -710,32 +710,32 @@ pub trait BuilderIncoming: Send + Sync { } ///////// - -pub trait Client { +#[async_trait] +pub trait Client: Send + Sync { // To Scheduler - fn do_alloc_job(&self, tc: Toolchain) -> SFuture; + async fn do_alloc_job(&self, tc: Toolchain) -> Result; // To Scheduler - fn do_get_status(&self) -> SFuture; + async fn do_get_status(&self) -> Result; // To Server - fn do_submit_toolchain( + async fn do_submit_toolchain( &self, job_alloc: JobAlloc, tc: Toolchain, - ) -> SFuture; + ) -> Result; // To Server - fn do_run_job( + async fn do_run_job( &self, job_alloc: JobAlloc, command: CompileCommand, outputs: Vec, inputs_packager: Box, - ) -> SFuture<(RunJobResult, PathTransformer)>; - fn put_toolchain( + ) -> Result<(RunJobResult, PathTransformer)>; + async fn put_toolchain( &self, - compiler_path: &Path, - weak_key: &str, + compiler_path: PathBuf, + weak_key: String, toolchain_packager: Box, - ) -> SFuture<(Toolchain, Option<(String, PathBuf)>)>; + ) -> Result<(Toolchain, Option<(String, PathBuf)>)>; fn rewrite_includes_only(&self) -> bool; fn get_custom_toolchain(&self, exe: &PathBuf) -> Option; } diff --git a/src/errors.rs b/src/errors.rs index bc33f963..7933ed1e 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -13,10 +13,6 @@ // limitations under the License. pub use anyhow::{anyhow, bail, Context, Error}; -use futures::future; -use futures::Future; -use std::boxed::Box; -use std::fmt::Display; use std::process; // We use `anyhow` for error handling. @@ -25,7 +21,6 @@ use std::process; // - The error types below (`BadHttpStatusError`, etc.) are internal ones that // need to be checked at points other than the outermost error-checking // layer. -// - There are some combinators below for working with futures. #[cfg(feature = "hyper")] #[derive(Debug)] @@ -64,73 +59,3 @@ impl std::fmt::Display for ProcessError { } pub type Result = anyhow::Result; - -pub type SFuture = Box>; -pub type SFutureSend = Box + Send>; - -pub trait FutureContext { - fn fcontext(self, context: C) -> SFuture - where - C: Display + Send + Sync + 'static; - - fn fwith_context(self, callback: CB) -> SFuture - where - CB: FnOnce() -> C + 'static, - C: Display + Send + Sync + 'static; -} - -impl FutureContext for F -where - F: Future + 'static, - F::Error: Into + Send + Sync, -{ - fn fcontext(self, context: C) -> SFuture - where - C: Display + Send + Sync + 'static, - { - Box::new(self.then(|r| r.map_err(F::Error::into).context(context))) - } - - fn fwith_context(self, callback: CB) -> SFuture - where - CB: FnOnce() -> C + 'static, - C: Display + Send + Sync + 'static, - { - Box::new(self.then(|r| r.map_err(F::Error::into).context(callback()))) - } -} - -/// Like `try`, but returns an SFuture instead of a Result. -macro_rules! ftry { - ($e:expr) => { - match $e { - Ok(v) => v, - Err(e) => return Box::new($crate::futures::future::err(e.into())) as SFuture<_>, - } - }; -} - -#[cfg(feature = "dist-client")] -macro_rules! ftry_send { - ($e:expr) => { - match $e { - Ok(v) => v, - Err(e) => return Box::new($crate::futures::future::err(e)) as SFutureSend<_>, - } - }; -} - -pub fn f_ok(t: T) -> SFuture -where - T: 'static, -{ - Box::new(future::ok(t)) -} - -pub fn f_err(e: E) -> SFuture -where - T: 'static, - E: Into, -{ - Box::new(future::err(e.into())) -} diff --git a/src/jobserver.rs b/src/jobserver.rs index 6d5f1aa4..03d8f97a 100644 --- a/src/jobserver.rs +++ b/src/jobserver.rs @@ -2,10 +2,9 @@ use std::io; use std::process::Command; use std::sync::Arc; -use futures::future; -use futures::prelude::*; -use futures::sync::mpsc; -use futures::sync::oneshot; +use futures::channel::mpsc; +use futures::channel::oneshot; +use futures::StreamExt; use crate::errors::*; @@ -38,14 +37,19 @@ impl Client { let (helper, tx) = if inherited { (None, None) } else { - let (tx, rx) = mpsc::unbounded::>(); - let mut rx = rx.wait(); + let (tx, mut rx) = mpsc::unbounded::>(); let helper = inner .clone() .into_helper_thread(move |token| { - if let Some(Ok(sender)) = rx.next() { - drop(sender.send(token)); - } + let mut rt = tokio::runtime::Builder::new() + .basic_scheduler() + .build() + .unwrap(); + rt.block_on(async { + if let Some(sender) = rx.next().await { + drop(sender.send(token)); + } + }); }) .expect("failed to spawn helper thread"); (Some(Arc::new(helper)), Some(tx)) @@ -64,18 +68,22 @@ impl Client { /// This should be invoked before any "work" is spawned (for whatever the /// definition of "work" is) to ensure that the system is properly /// rate-limiting itself. - pub fn acquire(&self) -> SFuture { + pub async fn acquire(&self) -> Result { let (helper, tx) = match (self.helper.as_ref(), self.tx.as_ref()) { (Some(a), Some(b)) => (a, b), - _ => return Box::new(future::ok(Acquired { _token: None })), + _ => return Ok(Acquired { _token: None }), }; let (mytx, myrx) = oneshot::channel(); helper.request_token(); tx.unbounded_send(mytx).unwrap(); - Box::new( - myrx.fcontext("jobserver helper panicked") - .and_then(|t| t.context("failed to acquire jobserver token")) - .map(|t| Acquired { _token: Some(t) }), - ) + + let acquired = myrx + .await + .context("jobserver helper panicked")? + .context("failed to acquire jobserver token")?; + + Ok(Acquired { + _token: Some(acquired), + }) } } diff --git a/src/lib.rs b/src/lib.rs index 94c802a0..b4dc65af 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,12 +15,12 @@ #![deny(rust_2018_idioms)] #![recursion_limit = "256"] +#[macro_use] +extern crate async_trait; #[macro_use] extern crate clap; #[macro_use] extern crate counted_array; -#[macro_use] -extern crate futures; #[cfg(feature = "jsonwebtoken")] use jsonwebtoken as jwt; #[macro_use] diff --git a/src/mock_command.rs b/src/mock_command.rs index 7171b714..38515838 100644 --- a/src/mock_command.rs +++ b/src/mock_command.rs @@ -47,7 +47,6 @@ use crate::errors::*; use crate::jobserver::{Acquired, Client}; -use futures::future::{self, Future}; use std::boxed::Box; use std::ffi::{OsStr, OsString}; use std::fmt; @@ -55,17 +54,18 @@ use std::io; use std::path::Path; use std::process::{Command, ExitStatus, Output, Stdio}; use std::sync::{Arc, Mutex}; -use tokio_io::{AsyncRead, AsyncWrite}; -use tokio_process::{self, ChildStderr, ChildStdin, ChildStdout, CommandExt}; +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio::process::{ChildStderr, ChildStdin, ChildStdout}; /// A trait that provides a subset of the methods of `std::process::Child`. +#[async_trait] pub trait CommandChild { /// The type of the process' standard input. - type I: AsyncWrite + Sync + Send + 'static; + type I: AsyncWrite + Unpin + Sync + Send + 'static; /// The type of the process' standard output. - type O: AsyncRead + Sync + Send + 'static; + type O: AsyncRead + Unpin + Sync + Send + 'static; /// The type of the process' standard error. - type E: AsyncRead + Sync + Send + 'static; + type E: AsyncRead + Unpin + Sync + Send + 'static; /// Take the stdin object from the process, if available. fn take_stdin(&mut self) -> Option; @@ -74,15 +74,16 @@ pub trait CommandChild { /// Take the stderr object from the process, if available. fn take_stderr(&mut self) -> Option; /// Wait for the process to complete and return its exit status. - fn wait(self) -> Box>; + async fn wait(self) -> io::Result; /// Wait for the process to complete and return its output. - fn wait_with_output(self) -> Box>; + async fn wait_with_output(self) -> io::Result; } /// A trait that provides a subset of the methods of `std::process::Command`. -pub trait RunCommand: fmt::Debug { +#[async_trait] +pub trait RunCommand: fmt::Debug + Send { /// The type returned by `spawn`. - type C: CommandChild + 'static; + type C: CommandChild + Send + 'static; /// Append `arg` to the process commandline. fn arg>(&mut self, arg: S) -> &mut Self; @@ -112,7 +113,7 @@ pub trait RunCommand: fmt::Debug { /// Set the process' stderr from `cfg`. fn stderr(&mut self, cfg: Stdio) -> &mut Self; /// Execute the process and return a process object. - fn spawn(&mut self) -> SFuture; + async fn spawn(&mut self) -> Result; } /// A trait that provides a means to create objects implementing `RunCommand`. @@ -132,7 +133,7 @@ pub trait CommandCreator { } /// A trait for simplifying the normal case while still allowing the mock case requiring mutability. -pub trait CommandCreatorSync: Clone + 'static { +pub trait CommandCreatorSync: Clone + Send + Sync + 'static { type Cmd: RunCommand; fn new(client: &Client) -> Self; @@ -141,40 +142,41 @@ pub trait CommandCreatorSync: Clone + 'static { } pub struct Child { - inner: tokio_process::Child, + inner: tokio::process::Child, token: Acquired, } /// Trivial implementation of `CommandChild` for `std::process::Child`. +#[async_trait] impl CommandChild for Child { type I = ChildStdin; type O = ChildStdout; type E = ChildStderr; fn take_stdin(&mut self) -> Option { - self.inner.stdin().take() + self.inner.stdin.take() } fn take_stdout(&mut self) -> Option { - self.inner.stdout().take() + self.inner.stdout.take() } fn take_stderr(&mut self) -> Option { - self.inner.stderr().take() + self.inner.stderr.take() } - fn wait(self) -> Box> { + async fn wait(self) -> io::Result { let Child { inner, token } = self; - Box::new(inner.map(|ret| { + inner.await.map(|ret| { drop(token); ret - })) + }) } - fn wait_with_output(self) -> Box> { + async fn wait_with_output(self) -> io::Result { let Child { inner, token } = self; - Box::new(inner.wait_with_output().map(|ret| { + inner.wait_with_output().await.map(|ret| { drop(token); ret - })) + }) } } @@ -197,6 +199,7 @@ impl AsyncCommand { } /// Trivial implementation of `RunCommand` for `std::process::Command`. +#[async_trait] impl RunCommand for AsyncCommand { type C = Child; @@ -259,21 +262,23 @@ impl RunCommand for AsyncCommand { self.inner().stderr(cfg); self } - fn spawn(&mut self) -> SFuture { + async fn spawn(&mut self) -> Result { let mut inner = self.inner.take().unwrap(); inner.env_remove("MAKEFLAGS"); inner.env_remove("MFLAGS"); inner.env_remove("CARGO_MAKEFLAGS"); self.jobserver.configure(&mut inner); - Box::new(self.jobserver.acquire().and_then(move |token| { - let child = inner - .spawn_async() - .with_context(|| format!("failed to spawn {:?}", inner))?; - Ok(Child { - inner: child, - token, - }) - })) + + let token = self.jobserver.acquire().await?; + let mut inner = tokio::process::Command::from(inner); + let child = inner + .spawn() + .with_context(|| format!("failed to spawn {:?}", inner))?; + + Ok(Child { + inner: child, + token, + }) } } @@ -377,6 +382,7 @@ impl MockChild { } } +#[async_trait] impl CommandChild for MockChild { type I = io::Cursor>; type O = io::Cursor>; @@ -392,23 +398,23 @@ impl CommandChild for MockChild { self.stderr.take() } - fn wait(mut self) -> Box> { - Box::new(future::result(self.wait_result.take().unwrap())) + async fn wait(mut self) -> io::Result { + self.wait_result.take().unwrap() } - fn wait_with_output(self) -> Box> { + async fn wait_with_output(self) -> io::Result { let MockChild { stdout, stderr, wait_result, .. } = self; - let result = wait_result.unwrap().map(|status| Output { + + wait_result.unwrap().map(|status| Output { status, stdout: stdout.map(|c| c.into_inner()).unwrap_or_else(Vec::new), stderr: stderr.map(|c| c.into_inner()).unwrap_or_else(Vec::new), - }); - Box::new(future::result(result)) + }) } } @@ -434,6 +440,7 @@ pub struct MockCommand { pub args: Vec, } +#[async_trait] impl RunCommand for MockCommand { type C = MockChild; @@ -479,10 +486,10 @@ impl RunCommand for MockCommand { fn stderr(&mut self, _cfg: Stdio) -> &mut MockCommand { self } - fn spawn(&mut self) -> SFuture { + async fn spawn(&mut self) -> Result { match self.child.take().unwrap() { - ChildOrCall::Child(c) => Box::new(future::result(c)), - ChildOrCall::Call(f) => Box::new(future::result(f(&self.args))), + ChildOrCall::Child(c) => c, + ChildOrCall::Call(f) => f(&self.args), } } } @@ -532,7 +539,7 @@ impl CommandCreator for MockCommandCreator { } /// To simplify life for using a `CommandCreator` across multiple threads. -impl CommandCreatorSync for Arc> { +impl CommandCreatorSync for Arc> { type Cmd = T::Cmd; fn new(client: &Client) -> Arc> { @@ -549,7 +556,6 @@ mod test { use super::*; use crate::jobserver::Client; use crate::test::utils::*; - use futures::Future; use std::ffi::OsStr; use std::io; use std::process::{ExitStatus, Output}; diff --git a/src/server.rs b/src/server.rs index 7364c3e8..7981c2df 100644 --- a/src/server.rs +++ b/src/server.rs @@ -12,9 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// For tokio_io::codec::length_delimited::Framed; -#![allow(deprecated)] - use crate::cache::{storage_from_config, Storage}; use crate::compiler::{ get_compiler_info, CacheControl, CompileResult, Compiler, CompilerArguments, CompilerHasher, @@ -30,39 +27,41 @@ use crate::protocol::{Compile, CompileFinished, CompileResponse, Request, Respon use crate::util; #[cfg(feature = "dist-client")] use anyhow::Context as _; +use bytes::{buf::ext::BufMutExt, Bytes, BytesMut}; use filetime::FileTime; -use futures::sync::mpsc; -use futures::{future, stream, Async, AsyncSink, Future, Poll, Sink, StartSend, Stream}; -use futures_03::compat::Compat; -use futures_03::executor::ThreadPool; +use futures::channel::mpsc; +use futures::future::FutureExt; +use futures::{future, stream, Sink, SinkExt, Stream, StreamExt, TryFutureExt}; +use futures_locks::RwLock; use number_prefix::NumberPrefix; -use std::cell::RefCell; use std::collections::HashMap; use std::env; use std::ffi::{OsStr, OsString}; use std::fs::metadata; +use std::future::Future; use std::io::{self, Write}; +use std::marker::Unpin; #[cfg(feature = "dist-client")] use std::mem; use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; use std::path::PathBuf; use std::pin::Pin; use std::process::{ExitStatus, Output}; -use std::rc::Rc; use std::sync::Arc; -#[cfg(feature = "dist-client")] use std::sync::Mutex; -use std::task::{Context, Waker}; +use std::task::{Context, Poll, Waker}; use std::time::Duration; +#[cfg(feature = "dist-client")] use std::time::Instant; use std::u64; -use tokio_compat::runtime::current_thread::Runtime; -use tokio_io::codec::length_delimited; -use tokio_io::codec::length_delimited::Framed; -use tokio_io::{AsyncRead, AsyncWrite}; -use tokio_serde_bincode::{ReadBincode, WriteBincode}; -use tokio_tcp::TcpListener; -use tokio_timer::{Delay, Timeout}; +use tokio::runtime::Runtime; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + net::TcpListener, + time::{self, delay_for, Delay}, +}; +use tokio_serde::Framed; +use tokio_util::codec::{length_delimited, LengthDelimitedCodec}; use tower::Service; use crate::errors::*; @@ -144,7 +143,7 @@ pub struct DistClientContainer { #[cfg(feature = "dist-client")] struct DistClientConfig { // Reusable items tied to an SccacheServer instance - pool: ThreadPool, + pool: tokio::runtime::Handle, // From the static dist configuration scheduler_url: Option, @@ -169,7 +168,7 @@ enum DistClientState { #[cfg(not(feature = "dist-client"))] impl DistClientContainer { #[cfg(not(feature = "dist-client"))] - fn new(config: &Config, _: &ThreadPool) -> Self { + fn new(config: &Config, _: &tokio::runtime::Handle) -> Self { if config.dist.scheduler_url.is_some() { warn!("Scheduler address configured but dist feature disabled, disabling distributed sccache") } @@ -182,7 +181,7 @@ impl DistClientContainer { pub fn reset_state(&self) {} - pub fn get_status(&self) -> DistInfo { + pub async fn get_status(&self) -> DistInfo { DistInfo::Disabled("dist-client feature not selected".to_string()) } @@ -193,7 +192,7 @@ impl DistClientContainer { #[cfg(feature = "dist-client")] impl DistClientContainer { - fn new(config: &Config, pool: &ThreadPool) -> Self { + fn new(config: &Config, pool: &tokio::runtime::Handle) -> Self { let config = DistClientConfig { pool: pool.clone(), scheduler_url: config.dist.scheduler_url.clone(), @@ -231,28 +230,41 @@ impl DistClientContainer { } } - pub fn get_status(&self) -> DistInfo { + pub fn get_status(&self) -> impl Future { + // This function can't be wholly async because we can't hold mutex guard + // across the yield point - instead, either return an immediately ready + // future or perform async query with the client cloned beforehand. let mut guard = self.state.lock(); let state = guard.as_mut().unwrap(); let state: &mut DistClientState = &mut **state; - match state { - DistClientState::Disabled => DistInfo::Disabled("disabled".to_string()), - DistClientState::FailWithMessage(cfg, _) => DistInfo::NotConnected( - cfg.scheduler_url.clone(), - "enabled, auth not configured".to_string(), - ), - DistClientState::RetryCreateAt(cfg, _) => DistInfo::NotConnected( - cfg.scheduler_url.clone(), - "enabled, not connected, will retry".to_string(), - ), - DistClientState::Some(cfg, client) => match client.do_get_status().wait() { - Ok(res) => DistInfo::SchedulerStatus(cfg.scheduler_url.clone(), res), - Err(_) => DistInfo::NotConnected( + let (client, scheduler_url) = match state { + DistClientState::Disabled => { + return Either::Left(future::ready(DistInfo::Disabled("disabled".to_string()))) + } + DistClientState::FailWithMessage(cfg, _) => { + return Either::Left(future::ready(DistInfo::NotConnected( + cfg.scheduler_url.clone(), + "enabled, auth not configured".to_string(), + ))) + } + DistClientState::RetryCreateAt(cfg, _) => { + return Either::Left(future::ready(DistInfo::NotConnected( cfg.scheduler_url.clone(), + "enabled, not connected, will retry".to_string(), + ))) + } + DistClientState::Some(cfg, client) => (Arc::clone(client), cfg.scheduler_url.clone()), + }; + + Either::Right(Box::pin(async move { + match client.do_get_status().await { + Ok(res) => DistInfo::SchedulerStatus(scheduler_url.clone(), res), + Err(_) => DistInfo::NotConnected( + scheduler_url.clone(), "could not communicate with scheduler".to_string(), ), - }, - } + } + })) } fn get_client(&self) -> Result>> { @@ -351,7 +363,7 @@ impl DistClientContainer { let dist_client = try_or_retry_later!(dist_client.context("failure during dist client creation")); use crate::dist::Client; - match dist_client.do_get_status().wait() { + match config.pool.block_on(dist_client.do_get_status()) { Ok(res) => { info!( "Successfully created dist client with {:?} cores across {:?} servers", @@ -390,27 +402,23 @@ impl DistClientContainer { pub fn start_server(config: &Config, port: u16) -> Result<()> { info!("start_server: port: {}", port); let client = unsafe { Client::new() }; - let runtime = Runtime::new()?; - let pool = ThreadPool::builder() - .pool_size(std::cmp::max(20, 2 * num_cpus::get())) - .create()?; + let runtime = tokio::runtime::Builder::new() + .enable_all() + .threaded_scheduler() + .core_threads(std::cmp::max(20, 2 * num_cpus::get())) + .build()?; + let pool = runtime.handle().clone(); let dist_client = DistClientContainer::new(config, &pool); let storage = storage_from_config(config, &pool); - let res = SccacheServer::::new( - port, - pool, - runtime, - client, - dist_client, - storage, - ); + let res = + SccacheServer::::new(port, runtime, client, dist_client, storage); let notify = env::var_os("SCCACHE_STARTUP_NOTIFY"); match res { Ok(srv) => { let port = srv.port(); info!("server started, listening on port {}", port); notify_server_startup(¬ify, ServerStartup::Ok { port })?; - srv.run(future::empty::<(), ()>())?; + srv.run(future::pending::<()>())?; Ok(()) } Err(e) => { @@ -441,19 +449,19 @@ pub struct SccacheServer { impl SccacheServer { pub fn new( port: u16, - pool: ThreadPool, - runtime: Runtime, + mut runtime: Runtime, client: Client, dist_client: DistClientContainer, storage: Arc, ) -> Result> { let addr = SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), port); - let listener = TcpListener::bind(&SocketAddr::V4(addr))?; + let listener = runtime.block_on(TcpListener::bind(&SocketAddr::V4(addr)))?; // Prepare the service which we'll use to service all incoming TCP // connections. let (tx, rx) = mpsc::channel(1); let (wait, info) = WaitUntilZero::new(); + let pool = runtime.handle().clone(); let service = SccacheService::new(dist_client, storage, &client, pool, tx, info); Ok(SccacheServer { @@ -480,8 +488,8 @@ impl SccacheServer { /// Returns a reference to a thread pool to run work on #[allow(dead_code)] - pub fn pool(&self) -> &ThreadPool { - &self.service.pool + pub fn pool(&self) -> &tokio::runtime::Handle { + &self.service.rt } /// Returns a reference to the command creator this server will use @@ -504,11 +512,8 @@ impl SccacheServer { pub fn run(self, shutdown: F) -> io::Result<()> where F: Future, + C: Send, { - self._run(Box::new(shutdown.then(|_| Ok(())))) - } - - fn _run<'a>(self, shutdown: Box + 'a>) -> io::Result<()> { let SccacheServer { mut runtime, listener, @@ -520,14 +525,16 @@ impl SccacheServer { // Create our "server future" which will simply handle all incoming // connections in separate tasks. - let server = listener.incoming().for_each(move |socket| { + let server = listener.try_for_each(move |socket| { trace!("incoming connection"); - tokio_compat::runtime::current_thread::TaskExecutor::current() - .spawn_local(Box::new(service.clone().bind(socket).map_err(|err| { - error!("{}", err); - }))) - .unwrap(); - Ok(()) + let conn = service.clone().bind(socket).map_err(|res| { + error!("Failed to bind socket: {}", res); + }); + + // We're not interested if the task panicked; immediately process + // another connection + let _ = tokio::spawn(conn); + async { Ok(()) } }); // Right now there's a whole bunch of ways to shut down this server for @@ -542,39 +549,37 @@ impl SccacheServer { // inactivity, and this is then select'd with the `shutdown` future // passed to this function. - let shutdown = shutdown.map(|a| { + let shutdown = shutdown.map(|_| { info!("shutting down due to explicit signal"); - a }); - let mut futures = vec![ - Box::new(server) as Box>, - Box::new( - shutdown - .map_err(|()| io::Error::new(io::ErrorKind::Other, "shutdown signal failed")), - ), - ]; - - let shutdown_idle = ShutdownOrInactive { - rx, - timeout: if timeout != Duration::new(0, 0) { - Some(Delay::new(Instant::now() + timeout)) - } else { - None - }, - timeout_dur: timeout, - }; - futures.push(Box::new(shutdown_idle.map(|a| { + let shutdown_idle = async { + ShutdownOrInactive { + rx, + timeout: if timeout != Duration::new(0, 0) { + Some(delay_for(timeout)) + } else { + None + }, + timeout_dur: timeout, + } + .await; info!("shutting down due to being idle or request"); - a - }))); + }; - let server = future::select_all(futures); - runtime.block_on(server).map_err(|p| p.0)?; + runtime.block_on(async { + futures::select! { + server = server.fuse() => server, + _res = shutdown.fuse() => Ok(()), + _res = shutdown_idle.fuse() => Ok(()), + } + })?; + const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(10); info!( - "moving into the shutdown phase now, waiting at most 10 seconds \ - for all client requests to complete" + "moving into the shutdown phase now, waiting at most {} seconds \ + for all client requests to complete", + SHUTDOWN_TIMEOUT.as_secs() ); // Once our server has shut down either due to inactivity or a manual @@ -584,15 +589,7 @@ impl SccacheServer { // // Note that we cap the amount of time this can take, however, as we // don't want to wait *too* long. - runtime - .block_on(Timeout::new(Compat::new(wait), Duration::new(30, 0))) - .map_err(|e| { - if e.is_inner() { - e.into_inner().unwrap() - } else { - io::Error::new(io::ErrorKind::Other, e) - } - })?; + runtime.block_on(async { time::timeout(SHUTDOWN_TIMEOUT, wait).await })?; info!("ok, fully shutting down now"); @@ -600,10 +597,12 @@ impl SccacheServer { } } +/// Maps a compiler proxy path to a compiler proxy and it's last modification time +type CompilerProxyMap = HashMap>, FileTime)>; type CompilerMap = HashMap>>; /// entry of the compiler cache -struct CompilerCacheEntry { +struct CompilerCacheEntry { /// compiler argument trait obj pub compiler: Box>, /// modification time of the compilers executable file @@ -612,10 +611,7 @@ struct CompilerCacheEntry { pub dist_info: Option<(PathBuf, FileTime)>, } -impl CompilerCacheEntry -where - C: CommandCreatorSync, -{ +impl CompilerCacheEntry { fn new( compiler: Box>, mtime: FileTime, @@ -630,28 +626,32 @@ where } /// Service implementation for sccache #[derive(Clone)] -struct SccacheService { +struct SccacheService +where + C: Send, +{ /// Server statistics. - stats: Rc>, + stats: Arc>, /// Distributed sccache client - dist_client: Rc, + dist_client: Arc, /// Cache storage. storage: Arc, /// A cache of known compiler info. - compilers: Rc>>, + compilers: Arc>>, /// map the cwd with compiler proxy path to a proxy resolver, which /// will dynamically resolve the input compiler for the current context /// (usually file or current working directory) /// the associated `FileTime` is the modification time of /// the compiler proxy, in order to track updates of the proxy itself - compiler_proxies: Rc>, FileTime)>>>, + compiler_proxies: Arc>>, - /// Thread pool to execute work in - pool: ThreadPool, + /// Task pool for blocking (used mostly for disk I/O-bound tasks) and + // non-blocking tasks + rt: tokio::runtime::Handle, /// An object for creating commands. /// @@ -685,13 +685,13 @@ pub enum ServerMessage { Shutdown, } -impl Service for SccacheService +impl Service for Arc> where - C: CommandCreatorSync + 'static, + C: CommandCreatorSync + Send + Sync + 'static, { type Response = SccacheResponse; type Error = Error; - type Future = SFuture; + type Future = Pin> + Send + 'static>>; fn call(&mut self, req: SccacheRequest) -> Self::Future { trace!("handle_client"); @@ -701,75 +701,91 @@ where // that every message is received. drop(self.tx.clone().start_send(ServerMessage::Request)); - let res: SFuture = match req.into_inner() { - Request::Compile(compile) => { - debug!("handle_client: compile"); - self.stats.borrow_mut().compile_requests += 1; - return self.handle_compile(compile); - } - Request::GetStats => { - debug!("handle_client: get_stats"); - Box::new(self.get_info().map(|i| Response::Stats(Box::new(i)))) - } - Request::DistStatus => { - debug!("handle_client: dist_status"); - Box::new(self.get_dist_status().map(Response::DistStatus)) - } - Request::ZeroStats => { - debug!("handle_client: zero_stats"); - self.zero_stats(); - Box::new(self.get_info().map(|i| Response::Stats(Box::new(i)))) - } - Request::Shutdown => { - debug!("handle_client: shutdown"); - let future = self - .tx - .clone() - .send(ServerMessage::Shutdown) - .then(|_| Ok(())); - let info_future = self.get_info(); - return Box::new(future.join(info_future).map(move |(_, info)| { - Message::WithoutBody(Response::ShuttingDown(Box::new(info))) - })); + let me = self.clone(); + Box::pin(async move { + match req.into_inner() { + Request::Compile(compile) => { + debug!("handle_client: compile"); + me.stats.write().await.compile_requests += 1; + me.handle_compile(compile).await + } + Request::GetStats => { + debug!("handle_client: get_stats"); + me.get_info() + .await + .map(|i| Response::Stats(Box::new(i))) + .map(Message::WithoutBody) + } + Request::DistStatus => { + debug!("handle_client: dist_status"); + me.get_dist_status() + .await + .map(Response::DistStatus) + .map(Message::WithoutBody) + } + Request::ZeroStats => { + debug!("handle_client: zero_stats"); + me.zero_stats().await; + me.get_info() + .await + .map(|i| Response::Stats(Box::new(i))) + .map(Message::WithoutBody) + } + Request::Shutdown => { + debug!("handle_client: shutdown"); + let mut tx = me.tx.clone(); + future::try_join( + async { + let _ = tx.send(ServerMessage::Shutdown).await; + Ok(()) + }, + me.get_info(), + ) + .await + .map(move |(_, info)| { + Message::WithoutBody(Response::ShuttingDown(Box::new(info))) + }) + } } - }; - - Box::new(res.map(Message::WithoutBody)) + }) } - fn poll_ready(&mut self) -> Poll<(), Self::Error> { - Ok(Async::Ready(())) + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) } } +use futures::future::Either; +use futures::TryStreamExt; + impl SccacheService where - C: CommandCreatorSync, + C: CommandCreatorSync + Clone + Send + Sync + 'static, { pub fn new( dist_client: DistClientContainer, storage: Arc, client: &Client, - pool: ThreadPool, + rt: tokio::runtime::Handle, tx: mpsc::Sender, info: ActiveInfo, ) -> SccacheService { SccacheService { - stats: Rc::new(RefCell::new(ServerStats::default())), - dist_client: Rc::new(dist_client), + stats: Arc::new(RwLock::new(ServerStats::default())), + dist_client: Arc::new(dist_client), storage, - compilers: Rc::new(RefCell::new(HashMap::new())), - compiler_proxies: Rc::new(RefCell::new(HashMap::new())), - pool, + compilers: Arc::new(RwLock::new(HashMap::new())), + compiler_proxies: Arc::new(RwLock::new(HashMap::new())), + rt, creator: C::new(client), tx, info, } } - fn bind(mut self, socket: T) -> impl Future + fn bind(self, socket: T) -> impl Future> + Send + Sized + 'static where - T: AsyncRead + AsyncWrite + 'static, + T: AsyncRead + AsyncWrite + Unpin + Send + 'static, { let mut builder = length_delimited::Builder::new(); if let Ok(max_frame_length_str) = env::var("SCCACHE_MAX_FRAME_LENGTH") { @@ -782,61 +798,56 @@ where let io = builder.new_framed(socket); let (sink, stream) = SccacheTransport { - inner: WriteBincode::new(ReadBincode::new(io)), + inner: Framed::new(io.sink_err_into().err_into(), BincodeCodec), } .split(); - let sink = sink.sink_from_err::(); + let sink = sink.sink_err_into::(); + let me = Arc::new(self); stream - .from_err::() - .and_then(move |input| self.call(input)) - .and_then(|message| { - let f: Box> = match message { - Message::WithoutBody(message) => Box::new(stream::once(Ok(Frame::Message { - message, - body: false, - }))), - Message::WithBody(message, body) => Box::new( - stream::once(Ok(Frame::Message { - message, - body: true, - })) - .chain(Compat::new(body).map(|chunk| Frame::Body { chunk: Some(chunk) })) - .chain(stream::once(Ok(Frame::Body { chunk: None }))), - ), + .err_into::() + .and_then(move |input| me.clone().call(input)) + .and_then(move |message| async move { + let fut = match message { + Message::WithoutBody(message) => { + let stream = stream::once(async move { Ok(Frame::Message { message }) }); + Either::Left(stream) + } + Message::WithBody(message, body) => { + let stream = stream::once(async move { Ok(Frame::Message { message }) }) + .chain(body.map_ok(|chunk| Frame::Body { chunk: Some(chunk) })) + .chain(stream::once(async move { Ok(Frame::Body { chunk: None }) })); + Either::Right(stream) + } }; - Ok(f.from_err::()) + Ok(Box::pin(fut)) }) - .flatten() + .try_flatten() .forward(sink) - .map(|_| ()) } /// Get dist status. - fn get_dist_status(&self) -> SFuture { - f_ok(self.dist_client.get_status()) + async fn get_dist_status(&self) -> Result { + Ok(self.dist_client.get_status().await) } /// Get info and stats about the cache. - fn get_info(&self) -> SFuture { - let stats = self.stats.borrow().clone(); + async fn get_info(&self) -> Result { + let stats = self.stats.read().await.clone(); let cache_location = self.storage.location(); - Box::new( - self.storage - .current_size() - .join(self.storage.max_size()) - .map(move |(cache_size, max_cache_size)| ServerInfo { - stats, - cache_location, - cache_size, - max_cache_size, - }), + futures::try_join!(self.storage.current_size(), self.storage.max_size()).map( + move |(cache_size, max_cache_size)| ServerInfo { + stats, + cache_location, + cache_size, + max_cache_size, + }, ) } /// Zero stats about the cache. - fn zero_stats(&self) { - *self.stats.borrow_mut() = ServerStats::default(); + async fn zero_stats(&self) { + *self.stats.write().await = ServerStats::default(); } /// Handle a compile request from a client. @@ -844,196 +855,175 @@ where /// This will handle a compile request entirely, generating a response with /// the inital information and an optional body which will eventually /// contain the results of the compilation. - fn handle_compile(&self, compile: Compile) -> SFuture { + async fn handle_compile(&self, compile: Compile) -> Result { let exe = compile.exe; let cmd = compile.args; let cwd: PathBuf = compile.cwd.into(); let env_vars = compile.env_vars; let me = self.clone(); - Box::new( - self.compiler_info(exe.into(), cwd.clone(), &env_vars) - .map(move |info| me.check_compiler(info, cmd, cwd, env_vars)), - ) + let info = self.compiler_info(exe.into(), cwd.clone(), &env_vars).await; + Ok(me.check_compiler(info, cmd, cwd, env_vars).await) } /// Look up compiler info from the cache for the compiler `path`. /// If not cached, determine the compiler type and cache the result. - fn compiler_info( + async fn compiler_info( &self, path: PathBuf, cwd: PathBuf, env: &[(OsString, OsString)], - ) -> SFuture>>> { + ) -> Result>> { trace!("compiler_info"); let me = self.clone(); let me1 = self.clone(); - // lookup if compiler proxy exists for the current compiler path let path2 = path.clone(); let path1 = path.clone(); let env = env.to_vec(); - let resolve_w_proxy = { - let compiler_proxies_borrow = self.compiler_proxies.borrow(); + let resolved_with_proxy = { + let compiler_proxies_borrow = self.compiler_proxies.read().await; + // Create an owned future - compiler proxy is not Send so we can't + // really await while borrowing the proxy since rustc is too conservative + let resolve_proxied_executable = + compiler_proxies_borrow + .get(&path) + .map(|(compiler_proxy, _filetime)| { + compiler_proxy.resolve_proxied_executable( + self.creator.clone(), + cwd.clone(), + env.as_slice(), + ) + }); - if let Some((compiler_proxy, _filetime)) = compiler_proxies_borrow.get(&path) { - let fut = compiler_proxy.resolve_proxied_executable( - self.creator.clone(), - cwd.clone(), - env.as_slice(), - ); - Box::new(fut.then(|res: Result<_>| Ok(res.ok()))) - } else { - f_ok(None) + match resolve_proxied_executable { + Some(fut) => fut.await.ok(), + None => None, } }; // use the supplied compiler path as fallback, lookup its modification time too - let w_fallback = resolve_w_proxy.then(move |res: Result>| { - let opt = match res { - Ok(Some(x)) => Some(x), // TODO resolve the path right away - _ => { - // fallback to using the path directly - metadata(&path2) - .map(|attr| FileTime::from_last_modification_time(&attr)) - .ok() - .map(move |filetime| (path2, filetime)) - } - }; - f_ok(opt) - }); + let (resolved_compiler_path, mtime) = match resolved_with_proxy { + Some(x) => x, // TODO resolve the path right away + _ => { + // fallback to using the path directly + metadata(&path2) + .map(|attr| FileTime::from_last_modification_time(&attr)) + .ok() + .map(move |filetime| (path2, filetime)) + .expect("Must contain sane data, otherwise mtime is not avail") + } + }; - let lookup_compiler = w_fallback.and_then(move |opt: Option<(PathBuf, FileTime)>| { - let (resolved_compiler_path, mtime) = - opt.expect("Must contain sane data, otherwise mtime is not avail"); - - let dist_info = match me1.dist_client.get_client() { - Ok(Some(ref client)) => { - if let Some(archive) = client.get_custom_toolchain(&resolved_compiler_path) { - match metadata(&archive) - .map(|attr| FileTime::from_last_modification_time(&attr)) - { - Ok(mtime) => Some((archive, mtime)), - _ => None, - } - } else { - None + let dist_info = match me1.dist_client.get_client() { + Ok(Some(ref client)) => { + if let Some(archive) = client.get_custom_toolchain(&resolved_compiler_path) { + match metadata(&archive) + .map(|attr| FileTime::from_last_modification_time(&attr)) + { + Ok(mtime) => Some((archive, mtime)), + _ => None, } + } else { + None } - _ => None, - }; + } + _ => None, + }; - let opt = match me1.compilers.borrow().get(&resolved_compiler_path) { - // It's a hit only if the mtime and dist archive data matches. - Some(&Some(ref entry)) => { - if entry.mtime == mtime && entry.dist_info == dist_info { - Some(entry.compiler.clone()) - } else { - None - } + let opt = match me1.compilers.read().await.get(&resolved_compiler_path) { + // It's a hit only if the mtime and dist archive data matches. + Some(&Some(ref entry)) => { + if entry.mtime == mtime && entry.dist_info == dist_info { + Some(entry.compiler.box_clone()) + } else { + None } - _ => None, - }; - f_ok((resolved_compiler_path, mtime, opt, dist_info)) - }); + } + _ => None, + }; - let obtain = lookup_compiler.and_then( - move |(resolved_compiler_path, mtime, opt, dist_info): ( - PathBuf, - FileTime, - Option>>, - Option<(PathBuf, FileTime)>, - )| { - match opt { - Some(info) => { - trace!("compiler_info cache hit"); - f_ok(Ok(info)) + match opt { + Some(info) => { + trace!("compiler_info cache hit"); + Ok(info) + } + None => { + trace!("compiler_info cache miss"); + // Check the compiler type and return the result when + // finished. This generally involves invoking the compiler, + // so do it asynchronously. + + // the compiler path might be compiler proxy, so it is important to use + // `path` (or its clone `path1`) to resolve using that one, not using `resolved_compiler_path` + let info = get_compiler_info::( + me.creator.clone(), + &path1, + &cwd, + env.as_slice(), + &me.rt, + dist_info.clone().map(|(p, _)| p), + ) + .await; + + let (c, proxy) = match info { + Ok((c, proxy)) => (c.clone(), proxy.clone()), + Err(err) => { + trace!("Inserting PLAIN cache map info for {:?}", &path); + me.compilers.write().await.insert(path, None); + + return Err(err); } - None => { - trace!("compiler_info cache miss"); - // Check the compiler type and return the result when - // finished. This generally involves invoking the compiler, - // so do it asynchronously. - - // the compiler path might be compiler proxy, so it is important to use - // `path` (or its clone `path1`) to resolve using that one, not using `resolved_compiler_path` - let x = get_compiler_info::( - me.creator.clone(), - &path1, - &cwd, - env.as_slice(), - &me.pool, - dist_info.clone().map(|(p, _)| p), - ); + }; - Box::new(x.then( - move |info: Result<( - Box>, - Option>>, - )>| { - match info { - Ok((ref c, ref proxy)) => { - // register the proxy for this compiler, so it will be used directly from now on - // and the true/resolved compiler will create table hits in the hash map - // based on the resolved path - if let Some(proxy) = proxy { - trace!( - "Inserting new path proxy {:?} @ {:?} -> {:?}", - &path, - &cwd, - resolved_compiler_path - ); - let proxy: Box> = - proxy.box_clone(); - me.compiler_proxies - .borrow_mut() - .insert(path, (proxy, mtime)); - } - // TODO add some safety checks in case a proxy exists, that the initial `path` is not - // TODO the same as the resolved compiler binary - - // cache - let map_info = - CompilerCacheEntry::new(c.clone(), mtime, dist_info); - trace!( - "Inserting POSSIBLY PROXIED cache map info for {:?}", - &resolved_compiler_path - ); - me.compilers - .borrow_mut() - .insert(resolved_compiler_path, Some(map_info)); - } - Err(_) => { - trace!("Inserting PLAIN cache map info for {:?}", &path); - me.compilers.borrow_mut().insert(path, None); - } - } - // drop the proxy information, response is compiler only - let r: Result>> = info.map(|info| info.0); - f_ok(r) - }, - )) - } + // register the proxy for this compiler, so it will be used directly from now on + // and the true/resolved compiler will create table hits in the hash map + // based on the resolved path + if let Some(proxy) = proxy { + trace!( + "Inserting new path proxy {:?} @ {:?} -> {:?}", + &path, + &cwd, + resolved_compiler_path + ); + me.compiler_proxies + .write() + .await + .insert(path, (proxy, mtime)); } - }, - ); + // TODO add some safety checks in case a proxy exists, that the initial `path` is not + // TODO the same as the resolved compiler binary + + // cache + let map_info = CompilerCacheEntry::new(c.clone(), mtime, dist_info); + trace!( + "Inserting POSSIBLY PROXIED cache map info for {:?}", + &resolved_compiler_path + ); + me.compilers + .write() + .await + .insert(resolved_compiler_path, Some(map_info)); - Box::new(obtain) + // drop the proxy information, response is compiler only + Ok(c) + } + } } /// Check that we can handle and cache `cmd` when run with `compiler`. /// If so, run `start_compile_task` to execute it. - fn check_compiler( + async fn check_compiler( &self, compiler: Result>>, cmd: Vec, cwd: PathBuf, env_vars: Vec<(OsString, OsString)>, ) -> SccacheResponse { - let mut stats = self.stats.borrow_mut(); + let mut stats = self.stats.write().await; match compiler { Err(e) => { debug!("check_compiler: Unsupported compiler: {}", e.to_string()); @@ -1089,7 +1079,7 @@ where arguments: Vec, cwd: PathBuf, env_vars: Vec<(OsString, OsString)>, - tx: mpsc::Sender>, + mut tx: mpsc::Sender>, ) { let force_recache = env_vars .iter() @@ -1101,27 +1091,39 @@ where }; let out_pretty = hasher.output_pretty().into_owned(); let color_mode = hasher.color_mode(); - let result = hasher.get_cached_or_compile( - self.dist_client.get_client(), - self.creator.clone(), - self.storage.clone(), - arguments, - cwd, - env_vars, - cache_control, - self.pool.clone(), - ); let me = self.clone(); let kind = compiler.kind(); - let task = result.then(move |result| { + let dist_client = self.dist_client.get_client(); + let creator = self.creator.clone(); + let storage = self.storage.clone(); + let pool = self.rt.clone(); + + let task = async move { + let result = match dist_client { + Ok(client) => { + hasher + .get_cached_or_compile( + client, + creator, + storage, + arguments, + cwd, + env_vars, + cache_control, + pool, + ) + .await + } + Err(e) => Err(e), + }; let mut cache_write = None; - let mut stats = me.stats.borrow_mut(); let mut res = CompileFinished { color_mode, ..Default::default() }; match result { Ok((compiled, out)) => { + let mut stats = me.stats.write().await; match compiled { CompileResult::Error => { stats.cache_errors.increment(&kind); @@ -1179,6 +1181,7 @@ where res.stderr = stderr; } Err(err) => { + let mut stats = me.stats.write().await; match err.downcast::() { Ok(ProcessError(output)) => { debug!("Compilation failed: {:?}", output); @@ -1219,37 +1222,43 @@ where } } }; - let send = tx.send(Ok(Response::CompileFinished(res))); + let send = tx + .send(Ok(Response::CompileFinished(res))) + .map_err(|e| anyhow!("send on finish failed").context(e)); let me = me.clone(); - let cache_write = cache_write.then(move |result| { - match result { - Err(e) => { - debug!("Error executing cache write: {}", e); - me.stats.borrow_mut().cache_write_errors += 1; - } - //TODO: save cache stats! - Ok(Some(info)) => { - debug!( - "[{}]: Cache write finished in {}", - info.object_file_pretty, - util::fmt_duration_as_secs(&info.duration) - ); - me.stats.borrow_mut().cache_writes += 1; - me.stats.borrow_mut().cache_write_duration += info.duration; + let cache_write = async move { + if let Some(cache_write) = cache_write { + match cache_write.await { + Err(e) => { + debug!("Error executing cache write: {}", e); + me.stats.write().await.cache_write_errors += 1; + } + //TODO: save cache stats! + Ok(info) => { + debug!( + "[{}]: Cache write finished in {}", + info.object_file_pretty, + util::fmt_duration_as_secs(&info.duration) + ); + let mut stats = me.stats.write().await; + stats.cache_writes += 1; + stats.cache_write_duration += info.duration; + } } - - Ok(None) => {} } Ok(()) - }); + }; - send.join(cache_write).then(|_| Ok(())) - }); + futures::future::try_join(send, cache_write).await?; - tokio_compat::runtime::current_thread::TaskExecutor::current() - .spawn_local(Box::new(task)) - .unwrap(); + Ok::<_, Error>(()) + }; + + self.rt.spawn(async move { + task.await + .unwrap_or_else(|e| warn!("Failed to execute task: {:?}", e)); + }); } } @@ -1555,7 +1564,7 @@ impl ServerInfo { enum Frame { Body { chunk: Option }, - Message { message: R, body: bool }, + Message { message: R }, } struct Body { @@ -1569,16 +1578,13 @@ impl Body { } } -impl futures_03::Stream for Body { +impl futures::Stream for Body { type Item = Result; fn poll_next( mut self: Pin<&mut Self>, - _cx: &mut Context<'_>, + cx: &mut Context<'_>, ) -> std::task::Poll> { - match Pin::new(&mut self.receiver).poll().unwrap() { - Async::Ready(item) => std::task::Poll::Ready(item), - Async::NotReady => std::task::Poll::Pending, - } + Pin::new(&mut self.receiver).poll_next(cx) } } @@ -1596,6 +1602,32 @@ impl Message { } } +struct BincodeCodec; +impl tokio_serde::Serializer for BincodeCodec +where + T: serde::Serialize, +{ + type Error = Error; + + fn serialize(self: Pin<&mut Self>, item: &T) -> std::result::Result { + let mut bytes = BytesMut::new(); + bincode::serialize_into((&mut bytes).writer(), item)?; + Ok(bytes.freeze()) + } +} + +impl tokio_serde::Deserializer for BincodeCodec +where + T: serde::de::DeserializeOwned, +{ + type Error = Error; + + fn deserialize(self: Pin<&mut Self>, buf: &BytesMut) -> std::result::Result { + let ret = bincode::deserialize(buf)?; + Ok(ret) + } +} + /// Implementation of `Stream + Sink` that tokio-proto is expecting /// /// This type is composed of a few layers: @@ -1611,51 +1643,53 @@ impl Message { /// `Sink` implementation to switch from `BytesMut` to `Response` meaning that /// all `Response` types pushed in will be converted to `BytesMut` and pushed /// below. -struct SccacheTransport { - inner: WriteBincode, Request>, Response>, +struct SccacheTransport { + inner: Framed< + futures::stream::ErrInto< + futures::sink::SinkErrInto< + tokio_util::codec::Framed, + Bytes, + Error, + >, + Error, + >, + Request, + Response, + BincodeCodec, + >, } -impl Stream for SccacheTransport { - type Item = Message>; - type Error = io::Error; +impl Stream for SccacheTransport { + type Item = Result>>; - fn poll(&mut self) -> Poll, io::Error> { - let msg = try_ready!(self.inner.poll().map_err(|e| { - error!("SccacheTransport::poll failed: {}", e); - io::Error::new(io::ErrorKind::Other, e) - })); - Ok(msg.map(Message::WithoutBody).into()) + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner) + .poll_next(cx) + .map(|r| r.map(|s| s.map(Message::WithoutBody))) } } -impl Sink for SccacheTransport { - type SinkItem = Frame; - type SinkError = io::Error; +impl Sink> for SccacheTransport { + type Error = Error; - fn start_send(&mut self, item: Self::SinkItem) -> StartSend { + fn poll_ready(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_ready(cx) + } + + fn start_send(mut self: Pin<&mut Self>, item: Frame) -> Result<()> { match item { - Frame::Message { message, body } => match self.inner.start_send(message)? { - AsyncSink::Ready => Ok(AsyncSink::Ready), - AsyncSink::NotReady(message) => { - Ok(AsyncSink::NotReady(Frame::Message { message, body })) - } - }, - Frame::Body { chunk: Some(chunk) } => match self.inner.start_send(chunk)? { - AsyncSink::Ready => Ok(AsyncSink::Ready), - AsyncSink::NotReady(chunk) => { - Ok(AsyncSink::NotReady(Frame::Body { chunk: Some(chunk) })) - } - }, - Frame::Body { chunk: None } => Ok(AsyncSink::Ready), + Frame::Message { message } => Pin::new(&mut self.inner).start_send(message), + Frame::Body { chunk: Some(chunk) } => Pin::new(&mut self.inner).start_send(chunk), + Frame::Body { chunk: None } => Ok(()), } } - fn poll_complete(&mut self) -> Poll<(), io::Error> { - self.inner.poll_complete() + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_flush(cx) } - fn close(&mut self) -> Poll<(), io::Error> { - self.inner.close() + fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_close(cx) } } @@ -1666,29 +1700,26 @@ struct ShutdownOrInactive { } impl Future for ShutdownOrInactive { - type Item = (); - type Error = io::Error; + type Output = (); - fn poll(&mut self) -> Poll<(), io::Error> { + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { loop { - match self.rx.poll().unwrap() { - Async::NotReady => break, + match Pin::new(&mut self.rx).poll_next(cx) { + Poll::Pending => break, // Shutdown received! - Async::Ready(Some(ServerMessage::Shutdown)) => return Ok(().into()), - Async::Ready(Some(ServerMessage::Request)) => { + Poll::Ready(Some(ServerMessage::Shutdown)) => return Poll::Ready(()), + Poll::Ready(Some(ServerMessage::Request)) => { if self.timeout_dur != Duration::new(0, 0) { - self.timeout = Some(Delay::new(Instant::now() + self.timeout_dur)); + self.timeout = Some(delay_for(self.timeout_dur)); } } // All services have shut down, in theory this isn't possible... - Async::Ready(None) => return Ok(().into()), + Poll::Ready(None) => return Poll::Ready(()), } } match self.timeout { - None => Ok(Async::NotReady), - Some(ref mut timeout) => timeout - .poll() - .map_err(|err| io::Error::new(io::ErrorKind::Other, err)), + None => Poll::Pending, + Some(ref mut timeout) => Pin::new(timeout).poll(cx), } } } @@ -1696,60 +1727,66 @@ impl Future for ShutdownOrInactive { /// Helper future which tracks the `ActiveInfo` below. This future will resolve /// once all instances of `ActiveInfo` have been dropped. struct WaitUntilZero { - info: Rc>, + info: std::sync::Weak>, } +#[derive(Clone)] struct ActiveInfo { - info: Rc>, + info: Arc>, } struct Info { - active: usize, waker: Option, } +impl Drop for Info { + fn drop(&mut self) { + if let Some(waker) = self.waker.as_ref() { + waker.wake_by_ref(); + } + } +} + impl WaitUntilZero { + #[rustfmt::skip] fn new() -> (WaitUntilZero, ActiveInfo) { - let info = Rc::new(RefCell::new(Info { - active: 1, - waker: None, - })); + let info = Arc::new(Mutex::new(Info { waker: None })); - (WaitUntilZero { info: info.clone() }, ActiveInfo { info }) + (WaitUntilZero { info: Arc::downgrade(&info) }, ActiveInfo { info }) } } -impl Clone for ActiveInfo { - fn clone(&self) -> ActiveInfo { - self.info.borrow_mut().active += 1; - ActiveInfo { - info: self.info.clone(), - } - } -} +impl std::future::Future for WaitUntilZero { + type Output = (); -impl Drop for ActiveInfo { - fn drop(&mut self) { - let mut info = self.info.borrow_mut(); - info.active -= 1; - if info.active == 0 { - if let Some(waker) = info.waker.take() { - waker.wake(); + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> std::task::Poll { + match self.info.upgrade() { + None => std::task::Poll::Ready(()), + Some(arc) => { + let mut info = arc.lock().expect("we can't panic when holding lock"); + info.waker = Some(cx.waker().clone()); + std::task::Poll::Pending } } } } -impl std::future::Future for WaitUntilZero { - type Output = io::Result<()>; +#[test] +fn waits_until_zero() { + let (wait, _active) = WaitUntilZero::new(); + assert_eq!(wait.now_or_never(), None); - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> std::task::Poll { - let mut info = self.info.borrow_mut(); - if info.active == 0 { - std::task::Poll::Ready(Ok(())) - } else { - info.waker = Some(cx.waker().clone()); - std::task::Poll::Pending - } - } + let (wait, active) = WaitUntilZero::new(); + let _active2 = active.clone(); + drop(active); + assert_eq!(wait.now_or_never(), None); + + let (wait, _) = WaitUntilZero::new(); + assert_eq!(wait.now_or_never(), Some(())); + + let (wait, active) = WaitUntilZero::new(); + let active2 = active.clone(); + drop(active); + drop(active2); + assert_eq!(wait.now_or_never(), Some(())); } diff --git a/src/simples3/credential.rs b/src/simples3/credential.rs index bc7c3adf..a6197eb2 100644 --- a/src/simples3/credential.rs +++ b/src/simples3/credential.rs @@ -4,24 +4,21 @@ use chrono::{offset, DateTime, Duration}; use directories::UserDirs; -use futures::future::{self, Shared}; -use futures::{Async, Future, Stream}; -use hyper::client::HttpConnector; -use hyper::{Client, Request}; +use futures::future; use hyperx::header::Connection; use regex::Regex; -use serde_json::{from_str, Value}; +use reqwest::Client; #[allow(unused_imports, deprecated)] use std::ascii::AsciiExt; -use std::cell::RefCell; use std::collections::HashMap; use std::env::*; use std::fs::{self, File}; +use std::future::Future; use std::io::prelude::*; use std::io::BufReader; use std::path::{Path, PathBuf}; -use std::time::Duration as StdDuration; -use tokio_timer::Timeout; +use std::pin::Pin; +use std::sync::Arc; use crate::errors::*; use crate::util::RequestExt; @@ -86,17 +83,20 @@ impl AwsCredentials { } /// A trait for types that produce `AwsCredentials`. -pub trait ProvideAwsCredentials { +pub trait ProvideAwsCredentials: Send + Sync { /// Produce a new `AwsCredentials`. - fn credentials(&self) -> SFuture; + fn credentials(&self) + -> Pin> + Send + 'static>>; } /// Provides AWS credentials from environment variables. pub struct EnvironmentProvider; impl ProvideAwsCredentials for EnvironmentProvider { - fn credentials(&self) -> SFuture { - Box::new(future::result(credentials_from_environment())) + fn credentials( + &self, + ) -> Pin> + Send + 'static>> { + Box::pin(async { credentials_from_environment() }) } } @@ -198,11 +198,17 @@ impl ProfileProvider { } impl ProvideAwsCredentials for ProfileProvider { - fn credentials(&self) -> SFuture { - let result = parse_credentials_file(self.file_path()); - let result = result - .and_then(|mut profiles| profiles.remove(self.profile()).context("profile not found")); - Box::new(future::result(result)) + fn credentials( + &self, + ) -> Pin> + Send + 'static>> { + let file_path = self.file_path().to_owned(); + let profile = self.profile.to_owned(); + + Box::pin(async move { + let mut profiles = parse_credentials_file(&file_path)?; + + profiles.remove(&profile).context("profile not found") + }) } } @@ -280,7 +286,7 @@ fn parse_credentials_file(file_path: &Path) -> Result, + client: Client, } impl IamProvider { @@ -290,182 +296,156 @@ impl IamProvider { } } - fn iam_role(&self) -> SFuture { + async fn iam_role(client: &Client) -> Result { // First get the IAM role let address = "http://169.254.169.254/latest/meta-data/iam/security-credentials/"; - let req = Request::get(address) + let response = client + .get(address) .set_header(Connection::close()) - .body("".into()) - .unwrap(); - let response = self.client.request(req).and_then(|response| { - response.into_body().fold(Vec::new(), |mut body, chunk| { - body.extend_from_slice(&chunk); - Ok::<_, hyper::Error>(body) - }) - }); - - Box::new( - response - .then(|res| { - let bytes = res.context("couldn't connect to metadata service")?; - String::from_utf8(bytes) - .context("Didn't get a parsable response body from metadata service") - }) - .map(move |body| { - let mut address = address.to_string(); - address.push_str(&body); - address - }), - ) + .body("") + .send() + .await + .context("couldn't connect to metadata service")?; + let bytes = response.bytes().await?; + let body = String::from_utf8(bytes.into_iter().collect()) + .context("Didn't get a parsable response body from metadata service")?; + + let mut address = address.to_string(); + address.push_str(&body); + Ok(address) } } impl ProvideAwsCredentials for IamProvider { - fn credentials(&self) -> SFuture { - let url = match var("AWS_IAM_CREDENTIALS_URL") { - Ok(url) => f_ok(url), - Err(_) => self.iam_role(), - }; - let url = url.and_then(|url| { - url.parse::() - .with_context(|| format!("failed to parse `{}` as url", url)) - }); - + fn credentials( + &self, + ) -> Pin> + Send + 'static>> { let client = self.client.clone(); - let response = url.and_then(move |address| { - debug!("Attempting to fetch credentials from {}", address); - let req = Request::get(address) - .set_header(Connection::close()) - .body("".into()) - .unwrap(); - client.request(req).fcontext("failed to send http request") - }); - let body = response.and_then(|response| { - response - .into_body() - .fold(Vec::new(), |mut body, chunk| { - body.extend_from_slice(&chunk); - Ok::<_, hyper::Error>(body) - }) - .fcontext("failed to read http body") - }); - let body = body - .map_err(|e| anyhow!("Failed to get IAM credentials: {}", e)) - .and_then(|body| String::from_utf8(body).context("failed to read iam role response")); - - let creds = body.and_then(|body| { - let json_object: Value; - match from_str(&body) { - Err(_) => bail!("Couldn't parse metadata response body."), - Ok(val) => json_object = val, + + Box::pin(async { + let url = match var("AWS_IAM_CREDENTIALS_URL") { + Ok(url) => url, + Err(_) => Self::iam_role(&client).await?, }; + let address = url + .parse::() + .with_context(|| format!("failed to parse `{}` as url", url))?; - let access_key; - match json_object.get("AccessKeyId") { - None => bail!("Couldn't find AccessKeyId in response."), - Some(val) => { - access_key = val + debug!("Attempting to fetch credentials from {}", address); + + let fetch_creds = async move { + let response = client + .get(address) + .set_header(Connection::close()) + .body("") + .send() + .await + .context("failed to send http request")?; + + let body: serde_json::Value = response + .json() + .await + .context("failed to read IAM role response")?; + + let access_key = match body.get("AccessKeyId") { + None => bail!("Couldn't find AccessKeyId in response."), + Some(val) => val .as_str() .expect("AccessKeyId value was not a string") .to_owned() - .replace("\"", "") - } - }; + .replace("\"", ""), + }; - let secret_key; - match json_object.get("SecretAccessKey") { - None => bail!("Couldn't find SecretAccessKey in response."), - Some(val) => { - secret_key = val + let secret_key = match body.get("SecretAccessKey") { + None => bail!("Couldn't find SecretAccessKey in response."), + Some(val) => val .as_str() .expect("SecretAccessKey value was not a string") .to_owned() - .replace("\"", "") - } - }; + .replace("\"", ""), + }; - let expiration; - match json_object.get("Expiration") { - None => bail!("Couldn't find Expiration in response."), - Some(val) => { - expiration = val + let expiration = match body.get("Expiration") { + None => bail!("Couldn't find Expiration in response."), + Some(val) => val .as_str() .expect("Expiration value was not a string") .to_owned() - .replace("\"", "") - } - }; + .replace("\"", ""), + }; - let expiration_time = expiration - .parse() - .context("failed to parse expiration time")?; + let expiration_time = expiration + .parse() + .context("failed to parse expiration time")?; - let token_from_response; - match json_object.get("Token") { - None => bail!("Couldn't find Token in response."), - Some(val) => { - token_from_response = val + let token_from_response = match body.get("Token") { + None => bail!("Couldn't find Token in response."), + Some(val) => val .as_str() .expect("Token value was not a string") .to_owned() - .replace("\"", "") - } + .replace("\"", ""), + }; + + Ok(AwsCredentials::new( + access_key, + secret_key, + Some(token_from_response), + expiration_time, + )) }; - Ok(AwsCredentials::new( - access_key, - secret_key, - Some(token_from_response), - expiration_time, - )) - }); - - //XXX: this is crappy, but this blocks on non-EC2 machines like - // our mac builders. - let timeout = Timeout::new(creds, StdDuration::from_secs(2)); - - Box::new(timeout.then(|result| match result { - Ok(creds) => Ok(creds), - Err(err) => match err.into_inner() { - None => bail!("took too long to fetch credentials"), - Some(e) => { + //XXX: this is crappy, but this blocks on non-EC2 machines like + // our mac builders. + match tokio::time::timeout(std::time::Duration::from_secs(2), fetch_creds).await { + Ok(Ok(creds)) => Ok(creds), + Ok(Err(e)) => { warn!("Failed to fetch IAM credentials: {}", e); Err(e) } - }, - })) + Err(_elased) => bail!("took too long to fetch credentials"), + } + }) } } /// Wrapper for ProvideAwsCredentials that caches the credentials returned by the /// wrapped provider. Each time the credentials are accessed, they are checked to see if /// they have expired, in which case they are retrieved from the wrapped provider again. -pub struct AutoRefreshingProvider

{ +pub struct AutoRefreshingProvider { credentials_provider: P, - cached_credentials: RefCell>>, + cached_credentials: Arc>>, } -impl AutoRefreshingProvider

{ +impl AutoRefreshingProvider

{ pub fn new(provider: P) -> AutoRefreshingProvider

{ AutoRefreshingProvider { - cached_credentials: RefCell::new(provider.credentials().shared()), + cached_credentials: Arc::new(futures_locks::Mutex::new(None)), credentials_provider: provider, } } } -impl ProvideAwsCredentials for AutoRefreshingProvider

{ - fn credentials(&self) -> SFuture { - let mut future = self.cached_credentials.borrow_mut(); - if let Ok(Async::Ready(creds)) = future.poll() { - if creds.credentials_are_expired() { - *future = self.credentials_provider.credentials().shared(); +impl ProvideAwsCredentials for AutoRefreshingProvider

{ + fn credentials( + &self, + ) -> Pin> + Send + 'static>> { + let cached_credentials = Arc::clone(&self.cached_credentials); + let new_creds = self.credentials_provider.credentials(); + + Box::pin(async move { + let mut cache = cached_credentials.lock().await; + + match *cache { + Some(ref creds) if !creds.credentials_are_expired() => Ok(creds.clone()), + _ => { + let new_creds = new_creds.await?; + + *cache = Some(new_creds.clone()); + Ok(new_creds) + } } - } - Box::new(future.clone().then(|result| match result { - Ok(e) => Ok((*e).clone()), - Err(e) => Err(anyhow!(e.to_string())), - })) + }) } } @@ -484,30 +464,38 @@ pub struct ChainProvider { } impl ProvideAwsCredentials for ChainProvider { - fn credentials(&self) -> SFuture { - let creds = EnvironmentProvider.credentials().map(|c| { - debug!("Using AWS credentials from environment"); - c - }); - let mut creds = Box::new(creds) as SFuture<_>; - for provider in self.profile_providers.iter() { - let alternate = provider.credentials(); - creds = Box::new(creds.or_else(|_| alternate)); - } - Box::new( - creds - .or_else(move |_| { - IamProvider::new().credentials().map(|c| { - debug!("Using AWS credentials from IAM"); - c - }) - }) - .map_err(|_| { - anyhow!( + fn credentials( + &self, + ) -> Pin> + Send + 'static>> { + use futures::FutureExt; + + let environment_provider = EnvironmentProvider + .credentials() + .inspect(|_| debug!("Using AWS credentials from environment")) + .boxed(); + let iam_provider = IamProvider::new() + .credentials() + .inspect(|_| debug!("Using AWS credentials from IAM")) + .boxed(); + + let providers: Vec<_> = std::iter::empty() + .chain(std::iter::once(environment_provider)) + .chain( + self.profile_providers + .iter() + .map(ProvideAwsCredentials::credentials), + ) + .chain(std::iter::once(iam_provider)) + .collect(); + + Box::pin(async move { + match future::select_ok(providers).await { + Ok((creds, _rest)) => Ok(creds), + Err(_) => bail!( "Couldn't find AWS credentials in environment, credentials file, or IAM role." - ) - }), - ) + ), + } + }) } } diff --git a/src/simples3/s3.rs b/src/simples3/s3.rs index 3b6359bd..8d020927 100644 --- a/src/simples3/s3.rs +++ b/src/simples3/s3.rs @@ -6,12 +6,11 @@ use std::ascii::AsciiExt; use std::fmt; use crate::simples3::credential::*; -use futures::{Future, Stream}; use hmac::{Hmac, Mac, NewMac}; use hyper::header::HeaderValue; use hyper::Method; use hyperx::header; -use reqwest::r#async::{Client, Request}; +use reqwest::{Client, Request}; use sha1::Sha1; use crate::errors::*; @@ -72,7 +71,7 @@ impl Bucket { }) } - pub fn get(&self, key: &str, creds: Option<&AwsCredentials>) -> SFuture> { + pub async fn get(&self, key: &str, creds: Option<&AwsCredentials>) -> Result> { let url = format!("{}{}", self.base_url, key); debug!("GET {}", url); let url2 = url.clone(); @@ -100,46 +99,23 @@ impl Bucket { ); } - Box::new( - self.client - .execute(request) - .fwith_context(move || format!("failed GET: {}", url)) - .and_then(|res| { - if res.status().is_success() { - let content_length = res - .headers() - .get_hyperx::() - .map(|header::ContentLength(len)| len); - Ok((res.into_body(), content_length)) - } else { - Err(BadHttpStatusError(res.status()).into()) - } - }) - .and_then(|(body, content_length)| { - body.fold(Vec::new(), |mut body, chunk| { - body.extend_from_slice(&chunk); - Ok::<_, reqwest::Error>(body) - }) - .fcontext("failed to read HTTP body") - .and_then(move |bytes| { - if let Some(len) = content_length { - if len != bytes.len() as u64 { - bail!(format!( - "Bad HTTP body size read: {}, expected {}", - bytes.len(), - len - )); - } else { - info!("Read {} bytes from {}", bytes.len(), url2); - } - } - Ok(bytes) - }) - }), - ) + let res = self + .client + .execute(request) + .await + .with_context(move || format!("failed GET: {}", url))?; + + if res.status().is_success() { + let body = res.bytes().await.context("failed to read HTTP body")?; + info!("Read {} bytes from {}", body.len(), url2); + + Ok(body.into_iter().collect()) + } else { + Err(BadHttpStatusError(res.status()).into()) + } } - pub fn put(&self, key: &str, content: Vec, creds: &AwsCredentials) -> SFuture<()> { + pub async fn put(&self, key: &str, content: Vec, creds: &AwsCredentials) -> Result<()> { let url = format!("{}{}", self.base_url, key); debug!("PUT {}", url); let mut request = Request::new(Method::PUT, url.parse().unwrap()); @@ -189,7 +165,7 @@ impl Bucket { ); *request.body_mut() = Some(content.into()); - Box::new(self.client.execute(request).then(|result| match result { + match self.client.execute(request).await { Ok(res) => { if res.status().is_success() { trace!("PUT succeeded"); @@ -203,7 +179,7 @@ impl Bucket { trace!("PUT failed with error: {:?}", e); Err(e.into()) } - })) + } } // http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html diff --git a/src/test/mock_storage.rs b/src/test/mock_storage.rs index 9a787550..2c22f9b9 100644 --- a/src/test/mock_storage.rs +++ b/src/test/mock_storage.rs @@ -14,48 +14,50 @@ use crate::cache::{Cache, CacheWrite, Storage}; use crate::errors::*; -use futures::future; -use std::cell::RefCell; +use futures::channel::mpsc; +use futures_locks::Mutex; +use std::sync::Arc; use std::time::Duration; /// A mock `Storage` implementation. pub struct MockStorage { - gets: RefCell>>, + rx: Arc>>>, + tx: mpsc::UnboundedSender>, } impl MockStorage { /// Create a new `MockStorage`. - pub fn new() -> MockStorage { - MockStorage { - gets: RefCell::new(vec![]), + pub(crate) fn new() -> MockStorage { + let (tx, rx) = mpsc::unbounded(); + Self { + tx, + rx: Arc::new(Mutex::new(rx)), } } /// Queue up `res` to be returned as the next result from `Storage::get`. - pub fn next_get(&self, res: SFuture) { - self.gets.borrow_mut().push(res) + pub(crate) fn next_get(&self, res: Result) { + self.tx.unbounded_send(res).unwrap(); } } +#[async_trait] impl Storage for MockStorage { - fn get(&self, _key: &str) -> SFuture { - let mut g = self.gets.borrow_mut(); - assert!( - g.len() > 0, - "MockStorage get called, but no get results available" - ); - g.remove(0) + async fn get(&self, _key: &str) -> Result { + let next = self.rx.lock().await.try_next().unwrap(); + + next.expect("MockStorage get called but no get results available") } - fn put(&self, _key: &str, _entry: CacheWrite) -> SFuture { - f_ok(Duration::from_secs(0)) + async fn put(&self, _key: &str, _entry: CacheWrite) -> Result { + Ok(Duration::from_secs(0)) } fn location(&self) -> String { "Mock Storage".to_string() } - fn current_size(&self) -> SFuture> { - Box::new(future::ok(None)) + async fn current_size(&self) -> Result> { + Ok(None) } - fn max_size(&self) -> SFuture> { - Box::new(future::ok(None)) + async fn max_size(&self) -> Result> { + Ok(None) } } diff --git a/src/test/tests.rs b/src/test/tests.rs index a36e232c..38c110f0 100644 --- a/src/test/tests.rs +++ b/src/test/tests.rs @@ -19,8 +19,7 @@ use crate::jobserver::Client; use crate::mock_command::*; use crate::server::{DistClientContainer, SccacheServer, ServerMessage}; use crate::test::utils::*; -use futures::sync::oneshot::{self, Sender}; -use futures_03::executor::ThreadPool; +use futures::channel::oneshot::{self, Sender}; use std::fs::File; use std::io::{Cursor, Write}; #[cfg(not(target_os = "macos"))] @@ -32,7 +31,7 @@ use std::sync::{mpsc, Arc, Mutex}; use std::thread; use std::time::Duration; use std::u64; -use tokio_compat::runtime::current_thread::Runtime; +use tokio::runtime::Runtime; /// Options for running the server in tests. #[derive(Default)] @@ -75,13 +74,12 @@ where let (tx, rx) = mpsc::channel(); let (shutdown_tx, shutdown_rx) = oneshot::channel(); let handle = thread::spawn(move || { - let pool = ThreadPool::sized(1); + let runtime = Runtime::new().unwrap(); let dist_client = DistClientContainer::new_disabled(); - let storage = Arc::new(DiskCache::new(&cache_dir, cache_size, &pool)); + let storage = Arc::new(DiskCache::new(&cache_dir, cache_size, runtime.handle())); - let runtime = Runtime::new().unwrap(); let client = unsafe { Client::new() }; - let srv = SccacheServer::new(0, pool, runtime, client, dist_client, storage).unwrap(); + let srv = SccacheServer::new(0, runtime, client, dist_client, storage).unwrap(); let mut srv: SccacheServer>> = srv; assert!(srv.port() > 0); if let Some(options) = options { diff --git a/src/test/utils.rs b/src/test/utils.rs index 725efd26..f879c35b 100644 --- a/src/test/utils.rs +++ b/src/test/utils.rs @@ -17,10 +17,10 @@ use std::collections::HashMap; use std::env; use std::ffi::OsString; use std::fs::{self, File}; +use std::future::Future; use std::io; use std::path::{Path, PathBuf}; -use futures_03::executor::ThreadPool; use std::sync::{Arc, Mutex}; use tempfile::TempDir; @@ -230,6 +230,33 @@ impl TestFixture { } } +pub fn single_threaded_runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new() + .enable_all() + .basic_scheduler() + .core_threads(1) + .build() + .unwrap() +} + +/// An add on trait, to allow calling `.wait()` for `futures::Future` +/// as it was possible for `futures` at `0.1`. +/// +/// Intended for test only! +pub(crate) trait Waiter { + fn wait(self) -> R; +} + +impl Waiter for T +where + T: Future, +{ + fn wait(self) -> O { + let mut rt = single_threaded_runtime(); + rt.block_on(self) + } +} + #[test] fn test_map_contains_ok() { let mut m = HashMap::new(); @@ -263,16 +290,3 @@ fn test_map_contains_wrong_value() { m.insert("b", 3); assert_map_contains!(m, ("a", 1), ("b", 2)); } - -pub trait ThreadPoolExt { - fn sized(size: usize) -> Self; -} - -impl ThreadPoolExt for ThreadPool { - fn sized(size: usize) -> Self { - ThreadPool::builder() - .pool_size(size) - .create() - .expect("Failed to start thread pool") - } -} diff --git a/src/util.rs b/src/util.rs index 1089a7d1..387ba775 100644 --- a/src/util.rs +++ b/src/util.rs @@ -15,10 +15,6 @@ use crate::mock_command::{CommandChild, RunCommand}; use blake3::Hasher as blake3_Hasher; use byteorder::{BigEndian, ByteOrder}; -use futures::{future, Future}; -use futures_03::executor::ThreadPool; -use futures_03::future::TryFutureExt; -use futures_03::task; use serde::Serialize; use std::ffi::{OsStr, OsString}; use std::fs::File; @@ -31,20 +27,6 @@ use std::time::Duration; use crate::errors::*; -pub trait SpawnExt: task::SpawnExt { - fn spawn_fn(&self, f: F) -> SFuture - where - F: FnOnce() -> Result + std::marker::Send + 'static, - T: std::marker::Send + 'static, - { - self.spawn_with_handle(async move { f() }) - .map(|f| Box::new(f.compat()) as _) - .unwrap_or_else(f_err) - } -} - -impl SpawnExt for S {} - #[derive(Clone)] pub struct Digest { inner: blake3_Hasher, @@ -59,11 +41,11 @@ impl Digest { /// Calculate the BLAKE3 digest of the contents of `path`, running /// the actual hash computation on a background thread in `pool`. - pub fn file(path: T, pool: &ThreadPool) -> SFuture + pub async fn file(path: T, pool: &tokio::runtime::Handle) -> Result where T: AsRef, { - Self::reader(path.as_ref().to_owned(), pool) + Self::reader(path.as_ref().to_owned(), pool).await } /// Calculate the BLAKE3 digest of the contents read from `reader`. @@ -84,12 +66,13 @@ impl Digest { /// Calculate the BLAKE3 digest of the contents of `path`, running /// the actual hash computation on a background thread in `pool`. - pub fn reader(path: PathBuf, pool: &ThreadPool) -> SFuture { - Box::new(pool.spawn_fn(move || -> Result<_> { + pub async fn reader(path: PathBuf, pool: &tokio::runtime::Handle) -> Result { + pool.spawn_blocking(move || { let reader = File::open(&path) .with_context(|| format!("Failed to open file for hashing: {:?}", path))?; Digest::reader_sync(reader) - })) + }) + .await? } pub fn update(&mut self, bytes: &[u8]) { @@ -125,26 +108,17 @@ pub fn hex(bytes: &[u8]) -> String { /// Calculate the digest of each file in `files` on background threads in /// `pool`. -pub fn hash_all(files: &[PathBuf], pool: &ThreadPool) -> SFuture> { +pub async fn hash_all(files: &[PathBuf], pool: &tokio::runtime::Handle) -> Result> { let start = time::Instant::now(); let count = files.len(); - let pool = pool.clone(); - Box::new( - future::join_all( - files - .iter() - .map(move |f| Digest::file(f, &pool)) - .collect::>(), - ) - .map(move |hashes| { - trace!( - "Hashed {} files in {}", - count, - fmt_duration_as_secs(&start.elapsed()) - ); - hashes - }), - ) + let iter = files.iter().map(move |f| Digest::file(f, &pool)); + let hashes = futures::future::try_join_all(iter).await?; + trace!( + "Hashed {} files in {}", + count, + fmt_duration_as_secs(&start.elapsed()) + ); + Ok(hashes) } /// Format `duration` as seconds with a fractional component. @@ -156,48 +130,69 @@ pub fn fmt_duration_as_secs(duration: &Duration) -> String { /// /// This was lifted from `std::process::Child::wait_with_output` and modified /// to also write to stdin. -fn wait_with_input_output(mut child: T, input: Option>) -> SFuture +async fn wait_with_input_output(mut child: T, input: Option>) -> Result where T: CommandChild + 'static, { - use tokio_io::io::{read_to_end, write_all}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; let stdin = input.and_then(|i| { - child - .take_stdin() - .map(|stdin| write_all(stdin, i).fcontext("failed to write stdin")) + child.take_stdin().map(|mut stdin| async move { + stdin.write_all(&i).await.context("failed to write stdin") + }) }); - let stdout = child - .take_stdout() - .map(|io| read_to_end(io, Vec::new()).fcontext("failed to read stdout")); - let stderr = child - .take_stderr() - .map(|io| read_to_end(io, Vec::new()).fcontext("failed to read stderr")); + let stdout = child.take_stdout(); + let stdout = async move { + match stdout { + Some(mut stdout) => { + let mut buf = Vec::new(); + stdout + .read_to_end(&mut buf) + .await + .context("failed to read stdout")?; + Result::Ok(Some(buf)) + } + None => Ok(None), + } + }; + + let stderr = child.take_stderr(); + let stderr = async move { + match stderr { + Some(mut stderr) => { + let mut buf = Vec::new(); + stderr + .read_to_end(&mut buf) + .await + .context("failed to read stderr")?; + Result::Ok(Some(buf)) + } + None => Ok(None), + } + }; // Finish writing stdin before waiting, because waiting drops stdin. - let status = Future::and_then(stdin, |io| { - drop(io); - child.wait().fcontext("failed to wait for child") - }); - - Box::new(status.join3(stdout, stderr).map(|(status, out, err)| { - let stdout = out.map(|p| p.1); - let stderr = err.map(|p| p.1); - process::Output { - status, - stdout: stdout.unwrap_or_default(), - stderr: stderr.unwrap_or_default(), + let status = async move { + if let Some(stdin) = stdin { + let _ = stdin.await; } - })) + + child.wait().await.context("failed to wait for child") + }; + + let (status, stdout, stderr) = futures::future::try_join3(status, stdout, stderr).await?; + + Ok(process::Output { + status, + stdout: stdout.unwrap_or_default(), + stderr: stderr.unwrap_or_default(), + }) } /// Run `command`, writing `input` to its stdin if it is `Some` and return the exit status and output. /// /// If the command returns a non-successful exit status, an error of `SccacheError::ProcessError` /// will be returned containing the process output. -pub fn run_input_output( - mut command: C, - input: Option>, -) -> impl Future +pub async fn run_input_output(mut command: C, input: Option>) -> Result where C: RunCommand, { @@ -210,17 +205,18 @@ where }) .stdout(Stdio::piped()) .stderr(Stdio::piped()) - .spawn(); + .spawn() + .await?; - child.and_then(|child| { - wait_with_input_output(child, input).and_then(|output| { + wait_with_input_output(child, input) + .await + .and_then(|output| { if output.status.success() { - f_ok(output) + Ok(output) } else { - f_err(ProcessError(output)) + Err(ProcessError(output).into()) } }) - }) } /// Write `data` to `writer` with bincode serialization, prefixed by a `u32` length. @@ -364,7 +360,7 @@ pub use self::http_extension::{HeadersExt, RequestExt}; #[cfg(feature = "hyperx")] mod http_extension { - use http::header::HeaderValue; + use reqwest::header::{HeaderMap, HeaderValue}; use std::fmt; pub trait HeadersExt { @@ -377,14 +373,14 @@ mod http_extension { H: hyperx::header::Header; } - impl HeadersExt for http::HeaderMap { + impl HeadersExt for HeaderMap { fn set(&mut self, header: H) where H: hyperx::header::Header + fmt::Display, { self.insert( H::header_name(), - HeaderValue::from_shared(header.to_string().into()).unwrap(), + HeaderValue::from_maybe_shared(header.to_string()).unwrap(), ); } @@ -404,53 +400,51 @@ mod http_extension { } impl RequestExt for http::request::Builder { - fn set_header(mut self, header: H) -> Self + fn set_header(self, header: H) -> Self where H: hyperx::header::Header + fmt::Display, { self.header( H::header_name(), - HeaderValue::from_shared(header.to_string().into()).unwrap(), - ); - self + HeaderValue::from_maybe_shared(header.to_string()).unwrap(), + ) } } impl RequestExt for http::response::Builder { - fn set_header(mut self, header: H) -> Self + fn set_header(self, header: H) -> Self where H: hyperx::header::Header + fmt::Display, { self.header( H::header_name(), - HeaderValue::from_shared(header.to_string().into()).unwrap(), - ); - self + HeaderValue::from_maybe_shared(header.to_string()).unwrap(), + ) } } #[cfg(feature = "reqwest")] - impl RequestExt for ::reqwest::r#async::RequestBuilder { + impl RequestExt for ::reqwest::RequestBuilder { fn set_header(self, header: H) -> Self where H: hyperx::header::Header + fmt::Display, { self.header( H::header_name(), - HeaderValue::from_shared(header.to_string().into()).unwrap(), + HeaderValue::from_maybe_shared(header.to_string()).unwrap(), ) } } #[cfg(feature = "reqwest")] - impl RequestExt for ::reqwest::RequestBuilder { + impl RequestExt for ::reqwest::blocking::RequestBuilder { fn set_header(self, header: H) -> Self where H: hyperx::header::Header + fmt::Display, { self.header( H::header_name(), - HeaderValue::from_shared(header.to_string().into()).unwrap(), + HeaderValue::from_maybe_shared(header.to_string()).unwrap(), ) } } diff --git a/tests/harness/mod.rs b/tests/harness/mod.rs index e451ab92..69678d52 100644 --- a/tests/harness/mod.rs +++ b/tests/harness/mod.rs @@ -5,6 +5,7 @@ use sccache::server::ServerInfo; #[cfg(feature = "dist-server")] use std::env; use std::fs; +use std::future::Future; use std::io::Write; use std::net::{self, IpAddr, SocketAddr}; use std::path::{Path, PathBuf}; @@ -297,7 +298,7 @@ impl DistSystem { || { let status = self.scheduler_status(); if matches!( - self.scheduler_status(), + status, SchedulerStatusResult { num_servers: 0, num_cpus: _, @@ -437,7 +438,7 @@ impl DistSystem { || { let status = self.scheduler_status(); if matches!( - self.scheduler_status(), + status, SchedulerStatusResult { num_servers: 1, num_cpus: _, @@ -461,7 +462,7 @@ impl DistSystem { } fn scheduler_status(&self) -> SchedulerStatusResult { - let res = reqwest::get(dist::http::urls::scheduler_status( + let res = reqwest::blocking::get(dist::http::urls::scheduler_status( &self.scheduler_url().to_url(), )) .unwrap(); @@ -646,8 +647,9 @@ fn wait_for_http(url: HTTPUrl, interval: Duration, max_wait: Duration) { // TODO: after upgrading to reqwest >= 0.9, use 'danger_accept_invalid_certs' and stick with that rather than tcp wait_for( || { - //match reqwest::get(url.to_url()) { - match net::TcpStream::connect(url.to_url()) { + let url = url.to_url(); + let url = url.socket_addrs(|| None).unwrap(); + match net::TcpStream::connect(url.as_slice()) { Ok(_) => Ok(()), Err(e) => Err(e.to_string()), }