From d9d79473d63f63f9600773b7249935933fa208dc Mon Sep 17 00:00:00 2001 From: shouyanwang Date: Wed, 24 Nov 2021 15:35:25 +0800 Subject: [PATCH] feat(init): release tat-agent from v0.1.12 Main feature includes: - Support shell, powershell and python commands. - Support both Linux and Windows. - Report command output. - Auto kill the timeout task. - Cancel the specified task. --- .gitignore | 14 + CHANGELOG.md | 111 + CODE_OF_CONDUCT.md | 3 + CONTRIBUTING.md | 56 + Cargo.lock | 3105 ++++++++++++++++++++++++++++ Cargo.toml | 46 + Cross.toml | 6 + Dockerfile | 5 + LICENSE | 208 ++ Makefile | 98 + README.md | 47 + install/build.bat | 3 + install/install.bat | 20 + install/install.sh | 119 ++ install/release.bat | 40 + install/release.sh | 50 + install/self_update.bat | 8 + install/self_update.sh | 4 + install/tat_agent.service | 16 + install/tat_agent_service | 92 + install/tat_agent_service.conf | 13 + install/test.bat | 1 + install/test.sh | 2 + install/uninstall.bat | 3 + install/uninstall.sh | 50 + src/common.rs | 6 + src/common/asserts.rs | 24 + src/common/consts.rs | 159 ++ src/common/envs.rs | 18 + src/common/logger.rs | 70 + src/common/option.rs | 44 + src/daemonizer.rs | 17 + src/daemonizer/unix.rs | 49 + src/daemonizer/windows.rs | 158 ++ src/executor.rs | 5 + src/executor/powershell_command.rs | 275 +++ src/executor/proc.rs | 910 ++++++++ src/executor/shell_command.rs | 373 ++++ src/http.rs | 9 + src/http/invoke_adapter.rs | 144 ++ src/http/requester.rs | 187 ++ src/http/store.rs | 240 +++ src/http/thread.rs | 572 +++++ src/lib.rs | 8 + src/main.rs | 61 + src/ontime.rs | 3 + src/ontime/self_update.rs | 345 ++++ src/ontime/thread.rs | 231 +++ src/ontime/timer.rs | 275 +++ src/types.rs | 20 + src/types/common.rs | 4 + src/types/error.rs | 43 + src/types/inner_msg.rs | 4 + src/types/task.rs | 434 ++++ src/types/ws_msg.rs | 13 + src/uname.rs | 13 + src/uname/common.rs | 57 + src/uname/unix.rs | 36 + src/uname/windows.rs | 16 + src/ws.rs | 1 + src/ws/thread.rs | 268 +++ tests/http_test.rs | 93 + tests/support/mod.rs | 2 + tests/support/response.rs | 138 ++ tests/support/server.rs | 115 ++ 65 files changed, 9560 insertions(+) create mode 100644 .gitignore create mode 100644 CHANGELOG.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 Cross.toml create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 install/build.bat create mode 100644 install/install.bat create mode 100755 install/install.sh create mode 100644 install/release.bat create mode 100755 install/release.sh create mode 100644 install/self_update.bat create mode 100755 install/self_update.sh create mode 100644 install/tat_agent.service create mode 100644 install/tat_agent_service create mode 100644 install/tat_agent_service.conf create mode 100644 install/test.bat create mode 100755 install/test.sh create mode 100644 install/uninstall.bat create mode 100755 install/uninstall.sh create mode 100644 src/common.rs create mode 100644 src/common/asserts.rs create mode 100644 src/common/consts.rs create mode 100644 src/common/envs.rs create mode 100644 src/common/logger.rs create mode 100644 src/common/option.rs create mode 100644 src/daemonizer.rs create mode 100644 src/daemonizer/unix.rs create mode 100644 src/daemonizer/windows.rs create mode 100644 src/executor.rs create mode 100644 src/executor/powershell_command.rs create mode 100644 src/executor/proc.rs create mode 100644 src/executor/shell_command.rs create mode 100644 src/http.rs create mode 100644 src/http/invoke_adapter.rs create mode 100644 src/http/requester.rs create mode 100644 src/http/store.rs create mode 100644 src/http/thread.rs create mode 100644 src/lib.rs create mode 100644 src/main.rs create mode 100644 src/ontime.rs create mode 100644 src/ontime/self_update.rs create mode 100644 src/ontime/thread.rs create mode 100644 src/ontime/timer.rs create mode 100644 src/types.rs create mode 100644 src/types/common.rs create mode 100644 src/types/error.rs create mode 100644 src/types/inner_msg.rs create mode 100644 src/types/task.rs create mode 100644 src/types/ws_msg.rs create mode 100644 src/uname.rs create mode 100644 src/uname/common.rs create mode 100644 src/uname/unix.rs create mode 100644 src/uname/windows.rs create mode 100644 src/ws.rs create mode 100644 src/ws/thread.rs create mode 100644 tests/http_test.rs create mode 100644 tests/support/mod.rs create mode 100644 tests/support/response.rs create mode 100644 tests/support/server.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ed53866 --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +/target +.vscode/ +.idea/ +log +install/tat_agent +tat_agent +install/tat_agent_*.zip +install/tat_agent_*.tar.gz +install/tat_agent_*install* + +# temp script generated by test. +.*.sh +*.exe +install/*.exe diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..adbd496 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,111 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +## [0.1.12] - 2021-11-16 + +### Changed + +- Optimize executor logic. +- Support task cancellation. +- Remove speculate. + +## [0.1.11] - 2021-10-12 + +### Changed + +- Agent support for windows. + +## [0.1.10] - 2021-10-12 + +### Changed + +- Modify the backend domain. + +## [0.1.9] - 2021-09-18 + +### Changed + +- Report `START_FAILED` during script file creating failed because of disk full or permission deny. +- Fix some warning, replace the deprecated func. +- Modify a test case of cache lib, make it more accurate. +- Reduce some redundant log. + +## [0.1.8] - 2021-08-19 + +### Changed + +- Reduce systemd restart seconds to 1s. + +## [0.1.7] - 2021-08-04 + +### Changed + +- Optimized for preload environment. +- Use `vendored` mode for building openssl, see: [openssl](https://docs.rs/openssl/0.10.35/openssl/#vendored). +- Optimize Makefile for cross-compile. + +## [0.1.6] - 2021-06-28 + +### Changed + +- Support agent run daemon tasks. + +## [0.1.5] - 2021-06-22 + +### Added + +- Support set `username` for invocation task. +- Support preload environment variables before running task. +- Add `err_info` to store the reason for task `START_FAILED`. + +## [0.1.4] - 2021-05-07 + +### Added + +- Support containerize agent, used for E2E environment. +- Support debug mode for mock vpc info, used for E2E environment. + +## [0.1.3] - 2021-04-25 + +### Added + +- Report dropped bytes of output. +- Add CHANGELOG & LICENSE. + +## [0.1.2] - 2021-01-05 + +### Added + +- Add integration tests for HTTP API. + +### Changed + +- Do CheckUpdate 10s after the agent started. +- Update install.sh to adapt some bash version which do not support [[ syntax. +- Fix bug of finish time return 0 when start failed. + +## [0.1.1] - 2020-12-14 + +### Added + +- Support install for CoreOS whose /usr is Read-only. + +### Changed + +- Report task start & finish support retry. +- Optimize several sleep operations. +- Fix sleep method in tokio, use await instead of thread sleep. +- Fix bug, commands local saving directory recreate each month. +- Set tat_agent auto-restart time to 5s by systemd. + +## [0.1.0] - 2020-11-16 + +### Added + +- First release of TAT agent. +- Including one WebSocket thread for task notify. +- Including one HTTP thread for task query & report. +- Including one On-time thread for some periodic & timer tasks. +- Commands spawned as an independent process. + diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..252289b --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,3 @@ +# TAT Agent Community Code of Conduct + +TAT Agent follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..9b8b197 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,56 @@ +# Prepare + +- Rust + +Install Rust with command: +``` +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +``` +See more at: +https://www.rust-lang.org/learn/get-started + +- Rust target + +Add target if you need, such as x86_64-unknown-linux-musl to build static target on linux: +``` +rustup target add x86_64-unknown-linux-musl +``` + +- IDE + +1. Linux command line; +2. IntelliJ IDEA with Rust plugin; +3. Visual Studio Code. + +# Build + +- Run directly + +cargo will run immediately after build. +``` +make run +``` + +- Build static target + +cargo will only build the target. +Need x86_64-unknown-linux-musl target added and docker installed. +``` +make build +``` + +# Run + +After build, you can run tat_agent anywhere with: +``` +# root user +./tat_agent +``` +The tat_agent will use flock() to lock the pid file /var/run/tat_agent.pid, +to ensure only one tat_agent process can start success on one machine. +The pid file will be auto unlocked after the tat_agent exit. +So you do not need to modify the pid file manually. + +# Contribute + +You are welcome to report bug, or contribute from github via issue, pull request or any discussions. diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..f926430 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,3105 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "adler" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" + +[[package]] +name = "ahash" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43bb833f0bf979d8475d38fbf09ed3b8a55e1885fe93ad3f93239fc6a4f17b98" +dependencies = [ + "getrandom 0.2.3", + "once_cell", + "version_check 0.9.2", +] + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "arc-swap" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034" + +[[package]] +name = "async-attributes" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efd3d156917d94862e779f356c5acae312b08fd3121e792c857d7928c8088423" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "async-channel" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59740d83946db6a5af71ae25ddf9562c2b176b2ca42cf99a455f09f4a220d6b9" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-executor" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d373d78ded7d0b3fa8039375718cde0aace493f2e34fb60f51cbf567562ca801" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "once_cell", + "vec-arena", +] + +[[package]] +name = "async-global-executor" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73079b49cd26b8fd5a15f68fc7707fc78698dc2a3d61430f2a7a9430230dfa04" +dependencies = [ + "async-executor", + "async-io", + "futures-lite", + "num_cpus", + "once_cell", +] + +[[package]] +name = "async-io" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54bc4c1c7292475efb2253227dbcfad8fe1ca4c02bc62c510cc2f3da5c4704e" +dependencies = [ + "concurrent-queue", + "fastrand", + "futures-lite", + "libc", + "log 0.4.11", + "nb-connect", + "once_cell", + "parking", + "polling", + "vec-arena", + "waker-fn", + "winapi 0.3.9", +] + +[[package]] +name = "async-mutex" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479db852db25d9dbf6204e6cb6253698f175c15726470f78af0d918e99d6156e" +dependencies = [ + "event-listener", +] + +[[package]] +name = "async-std" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7e82538bc65a25dbdff70e4c5439d52f068048ab97cdea0acd73f131594caa1" +dependencies = [ + "async-global-executor", + "async-io", + "async-mutex", + "blocking", + "crossbeam-utils 0.8.5", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite", + "gloo-timers", + "kv-log-macro", + "log 0.4.11", + "memchr", + "num_cpus", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + +[[package]] +name = "async-task" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91831deabf0d6d7ec49552e489aed63b7456a7a3c46cff62adad428110b0af0" + +[[package]] +name = "async-trait" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b98e84bbb4cbcdd97da190ba0c58a1bb0de2c1fdf67d159e192ed766aeca722" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "base64" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "489d6c0ed21b11d038c31b6ceccca973e65d73ba3bd8ecb9a2babf5546164643" +dependencies = [ + "byteorder", + "safemem", +] + +[[package]] +name = "base64" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" +dependencies = [ + "byteorder", +] + +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] + +[[package]] +name = "blocking" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5e170dbede1f740736619b776d7251cb1b9095c435c34d8ca9f57fcd2f335e9" +dependencies = [ + "async-channel", + "async-task", + "atomic-waker", + "fastrand", + "futures-lite", + "once_cell", +] + +[[package]] +name = "boxfnonce" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5988cb1d626264ac94100be357308f29ff7cbdd3b36bda27f450a4ee3f713426" + +[[package]] +name = "bumpalo" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" + +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + +[[package]] +name = "byteorder" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" + +[[package]] +name = "bytes" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +dependencies = [ + "byteorder", + "either", + "iovec", +] + +[[package]] +name = "bytes" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" + +[[package]] +name = "bzip2" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42b7c3cbf0fa9c1b82308d57191728ca0256cb821220f4e2fd410a72ade26e3b" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.9+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad3b39a260062fca31f7b0b12f207e8f2590a67d32ec7d59c20484b07ea7285e" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "cache-padded" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" + +[[package]] +name = "cc" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed67cbde08356238e75fc4656be4749481eeffb09e19f320a25237d5221c985d" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "time 0.1.44", + "winapi 0.3.9", +] + +[[package]] +name = "clap" +version = "2.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim 0.8.0", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "cloudabi" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467" +dependencies = [ + "bitflags", +] + +[[package]] +name = "codepage" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b0e9222c0cdf2c6ac27d73f664f9520266fa911c3106329d359f8861cb8bde9" +dependencies = [ + "encoding_rs", +] + +[[package]] +name = "codepage-strings" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f25fc635bc3c98b4a0c707c99b7ffe7f51cb98b904d814e9c24441fad8c52" +dependencies = [ + "codepage", + "encoding_rs", + "oem_cp", +] + +[[package]] +name = "concurrent-queue" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" +dependencies = [ + "cache-padded", +] + +[[package]] +name = "const_fn" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f92cfa0fd5690b3cf8c1ef2cabbd9b7ef22fa53cf5e1f92b05103f6d5d1cf6e7" + +[[package]] +name = "core-foundation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" + +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "crossbeam-deque" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils 0.7.2", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +dependencies = [ + "autocfg 1.0.1", + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "lazy_static", + "maybe-uninit", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +dependencies = [ + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg 1.0.1", + "cfg-if 0.1.10", + "lazy_static", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db" +dependencies = [ + "cfg-if 1.0.0", + "lazy_static", +] + +[[package]] +name = "daemonize" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70c24513e34f53b640819f0ac9f705b673fcf4006d7aab8778bee72ebfc89815" +dependencies = [ + "boxfnonce", + "libc", +] + +[[package]] +name = "darling" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f2c43f534ea4b0b049015d00269734195e6d3f0f6635cb692251aca6f9f8b3c" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e91455b86830a1c21799d94524df0845183fa55bafd9aa137b01c7d1065fa36" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29b5acf0dea37a7f66f7b25d2c5e93fd46f8f6968b1a5d7a3e02e97768afc95a" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array", +] + +[[package]] +name = "dtoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "encoding" +version = "0.2.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec" +dependencies = [ + "encoding-index-japanese", + "encoding-index-korean", + "encoding-index-simpchinese", + "encoding-index-singlebyte", + "encoding-index-tradchinese", +] + +[[package]] +name = "encoding-index-japanese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-korean" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-simpchinese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d87a7194909b9118fc707194baa434a4e3b0fb6a5a757c73c3adb07aa25031f7" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-singlebyte" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-tradchinese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding_index_tests" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" + +[[package]] +name = "encoding_rs" +version = "0.8.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "event-listener" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59" + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + +[[package]] +name = "fastrand" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca5faf057445ce5c9d4329e382b2ce7ca38550ef3b73a5348362d5f24e0c7fe3" +dependencies = [ + "instant", +] + +[[package]] +name = "faux" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1266501defc36302f75d5efe43e0e7442b08c04290c189e1f2d307eaa8c2c3ef" +dependencies = [ + "faux_macros", + "paste", +] + +[[package]] +name = "faux_macros" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2f9117c3e82bcac72fd7c2c181e38c116bcfb420a8f26c5966619e319b512f7" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "flate2" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7411863d55df97a419aa64cb4d2f167103ea9d767e2c54a1868b7ac3f6b47129" +dependencies = [ + "cfg-if 1.0.0", + "crc32fast", + "libc", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "futures" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7e4c2612746b0df8fed4ce0c69156021b704c9aefa360311c04e6e9e002eed" + +[[package]] +name = "futures" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95314d38584ffbfda215621d723e0a3906f032e03ae5551e650058dac83d4797" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0448174b01148032eed37ac4aed28963aaaa8cfa93569a08e5b479bbc6c2c151" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18eaa56102984bed2c88ea39026cff3ce3b4c7f508ca970cedf2450ea10d4e46" + +[[package]] +name = "futures-cpupool" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" +dependencies = [ + "futures 0.1.30", + "num_cpus", +] + +[[package]] +name = "futures-executor" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5f8e0c9258abaea85e78ebdda17ef9666d390e987f006be6080dfe354b708cb" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1798854a4727ff944a7b12aa999f58ce7aa81db80d2dfaaf2ba06f065ddd2b" + +[[package]] +name = "futures-lite" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6c079abfac3ab269e2927ec048dabc89d009ebfdda6b8ee86624f30c689658" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + +[[package]] +name = "futures-macro" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e36fccf3fc58563b4a14d265027c627c3b665d7fed489427e88e7cc929559efe" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e3ca3f17d6e8804ae5d3df7a7d35b2b3a6fe89dac84b31872720fc3060a0b11" + +[[package]] +name = "futures-task" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d502af37186c4fef99453df03e374683f8a1eec9dcc1e66b3b82dc8278ce3c" +dependencies = [ + "once_cell", +] + +[[package]] +name = "futures-util" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abcb44342f62e6f3e8ac427b8aa815f724fd705dfad060b18ac7866c15bb8e34" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project 1.0.1", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +dependencies = [ + "typenum", +] + +[[package]] +name = "getrandom" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", +] + +[[package]] +name = "gloo-timers" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47204a46aaff920a1ea58b11d03dec6f704287d27561724a4631e450654a891f" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "h2" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5b34c246847f938a410a03c5458c7fee2274436675e76d8b903c08efc29c462" +dependencies = [ + "byteorder", + "bytes 0.4.12", + "fnv", + "futures 0.1.30", + "http 0.1.21", + "indexmap", + "log 0.4.11", + "slab", + "string", + "tokio-io", +] + +[[package]] +name = "h2" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" +dependencies = [ + "bytes 0.5.6", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.1", + "indexmap", + "slab", + "tokio 0.2.22", + "tokio-util", + "tracing", + "tracing-futures", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + +[[package]] +name = "hermit-abi" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +dependencies = [ + "libc", +] + +[[package]] +name = "http" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ccf5ede3a895d8856620237b2f02972c1bbc78d2965ad7fe8838d4a0ed41f0" +dependencies = [ + "bytes 0.4.12", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" +dependencies = [ + "bytes 0.5.6", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.30", + "http 0.1.21", + "tokio-buf", +] + +[[package]] +name = "http-body" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +dependencies = [ + "bytes 0.5.6", + "http 0.2.1", +] + +[[package]] +name = "httparse" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" + +[[package]] +name = "httpdate" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + +[[package]] +name = "hyper" +version = "0.10.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a0652d9a2609a968c14be1a9ea00bf4b1d64e2e1f53a1b51b6fff3a6e829273" +dependencies = [ + "base64 0.9.3", + "httparse", + "language-tags", + "log 0.3.9", + "mime 0.2.6", + "num_cpus", + "time 0.1.44", + "traitobject", + "typeable", + "unicase 1.4.2", + "url 1.7.2", +] + +[[package]] +name = "hyper" +version = "0.12.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dbe6ed1438e1f8ad955a4701e9a944938e9519f6888d12d8558b645e247d5f6" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.30", + "futures-cpupool", + "h2 0.1.26", + "http 0.1.21", + "http-body 0.1.0", + "httparse", + "iovec", + "itoa", + "log 0.4.11", + "net2", + "rustc_version", + "time 0.1.44", + "tokio 0.1.22", + "tokio-buf", + "tokio-executor", + "tokio-io", + "tokio-reactor", + "tokio-tcp", + "tokio-threadpool", + "tokio-timer", + "want 0.2.0", +] + +[[package]] +name = "hyper" +version = "0.13.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf" +dependencies = [ + "bytes 0.5.6", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.2.7", + "http 0.2.1", + "http-body 0.3.1", + "httparse", + "httpdate", + "itoa", + "pin-project 1.0.1", + "socket2", + "tokio 0.2.22", + "tower-service", + "tracing", + "want 0.3.0", +] + +[[package]] +name = "hyper-tls" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" +dependencies = [ + "bytes 0.5.6", + "hyper 0.13.9", + "native-tls", + "tokio 0.2.22", + "tokio-tls 0.3.1", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "idna" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55e2e4c765aa53a0424761bf9f41aa7a6ac1efa87238f59560640e27fca028f2" +dependencies = [ + "autocfg 1.0.1", + "hashbrown", +] + +[[package]] +name = "instant" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb1fc4429a33e1f80d41dc9fea4d108a88bec1de8053878898ae448a0b52f613" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "ipnet" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" + +[[package]] +name = "itoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" + +[[package]] +name = "js-sys" +version = "0.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca059e81d9486668f12d455a4ea6daa600bd408134cd17e3d3fb5a32d1f016f8" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log 0.4.11", +] + +[[package]] +name = "language-tags" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1fa8cddc8fbbee11227ef194b5317ed014b8acbf15139bd716a18ad3fe99ec5" + +[[package]] +name = "linked-hash-map" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" + +[[package]] +name = "lock_api" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "lock_api" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" +dependencies = [ + "log 0.4.11", +] + +[[package]] +name = "log" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +dependencies = [ + "cfg-if 0.1.10", + "serde", +] + +[[package]] +name = "log-mdc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a94d21414c1f4a51209ad204c1776a3d0765002c76c6abcb602a6f09f1e881c7" + +[[package]] +name = "log4rs" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e1ad45e4584824d760c35d71868dd7e6e5acd8f5195a9573743b369fc86cd6" +dependencies = [ + "arc-swap", + "chrono", + "flate2", + "fnv", + "humantime", + "libc", + "log 0.4.11", + "log-mdc", + "parking_lot 0.11.0", + "serde", + "serde-value", + "serde_derive", + "serde_json", + "serde_yaml", + "thread-id", + "typemap", + "winapi 0.3.9", +] + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + +[[package]] +name = "memchr" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" + +[[package]] +name = "memoffset" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +dependencies = [ + "autocfg 1.0.1", +] + +[[package]] +name = "mime" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba626b8a6de5da682e1caa06bdb42a335aee5a84db8e5046a3e8ab17ba0a3ae0" +dependencies = [ + "log 0.3.9", +] + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +dependencies = [ + "mime 0.3.16", + "unicase 2.6.0", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" +dependencies = [ + "adler", + "autocfg 1.0.1", +] + +[[package]] +name = "mio" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" +dependencies = [ + "cfg-if 0.1.10", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log 0.4.11", + "miow 0.2.1", + "net2", + "slab", + "winapi 0.2.8", +] + +[[package]] +name = "mio-named-pipes" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656" +dependencies = [ + "log 0.4.11", + "mio", + "miow 0.3.5", + "winapi 0.3.9", +] + +[[package]] +name = "mio-uds" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" +dependencies = [ + "iovec", + "libc", + "mio", +] + +[[package]] +name = "miow" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + +[[package]] +name = "miow" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07b88fb9795d4d36d62a012dfbf49a8f5cf12751f36d31a9dbe66d528e58979e" +dependencies = [ + "socket2", + "winapi 0.3.9", +] + +[[package]] +name = "msdos_time" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aad9dfe950c057b1bfe9c1f2aa51583a8468ef2a5baba2ebbe06d775efeb7729" +dependencies = [ + "time 0.1.44", + "winapi 0.3.9", +] + +[[package]] +name = "native-tls" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a1cda389c26d6b88f3d2dc38aa1b750fe87d298cc5d795ec9e975f402f00372" +dependencies = [ + "lazy_static", + "libc", + "log 0.4.11", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nb-connect" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8123a81538e457d44b933a02faf885d3fe8408806b23fa700e8f01c6c3a98998" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "net2" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ebc3ec692ed7c9a255596c67808dee269f64655d8baf7b4f0638e51ba1d6853" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "nom" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf51a729ecf40266a2368ad335a5fdde43471f545a967109cd62146ecf8b66ff" + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg 1.0.1", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg 1.0.1", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "oem_cp" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56428601ad4b928c5b705e89eb2db6ae070de7bad96f0e1c2ae07288fdfff082" +dependencies = [ + "ahash", + "lazy_static", +] + +[[package]] +name = "once_cell" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" + +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + +[[package]] +name = "openssl" +version = "0.10.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "549430950c79ae24e6d02e0b7404534ecf311d94cc9f861e9e4020187d13d885" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" + +[[package]] +name = "openssl-src" +version = "111.15.0+1.1.1k" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1a5f6ae2ac04393b217ea9f700cd04fa9bf3d93fae2872069f3d15d908af70a" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a7907e3bfa08bb85105209cdfcb6c63d109f8f6c1ed6ca318fff5c1853fbc1d" +dependencies = [ + "autocfg 1.0.1", + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "ordered-float" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3741934be594d77de1c8461ebcbbe866f585ea616a9753aa78f2bdc69f0e4579" +dependencies = [ + "num-traits", +] + +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + +[[package]] +name = "parking_lot" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" +dependencies = [ + "lock_api 0.3.4", + "parking_lot_core 0.6.2", + "rustc_version", +] + +[[package]] +name = "parking_lot" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4893845fa2ca272e647da5d0e46660a314ead9c2fdd9a883aabc32e481a8733" +dependencies = [ + "instant", + "lock_api 0.4.1", + "parking_lot_core 0.8.0", +] + +[[package]] +name = "parking_lot_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b" +dependencies = [ + "cfg-if 0.1.10", + "cloudabi 0.0.3", + "libc", + "redox_syscall", + "rustc_version", + "smallvec 0.6.13", + "winapi 0.3.9", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" +dependencies = [ + "cfg-if 0.1.10", + "cloudabi 0.1.0", + "instant", + "libc", + "redox_syscall", + "smallvec 1.4.2", + "winapi 0.3.9", +] + +[[package]] +name = "paste" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58" + +[[package]] +name = "percent-encoding" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pin-project" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" +dependencies = [ + "pin-project-internal 0.4.27", +] + +[[package]] +name = "pin-project" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee41d838744f60d959d7074e3afb6b35c7456d0f61cad38a24e35e6553f73841" +dependencies = [ + "pin-project-internal 1.0.1", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81a4ffa594b66bff340084d4081df649a7dc049ac8d7fc458d8e628bfbbb2f86" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "podio" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b18befed8bc2b61abc79a457295e7e838417326da1586050b919414073977f19" + +[[package]] +name = "polling" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2a7bc6b2a29e632e45451c941832803a18cce6781db04de8a04696cdca8bde4" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "log 0.4.11", + "wepoll-sys", + "winapi 0.3.9", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro-nested" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" + +[[package]] +name = "proc-macro2" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "procinfo" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ab1427f3d2635891f842892dda177883dca0639e05fe66796a62c9d2f23b49c" +dependencies = [ + "byteorder", + "libc", + "nom", + "rustc_version", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.7", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc 0.1.0", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi 0.3.9", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.15", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.3.1", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.15", +] + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi 0.3.9", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi 0.0.3", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "winapi 0.3.9", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "reqwest" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9eaa17ac5d7b838b7503d118fa16ad88f440498bf9ffe5424e621f93190d61e" +dependencies = [ + "base64 0.12.3", + "bytes 0.5.6", + "encoding_rs", + "futures-core", + "futures-util", + "http 0.2.1", + "http-body 0.3.1", + "hyper 0.13.9", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log 0.4.11", + "mime 0.3.16", + "mime_guess", + "native-tls", + "percent-encoding 2.1.0", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio 0.2.22", + "tokio-tls 0.3.1", + "url 2.1.1", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "safemem" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" + +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi 0.3.9", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "security-framework" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1759c2e3c8580017a484a7ac56d3abc5a6c1feadf88db2f3633f12ae4268c69" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f99b9d5e26d2a71633cc4f2ebae7cc9f874044e0c351a27e17892d76dce5678b" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "serde" +version = "1.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b88fa983de7720629c9387e9f517353ed404164b1e482c970a90c1a4aaf7dc1a" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde-value" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a65a7291a8a568adcae4c10a677ebcedbc6c9cec91c054dee2ce40b0e3290eb" +dependencies = [ + "ordered-float", + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcac07dbffa1c65e7f816ab9eba78eb142c6d44410f4eeba1e26e4f5dfa56b95" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97" +dependencies = [ + "dtoa", + "itoa", + "serde", + "url 2.1.1", +] + +[[package]] +name = "serde_yaml" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7baae0a99f1a324984bcdc5f0718384c1f69775f1c7eec8b859b71b443e3fd7" +dependencies = [ + "dtoa", + "linked-hash-map", + "serde", + "yaml-rust", +] + +[[package]] +name = "sha-1" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" +dependencies = [ + "block-buffer", + "digest", + "fake-simd", + "opaque-debug", +] + +[[package]] +name = "signal-hook-registry" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce32ea0c6c56d5eacaeb814fbed9960547021d3edd010ded1425f180536b20ab" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "smallvec" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6" +dependencies = [ + "maybe-uninit", +] + +[[package]] +name = "smallvec" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" + +[[package]] +name = "socket2" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1fa70dc5c8104ec096f4fe7ede7a221d35ae13dcd19ba1ad9a81d2cab9a1c44" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "redox_syscall", + "winapi 0.3.9", +] + +[[package]] +name = "standback" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" +dependencies = [ + "version_check 0.9.2", +] + +[[package]] +name = "string" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24114bfcceb867ca7f71a0d3fe45d45619ec47a6fbfa98cb14e14250bfa5d6d" +dependencies = [ + "bytes 0.4.12", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tat_agent" +version = "0.1.12" +dependencies = [ + "async-attributes", + "async-std", + "async-trait", + "base64 0.13.0", + "bytes 0.5.6", + "cfg-if 1.0.0", + "chrono", + "clap", + "codepage-strings", + "daemonize", + "encoding", + "faux", + "futures 0.1.30", + "futures 0.3.7", + "hyper 0.12.35", + "libc", + "log 0.4.11", + "log4rs", + "md5", + "openssl", + "procinfo", + "rand 0.7.3", + "reqwest", + "serde", + "serde_json", + "time 0.2.27", + "tokio 0.1.22", + "tokio 0.2.22", + "tokio-test", + "unzip", + "url 2.1.1", + "users", + "websocket", + "winapi 0.3.9", +] + +[[package]] +name = "tempfile" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "rand 0.7.3", + "redox_syscall", + "remove_dir_all", + "winapi 0.3.9", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thread-id" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7fbf4c9d56b320106cd64fd024dadfa0be7cb4706725fc44a7d7ce952d820c1" +dependencies = [ + "libc", + "redox_syscall", + "winapi 0.3.9", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi 0.3.9", +] + +[[package]] +name = "time" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242" +dependencies = [ + "const_fn", + "standback", + "time-macros", + "version_check 0.9.2", +] + +[[package]] +name = "time-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" +dependencies = [ + "proc-macro-hack", + "time-macros-impl", +] + +[[package]] +name = "time-macros-impl" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "standback", + "syn", +] + +[[package]] +name = "tinyvec" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "238ce071d267c5710f9d31451efec16c5ee22de34df17cc05e56cbc92e967117" + +[[package]] +name = "tokio" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.30", + "mio", + "num_cpus", + "tokio-codec", + "tokio-current-thread", + "tokio-executor", + "tokio-fs", + "tokio-io", + "tokio-reactor", + "tokio-sync", + "tokio-tcp", + "tokio-threadpool", + "tokio-timer", + "tokio-udp", + "tokio-uds", +] + +[[package]] +name = "tokio" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d34ca54d84bf2b5b4d7d31e901a8464f7b60ac145a284fba25ceb801f2ddccd" +dependencies = [ + "bytes 0.5.6", + "fnv", + "futures-core", + "iovec", + "lazy_static", + "libc", + "memchr", + "mio", + "mio-named-pipes", + "mio-uds", + "num_cpus", + "pin-project-lite", + "signal-hook-registry", + "slab", + "tokio-macros", + "winapi 0.3.9", +] + +[[package]] +name = "tokio" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ca08accbcb46f11fd8d2d1c6158c348b7888009a1f39260bcad66f6a454250" +dependencies = [ + "autocfg 1.0.1", + "futures-core", + "pin-project-lite", + "slab", +] + +[[package]] +name = "tokio-buf" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fb220f46c53859a4b7ec083e41dec9778ff0b1851c0942b211edb89e0ccdc46" +dependencies = [ + "bytes 0.4.12", + "either", + "futures 0.1.30", +] + +[[package]] +name = "tokio-codec" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25b2998660ba0e70d18684de5d06b70b70a3a747469af9dea7618cc59e75976b" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.30", + "tokio-io", +] + +[[package]] +name = "tokio-current-thread" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" +dependencies = [ + "futures 0.1.30", + "tokio-executor", +] + +[[package]] +name = "tokio-executor" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.30", +] + +[[package]] +name = "tokio-fs" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297a1206e0ca6302a0eed35b700d292b275256f596e2f3fea7729d5e629b6ff4" +dependencies = [ + "futures 0.1.30", + "tokio-io", + "tokio-threadpool", +] + +[[package]] +name = "tokio-io" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.30", + "log 0.4.11", +] + +[[package]] +name = "tokio-macros" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-reactor" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.30", + "lazy_static", + "log 0.4.11", + "mio", + "num_cpus", + "parking_lot 0.9.0", + "slab", + "tokio-executor", + "tokio-io", + "tokio-sync", +] + +[[package]] +name = "tokio-sync" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" +dependencies = [ + "fnv", + "futures 0.1.30", +] + +[[package]] +name = "tokio-tcp" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.30", + "iovec", + "mio", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-test" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edc5a2b13a713b9134debc9aa1265b0a1ccf6a81fa05af2ba26e4aee56cdf1a" +dependencies = [ + "bytes 0.5.6", + "futures-core", + "tokio 0.3.3", +] + +[[package]] +name = "tokio-threadpool" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" +dependencies = [ + "crossbeam-deque", + "crossbeam-queue", + "crossbeam-utils 0.7.2", + "futures 0.1.30", + "lazy_static", + "log 0.4.11", + "num_cpus", + "slab", + "tokio-executor", +] + +[[package]] +name = "tokio-timer" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.30", + "slab", + "tokio-executor", +] + +[[package]] +name = "tokio-tls" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "354b8cd83825b3c20217a9dc174d6a0c67441a2fae5c41bcb1ea6679f6ae0f7c" +dependencies = [ + "futures 0.1.30", + "native-tls", + "tokio-io", +] + +[[package]] +name = "tokio-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343" +dependencies = [ + "native-tls", + "tokio 0.2.22", +] + +[[package]] +name = "tokio-udp" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2a0b10e610b39c38b031a2fcab08e4b82f16ece36504988dcbd81dbba650d82" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.30", + "log 0.4.11", + "mio", + "tokio-codec", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-uds" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab57a4ac4111c8c9dbcf70779f6fc8bc35ae4b2454809febac840ad19bd7e4e0" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.30", + "iovec", + "libc", + "log 0.4.11", + "mio", + "mio-uds", + "tokio-codec", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +dependencies = [ + "bytes 0.5.6", + "futures-core", + "futures-sink", + "log 0.4.11", + "pin-project-lite", + "tokio 0.2.22", +] + +[[package]] +name = "tower-service" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" + +[[package]] +name = "tracing" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0987850db3733619253fe60e17cb59b82d37c7e6c0236bb81e4d6b87c879f27" +dependencies = [ + "cfg-if 0.1.10", + "log 0.4.11", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-futures" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +dependencies = [ + "pin-project 0.4.27", + "tracing", +] + +[[package]] +name = "traitobject" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efd1f82c56340fdf16f2a953d7bda4f8fdffba13d93b00844c25572110b26079" + +[[package]] +name = "transformation-pipeline" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1d17c634c5196b37a2680f31627ab35b9e84e5f7f1abd13731c3fbd71dea36" + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "typeable" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1410f6f91f21d1612654e7cc69193b0334f909dcf2c790c4826254fbb86f8887" + +[[package]] +name = "typemap" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "653be63c80a3296da5551e1bfd2cca35227e13cdd08c6668903ae2f4f77aa1f6" +dependencies = [ + "unsafe-any", +] + +[[package]] +name = "typenum" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" + +[[package]] +name = "unicase" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4765f83163b74f957c797ad9253caf97f103fb064d3999aea9568d09fc8a33" +dependencies = [ + "version_check 0.1.5", +] + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check 0.9.2", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb19cf769fa8c6a80a162df694621ebeb4dafb606470b2b2fce0be40a98a977" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "unsafe-any" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30360d7979f5e9c6e6cea48af192ea8fab4afb3cf72597154b8f08935bc9c7f" +dependencies = [ + "traitobject", +] + +[[package]] +name = "unzip" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90e59a2baae95b4b60ecd5a1ba39e9654b09b09bf813989c70f52c957c213dfd" +dependencies = [ + "time 0.1.44", + "transformation-pipeline", + "zip", +] + +[[package]] +name = "url" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" +dependencies = [ + "idna 0.1.5", + "matches", + "percent-encoding 1.0.1", +] + +[[package]] +name = "url" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb" +dependencies = [ + "idna 0.2.0", + "matches", + "percent-encoding 2.1.0", +] + +[[package]] +name = "users" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24cc0f6d6f267b73e5a2cadf007ba8f9bc39c6a6f9666f8cf25ea809a153b032" +dependencies = [ + "libc", + "log 0.4.11", +] + +[[package]] +name = "vcpkg" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6454029bf181f092ad1b853286f23e2c507d8e8194d01d92da4a55c274a5508c" + +[[package]] +name = "vec-arena" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eafc1b9b2dfc6f5529177b62cf806484db55b32dc7c9658a118e11bbeb33061d" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" + +[[package]] +name = "version_check" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" + +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + +[[package]] +name = "want" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6395efa4784b027708f7451087e647ec73cc74f5d9bc2e418404248d679a230" +dependencies = [ + "futures 0.1.30", + "log 0.4.11", + "try-lock", +] + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log 0.4.11", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasm-bindgen" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42" +dependencies = [ + "cfg-if 0.1.10", + "serde", + "serde_json", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f22b422e2a757c35a73774860af8e112bff612ce6cb604224e8e47641a9e4f68" +dependencies = [ + "bumpalo", + "lazy_static", + "log 0.4.11", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7866cab0aa01de1edf8b5d7936938a7e397ee50ce24119aef3e1eaa3b6171da" +dependencies = [ + "cfg-if 0.1.10", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b13312a745c08c469f0b292dd2fcd6411dba5f7160f593da6ef69b64e407038" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f249f06ef7ee334cc3b8ff031bfc11ec99d00f34d86da7498396dc1e3b1498fe" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307" + +[[package]] +name = "web-sys" +version = "0.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bf6ef87ad7ae8008e15a355ce696bed26012b7caa21605188cfd8214ab51e2d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "websocket" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "723abe6b75286edc51d8ecabb38a2353f62a9e9b0588998b59111474f1dcd637" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.30", + "hyper 0.10.16", + "native-tls", + "rand 0.6.5", + "tokio-codec", + "tokio-io", + "tokio-reactor", + "tokio-tcp", + "tokio-tls 0.2.1", + "unicase 1.4.2", + "url 1.7.2", + "websocket-base", +] + +[[package]] +name = "websocket-base" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403f3fd505ff930da84156389639932955fb09705b3dccd1a3d60c8e7ff62776" +dependencies = [ + "base64 0.10.1", + "bitflags", + "byteorder", + "bytes 0.4.12", + "futures 0.1.30", + "native-tls", + "rand 0.6.5", + "sha-1", + "tokio-codec", + "tokio-io", + "tokio-tcp", + "tokio-tls 0.2.1", +] + +[[package]] +name = "wepoll-sys" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fcb14dea929042224824779fbc82d9fab8d2e6d3cbc0ac404de8edf489e77ff" +dependencies = [ + "cc", +] + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "yaml-rust" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39f0c922f1a334134dc2f7a8b67dc5d25f0735263feec974345ff706bcf20b0d" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "zip" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7341988e4535c60882d5e5f0b7ad0a9a56b080ade8bdb5527cb512f7b2180e0" +dependencies = [ + "bzip2", + "flate2", + "msdos_time", + "podio", + "time 0.1.44", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..a90501c --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "tat_agent" +version = "0.1.12" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +tokio = { version = "0.2.22", features = ["full"]} +futures = "0.3" +futures01 = { version = "0.1", package = "futures"} +tokio01 = { version = "0.1.22", package = "tokio"} +reqwest = { version = "0.10", features = ["blocking", "json"] } +serde_json = "1.0.57" +serde = {version = "1.0.115", features = ["derive"]} +url = "2.1.1" +clap = "2.33.3" +async-attributes = "1.0.0" +tokio-test = "*" +log = "0.4.11" +log4rs = "0.13.0" +websocket = "0.26.2" +libc = "0.2.77" +rand = "0.7.3" +base64 = "0.13.0" +chrono = "0.4.18" +md5 = "0.7.0" +bytes = "0.5.6" +unzip = "0.1.0" +async-std = "1.6.5" +hyper = { version = "^0.12"} +async-trait = "0.1.50" +encoding = "0.2.33" +cfg-if = "1.0" +time = { version = "0.2", default-features = false } +faux = "0.1.5" + +[target.'cfg(target_family="unix")'.dependencies] +daemonize = "0.4.1" +users = { version = "0.11.0"} +openssl = { version = '0.10.35', features = ["vendored"] } +procinfo = { version = "0.4.2" } + +[target.'cfg(windows)'.dependencies] +codepage-strings = "1.0.1" +winapi = { version = "0.3", features = ["winsvc","winbase","winnt","stringapiset","winnls","wow64apiset","synchapi","namedpipeapi"] } diff --git a/Cross.toml b/Cross.toml new file mode 100644 index 0000000..5ad5d96 --- /dev/null +++ b/Cross.toml @@ -0,0 +1,6 @@ +# TODO: remove the whole file when `cross` new version is released. +# +# i686-pc-windows-gnu has been supported by `cross`, but not released yet. +# See https://github.com/rust-embedded/cross/pull/471 for detail. +[target.i686-pc-windows-gnu] +image = "rustembedded/cross:i686-pc-windows-gnu" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..99fc0a7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,5 @@ +FROM csighub.tencentyun.com/tat-develop/library:centos-base + +ADD tat_agent /usr/local/bin + +CMD ["/usr/local/bin/tat_agent"] \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a626264 --- /dev/null +++ b/LICENSE @@ -0,0 +1,208 @@ +MIT License +-------------------------------------------------------------------- +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of +the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT +LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +TATAgent-腾讯云自动化助手客户端组件 is licensed under the MIT License except for the third-party components listed below. +Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. The below software in this distribution may have been modified by THL A29 Limited (“Tencent Modifications”). All Tencent Modifications are Copyright (C) 2021 THL A29 Limited. + + +Other dependencies and licenses: + + +Open Source Software Licensed under the Apache License Version 2.0: +-------------------------------------------------------------------- +1. openssl +Copyright 2011-2017 Google Inc. +2013 Jack Lloyd +2013-2014 Steven Fackler + +2. time +Copyright 2021 Jacob Pratt et al. + + +Terms of the Apache License Version 2.0: +-------------------------------------------------------------------- +Apache License + +Version 2.0, January 2004 + +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of this License; and + +You must cause any modified files to carry prominent notices stating that You changed the files; and + +You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + +If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + +You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + + + +Open Source Software Licensed under the MIT License: +-------------------------------------------------------------------- +1. tokio (v0.2.22) +Copyright (c) 2019 Tokio Contributors + +2. tokio (v0.1.22) +Copyright (c) 2019 Tokio Contributors + +3. futures (v0.3) +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors + +4. futures (v0.1) +Copyright (c) 2016 Alex Crichton + +5. reqwest +Copyright (c) 2016 Sean McArthur + +6. serde_json +Copyright (c) "Erick Tryzelaar ", "David Tolnay " + +7. serde +Copyright (c) "Erick Tryzelaar ", "David Tolnay " + +8. url +Copyright (c) 2013-2016 The rust-url developers + +9. clap +Copyright (c) 2015-2016 Kevin B. Knapp + +10. async-attributes +Copyright (c) 2019 Yoshua Wuyts + +11. tokio-test +Copyright (c) 2019 Tokio Contributors + +12. log +Copyright (c) 2014 The Rust Project Developers + +13. log4rs +Copyright (c) 2015-2016 Steven Fackler + +14. daemonize +Copyright (c) 2016 Fedor Gogolev + +15. websocket +Copyright (c) 2014 Rust Websockets Developers + +16. libc +Copyright (c) 2014-2020 The Rust Project Developers + +17. bytes +Copyright (c) 2018 Carl Lerche + +18. unzip +Copyright (c) “Ryan Leonard (http://ryanleonard.us)” + +19. async-std +Copyright (c) Stjepan Glavina ", "Yoshua Wuyts ", "Friedel Ziegelmayer ", "Contributors to async-std" + +20. hyper +Copyright (c) 2014-2018 Sean McArthur + +21. users +Copyright (c) 2019 Benjamin Sago + +22. async-trait +Copyright (c) David Tolnay + +23. rand +Copyright 2018 Developers of the Rand project +Copyright (c) 2014 The Rust Project Developers + +24. base64 +Copyright (c) 2015 Alice Maz + +25. chrono +Copyright (c) 2014--2017, Kang Seonghoon and +contributors. + +26. md5 +Copyright 2015–2019 The md5 Developers + +27. encoding +Copyright (c) 2013, Kang Seonghoon + +28. cfg-if +Copyright (c) 2014 Alex Crichton + +29. procinfo +Copyright (c) 2015 The Rust Project Developers + +30. codepage-strings +Copyright (c) 2021 Bart Massey + +31. winapi +Copyright (c) 2015-2018 The winapi-rs Developers + +32. faux +Copyright (c) 2020 Andres Medina + + +Terms of the MIT License: +-------------------------------------------------------------------- +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ba4371e --- /dev/null +++ b/Makefile @@ -0,0 +1,98 @@ +.PHONY: test static release stop run t clean build +ifndef QCI_TRIGGER + QCI_TRIGGER = root +endif + +VERSION ?= $(shell git rev-parse --short HEAD || echo "GitNotFound") +DOCKER_REGISTRY=csighub.tencentyun.com +AGENT_IMG=$(DOCKER_REGISTRY)/tat-develop/tat_agent + +# test all case +lib-test: + cargo test --lib -- --nocapture --skip ontime --skip executor::proc::tests::test_shell_cmd_timeout + +integration-test: + cargo test --test http_test + +arch ?= x86_64 +rust_target = +in_docker ?= true +ifeq ($(arch),x86_64) + rust_target = x86_64-unknown-linux-musl +else ifeq ($(arch), i686) + rust_target = i686-unknown-linux-musl +else ifeq ($(arch), i586) + rust_target = i586-unknown-linux-musl +endif + +# ensure cross installed. we use it for cross-compile. +ifeq (, $(shell which cross)) +$(info "cross not found, install it now.") +$(shell cargo install cross) +endif + +# build a pure static binary +static: +ifeq ($(rust_target), ) +$(error `$(arch)` not exists or not supported yet.) +endif + +ifeq ($(in_docker),true) + cross build --target=$(rust_target) +else + cargo build --target=$(rust_target) +endif + + ln -f target/$(rust_target)/debug/tat_agent tat_agent + +# build a pure static binary in release +release: +ifeq ($(in_docker),true) + cross build --release --target=x86_64-unknown-linux-musl + cross build --release --target=i686-unknown-linux-musl +else + cargo build --release --target=x86_64-unknown-linux-musl + cargo build --release --target=i686-unknown-linux-musl +endif + + ln -f target/x86_64-unknown-linux-musl/release/tat_agent tat_agent + ln -f target/i686-unknown-linux-musl/release/tat_agent tat_agent32 + install/release.sh + +# stop the daemon by pid +stop: + kill -9 `cat /var/run/tat_agent.pid` + +# run a binary for debugging +run: + cargo run + +# build a pure static binary for debugging +build: + cargo build --target=x86_64-unknown-linux-musl + ln -f target/x86_64-unknown-linux-musl/debug/tat_agent tat_agent + +# a shortcut for fuzzy matching +# usage: make t m=partial_of_testcase_name +t: + cargo test $(m) --lib -- --nocapture + +# notice:agent image only used for E2E environment. +build-img: + $(info VERSION: $(VERSION)) + mkdir -p ./bin + ln -f tat_agent ./bin/tat_agent + docker build --tag $(AGENT_IMG):$(VERSION) -f Dockerfile ./bin + rm -r ./bin + +# notice:agent image only used for E2E environment. +push-img: + $(info image: $(AGENT_IMG):$(VERSION)) + docker push $(AGENT_IMG):$(VERSION) + +# notice:agent image only used for E2E environment. +all-img: static build-img push-img + +clean: + rm .*.sh 2> /dev/null || true + rm tasks/* 2> /dev/null || true diff --git a/README.md b/README.md new file mode 100644 index 0000000..1cb1a91 --- /dev/null +++ b/README.md @@ -0,0 +1,47 @@ +# TAT Agent +TAT agent is an agent written in Rust, which run in CVM or Lighthouse instances. +Its role is to run commands remotely without ssh login, invoked from TencentCloud Console/API. +Commands include but not limited to: shell, python, php, you can provide any script interpreter +at first line, such as: #!/bin/bash, #!/usr/bin/env python3.8. +See more info at https://cloud.tencent.com/product/tat . + + +## prerequisites + +- Rust environment, such as cargo, rustc, rustup. See more info at https://www.rust-lang.org/learn/get-started . +- Docker, some binary need to be compiled in docker. + +## run +``` +make run +``` +Run directly by cargo in debug mode with the test domain. + +## build +``` +make release # on linux +``` +Build a pure static binary in release mode with the real domain, need docker installed. +``` +.\install\build.bat # on windows +``` + +## stop +``` +make stop +``` +Stop the daemon by pid which was written in a pidfile. + +## other +See more details at Makefile. + +## supported OS +Binary can run at both Linux & Windows Distributions, including but not limited to: +- Tencent Linux +- CentOS +- Ubuntu +- Debian +- openSUSE +- SUSE +- CoreOS + diff --git a/install/build.bat b/install/build.bat new file mode 100644 index 0000000..be88b0e --- /dev/null +++ b/install/build.bat @@ -0,0 +1,3 @@ +SET RUSTFLAGS=-C target-feature=+crt-static +rustup target add i686-pc-windows-msvc +cargo build --release --target i686-pc-windows-msvc \ No newline at end of file diff --git a/install/install.bat b/install/install.bat new file mode 100644 index 0000000..caf4d17 --- /dev/null +++ b/install/install.bat @@ -0,0 +1,20 @@ +@echo off +cd /d %~dp0 + +if not exist "C:\Program Files\qcloud\tat_agent" ( + md "C:\Program Files\qcloud\tat_agent" +) + +if not exist "C:\Program Files\qcloud\tat_agent\workdir" ( + md "C:\Program Files\qcloud\tat_agent\workdir" +) + + +sc query tatsvc | find "STATE" >nul && sc stop tatsvc >nul ||^ +sc create tatsvc binPath= "C:\Program Files\qcloud\tat_agent\tat_agent.exe" start= auto + +copy /Y tat_agent.exe "C:\Program Files\qcloud\tat_agent\tat_agent.exe" + +sc failure tatsvc actions= restart/1000 reset= -1 >nul + +sc start tatsvc \ No newline at end of file diff --git a/install/install.sh b/install/install.sh new file mode 100755 index 0000000..31a0eea --- /dev/null +++ b/install/install.sh @@ -0,0 +1,119 @@ +#!/bin/bash +cd `dirname $0` + +need_restart=true +if [ "only_update" = ""$1 ]; then + need_restart=false +fi + +PID_FILE="/var/run/tat_agent.pid" + +# install the agent binary +SERVICE_DIR="/usr/local/qcloud/tat_agent/" +PATH_DIR="/usr/sbin/" +TAT_AGENT="tat_agent" +TAT_AGENT32="tat_agent32" + +# if arch is 32bit and 32bit bin exists, rename `tat_agent32` to `tat_agent` +machine=$(uname -m) +if [ "$machine" != "x86_64" ] && [ -f "$TAT_AGENT32" ]; then + mv ${TAT_AGENT} -f ${TAT_AGENT}64 + mv ${TAT_AGENT32} -f ${TAT_AGENT} +fi + +# check if agent runnable +chmod +x ${TAT_AGENT} +if ! ./${TAT_AGENT} -V; then + echo "tat_agent not runnable, exit." + exit 1 +fi + +mkdir -p ${SERVICE_DIR} +if [ $? -ne 0 ]; then + # handle special case for CoreOS whose /usr is Read-only + grep -q CoreOS /etc/os-release + if [ $? -eq 0 ]; then + SERVICE_DIR="/var/lib/qcloud/tat_agent/" + mkdir -p ${SERVICE_DIR} + PATH_DIR="/opt/bin/" + sed -i 's/\/usr\/local\/qcloud/\/var\/lib\/qcloud/g' tat_agent.service tat_agent_service.conf tat_agent_service uninstall.sh + sed -i 's/\/usr\/sbin/\/opt\/bin/g' uninstall.sh + else + echo 'Install fail, has no permission, may not root.' + exit 1 + fi +fi +cp -f ${TAT_AGENT} ${SERVICE_DIR} +ln -sf ${SERVICE_DIR}${TAT_AGENT} ${PATH_DIR}${TAT_AGENT} + +has_systemd() { + [[ `systemctl` =~ -\.mount ]] > /dev/null 2>&1 && return 0 + if systemctl 2>/dev/null | grep -e "-\.mount" > /dev/null 2>&1; then + return 0 + fi + return 1 +} + +has_sysvinit() { + if [ -f /etc/init.d/cron ] && [ ! -h /etc/init.d/cron ]; then + return 0 + fi + if [ -f /etc/init.d/crond ] && [ ! -h /etc/init.d/crond ]; then + return 0 + fi + which chkconfig > /dev/null 2>&1 && return 0 + which update-rc.d > /dev/null 2>&1 && return 0 + return 1 +} + +has_upstart() { + which initctl > /dev/null 2>&1 || return 1 + if /sbin/init --version 2>/dev/null | grep upstart > /dev/null 2>&1; then + return 0 + fi + return 1 +} + +if has_systemd; then + echo "use systemd to manage service" + SYSTEMD_DIR="/etc/systemd/system/" + cp -f tat_agent.service ${SYSTEMD_DIR} + systemctl daemon-reload + systemctl enable tat_agent.service + if test "${need_restart}" = true; then + systemctl restart tat_agent.service + fi +elif has_upstart; then + echo "use upstart(initctl) to manage service" + cp -f tat_agent_service.conf /etc/init/ + if test "${need_restart}" = true; then + initctl stop tat_agent_service + initctl start tat_agent_service + fi +elif has_sysvinit; then + cp -f tat_agent_service /etc/init.d/ + chmod 755 /etc/init.d/tat_agent_service + /etc/init.d/tat_agent_service restart + which chkconfig > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo "use chkconfig to manage service" + chkconfig --add tat_agent_service + chkconfig tat_agent_service on + else + which update-rc.d > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo "use update-rc.d to manage service" + update-rc.d tat_agent_service defaults + else + echo "no proper daemon manager found, tat_agent can not auto start" + fi + fi +else + echo "no proper daemon manager found, tat_agent can not auto start" + kill `cat ${PID_FILE}` > /dev/null 2>&1 + sleep 1 + rm -f ${PID_FILE} + cd ${SERVICE_DIR} + ./${TAT_AGENT} + echo "tat_agent started" +fi diff --git a/install/release.bat b/install/release.bat new file mode 100644 index 0000000..4731693 --- /dev/null +++ b/install/release.bat @@ -0,0 +1,40 @@ +@echo off +cd /d %~dp0 + +set SIGNED_AGENT="..\tat_agent.exe" +FOR /F "tokens=2*" %%g IN ('%SIGNED_AGENT% --version') do (SET VERSION=%%g) + +if not exist "C:\Program Files\7-Zip\7z.exe" ( + echo "7z.exe not found, which is used for tar release files, exit now" + exit +) + +set COMPRESS_PROC="C:\Program Files\7-Zip\7z.exe" + +:: generate self update file for release +SET FILE="tat_agent_windows_update_%VERSION%.zip" +%COMPRESS_PROC% a %FILE% %SIGNED_AGENT% install.bat uninstall.bat self_update.bat test.bat + +:: generate install file for release +SET INSTALL_DIR="tat_agent_windows_install_%VERSION%" +if not exist %INSTALL_DIR% ( + md %INSTALL_DIR% +) +for %%i in (%SIGNED_AGENT% install.bat uninstall.bat test.bat) do ( copy %%i %INSTALL_DIR% ) +set FILE="%INSTALL_DIR%.tar.gz" +:: del old file +del %FILE% >nul 2>&1 +%COMPRESS_PROC% a -ttar -so -an %INSTALL_DIR% | %COMPRESS_PROC% a -si %FILE% +rd /s/q %INSTALL_DIR% + +:: generate uninstall file for release +SET UNINSTALL_DIR=".\tat_agent_windows_uninstall_%VERSION%" +if not exist %UNINSTALL_DIR% ( + md %UNINSTALL_DIR% +) +copy uninstall.bat %UNINSTALL_DIR% +set FILE="%UNINSTALL_DIR%.tar.gz" +:: del old file +del %FILE% >nul 2>&1 +%COMPRESS_PROC% a -ttar -so -an %UNINSTALL_DIR% | %COMPRESS_PROC% a -si %FILE% +rd /s/q %UNINSTALL_DIR% diff --git a/install/release.sh b/install/release.sh new file mode 100755 index 0000000..409dd0a --- /dev/null +++ b/install/release.sh @@ -0,0 +1,50 @@ +#!/bin/bash +cd `dirname $0` +ARCH=$1 +TAT_AGENT="tat_agent" +TAT_AGENT_32="tat_agent32" +ln -f ../${TAT_AGENT} ../${TAT_AGENT_32} . + +VERSION=`./${TAT_AGENT} --version | awk '{print $2}'` +if [ -z "${VERSION}" ]; then + echo "${TAT_AGENT} version get fail, now exit" + exit 1 +fi +echo "${TAT_AGENT} version: ${VERSION}" + +FILE_SUFFIX=".tar.gz" +INSTALL_FILE=${TAT_AGENT}_linux_install_${VERSION}${FILE_SUFFIX} +UNINSTALL_FILE=${TAT_AGENT}_linux_uninstall_${VERSION}${FILE_SUFFIX} + +# clean old files. +rm -rf "${INSTALL_FILE}" "${UNINSTALL_FILE}" "${UPDATE_FILE}" "${UPDATE_FILE_32}" + +# NOTE: mac tar do not support `--transform`, use gtar instead. +# generate install file for release +tar czf "${INSTALL_FILE}" install.sh ${TAT_AGENT} ${TAT_AGENT_32} uninstall.sh test.sh tat_agent_service \ +tat_agent.service tat_agent_service.conf --transform "s,^,${TAT_AGENT}_linux_install_${VERSION}/," + +# generate uninstall file for release +tar czf "${UNINSTALL_FILE}" uninstall.sh test.sh --transform "s,^,${TAT_AGENT}_linux_uninstall_${VERSION}/," + +# generate self update file for release (64bit) +ARCH=x86_64 +UPDATE_FILE="${TAT_AGENT}_linux_update_${ARCH}_${VERSION}.zip" +zip "${UPDATE_FILE}" install.sh ${TAT_AGENT} uninstall.sh tat_agent_service tat_agent.service tat_agent_service.conf \ +self_update.sh + +# generate self update file for release (32bit) +ARCH=i686 +UPDATE_FILE_32="${TAT_AGENT}_linux_update_${ARCH}_${VERSION}.zip" +# rename tat_agent_32 to tat_agent. +mv -f ${TAT_AGENT_32} ${TAT_AGENT} +zip "${UPDATE_FILE_32}" install.sh ${TAT_AGENT} uninstall.sh tat_agent_service tat_agent.service tat_agent_service.conf self_update.sh + +# clean +rm -rf ${TAT_AGENT} ${TAT_AGENT_32} +echo "release file generated: +${INSTALL_FILE} +${UNINSTALL_FILE} +${UPDATE_FILE} +${UPDATE_FILE_32} +" diff --git a/install/self_update.bat b/install/self_update.bat new file mode 100644 index 0000000..3d44f38 --- /dev/null +++ b/install/self_update.bat @@ -0,0 +1,8 @@ +cd /d %~dp0 +set temp=%TIME% +set "temp=%temp::=%" +set "temp=%temp:.=%" +set newname=temp_%temp%.exe +rename "C:\Program Files\qcloud\tat_agent\tat_agent.exe" %newname% +copy /Y tat_agent.exe "C:\Program Files\qcloud\tat_agent\" + diff --git a/install/self_update.sh b/install/self_update.sh new file mode 100755 index 0000000..43aa885 --- /dev/null +++ b/install/self_update.sh @@ -0,0 +1,4 @@ +#!/bin/bash +cd `dirname $0` +chmod +x ./install.sh +./install.sh only_update diff --git a/install/tat_agent.service b/install/tat_agent.service new file mode 100644 index 0000000..a5225fb --- /dev/null +++ b/install/tat_agent.service @@ -0,0 +1,16 @@ +[Unit] +Description=tat_agent +After=network-online.target + +[Service] +Type=forking +PIDFile=/var/run/tat_agent.pid +WorkingDirectory=/usr/local/qcloud/tat_agent/ +ExecStart=/usr/local/qcloud/tat_agent/tat_agent +ExecStartPost=/bin/sleep 0.2 +KillMode=process +Restart=always +RestartSec=1s + +[Install] +WantedBy=multi-user.target diff --git a/install/tat_agent_service b/install/tat_agent_service new file mode 100644 index 0000000..7c6e49f --- /dev/null +++ b/install/tat_agent_service @@ -0,0 +1,92 @@ +#!/bin/bash + +# chkconfig: 2345 88 66 +# description: TAT agent + +### BEGIN INIT INFO +# Provides: tat_agent +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: TAT agent +### END INIT INFO + +RET=0 + +AGENT="tat_agent" +AGENT_PATH="/usr/local/qcloud/tat_agent/tat_agent" +PID_FILE="/var/run/tat_agent.pid" + +start() { + test -x ${AGENT_PATH} || exit 1 + + ${AGENT_PATH} + RET=$? + if [ ${RET} -eq 0 ]; then + echo "tat_agent start succ" + sleep 0.2 + agent_status + fi + return ${RET} +} + +stop() { + if [ -f ${PID_FILE} ]; then + PID=`cat ${PID_FILE}` + ps ${PID} | grep ${AGENT} + if [ $? -eq 0 ]; then + echo "tat_agent found, now kill it" + kill -9 ${PID} + RET=$? + else + echo "tat_agent not found, nothing to stop" + RET=2 + fi + else + echo "pid file of tat_agent not exist" + RET=3 + fi + return ${RET} +} + +agent_status() { + if [ -f ${PID_FILE} ]; then + PID=`cat ${PID_FILE}` + ps ${PID} | grep ${AGENT} + if [ $? -eq 0 ]; then + echo "tat_agent running with pid:${PID}" + RET=0 + else + echo "tat_agent not running" + RET=0 + fi + else + echo "pid file of tat_agent not exist" + RET=4 + fi + return ${RET} +} + +case "$1" in +start) + start + ;; +stop) + stop + ;; +status) + agent_status + RET=$? + ;; +restart) + stop + sleep 0.2 + start + ;; +*) + echo $"Usage: $0 {start|stop|restart|status}" + RET=4 +esac + +exit ${RET} diff --git a/install/tat_agent_service.conf b/install/tat_agent_service.conf new file mode 100644 index 0000000..e77b404 --- /dev/null +++ b/install/tat_agent_service.conf @@ -0,0 +1,13 @@ +description "TAT agent" +author "Tencent Cloud" + +start on runlevel [2345] +stop on runlevel [!2345] + +respawn +respawn limit unlimited + +expect daemon + +pre-start exec sleep 0.2 +exec /usr/local/qcloud/tat_agent/tat_agent diff --git a/install/test.bat b/install/test.bat new file mode 100644 index 0000000..83cb140 --- /dev/null +++ b/install/test.bat @@ -0,0 +1 @@ +@echo off diff --git a/install/test.sh b/install/test.sh new file mode 100755 index 0000000..3617d42 --- /dev/null +++ b/install/test.sh @@ -0,0 +1,2 @@ +#!/bin/bash +: diff --git a/install/uninstall.bat b/install/uninstall.bat new file mode 100644 index 0000000..d71d0f6 --- /dev/null +++ b/install/uninstall.bat @@ -0,0 +1,3 @@ +sc stop tatsvc +sc delete tatsvc +rd /s/q "C:\Program Files\qcloud\tat_agent" \ No newline at end of file diff --git a/install/uninstall.sh b/install/uninstall.sh new file mode 100755 index 0000000..a3f4478 --- /dev/null +++ b/install/uninstall.sh @@ -0,0 +1,50 @@ +#!/bin/bash +cd `dirname $0` + +PID_FILE="/var/run/tat_agent.pid" +PID=0 +if [ -e ${PID_FILE} ]; then + PID=`cat ${PID_FILE}` +fi +SERVICE_DIR="/usr/local/qcloud/tat_agent/" +TAT_AGENT="tat_agent" +SYSTEMD_DIR="/etc/systemd/system/" + +if [ -e ${SYSTEMD_DIR}tat_agent.service ]; then + systemctl stop ${TAT_AGENT} + rm -f ${SYSTEMD_DIR}tat_agent.service + rm -f ${SYSTEMD_DIR}multi-user.target.wants/tat_agent.service + systemctl daemon-reload +fi + +if [ -e /etc/init/tat_agent_service.conf ]; then + initctl stop tat_agent_service + rm -f /etc/init/tat_agent_service.conf +fi + +if [ -e /etc/init.d/tat_agent_service ]; then + which chkconfig > /dev/null 2>&1 + if [ $? -eq 0 ]; then + chkconfig tat_agent_service off + chkconfig --del tat_agent_service + else + which update-rc.d > /dev/null 2>&1 + if [ $? -eq 0 ]; then + update-rc.d -f tat_agent_service remove + fi + fi + /etc/init.d/tat_agent_service stop + rm -f /etc/init.d/tat_agent_service +fi + +if [ ${PID} -ne 0 ]; then + ps ${PID} | grep ${TAT_AGENT} + if [ $? -eq 0 ]; then + kill -9 ${PID} + fi +fi + +rm -f /usr/sbin/${TAT_AGENT} +rm -f ${PID_FILE} +rm -rf ${SERVICE_DIR} +echo "uninstall finished" diff --git a/src/common.rs b/src/common.rs new file mode 100644 index 0000000..6312287 --- /dev/null +++ b/src/common.rs @@ -0,0 +1,6 @@ +pub mod asserts; +pub mod consts; +pub mod logger; +pub mod option; +pub mod envs; +pub use option::Opts; diff --git a/src/common/asserts.rs b/src/common/asserts.rs new file mode 100644 index 0000000..45a1a8d --- /dev/null +++ b/src/common/asserts.rs @@ -0,0 +1,24 @@ +use log::info; +use log::error; +use std::process::exit; + + +pub trait GracefulUnwrap { + fn unwrap_or_exit(self, desc: &str) -> T; + fn or_log(self, desc: &str); +} + +impl GracefulUnwrap for Result { + fn unwrap_or_exit(self, desc: &str) -> T { + self.unwrap_or_else(|e| { + error!("Result:{:#?}, Desc:{}, exit program now", e, desc); + exit(1); + }) + } + + fn or_log(self, desc: &str) { + if let Err(e) = self { + info!("Result:{:#?}, Desc:{}, program continue", e, desc); + } + } +} diff --git a/src/common/consts.rs b/src/common/consts.rs new file mode 100644 index 0000000..a9e55ee --- /dev/null +++ b/src/common/consts.rs @@ -0,0 +1,159 @@ +use std::env; + +// log related +pub const LOG_PATTERN: &str = "{d}|{f}:{L}|{l}|{m}{n}"; +pub const LOG_FILE_NAME: &str = "log/tat_agent.log"; +pub const LOG_FILE_NAME_WHEN_ROLL: &str = "log/tat_agent_{}.log"; +pub const LOG_FILE_SIZE: u64 = 10 * 1024 * 1024; +pub const LOG_FILE_BASE_INDEX: u32 = 0; +pub const MAX_LOG_FILE_COUNT: u32 = 2; +pub const LOG_LEVEL: log::LevelFilter = log::LevelFilter::Info; +pub const LOG_LEVEL_DEBUG: log::LevelFilter = log::LevelFilter::Debug; + +// http headers used for e2e test. +pub const VPCID_HEADER: &str = "Tat-Vpcid"; +pub const VIP_HEADER: &str = "Tat-Vip"; + +pub const SELF_UPDATE_FILENAME: &str = "agent_update.zip"; +cfg_if::cfg_if! { + if #[cfg(unix)] { + // daemon related + pub const PID_FILE: &str = "/var/run/tat_agent.pid"; + + pub const TASK_STORE_PATH: &str = "/tmp/tat_agent/commands/"; + pub const SELF_UPDATE_PATH: &str = "/tmp/tat_agent/self_update/"; + pub const SELF_UPDATE_SCRIPT: &str = "self_update.sh"; + pub const AGENT_DEFAULT_WORK_DIRECTORY: &str = "/root"; + + // pre_exec fn name for cmd + pub const OWN_PROCESS_GROUP: &str = "OWN_PROCESS_GROUP"; + pub const DUP2_1_2: &str = "DUP2_1_2"; + + pub const FILE_EXECUTE_PERMISSION_MODE: u32 = 0o755; + pub const PIPE_BUF_DEFAULT_SIZE: usize = 64 * 4096; + } else if #[cfg(windows)] { + pub const TASK_STORE_PATH: &str = "C:\\Program Files\\qcloud\\tat_agent\\tmp\\commands\\"; + pub const SELF_UPDATE_PATH: &str = "C:\\Program Files\\qcloud\\tat_agent\\tmp\\self_update"; + pub const SELF_UPDATE_SCRIPT: &str = "self_update.bat"; + } +} + +// ws related +pub const WS_VERSION_HEADER: &str = "Tat-Version"; +pub const WS_KERNEL_NAME_HEADER: &str = "Tat-KernelName"; +#[cfg(not(debug_assertions))] +pub const WS_URL: &str = "ws://notify.tat-tc.tencent.cn:8086/ws"; +#[cfg(debug_assertions)] +pub const WS_URL: &str = "ws://proxy:8086/ws"; +pub const WS_PASSIVE_CLOSE: &str = "cli_passive_close"; +pub const WS_PASSIVE_CLOSE_CODE: u16 = 3001; +pub const WS_ACTIVE_CLOSE: &str = "cli_active_close"; +pub const WS_ACTIVE_CLOSE_CODE: u16 = 3002; +pub const MAX_PING_FROM_LAST_PONG: usize = 3; +pub const WS_RECONNECT_INTERVAL: u64 = 3; +pub const WS_LAST_CLOSE_INTERVAL: u64 = 1; +pub const WS_CONNECT_TIMEOUT: u64 = 5; +// ws msg +pub const WS_MSG_TYPE_KICK: &str = "kick"; +pub const WS_MSG_TYPE_ACK: &str = "ack"; +// http related +pub const HTTP_REQUEST_TIME_OUT: u64 = 5; +pub const HTTP_REQUEST_RETRIES: u64 = 3; +pub const HTTP_REQUEST_NO_RETRIES: u64 = 1; +pub const HTTP_REQUEST_RETRY_INTERVAL: u64 = 5; + +pub const TASK_STORE_PREFIX: &str = "task"; +pub const FINISH_RESULT_TIMEOUT: &str = "TIMEOUT"; +pub const FINISH_RESULT_SUCCESS: &str = "SUCCESS"; +pub const FINISH_RESULT_FAILED: &str = "FAILED"; +pub const FINISH_RESULT_START_FAILED: &str = "START_FAILED"; +pub const FINISH_RESULT_TERMINATED: &str = "TERMINATED"; + +// agent +pub const AGENT_VERSION: &str = env!("CARGO_PKG_VERSION"); + +// task start failed errInfo +#[macro_export] +macro_rules! start_failed_err_info { + (ERR_WORKING_DIRECTORY_NOT_EXISTS, $working_directory:expr) => { + format!("DirectoryNotExists: working_directory `{}` not exists", $working_directory); + }; + (ERR_USER_NOT_EXISTS, $user:expr) => { + format!("UserNotExists: user `{}` not exists", $user); + }; + (ERR_USER_NO_PERMISSION_OF_WORKING_DIRECTORY, $user:expr, $working_directory:expr) => { + format!("DirectoryPermissionDeny: user `{}` has no permission of working_directory `{}`", + $user, $working_directory); + }; + (ERR_SUDO_NOT_EXISTS) => { + format!("SudoNotExists: command sudo not exists"); + }; + (ERR_SCRIPT_FILE_STORE_FAILED, $store_path:expr) => { + format!("ScriptStoreFailed: script file store failed at `{}`, please check disk space or permission", + $store_path); + }; +} + +// service related +#[cfg(not(debug_assertions))] +pub const INVOKE_API: &str = "https://invoke.tat-tc.tencent.cn"; +#[cfg(debug_assertions)] +pub const INVOKE_API: &str = "http://proxy-invoke"; + +#[allow(dead_code)] +pub const MOCK_INVOKE_API: &str = "http://127.0.0.1:8080"; + +// cmd related +pub const CMD_TYPE_SHELL: &str = "SHELL"; +pub const CMD_TYPE_POWERSHELL: &str = "POWERSHELL"; +pub const CMD_TYPE_BAT: &str = "BAT"; + +pub const SUFFIX_SHELL: &str = ".sh"; +pub const SUFFIX_BAT: &str = ".bat"; +pub const SUFFIX_PS1: &str = ".ps1"; + +pub const OUTPUT_BYTE_LIMIT_EACH_REPORT: usize = 30 * 1024; +pub const DEFAULT_OUTPUT_BYTE: u64 = 24 * 1024; + +// ontime related +pub const ONTIME_KICK_SOURCE: &str = "ONTIME_KICK"; +pub const ONTIME_KICK_INTERVAL: u64 = 3600 * 24; +pub const ONTIME_PING_INTERVAL: u64 = 2 * 60; +pub const ONTIME_UPDATE_INTERVAL: u64 = 2 * 60 * 60; +pub const ONTIME_THREAD_INTERVAL: u64 = 1; +pub const ONTIME_CHECK_TASK_NUM: u64 = 10; + +// self update related +pub const UPDATE_FILE_UNZIP_DIR: &str = "agent_update_unzip"; + +pub const AGENT_FILENAME: &str = "tat_agent"; +pub const UPDATE_DOWNLOAD_TIMEOUT: u64 = 20 * 60; + +#[cfg(test)] +mod tests { + #[test] + fn test_err_info() { + let dir = "/tmp/"; + let user = "worker"; + let s = start_failed_err_info!(ERR_WORKING_DIRECTORY_NOT_EXISTS, dir); + assert_err_info(&s, "DirectoryNotExists"); + + let s = start_failed_err_info!(ERR_USER_NOT_EXISTS, user); + assert_err_info(&s, "UserNotExists"); + + let s = start_failed_err_info!(ERR_USER_NO_PERMISSION_OF_WORKING_DIRECTORY, user, dir); + assert_err_info(&s, "DirectoryPermissionDeny"); + + let s = start_failed_err_info!(ERR_SUDO_NOT_EXISTS); + assert_err_info(&s, "SudoNotExists"); + + let s = start_failed_err_info!(ERR_SCRIPT_FILE_STORE_FAILED, dir); + assert_err_info(&s, "ScriptStoreFailed"); + } + + fn assert_err_info(err_info: &String, err_code: &str) { + println!("err_code: {}, err_info: {}", err_code, err_info); + let s = format!("{}{}", err_code, ":"); + assert!(err_info.starts_with(s.as_str())); + } +} diff --git a/src/common/envs.rs b/src/common/envs.rs new file mode 100644 index 0000000..fe1bd0a --- /dev/null +++ b/src/common/envs.rs @@ -0,0 +1,18 @@ +use std::env; + +pub fn enable_test() -> bool { + let enable = match env::var("TEST_ENABLE") { + Ok(val) => str::to_lowercase(&*val) == "true", + Err(_e) => false, + }; + return enable +} + +pub fn test_vpcid() -> String { + return env::var("MOCK_VPCID").unwrap(); +} + +pub fn test_vip() -> String { + return env::var("MOCK_VIP").unwrap(); +} + diff --git a/src/common/logger.rs b/src/common/logger.rs new file mode 100644 index 0000000..a749eac --- /dev/null +++ b/src/common/logger.rs @@ -0,0 +1,70 @@ +use log::info; +use log4rs::append::rolling_file::policy::compound::roll::fixed_window::FixedWindowRoller; +use log4rs::append::rolling_file::policy::compound::trigger::size::SizeTrigger; +use log4rs::append::rolling_file::policy::compound::CompoundPolicy; +use log4rs::append::rolling_file::RollingFileAppender; +use log4rs::config::Appender; +use log4rs::config::Config; +use log4rs::config::Root; +use log4rs::encode::pattern::PatternEncoder; +use log4rs::filter::threshold::ThresholdFilter; + +use crate::common::consts::{ + LOG_FILE_BASE_INDEX, LOG_FILE_NAME, LOG_FILE_NAME_WHEN_ROLL, LOG_FILE_SIZE, LOG_LEVEL, + LOG_LEVEL_DEBUG, LOG_PATTERN, MAX_LOG_FILE_COUNT, +}; +use crate::common::Opts; + +pub fn init() { + let opts = Opts::get_opts(); + let option_log_level = opts.debug(); + let mut log_level: log::LevelFilter = LOG_LEVEL; + + let logfile = RollingFileAppender::builder() + .encoder(Box::new(PatternEncoder::new(LOG_PATTERN))) + .build( + LOG_FILE_NAME, + Box::new(CompoundPolicy::new( + Box::new(SizeTrigger::new(LOG_FILE_SIZE)), + Box::new( + FixedWindowRoller::builder() + .base(LOG_FILE_BASE_INDEX) + .build(LOG_FILE_NAME_WHEN_ROLL, MAX_LOG_FILE_COUNT) + .unwrap(), + ), + )), + ) + .unwrap(); + if let Some(true) = option_log_level { + log_level = LOG_LEVEL_DEBUG; + } + let config = Config::builder() + .appender( + Appender::builder() + .filter(Box::new(ThresholdFilter::new(log_level))) + .build("logfile", Box::new(logfile)), + ) + .build(Root::builder().appender("logfile").build(log_level)) + .unwrap(); + let config_log = format!("{:?}", config); + log4rs::init_config(config).unwrap(); + info!("logger init succ, config: {}", config_log); +} + +#[allow(dead_code)] +pub fn init_test_log() { + use log4rs::append::console::ConsoleAppender; + + let stdout: ConsoleAppender = ConsoleAppender::builder() + .encoder(Box::new(PatternEncoder::new(LOG_PATTERN))) + .build(); + let log_config = log4rs::config::Config::builder() + .appender(Appender::builder().build("stdout", Box::new(stdout))) + .build(Root::builder().appender("stdout").build(LOG_LEVEL_DEBUG)) + .unwrap(); + match log4rs::init_config(log_config) { + Ok(_) => (), + Err(why) => info!("init test log failed: {}", why), + }; + info!("logger init succ"); +} diff --git a/src/common/option.rs b/src/common/option.rs new file mode 100644 index 0000000..5279e40 --- /dev/null +++ b/src/common/option.rs @@ -0,0 +1,44 @@ +use std::sync::Arc; + +use clap::{App, Arg}; + +use crate::common::consts::AGENT_VERSION; + +#[derive(Debug, Clone)] +pub struct Opts { + debug: Option, +} + +impl Opts { + pub fn get_opts() -> Arc { + static mut INS: Option> = None; + let &mut ins; + unsafe { + ins = INS.get_or_insert(Arc::new(Self::generate_opts())); + } + ins.clone() + } + + pub fn debug(&self) -> &Option { + &self.debug + } + + fn generate_opts() -> Self { + let matches = App::new("tat_agent") + .version(AGENT_VERSION) + .arg( + Arg::with_name("debug") + .long("debug") + .help("Set log level to debug") + .required(false), + ) + .get_matches(); + + let mut debug = None; + if matches.is_present("debug") { + debug = Some(true); + } + // return Opts + Opts { debug } + } +} diff --git a/src/daemonizer.rs b/src/daemonizer.rs new file mode 100644 index 0000000..098543c --- /dev/null +++ b/src/daemonizer.rs @@ -0,0 +1,17 @@ +cfg_if::cfg_if! { + if #[cfg(unix)] { + mod unix; + pub use unix::daemonize; + } else if #[cfg(windows)] { + mod windows; + pub use windows::daemonize; + pub use windows::wow64_disable_exc; + } else { + // not supported platform. + use log::warn; + pub fn daemonize(entry: fn()) { + warn!("unsupported platform, daemonize will do nothing.") + entry(); + } + } +} diff --git a/src/daemonizer/unix.rs b/src/daemonizer/unix.rs new file mode 100644 index 0000000..c41758c --- /dev/null +++ b/src/daemonizer/unix.rs @@ -0,0 +1,49 @@ +use log::debug; +use log::info; +use std::path::PathBuf; +use daemonize::Daemonize; +use daemonize::DaemonizeError; + +use crate::common::consts::{AGENT_DEFAULT_WORK_DIRECTORY, PID_FILE}; + +pub fn daemonize(entry: fn()) { + let agent_dir = get_agent_dir(); + let daemonize = Daemonize::new() + // set working dir of the daemon to where agent program is + .working_directory(agent_dir) + .pid_file(PID_FILE); + match daemonize.start() { + Ok(_) => info!("daemonize succ"), + Err(e) => { + let mut reason = format!("Daemonize failed because: {}.", e); + if let DaemonizeError::LockPidfile(_errno) = e { + reason.push_str(" Another daemon agent may be already running."); + } + panic!("{}", reason); + } + }; + entry(); +} + +fn get_agent_dir() -> PathBuf { + let mut dir = std::env::current_exe().unwrap(); + debug!("agent path:{:?}", dir); + if dir.pop() { + debug!("agent dir:{:?}", dir); + dir + } else { + PathBuf::from(AGENT_DEFAULT_WORK_DIRECTORY) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::common::logger; + + #[test] + fn test_get_agent_dir() { + logger::init_test_log(); + get_agent_dir(); + } +} diff --git a/src/daemonizer/windows.rs b/src/daemonizer/windows.rs new file mode 100644 index 0000000..9c34a37 --- /dev/null +++ b/src/daemonizer/windows.rs @@ -0,0 +1,158 @@ +use std::env; +use std::process::Command; +use std::ptr; +use winapi::shared::minwindef::{DWORD, LPVOID, FALSE}; +use winapi::shared::ntdef::NULL; +use winapi::um::winnt::{LPWSTR, PVOID, SERVICE_WIN32_OWN_PROCESS}; +use winapi::um::winsvc::*; +use winapi::um::wow64apiset::*; +use winapi::um::synchapi::*; +use winapi::um::minwinbase::LPSECURITY_ATTRIBUTES; +use winapi::um::errhandlingapi::GetLastError; +use log::error; + +//static var if not start with upper case, cargo build will report warn +static mut HANDLE: SERVICE_STATUS_HANDLE = 0 as SERVICE_STATUS_HANDLE; +static mut TAT_ENTRY: fn() = || {}; + +fn str2wstr(name: &str) -> Vec { + let mut result: Vec = name.chars().map(|c| c as u16).collect(); + result.push(0); + result +} + +fn create_service_status(current_state: DWORD) -> SERVICE_STATUS { + SERVICE_STATUS { + dwServiceType: SERVICE_WIN32_OWN_PROCESS, + dwCurrentState: current_state, + dwControlsAccepted: SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN, + dwWin32ExitCode: 0, + dwServiceSpecificExitCode: 0, + dwCheckPoint: 0, + dwWaitHint: 0, + } +} + +unsafe extern "system" fn service_main( + _: DWORD, // dw_num_services_args + _: *mut LPWSTR, // lp_service_arg_vectors +) { + let service_name = str2wstr("TAT_AGENT"); + HANDLE = RegisterServiceCtrlHandlerExW( + service_name.as_ptr(), + Some(service_handler), + ptr::null_mut(), + ); + std::thread::spawn(TAT_ENTRY); + let service_status = &mut create_service_status(SERVICE_RUNNING); + SetServiceStatus(HANDLE, service_status); +} + +unsafe extern "system" fn service_handler( + dw_control: DWORD, + _: DWORD, + _: LPVOID, + _: LPVOID, +) -> DWORD { + match dw_control { + SERVICE_CONTROL_STOP | SERVICE_CONTROL_SHUTDOWN => { + SetServiceStatus(HANDLE, &mut create_service_status(SERVICE_STOPPED)); + // exit after this function return,void sc report err + std::thread::spawn(|| { + std::thread::sleep(std::time::Duration::from_millis(10)); + std::process::exit(0); + }); + } + _ => {} + }; + return 0; +} + +fn try_start_service(entry: fn()) { + unsafe { + TAT_ENTRY = entry; + let service_name = str2wstr("TAT_AGENT"); + let service_table: &[*const SERVICE_TABLE_ENTRYW] = &[ + &SERVICE_TABLE_ENTRYW { + lpServiceName: service_name.as_ptr(), + lpServiceProc: Some(service_main), + }, + ptr::null(), + ]; + + match StartServiceCtrlDispatcherW(*service_table.as_ptr()) { + 0 => { + TAT_ENTRY(); + } + _ => {} + }; + + let service_table: &[*const SERVICE_TABLE_ENTRYW] = &[ + &SERVICE_TABLE_ENTRYW { + lpServiceName: service_name.as_ptr(), + lpServiceProc: Some(service_main), + }, + ptr::null(), + ]; + StartServiceCtrlDispatcherW(*service_table.as_ptr()); + return; + } +} + +fn clean_update_files() { + wow64_disable_exc(|| { + Command::new("cmd.exe") + .args(&[ + "/C", + "del", + "C:\\Program Files\\qcloud\\tat_agent\\temp_*.exe", + ]) + .spawn().map_err(|_|error!("clean_update_files fail")).ok(); + }) +} + +fn set_work_dir() { + let exe_path = env::current_exe().unwrap(); + let work_dir = exe_path.parent().unwrap(); + wow64_disable_exc(|| { + env::set_current_dir(work_dir).map_err(|_| error!("set_work_dir fail")).ok(); + }) +} + +pub fn wow64_disable_exc(func: F) -> T +where + F: Fn() -> T, +{ + let result: T; + let mut old: PVOID = NULL; + unsafe { + if Wow64DisableWow64FsRedirection(&mut old) != 0 { + result = func(); + Wow64RevertWow64FsRedirection(old); + } else { + result = func(); + }; + } + result +} + + fn already_start()->bool { + unsafe { + let event_name = str2wstr("Global\\tatsvc"); + CreateEventW(NULL as LPSECURITY_ATTRIBUTES, + FALSE,FALSE, event_name.as_ptr()); + //ERROR_ALREADY_EXISTS=183, ERROR_ACCESS_DENIED=05 get these value from vs winerror.h + let err= GetLastError(); + return err == 183 || err == 5 + } +} + +pub fn daemonize(entry: fn()) { + clean_update_files(); + set_work_dir(); + + if already_start() { + std::process::exit(183); + } + try_start_service(entry); +} diff --git a/src/executor.rs b/src/executor.rs new file mode 100644 index 0000000..36e3bb4 --- /dev/null +++ b/src/executor.rs @@ -0,0 +1,5 @@ +pub mod proc; +#[cfg(unix)] +pub mod shell_command; +#[cfg(windows)] +pub mod powershell_command; diff --git a/src/executor/powershell_command.rs b/src/executor/powershell_command.rs new file mode 100644 index 0000000..3dc26e4 --- /dev/null +++ b/src/executor/powershell_command.rs @@ -0,0 +1,275 @@ +use crate::daemonizer::wow64_disable_exc; +use crate::executor::proc::{BaseCommand, MyCommand}; +use crate::start_failed_err_info; +use async_trait::async_trait; +use codepage_strings::Coding; +use core::mem; +use log::{debug, error, info}; +use rand::distributions::Alphanumeric; +use rand::{thread_rng, Rng}; +use std::ffi::OsStr; +use std::fmt; +use std::fmt::{Debug, Formatter}; +use std::fs::File; +use std::fs::OpenOptions; +use std::os::windows::ffi::OsStrExt; +use std::path::Path; +use std::process::Stdio; +use std::ptr::null_mut; +use std::sync::Arc; +use tokio::io::AsyncReadExt; +use tokio::process::Command; +use winapi::shared::winerror::ERROR_ACCESS_DENIED; +use winapi::um::errhandlingapi::GetLastError; +use winapi::um::handleapi::INVALID_HANDLE_VALUE; +use winapi::um::namedpipeapi::CreateNamedPipeW; +use winapi::um::processthreadsapi::GetCurrentProcessId; +use winapi::um::winbase::{ + FILE_FLAG_FIRST_PIPE_INSTANCE, FILE_FLAG_OVERLAPPED, PIPE_ACCESS_INBOUND, PIPE_ACCESS_OUTBOUND, + PIPE_READMODE_BYTE, PIPE_TYPE_BYTE, PIPE_WAIT, +}; + +use winapi::um::winnls::GetOEMCP; +use winapi::um::winnt::HANDLE; + +pub struct PowerShellCommand { + base: Arc, +} + +impl PowerShellCommand { + pub fn new( + cmd_path: &str, + username: &str, + work_dir: &str, + timeout: u64, + bytes_max_report: u64, + ) -> PowerShellCommand { + let cmd_path = String::from(cmd_path).replace(" ", "` "); + PowerShellCommand { + base: Arc::new(BaseCommand::new( + cmd_path.as_str(), + username, + work_dir, + timeout, + bytes_max_report, + )), + } + } + + async fn set_ps1_policy(&self) { + let mut cmd = Command::new("PowerShell.exe"); + cmd.args(&["set-ExecutionPolicy", "RemoteSigned"]); + let child = cmd.spawn().unwrap(); + child.await.map_err(|_| error!("set_ps1_policy fail")).ok(); + } + + fn work_dir_check(&self) -> Result<(), String> { + if !wow64_disable_exc(|| Path::new(self.base.work_dir.as_str()).exists()) { + let ret = format!( + "PowerShellCommand {} start fail, working_directory:{}, username: {}: working directory not exists", + self.base.cmd_path, self.base.work_dir, self.base.username + ); + *self.base.err_info.lock().unwrap() = + start_failed_err_info!(ERR_WORKING_DIRECTORY_NOT_EXISTS, self.base.work_dir); + return Err(ret); + }; + Ok(()) + } + + fn prepare_cmd(&self, theirs: File) -> Result { + let std_out = theirs.try_clone().map_err(|e| { + error!("prepare_cmd,clone pipe std_out fail {}", e); + e.to_string() + })?; + + let std_err = theirs.try_clone().map_err(|e| { + error!("prepare_cmd,clone pipe std_err fail {}", e); + e.to_string() + })?; + + let mut comand = Command::new("PowerShell.exe"); + comand + .args(&[self.base.cmd_path.as_str()]) + .stdin(Stdio::null()) + .stdout(std_out) + .stderr(std_err) + .current_dir(self.base.work_dir.as_str()); + + Ok(comand) + } +} + +#[async_trait] +impl MyCommand for PowerShellCommand { + /* TODO: + 1. support set username + 2. support kill process when cancelled or timout. + 3. support set process group. + */ + async fn run(&mut self) -> Result<(), String> { + //work dir check + self.work_dir_check()?; + //set policy + self.set_ps1_policy().await; + + //create pipe + let (our_pipe, their_pipe) = anon_pipe(true)?; + + //start child + let mut cmd = self.prepare_cmd(their_pipe)?; + let mut child = cmd.spawn().map_err(|e| { + *self.base.err_info.lock().unwrap() = e.to_string(); + format!( + "PowerShellCommand {}, working_directory:{}, start fail: {}", + self.base.cmd_path, self.base.work_dir, e + ) + })?; + + *self.base.pid.lock().unwrap() = Some(child.id()); + let base = self.base.clone(); + // async read output. + tokio::spawn(async move { + base.add_timeout_timer(); + base.read_ps1_output(our_pipe).await; + base.del_timeout_timer(); + base.process_finish(&mut child).await; + }); + Ok(()) + } + + fn debug(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.fmt(f) + } + fn get_base(&self) -> Arc { + self.base.clone() + } +} + +impl BaseCommand { + async fn read_ps1_output(&self, file: File) { + let pid = self.pid.lock().unwrap().unwrap(); + const BUF_SIZE: usize = 1024; + let mut buffer: [u8; BUF_SIZE] = [0; BUF_SIZE]; + + let mut file = tokio::fs::File::from_std(file); + let codepage = unsafe { GetOEMCP() }; + loop { + let size = file.read(&mut buffer[..]).await; + if size.is_err() { + error!("read output err:{}, pid:{}", size.unwrap_err(), pid); + break; + } + let len = size.unwrap(); + if len > 0 { + let decoded_string = Coding::new(codepage as u16) + .unwrap() + .decode(&buffer[..len]) + .unwrap(); + debug!("output:[{}], pid:{}, len:{}", decoded_string, pid, len); + unsafe { + self.append_output(String::from(decoded_string).as_mut_vec()); + } + } else { + info!("read output finished normally, pid:{}", pid); + break; + } + } + } + + pub fn kill_process_group(pid: u32) { + let pid = pid.to_string(); + let mut child = std::process::Command::new("TASKKILL") + .args(&["/F", "/PID", pid.as_str(), "/T"]) + .spawn() + .expect("failed kill_process_group"); + + child + .wait() + .map_err(|_| error!("kill_process_group fail")) + .ok(); + } +} + +impl Debug for PowerShellCommand { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + self.base.fmt(f) + } +} + +fn anon_pipe(ours_readable: bool) -> Result<(File, File), String> { + unsafe { + let mut tries = 0; + let mut name; + let ours: File; + loop { + tries += 1; + name = format!( + r"\\.\pipe\__tat_anon_pipe__.{}.{}", + GetCurrentProcessId(), + thread_rng() + .sample_iter(&Alphanumeric) + .take(10) + .collect::(), + ); + + let wide_name = OsStr::new(&name) + .encode_wide() + .chain(Some(0)) + .collect::>(); + + let mut flags = FILE_FLAG_FIRST_PIPE_INSTANCE | FILE_FLAG_OVERLAPPED; + if ours_readable { + flags |= PIPE_ACCESS_INBOUND; + } else { + flags |= PIPE_ACCESS_OUTBOUND; + } + + let handle = CreateNamedPipeW( + wide_name.as_ptr(), + flags, + PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT, + 1, + 4096, + 4096, + 0, + null_mut(), + ); + + if handle == INVALID_HANDLE_VALUE { + let err = GetLastError(); + if tries < 10 { + if err == ERROR_ACCESS_DENIED { + continue; + } + } + error!("creat namepipe fail,{}", err); + return Err(format!("creat namepipe fail,{}", err)); + } + ours = mem::transmute::(handle); + break; + } + + let mut opts = OpenOptions::new(); + opts.write(ours_readable); + opts.read(!ours_readable); + + let theirs = opts.open(Path::new(&name)).map_err(|e| e.to_string())?; + Ok((ours, theirs)) + } +} + +#[cfg(test)] +mod test { + use crate::executor::powershell_command::anon_pipe; + use std::io::{Read, Write}; + #[test] + fn test() { + let (mut ours, mut theirs) = anon_pipe(true).unwrap(); + theirs.write("test".as_bytes()).unwrap(); + let mut buffer = [0; 1024]; + let size = ours.read(&mut buffer).unwrap(); + let result = String::from_utf8_lossy(&buffer[0..size]); + assert_eq!(result, "test".to_string()); + return; + } +} diff --git a/src/executor/proc.rs b/src/executor/proc.rs new file mode 100644 index 0000000..e0b32e4 --- /dev/null +++ b/src/executor/proc.rs @@ -0,0 +1,910 @@ +cfg_if::cfg_if! { + if #[cfg(unix)] { + use crate::common::consts::CMD_TYPE_SHELL; + use crate::executor::shell_command::ShellCommand; + } else if #[cfg(windows)] { + use crate::common::consts::CMD_TYPE_POWERSHELL; + use crate::executor::powershell_command::PowerShellCommand; + } +} +use crate::common::asserts::GracefulUnwrap; +use crate::common::consts::{ + FINISH_RESULT_FAILED, FINISH_RESULT_START_FAILED, FINISH_RESULT_SUCCESS, + FINISH_RESULT_TERMINATED, FINISH_RESULT_TIMEOUT, OUTPUT_BYTE_LIMIT_EACH_REPORT, +}; +use crate::ontime::timer::Timer; +use async_trait::async_trait; +use log::{debug, info}; +use std::fmt; +use std::sync::atomic::Ordering::SeqCst; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::SystemTime; +use tokio::process::Child; + +#[async_trait] +pub trait MyCommand { + async fn run(&mut self) -> Result<(), String>; + fn cancel(&self) -> Result<(), String> { + self.get_base().cancel() + } + fn cur_output_len(&self) -> usize { + self.get_base().cur_output_len() + } + + fn next_output(&mut self) -> (Vec, u32, u64) { + self.get_base().next_output() + } + + fn is_finished(&self) -> bool { + self.get_base().is_finished() + } + + fn is_started(&self) -> bool { + self.get_base().is_started() + } + + fn exit_code(&self) -> i32 { + self.get_base().exit_code() + } + + fn pid(&self) -> u32 { + self.get_base().pid() + } + + fn is_timeout(&self) -> bool { + self.get_base().is_timeout() + } + + fn finish_result(&self) -> String { + self.get_base().finish_result() + } + + fn err_info(&self) -> String { + self.get_base().err_info() + } + fn finish_time(&self) -> u64 { + self.get_base().finish_time() + } + fn debug(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result; + fn get_base(&self) -> Arc; +} + +pub fn new( + cmd_path: &str, + username: &str, + cmd_type: &str, + work_dir: &str, + timeout: u64, + bytes_max_report: u64, +) -> Result, String> { + match cmd_type { + #[cfg(unix)] + CMD_TYPE_SHELL => Ok(Box::new(ShellCommand::new( + cmd_path, + username, + work_dir, + timeout, + bytes_max_report, + ))), + #[cfg(windows)] + CMD_TYPE_POWERSHELL => Ok(Box::new(PowerShellCommand::new( + cmd_path, + username, + work_dir, + timeout, + bytes_max_report, + ))), + _ => Err(format!("invalid cmd_type:{}", cmd_type)), + } +} + +impl std::fmt::Debug for std::boxed::Box { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.debug(f) + } +} + +pub struct BaseCommand { + pub cmd_path: String, + pub username: String, + pub work_dir: String, + // the whole process group will be killed after timeout + pub timeout: u64, + pub bytes_max_report: u64, + + pub bytes_reported: AtomicU64, + pub bytes_dropped: AtomicU64, + + // it's true after finish + pub finished: Arc, + // Only read this value if self.finished==true + pub exit_code: Arc>>, + // current output which ready to report + pub output: Arc>>, + // current output report index + pub output_idx: AtomicU32, + + // it's None before start, will be Some after self.run() + pub pid: Mutex>, + // if child has been killed by kill -9 + pub killed: Arc, + // if child is timeout + pub is_timeout: Arc, + // the time command process finished + pub finish_time: Arc, + // err_info when command start fail + pub err_info: Mutex, + //timer_key and timer_id + pub timer_info: Mutex<(u128, u64)>, +} + +impl BaseCommand { + pub fn new( + cmd_path: &str, + username: &str, + work_dir: &str, + timeout: u64, + bytes_max_report: u64, + ) -> BaseCommand { + let timestamp = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_exit("sys time may before 1970") + .as_secs(); + BaseCommand { + cmd_path: cmd_path.to_string(), + username: username.to_string(), + work_dir: work_dir.to_string(), + timeout, + bytes_max_report, + bytes_reported: AtomicU64::new(0), + bytes_dropped: AtomicU64::new(0), + finished: Arc::new(AtomicBool::new(false)), + exit_code: Arc::new(Mutex::new(None)), + output: Arc::new(Mutex::new(Default::default())), + output_idx: AtomicU32::new(0), + pid: Mutex::new(None), + killed: Arc::new(AtomicBool::new(false)), + is_timeout: Arc::new(AtomicBool::new(false)), + finish_time: Arc::new(AtomicU64::new(timestamp)), + err_info: Mutex::new("".to_string()), + timer_info: Mutex::new((0, 0)), + } + } + // length of bytes, not chars + pub fn cur_output_len(&self) -> usize { + let output = self.output.lock().unwrap_or_exit("lock failed"); + output.len() + } + + pub fn append_output(&self, data: &mut Vec) { + let mut output = self.output.lock().unwrap_or_exit("lock failed"); + output.append(data); + } + + pub fn next_output(&self) -> (Vec, u32, u64) { + let mut output = self.output.lock().unwrap_or_exit("lock failed"); + let len = output.len(); + // current output is empty, return. + if len == 0 { + return ( + vec![], + self.output_idx.fetch_add(1, Ordering::SeqCst), + self.bytes_dropped.load(Ordering::SeqCst), + ); + } + + // already exceed max report, update dropped, index then return. + let dropped_pre = self.bytes_dropped.load(Ordering::SeqCst); + if dropped_pre > 0 { + self.bytes_dropped.fetch_add(len as u64, Ordering::SeqCst); + output.clear(); + return ( + vec![], + self.output_idx.fetch_add(1, Ordering::SeqCst), + dropped_pre + len as u64, + ); + } + + // not exceed max report, continue report output. + let mut ret; + if len <= OUTPUT_BYTE_LIMIT_EACH_REPORT { + // copy and move out + ret = output.clone(); + output.clear(); + } else { + debug!("output origin:{:?}", output); + // move [0..OUTPUT_BYTE_LIMIT_EACH_REPORT] out to ret + output.rotate_left(OUTPUT_BYTE_LIMIT_EACH_REPORT); + ret = output.split_off(len - OUTPUT_BYTE_LIMIT_EACH_REPORT); + debug!("output to ret:{:?}", ret); + debug!("output left:{:?}", output); + } + + let ret_len = ret.len() as u64; + let bytes_pre = self.bytes_reported.fetch_add(ret_len, Ordering::SeqCst); + // current output exceeds max report, init bytes_dropped and clear output. + if ret_len + bytes_pre >= self.bytes_max_report { + let need_report_len = self.bytes_max_report - bytes_pre; + ret.truncate(need_report_len as usize); + self.bytes_dropped + .store(ret_len - need_report_len, Ordering::SeqCst); + output.clear(); + } + + ( + ret, + self.output_idx.fetch_add(1, Ordering::SeqCst), + self.bytes_dropped.load(Ordering::SeqCst), + ) + } + + pub fn is_started(&self) -> bool { + return match *self.pid.lock().unwrap() { + None => false, + _ => true, + }; + } + + pub fn is_finished(&self) -> bool { + self.finished.load(Ordering::SeqCst) + } + + pub fn exit_code(&self) -> i32 { + let pid = *self.pid.lock().unwrap(); + if let None = pid { + return 0; + } + let exit_code = self + .exit_code + .lock() + .unwrap_or_exit("exit_code get lock fail"); + match *exit_code { + Some(code) => code, + None => -1, + } + } + + pub fn cmd_path(&self) -> &String { + &self.cmd_path + } + + pub fn pid(&self) -> u32 { + let pid = *self.pid.lock().unwrap(); + match pid { + Some(pid) => pid, + None => 0, + } + } + + pub fn is_timeout(&self) -> bool { + self.is_timeout.load(Ordering::SeqCst) + } + + pub fn finish_result(&self) -> String { + let pid = *self.pid.lock().unwrap(); + if let None = pid { + return FINISH_RESULT_START_FAILED.to_string(); + } + if self.is_timeout() { + return FINISH_RESULT_TIMEOUT.to_string(); + } + if self.killed.load(Ordering::SeqCst) { + return FINISH_RESULT_TERMINATED.to_string(); + } + if self.is_finished() { + let code = self.exit_code(); + if 0 == code { + return FINISH_RESULT_SUCCESS.to_string(); + } + } + FINISH_RESULT_FAILED.to_string() + } + + pub fn err_info(&self) -> String { + let err_info = self.err_info.lock().unwrap(); + return String::from(err_info.as_str()); + } + + pub fn finish_time(&self) -> u64 { + self.finish_time.load(Ordering::SeqCst) + } + + pub fn add_timeout_timer(&self) { + let pid = self.pid.lock().unwrap().unwrap(); + let timeout = self.timeout; + let killed = self.killed.clone(); + let is_timeout = self.is_timeout.clone(); + + *self.timer_info.lock().unwrap() = + Timer::get_instance() + .lock() + .unwrap() + .add_task(timeout, move || { + info!("process {} timeout,timeout value is {}", pid, timeout); + let ret = killed.compare_exchange(false, true, SeqCst, SeqCst); + if ret.is_err() { + info!("pid:{} already killed, ignore this timer task", pid); + } else { + is_timeout.store(true, Ordering::SeqCst); + BaseCommand::kill_process_group(pid); + info!("pid:{} killed because of timeout", pid); + } + }); + } + + pub fn del_timeout_timer(&self) { + let timer_info = *self.timer_info.lock().unwrap(); + let deleted = Timer::get_instance() + .lock() + .unwrap() + .del_task(timer_info.0, timer_info.1); + if deleted { + debug!( + "timer task deleted, task_key:{}, task_id:{}", + timer_info.0, timer_info.1 + ); + } else { + debug!( + "timer task NOT deleted, maybe task already scheduled, task_key:{}, task_id:{}", + timer_info.0, timer_info.1 + ); + } + } + + pub async fn process_finish(&self, child: &mut Child) { + let pid = child.id(); + info!("=>process {} finish", pid); + let status = child.await.expect("child process encountered an error"); + let mut exit_code = self + .exit_code + .lock() + .unwrap_or_exit("exit_code get lock fail"); + match status.code() { + Some(code) => { + exit_code.replace(code); + } + None => { + info!("Process terminated by signal: {}", pid); + exit_code.replace(-1); + } + } + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_exit("sys time may before 1970") + .as_secs(); + self.finished.store(true, Ordering::SeqCst); + self.finish_time.store(now, Ordering::SeqCst); + } + + pub fn cancel(&self) -> Result<(), String> { + let pid = *self.pid.lock().unwrap(); + match pid { + Some(pid) => { + let ret = + self.killed + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst); + if ret.is_err() { + info!("pid:{} already killed, ignore cancel request", pid); + } else { + BaseCommand::kill_process_group(pid); + info!("pid:{} killed because of cancel", pid); + } + Ok(()) + } + None => Err("Process not running, no pid to kill".to_string()), + } + } + + pub fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let output_clone; + { + let output = self.output.lock().unwrap_or_exit("lock failed"); + output_clone = output.clone(); + } + let output_debug = String::from_utf8_lossy(output_clone.as_slice()); + let may_contain_binary = String::from_utf8(output_clone.clone()).is_err(); + + f.debug_struct("ShellCommand") + .field("cmd_path", &self.cmd_path) + .field("work_dir", &self.work_dir) + .field("timeout", &self.timeout) + .field("bytes_max_report", &self.bytes_max_report) + .field("bytes_reported", &self.bytes_reported) + .field("finished", &self.finished) + .field("exit_code", &self.exit_code) + .field("output", &self.output) + .field("output_debug", &output_debug) + .field("may_contain_binary", &may_contain_binary) + .field("output_idx", &self.output_idx) + .field("pid", &self.pid) + .field("killed", &self.killed) + .field("is_timeout", &self.is_timeout) + .field("finish_time", &self.finish_time) + .finish() + } +} +#[cfg(test)] +mod tests { + use std::fs::File; + use std::io::Write; + use std::time::{Duration, Instant, SystemTime}; + use std::{fs, thread}; + + use log::info; + use rand::distributions::Alphanumeric; + use rand::{thread_rng, Rng}; + + use super::*; + use crate::common::asserts::GracefulUnwrap; + use crate::common::consts::FINISH_RESULT_START_FAILED; + use crate::ontime::timer::Timer; + + cfg_if::cfg_if! { + if #[cfg(unix)] { + use std::fs::read_dir; + use std::process::Command; + use users::get_current_username; + } else if #[cfg(windows)] { + use crate::common::consts::CMD_TYPE_POWERSHELL; + } + } + + #[cfg(unix)] + static CMD_TYPE: &str = CMD_TYPE_SHELL; + #[cfg(windows)] + static CMD_TYPE: &str = CMD_TYPE_POWERSHELL; + + #[cfg(unix)] + static CMD_PATH: &str = "./a.sh"; + #[cfg(windows)] + static CMD_PATH: &str = "./a.ps1"; + + #[cfg(unix)] + fn username() -> String { + String::from(get_current_username().unwrap().to_str().unwrap()) + } + + #[cfg(windows)] + fn username() -> String { + String::from("System") + } + + #[test] + fn test_valid_type_shell() { + let ret = new(CMD_PATH, &username(), CMD_TYPE, "./", 1024, 1024); + assert!(ret.is_ok()); + } + + #[test] + fn test_invalid_type() { + init_log(); + let ret = new(CMD_PATH, &username(), "xxx", "./", 1024, 1024); + match ret { + Ok(_) => panic!(), + Err(e) => info!("OK, ret:{}", e), + } + } + + #[test] + fn test_run_then_sleep() { + let mut rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + init_log(); + // it doesn't matter even if ./a.sh not exist + let ret = new(CMD_PATH, &username(), CMD_TYPE, "./", 1024, 1024); + let mut cmd = ret.unwrap(); + let ret = cmd.run().await; + assert!(ret.is_ok()); + info!("cmd running, pid:{}", cmd.pid()); + tokio::time::delay_for(Duration::from_secs(4)).await; + // now it's NOT a defunct process, cmd will be auto-waited + assert!(!is_process_exist(cmd.pid())); + //thread::sleep(Duration::new(10, 0)); + }); + } + + #[test] + fn test_run_start_fail_working_directory() { + let mut rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + init_log(); + let ret = new( + CMD_PATH, + &username(), + CMD_TYPE, + "./dir_not_exist", + 1024, + 1024, + ); + let mut cmd = ret.unwrap(); + let ret = cmd.run().await; + info!("cmd run ret:[{}]", ret.unwrap_err()); + assert_eq!(cmd.pid(), 0); + assert_eq!(cmd.finish_result(), FINISH_RESULT_START_FAILED); + assert_eq!(cmd.exit_code(), 0); + assert!(cmd.err_info().starts_with("DirectoryNotExists")); + }); + } + + #[cfg(unix)] + #[test] + fn test_run_start_fail_user_not_exists() { + let mut rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + init_log(); + + let ret = new(CMD_PATH, "hacker-neo", CMD_TYPE, "./", 1024, 1024); + let mut cmd = ret.unwrap(); + let ret = cmd.run().await; + info!("cmd run ret:[{}]", ret.unwrap_err()); + assert_eq!(cmd.pid(), 0); + assert_eq!(cmd.finish_result(), FINISH_RESULT_START_FAILED); + assert_eq!(cmd.exit_code(), 0); + assert!(cmd.err_info().starts_with("UserNotExists")); + }); + } + + fn gen_rand_str() -> String { + thread_rng().sample_iter(&Alphanumeric).take(10).collect() + } + + #[cfg(unix)] + fn run_shell(cmd: &str, args: &[&str]) { + Command::new(cmd).args(args).status().unwrap(); + } + + fn create_file(content: &str, filename: &str) { + let mut file = File::create(filename).unwrap(); + file.write_all(content.as_bytes()).unwrap(); + #[cfg(unix)] + run_shell("chmod", &["+x", filename]); + } + + fn init_log() { + use crate::common::logger; + logger::init_test_log(); + } + + #[cfg(unix)] + fn is_process_exist(pid: u32) -> bool { + // maybe need a time to clear the dir + thread::sleep(Duration::from_millis(2000)); + let path = format!("/proc/{}", pid); + let ret = read_dir(path); + let exist = ret.is_ok(); + info!("pid:{} is_exist:{}", pid, exist); + exist + } + + #[cfg(windows)] + fn is_process_exist(pid: u32) -> bool { + let pid_str = format!("PID eq {}", pid); + let output = std::process::Command::new("TASKLIST") + .args(&["/FI", pid_str.as_str()]) + .output() + .expect("failed find process"); + String::from_utf8_lossy(&output.stdout).contains(&pid.to_string()) + } + + #[cfg(unix)] + #[test] + fn test_pid_exist() { + let ret = is_process_exist(1); + assert!(ret); + } + + #[cfg(unix)] + #[test] + fn test_pid_not_exist() { + let ret = is_process_exist(0); + assert!(!ret); + } + + #[test] + fn test_cancel() { + let mut rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + init_log(); + + cfg_if::cfg_if! { + if #[cfg(unix)] { + let filename = format!("./.{}.sh", gen_rand_str()); + create_file("sleep 15", filename.as_str()); + } else if #[cfg(windows)] { + let filename = format!("./{}.ps1", gen_rand_str()); + create_file("Start-Sleep -s 1500", filename.as_str()); + } + } + + let ret = new(filename.as_str(), &username(), CMD_TYPE, "./", 1024, 1024); + let mut cmd = ret.unwrap(); + let ret = cmd.run().await; + assert!(ret.is_ok()); + info!("{} running, pid:{}", filename, cmd.pid()); + // now it's a still running + thread::sleep(Duration::new(10, 0)); + assert_eq!(cmd.is_started(), true); + assert!(is_process_exist(cmd.pid())); + + let ret = cmd.cancel(); + assert!(ret.is_ok()); + thread::sleep(Duration::new(1, 0)); + assert!(!is_process_exist(cmd.pid())); + // cmd.cancel() called twice is OK and safe + let ret = cmd.cancel(); + assert!(ret.is_ok()); + // Now it's killed & waited, check it's NOT a defunct. + // Even after killed, call cmd.pid() is OK + info!("{} killed, pid:{}", filename, cmd.pid()); + info!("cmd:{:?}", cmd); + thread::sleep(Duration::new(5, 0)); + + fs::remove_file(filename.as_str()).unwrap(); + }); + } + + #[test] + fn test_output() { + let mut rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + init_log(); + + cfg_if::cfg_if! { + if #[cfg(unix)] { + let filename = format!("./.{}.sh", gen_rand_str()); + create_file("yes | head -10 && sleep 3 && yes | head -5", filename.as_str()); + } else if #[cfg(windows)] { + let filename = format!("./{}.ps1", gen_rand_str()); + create_file( + "foreach ($i in 1..10) { Write-Host '00' -NoNewLine };\ + Start-Sleep -s 3; \ + foreach ($i in 1..5) { Write-Host '11' -NoNewLine };", + filename.as_str(), + ); + } + } + + let ret = new(filename.as_str(), &username(), CMD_TYPE, "./", 1024, 18); + let mut cmd = ret.unwrap(); + let ret = cmd.run().await; + assert!(ret.is_ok()); + info!("{} running, pid:{}", filename, cmd.pid()); + let mut cur_dropped = 0 as u64; + + // usage of read output + loop { + tokio::time::delay_for(Duration::from_secs(1)).await; + let len = cmd.cur_output_len(); + // is_finished() MUST be called after cur_output_len() + let finished = cmd.is_finished(); + if 0 != len && 0 == cur_dropped { + let (out, idx, dropped) = cmd.next_output(); + info!( + "ready to report output:{:?}, output_debug:{}, idx:{}, dropped:{}", + out, + String::from_utf8_lossy(&out[..]), + idx, + dropped + ); + assert_eq!(idx, 0); + assert_eq!(dropped, 2); + + // Do output report task here + // do_report(out, idx, dropped); + if dropped > 0 { + // max report exceeds, get dropped and idx during sleep + let (out, idx, dropped_new) = cmd.next_output(); + info!("during sleep: idx: {}, drop {}, ", idx, dropped); + assert_eq!(idx, 1); + assert_eq!(dropped_new, dropped); + assert_eq!(0, out.len()); + info!("dropped, not report output any more"); + + cur_dropped = dropped_new; + } + } + + if finished { + let (out, idx, dropped) = cmd.next_output(); + info!("after sleep: idx: {}, drop {}", idx, dropped); + assert_eq!(idx, 2); + assert_eq!(dropped, 12); + assert_eq!(0, out.len()); + // do_report(out, idx, dropped); + info!("finished, report final dropped bytes of output."); + break; + } + } + // will see the output bytes in cmd.output + info!("cmd:{:?}", cmd); + thread::sleep(Duration::new(1, 0)); + assert!(!is_process_exist(cmd.pid())); + + fs::remove_file(filename.as_str()).unwrap(); + }); + } + + #[test] + fn test_base64() { + let mut rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + init_log(); + + cfg_if::cfg_if! { + if #[cfg(unix)] { + let filename = format!("./.{}.sh", gen_rand_str()); + create_file("echo -n 'hello world'", filename.as_str()); + } else if #[cfg(windows)] { + let filename = format!("./{}.ps1", gen_rand_str()); + create_file( + "Write-Host 'hello world' -NoNewLine", + filename.as_str(), + ); + } + } + let ret = new(filename.as_str(), &username(), CMD_TYPE, "./", 10, 1024); + let mut cmd = ret.unwrap(); + let ret = cmd.run().await; + assert!(ret.is_ok()); + info!("{} running, pid:{}", filename, cmd.pid()); + + while !cmd.is_finished() { + thread::sleep(Duration::new(1, 0)); + } + + let (out, idx, dropped) = cmd.next_output(); + let out = base64::encode(out); + + assert_eq!(dropped, 0); + assert_eq!(0, idx); + assert_eq!(out, "aGVsbG8gd29ybGQ="); + info!("out:{}", out); + info!("cmd:{:?}", cmd); + thread::sleep(Duration::new(1, 0)); + assert!(!is_process_exist(cmd.pid())); + + fs::remove_file(filename.as_str()).unwrap(); + }); + } + + #[test] + // NOTICE: This testcase has use singleton of timer, + // All testcase share the same one timer, so: + // This testcase can NOT run together with test_timer_in_one_case + fn test_shell_cmd_timeout() { + let mut rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + init_log(); + + cfg_if::cfg_if! { + if #[cfg(unix)] { + let filename = format!("./.{}.sh", gen_rand_str()); + create_file("sleep 10240", filename.as_str()); + } else if #[cfg(windows)] { + let filename = format!("./{}.ps1", gen_rand_str()); + create_file( + "Start-Sleep -s 10240", + filename.as_str(), + ); + } + } + + let start_time = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + info!("start_time:{}", start_time); + + let ret = new(filename.as_str(), &username(), CMD_TYPE, "./", 2, 1024); + let mut cmd = ret.unwrap(); + let ret = cmd.run().await; + assert!(ret.is_ok()); + assert!(is_process_exist(cmd.pid())); + let instant = Instant::now(); + info!("{} running, pid:{}", filename, cmd.pid()); + thread::sleep(Duration::new(1, 0)); + + let mut cnt = 0; + loop { + { + let timer = Timer::get_instance(); + let mut timer = timer.lock().unwrap_or_exit(""); + info!("timer:{:?}", timer); + let tasks = timer.tasks_to_schedule(); + cnt += tasks.len(); + for task in tasks { + task.run_task(); + } + } + info!("total {} tasks run", cnt); + thread::sleep(Duration::new(0, 500_000_000)); + let finished = cmd.is_finished(); + if finished { + break; + } + } + info!("cmd:{:?}", cmd); + info!("finish result:{}", cmd.finish_result()); + assert!(cmd.is_timeout()); + assert!(instant.elapsed() <= Duration::from_secs(5)); + assert!(0 < cmd.finish_time()); + assert!(cmd.finish_time() < start_time + 5); + assert!(!is_process_exist(cmd.pid())); + fs::remove_file(filename.as_str()).unwrap(); + }); + } + + #[test] + #[cfg(unix)] + fn test_daemon() { + let mut rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + init_log(); + let filename = format!("./.{}.sh", gen_rand_str()); + create_file("echo 'hello world'\nsleep 10 &\ndate", filename.as_str()); + + let ret = new(filename.as_str(), &username(), CMD_TYPE, "./", 6, 1024); + let mut cmd = ret.unwrap(); + let ret = cmd.run().await; + assert!(ret.is_ok()); + + loop { + { + let timer = Timer::get_instance(); + let mut timer = timer.lock().unwrap_or_exit(""); + info!("timer:{:?}", timer); + let tasks = timer.tasks_to_schedule(); + for task in tasks { + task.run_task(); + } + } + thread::sleep(Duration::from_secs(1)); + let finished = cmd.is_finished(); + if finished { + break; + } + } + assert_eq!(cmd.is_timeout(), false); + fs::remove_file(filename.as_str()).unwrap(); + }); + } + + #[test] + #[cfg(unix)] + fn test_daemon_output() { + let mut rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + init_log(); + let filename = format!("./.{}.sh", gen_rand_str()); + create_file( + "yes | head -1024 \nsleep 1200 &\n yes | head -1025", + filename.as_str(), + ); + + let ret = new(filename.as_str(), &username(), CMD_TYPE, "./", 1200, 10240); + let mut cmd = ret.unwrap(); + let ret = cmd.run().await; + assert!(ret.is_ok()); + + loop { + { + let timer = Timer::get_instance(); + let mut timer = timer.lock().unwrap_or_exit(""); + //info!("timer:{:?}", timer); + let tasks = timer.tasks_to_schedule(); + for task in tasks { + task.run_task(); + } + } + thread::sleep(Duration::from_secs(1)); + let finished = cmd.is_finished(); + if finished { + break; + } + } + assert_eq!(cmd.is_timeout(), false); + fs::remove_file(filename.as_str()).unwrap(); + }); + } +} diff --git a/src/executor/shell_command.rs b/src/executor/shell_command.rs new file mode 100644 index 0000000..0455e67 --- /dev/null +++ b/src/executor/shell_command.rs @@ -0,0 +1,373 @@ +use std::env; +use std::fmt::Debug; +use std::path::Path; +use std::process::Stdio; +use std::sync::Arc; +use std::time::Duration; +use std::{fmt, io}; + +use async_trait::async_trait; +use libc; +use log::{debug, error, info}; +use tokio::io::{AsyncReadExt, BufReader}; +use tokio::process::{Child, Command}; +use tokio::time::timeout; +use users::get_user_by_name; + +use crate::common::asserts::GracefulUnwrap; +use crate::common::consts::{DUP2_1_2, OWN_PROCESS_GROUP, TASK_STORE_PATH,PIPE_BUF_DEFAULT_SIZE}; +use crate::executor::proc::{BaseCommand, MyCommand}; +use crate::start_failed_err_info; + +pub struct ShellCommand { + base: Arc, +} +impl ShellCommand { + pub fn new( + cmd_path: &str, + username: &str, + work_dir: &str, + timeout: u64, + bytes_max_report: u64, + ) -> ShellCommand { + ShellCommand { + base: Arc::new(BaseCommand::new( + cmd_path, + username, + work_dir, + timeout, + bytes_max_report, + )), + } + } + + fn cmd_path_check(&self) -> Result<(), String> { + if self.base.cmd_path.is_empty() { + let ret = format!("ShellCommand start fail because script file store failed."); + *self.base.err_info.lock().unwrap() = + start_failed_err_info!(ERR_SCRIPT_FILE_STORE_FAILED, TASK_STORE_PATH); + return Err(ret); + } + Ok(()) + } + + fn sudo_check(&self) -> Result<(), String> { + if !cmd_exists("sudo") { + let ret = format!( + "ShellCommand {} start fail, working_directory:{}, username: {}: sudo not exists", + self.base.cmd_path, self.base.work_dir, self.base.username + ); + *self.base.err_info.lock().unwrap() = start_failed_err_info!(ERR_SUDO_NOT_EXISTS); + return Err(ret); + } + Ok(()) + } + + fn user_check(&self) -> Result<(), String> { + if !user_exists(self.base.username.as_str()) { + let ret = format!( + "ShellCommand {} start fail, working_directory:{}, username: {}: user not exists", + self.base.cmd_path, self.base.work_dir, self.base.username + ); + *self.base.err_info.lock().unwrap() = + start_failed_err_info!(ERR_USER_NOT_EXISTS, self.base.username); + return Err(ret); + } + Ok(()) + } + + fn work_dir_check(&self) -> Result<(), String> { + if !working_directory_exists(self.base.work_dir.as_str()) { + let ret = format!( + "ShellCommand {} start fail, working_directory:{}, username: {}: working directory not exists", + self.base.cmd_path, self.base.work_dir, self.base.username + ); + *self.base.err_info.lock().unwrap() = + start_failed_err_info!(ERR_WORKING_DIRECTORY_NOT_EXISTS, self.base.work_dir); + return Err(ret); + } + Ok(()) + } + + fn work_dir_permission_check(&self) -> Result<(), String> { + if !working_directory_permission(self.base.work_dir.as_str(), self.base.username.as_str()) { + let ret = format!( + "ShellCommand {} start fail, working_directory:{}, username: {}: user has no permission to working_directory.", + self.base.cmd_path, self.base.work_dir, self.base.username + ); + *self.base.err_info.lock().unwrap() = start_failed_err_info!( + ERR_USER_NO_PERMISSION_OF_WORKING_DIRECTORY, + self.base.username, + self.base.work_dir + ); + return Err(ret); + } + Ok(()) + } + + fn prepare_cmd(&self) -> Command { + let mut shell = "sh"; + let mut entrypoint = format!( + "cd {} && {}", + self.base.work_dir.as_str(), + self.base.cmd_path.as_str() + ); + if cmd_exists("bash") { + shell = "bash"; + entrypoint = format!( + ". ~/.bash_profile 2> /dev/null || . ~/.bashrc 2> /dev/null ; {}", + entrypoint + ); + } + let mut cmd = Command::new("sudo"); + cmd.args(&[ + "-Hu", + self.base.username.as_str(), + shell, + "-c", + entrypoint.as_str(), + ]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + // Redirect stderr to stdout, thus the output order will be exactly same with origin + pre_exec_for_cmd(&mut cmd, DUP2_1_2); + // The command and its sub-processes will be in an independent process group, + // thus we can kill them cleanly by kill the whole process group when we need. + pre_exec_for_cmd(&mut cmd, OWN_PROCESS_GROUP); + cmd + } +} + +#[async_trait] +impl MyCommand for ShellCommand { + async fn run(&mut self) -> Result<(), String> { + // pre check before spawn cmd + self.cmd_path_check()?; + + self.sudo_check()?; + + self.user_check()?; + + self.work_dir_check()?; + + self.work_dir_permission_check()?; + + // start the process async + let mut child = self.prepare_cmd().spawn().map_err(|e| { + *self.base.err_info.lock().unwrap() = e.to_string(); + format!( + "ShellCommand {}, working_directory:{}, start fail: {}", + self.base.cmd_path, self.base.work_dir, e + ) + })?; + + *self.base.pid.lock().unwrap() = Some(child.id()); + let base = self.base.clone(); + // async read output. + tokio::spawn(async move { + base.add_timeout_timer(); + base.read_shl_output(&mut child).await; + base.del_timeout_timer(); + base.process_finish(&mut child).await; + }); + Ok(()) + } + + fn debug(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.fmt(f) + } + + fn get_base(&self) -> Arc { + self.base.clone() + } +} + +impl Debug for ShellCommand { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.base.fmt(f) + } +} + +impl BaseCommand { + async fn read_shl_output(&self, child: &mut Child) { + let pid = child.id(); + const BUF_SIZE: usize = 1024; + let mut buffer: [u8; BUF_SIZE] = [0; BUF_SIZE]; + let stdout = child.stdout.take(); + let mut reader = BufReader::new(stdout.unwrap()); + let mut byte_after_finish = 0; + loop { + let process_finish = is_process_finish(pid); + let timeout_read = + timeout(Duration::from_millis(100), reader.read(&mut buffer[..])).await; + + if timeout_read.is_err() { + if process_finish { + info!("read time out ,and process already finish,break"); + break; + } + continue; + } + + let read_size = timeout_read.unwrap(); + if read_size.is_err() { + error!("read output err:{} , pid:{}", read_size.unwrap_err(), pid); + break; + } + + let len = read_size.unwrap(); + if len > 0 { + if process_finish { + byte_after_finish = byte_after_finish + len + } + debug!( + "output:[{}], may_contain_binary:{}, pid:{}, len:{}", + String::from_utf8_lossy(&buffer[..len]), + String::from_utf8(Vec::from(&buffer[..len])).is_err(), + pid, + len + ); + // type convert + let mut new_out: Vec = Vec::from(&buffer[..len]); + self.append_output(&mut new_out); + if process_finish && len < BUF_SIZE { + info!("process finish and len < BUF_SIZE,break"); + break; + } + } else { + info!("read output finished normally, pid:{}", pid); + break; + } + if process_finish && byte_after_finish > PIPE_BUF_DEFAULT_SIZE { + info!("byte_after_finish > PIPE_BUF_DEFAULT_SIZE,break"); + break; + } + } + } + + pub fn kill_process_group(pid: u32) { + let pid = pid as i32; + unsafe { + // send SIGKILL to the process group of pid, note the -1 + // see more details at man 2 kill + libc::kill(pid * -1, 9); + } + } +} + +fn is_process_finish(pid: u32) -> bool { + let result = procinfo::pid::stat(pid as i32); + return match result { + Ok(stat) => { + if stat.state == procinfo::pid::State::Zombie { + true + } else { + false + } + } + Err(_) => true, + }; +} + +fn pre_exec_for_cmd(cmd: &mut Command, func_name: &str) { + let func = match func_name { + DUP2_1_2 => dup2_1_2, + OWN_PROCESS_GROUP => own_process_group, + _ => Err("").unwrap_or_exit( + format!("invalid func_name of pre_exec_for_cmd: {}", func_name).as_str(), + ), + }; + unsafe { + cmd.pre_exec(func); + } +} + +fn working_directory_exists(path: &str) -> bool { + return Path::new(path).exists(); +} + +fn working_directory_permission(dir: &str, username: &str) -> bool { + let ret = std::process::Command::new("sudo") + .args(&["-u", username, "sh", "-c", "cd", dir]) + .status(); + match ret { + Ok(status) => return status.success(), + Err(e) => { + error!( + "check working_directory permission err:{}, username:{}, dir: {}", + e, username, dir + ); + false + } + } +} + +fn user_exists(username: &str) -> bool { + if let None = get_user_by_name(username) { + return false; + } + true +} + +fn cmd_exists(cmd: &str) -> bool { + if let Ok(path) = env::var("PATH") { + for p in path.split(":") { + let p_str = format!("{}/{}", p, cmd); + if Path::new(&p_str).exists() { + return true; + } + } + } + false +} + +fn dup2_1_2() -> Result<(), io::Error> { + unsafe { + if libc::dup2(1, 2) != -1 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } + } +} + +fn own_process_group() -> Result<(), io::Error> { + unsafe { + if libc::setpgid(0, 0) == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_fmt_cmd() { + let cmd = ShellCommand::new("./a.sh", "root", "./", 60, 10240); + println!("fmt cmd:{:?}", cmd); + } + + #[test] + fn test_user_exists() { + assert_eq!(user_exists("root"), true); + assert_eq!(user_exists("hacker-neo"), false); + } + + #[test] + fn test_working_directory_exists() { + assert_eq!(working_directory_exists("/etc"), true); + assert_eq!(user_exists("/etcdefg"), false); + } + + #[test] + fn test_cmd_exists() { + assert_eq!(cmd_exists("pwd"), true); + assert_eq!(cmd_exists("pwd110"), false); + } +} diff --git a/src/http.rs b/src/http.rs new file mode 100644 index 0000000..8df7628 --- /dev/null +++ b/src/http.rs @@ -0,0 +1,9 @@ +mod invoke_adapter; +mod requester; + +pub mod store; +pub mod thread; + +pub use invoke_adapter::InvokeAPIAdapter; +pub use requester::HttpRequester; +pub use requester::Requester; diff --git a/src/http/invoke_adapter.rs b/src/http/invoke_adapter.rs new file mode 100644 index 0000000..dc873bf --- /dev/null +++ b/src/http/invoke_adapter.rs @@ -0,0 +1,144 @@ +// 用于封装访问HTTP API的方法 +use log::{error, info}; +use serde::{de::DeserializeOwned, Serialize}; +use serde_json::from_str; + + +use crate::common::consts::{HTTP_REQUEST_NO_RETRIES, HTTP_REQUEST_RETRIES}; +use crate::http::{HttpRequester, Requester}; +use crate::types::{ + AgentError, AgentErrorCode, AgentRequest, CheckUpdateRequest, CheckUpdateResponse, + DescribeTasksRequest, DescribeTasksResponse, HttpMethod, ReportTaskFinishRequest, + ReportTaskFinishResponse, ReportTaskStartRequest, ReportTaskStartResponse, ServerRawResponse, + UploadTaskLogRequest, UploadTaskLogResponse, +}; + +#[cfg_attr(test, faux::create)] +pub struct InvokeAPIAdapter { + requester: HttpRequester, +} + +#[cfg_attr(test, faux::methods)] +impl InvokeAPIAdapter { + pub fn build(url: &str) -> Self { + let mut req = HttpRequester::new(); + req.initialize(url); + InvokeAPIAdapter { requester: req } + } + + pub async fn describe_tasks(&self) -> Result> { + let req = DescribeTasksRequest {}; + self.send("DescribeTasks", req, HTTP_REQUEST_RETRIES).await + } + + pub async fn report_task_start( + &self, + invocation_task_id: &str, + start_timestamp: u64 + ) -> Result> { + + let req = ReportTaskStartRequest { + invocation_task_id: invocation_task_id.to_string(), + time_stamp: start_timestamp, + }; + self.send("ReportTaskStart", req, HTTP_REQUEST_RETRIES) + .await + } + + pub async fn report_task_finish( + &self, + invocation_task_id: &str, + result: &str, + err_info: &str, + exit_code: i32, + final_log_index: u32, + finish_timestamp: u64, + ) -> Result> { + let req = ReportTaskFinishRequest { + invocation_task_id: invocation_task_id.to_string(), + time_stamp: finish_timestamp, + result: result.to_string(), + error_info: err_info.to_string(), + exit_code, + final_log_index, + }; + self.send("ReportTaskFinish", req, HTTP_REQUEST_RETRIES) + .await + } + + pub async fn upload_task_log( + &self, + task_id: &str, + idx: u32, + output: Vec, + dropped: u64, + ) -> Result> { + let task_log = UploadTaskLogRequest::new(task_id, idx, output, dropped); + self.send("UploadTaskLog", task_log, HTTP_REQUEST_RETRIES) + .await + } + + pub async fn check_update(&self) -> Result> { + let body = CheckUpdateRequest::new(); + self.send("CheckUpdate", body, HTTP_REQUEST_NO_RETRIES) + .await + } + + // parse standard formatted response to custom type + async fn send( + &self, + action: &str, + request: T, + retries: u64, + ) -> Result> { + let body = AgentRequest::new(action, request); + let reqwest_resp_result = self + .requester + .with_time_out(10) + .with_retrying(retries) + .send_request::>(HttpMethod::POST, "/", Some(body)) + .await; + match reqwest_resp_result { + Ok(reqwest_resp) => { + let txt = match reqwest_resp.text().await { + Ok(txt) => txt, + Err(e) => { + error!("failed to read response {:?}", e); + return Err(AgentError::new( + AgentErrorCode::ResponseReadError, + &format!("failed to read response {:?}", e), + )); + } + }; + info!("response text {:?}", txt); + let raw_resp_result: Result, _> = from_str(&txt); + match raw_resp_result { + Ok(raw_resp) => match raw_resp.into_response() { + Ok(resp_content) => Ok(resp_content), + Err(resp_err) => { + let agent_err = AgentError::wrap( + AgentErrorCode::ResponseEmptyError, + "empty response content", + format!("response error {:?}", resp_err), + ); + error!("{:?}", agent_err); + return Err(agent_err); + } + }, + Err(e) => { + let agent_err = AgentError::new( + AgentErrorCode::JsonDecodeError, + &format!("failed to parse json response {:?}", e), + ); + error!("{:?}", agent_err); + return Err(agent_err); + } + } + } + Err(e) => { + error!("request error: {:?}", e); + Err(e) + } + } + } +} diff --git a/src/http/requester.rs b/src/http/requester.rs new file mode 100644 index 0000000..d57a099 --- /dev/null +++ b/src/http/requester.rs @@ -0,0 +1,187 @@ +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use async_std::task; +use log::{debug, error, info}; +use reqwest::{Client, ClientBuilder, header, Response}; +use serde::Serialize; +use serde_json::to_string; +use url::Url; + +use crate::common::asserts::GracefulUnwrap; +use crate::common::consts::{HTTP_REQUEST_RETRY_INTERVAL, HTTP_REQUEST_TIME_OUT, VIP_HEADER, VPCID_HEADER}; +use crate::common::envs; +use crate::types::{AgentError, AgentErrorCode, HttpMethod}; + +pub trait Requester: std::marker::Sized { + fn initialize(&mut self, url: &str) -> Option<&Self>; +} + +pub struct HttpRequester { + url: String, + time_out: AtomicU64, + retries: AtomicU64, + client: Option, +} + +impl Requester for HttpRequester { + fn initialize(&mut self, url: &str) -> Option<&Self> { + match Url::parse(url) { + Ok(_) => { + let cli_builder = ClientBuilder::new(); + // use for e2e test. + let mut headers = header::HeaderMap::new(); + if envs::enable_test() { + headers.insert(VPCID_HEADER, header::HeaderValue::from_str(&*envs::test_vpcid()).unwrap()); + headers.insert(VIP_HEADER, header::HeaderValue::from_str(&*envs::test_vip()).unwrap()); + } + + let cli = cli_builder + .default_headers(headers) + .pool_max_idle_per_host(1) + .build() + .unwrap_or_exit("fail to create http client"); + self.url = url.to_string(); + self.client = Option::from(cli); + Some(self) + } + Err(err) => { + error!("fail to create http client, error: {}", err); + None + } + } + } +} + +impl HttpRequester { + pub fn new() -> HttpRequester { + HttpRequester { + url: String::from(""), + client: Option::None, + time_out: AtomicU64::new(HTTP_REQUEST_TIME_OUT), + retries: AtomicU64::new(1), + } + } + + pub fn with_time_out(&self, time_out: u64) -> &Self { + self.time_out.store(time_out, Ordering::SeqCst); + self + } + + pub fn with_retrying(&self, retries: u64) -> &Self { + self.retries.store(retries, Ordering::SeqCst); + self + } + + pub async fn send_request( + &self, + method: HttpMethod, + path: &str, + body: Option, + ) -> Result> { + match method { + HttpMethod::POST => match body { + Some(b) => self.call_post(path, b).await, + None => Err(AgentError::new( + AgentErrorCode::RequestEmptyError, + "empty request body", + )), + }, + HttpMethod::GET => self.call_get(path).await, + } + } + + async fn call_get(&self, path: &str) -> Result> { + if self.client.is_some() { + let cli = self.client.as_ref().unwrap(); + let url = format!("{}{}", self.url, path); + info!("send request to :{}", url); + let time_out = self.time_out.load(Ordering::SeqCst); + let request_builder = cli + .get(&url) + .header(header::CONNECTION, "close") + .timeout(std::time::Duration::from_secs(time_out)); + let resp_res = request_builder.send().await; + match resp_res { + Ok(resp) => { + debug!("recv response: {:?}", resp); + Ok(resp) + } + Err(err) => { + let agent_err = AgentError::wrap( + AgentErrorCode::ResponseEmptyError, + &format!("request error: {}", err), + format!("{:?}", err), + ); + error!("{:?}", agent_err); + Err(agent_err) + } + } + } else { + Err(AgentError::new( + AgentErrorCode::ClientNotInitialized, + "empty client", + )) + } + } + + async fn call_post( + &self, + path: &str, + body: T, + ) -> Result> { + if self.client.is_some() { + let cli = self.client.as_ref().unwrap(); + let url = format!("{}{}", self.url, path); + info!( + "send request to {}, request body: {}", + url, + to_string(&body).unwrap() + ); + let time_out = self.time_out.load(Ordering::SeqCst); + let mut have_retries = 0; + let max_retries = self.retries.load(Ordering::SeqCst); + while have_retries < max_retries { + have_retries += 1; + let request_builder = cli + .post(&url) + .header(header::CONNECTION, "close") + .timeout(std::time::Duration::from_secs(time_out)); + let resp_res = request_builder.body(to_string(&body).unwrap()).send().await; + let res = match resp_res { + Ok(resp) => { + debug!("recv response: {:?}", resp); + Ok(resp) + } + Err(err) => { + let agent_err = AgentError::wrap( + AgentErrorCode::ResponseEmptyError, + &format!("request error: {}", err), + format!("{:?}", err), + ); + error!("{:?}", agent_err); + Err(agent_err) + } + }; + if res.is_ok() { + return res; + } else { + debug!( + "have tried for {} times, max retry {} times", + have_retries, max_retries + ); + task::sleep(Duration::from_secs(HTTP_REQUEST_RETRY_INTERVAL)).await; + } + } + return Err(AgentError::new( + AgentErrorCode::MaxRetryFailures, + "retry times exceeded", + )); + } else { + return Err(AgentError::new( + AgentErrorCode::ClientNotInitialized, + "empty client", + )); + } + } +} diff --git a/src/http/store.rs b/src/http/store.rs new file mode 100644 index 0000000..9bf20c5 --- /dev/null +++ b/src/http/store.rs @@ -0,0 +1,240 @@ +use log::{error, info}; +use std::fs::{create_dir_all, remove_file, File}; +use std::io::prelude::*; +use std::path::{Path, PathBuf}; + +use chrono::{DateTime, Local}; +use rand::distributions::Alphanumeric; +use rand::{thread_rng, Rng}; + +use crate::common::consts; +#[cfg(test)] +use crate::common::logger; +use crate::types::InvocationNormalTask; + +cfg_if::cfg_if! { + if #[cfg(unix)] { + use std::fs::{set_permissions, Permissions}; + use std::os::unix::fs::PermissionsExt; + use std::io; + use crate::common::asserts::GracefulUnwrap; + } +} + +pub struct TaskFileStore { + store_path: PathBuf, + prefix: String, +} + +impl TaskFileStore { + pub fn new() -> TaskFileStore { + let t = TaskFileStore { + store_path: Path::new(consts::TASK_STORE_PATH).to_path_buf(), + prefix: String::from(consts::TASK_STORE_PREFIX), + }; + t + } + + pub fn get_store_path(&self) -> PathBuf { + self.store_path.clone() + } + + fn get_suffix(&self, command_type: &String) -> &str { + return match command_type.as_str() { + consts::CMD_TYPE_SHELL => consts::SUFFIX_SHELL, + consts::CMD_TYPE_BAT => consts::SUFFIX_BAT, + consts::CMD_TYPE_POWERSHELL => consts::SUFFIX_PS1, + _ => consts::SUFFIX_SHELL, + }; + } + + fn get_task_file_path(&self, t: &InvocationNormalTask) -> PathBuf { + // use YYYYmm as task directory name + // use random string as postfix + let now: DateTime = Local::now(); + let rand_str: String = thread_rng().sample_iter(&Alphanumeric).take(10).collect(); + + let suffix = self.get_suffix(&t.command_type); + let file_name = format!( + "{}_{}_{}{}", + self.prefix, t.invocation_task_id, rand_str, suffix + ); + + let file_path = self + .get_store_path() + .join(format!("{}", now.format("%Y%m"))) + .join(file_name); + file_path + } + + fn create_file( + &self, + path: &str, + ignore_exists: bool, + executable: bool, + ) -> Result { + let file_path = Path::new(path); + if file_path.exists() { + if ignore_exists { + match remove_file(path) { + Err(e) => { + info!("failed to remove exist file {}: {}", path, e); + return Err(format!("failed to remove exist file {}: {}", path, e)); + } + _ => {} + } + } else { + return Err(format!("file {} already exists", path)); + } + } + let dir = match Path::parent(file_path) { + Some(p) => p, + None => return Err(format!("cannot find parent directory for {:?}", file_path)), + }; + let ret = match File::create(path) { + Err(why) => { + info!( + "couldn't create file {}, try to create directory {:?}", + why, dir + ); + match create_dir_all(dir) { + Err(why) => Err(format!("couldn't create directory: {}", why)), + Ok(_) => match File::create(path) { + Err(why) => Err(format!("couldn't create file: {}", why)), + Ok(file) => Ok(file), + }, + } + } + Ok(file) => Ok(file), + }; + + #[cfg(unix)] + if executable { + // set permissions for path recursively, to make task-xxx.sh available for non-root user. + match self.set_permissions_recursively(path.as_ref()) { + Err(e) => { + info!("failed to chmod path recursively {}: {}", path, e); + return Err(format!("failed to chmod path recursively {}: {}", path, e)); + } + _ => {} + }; + } + + ret + } + + #[cfg(unix)] + fn set_permissions_recursively(&self, path: &Path) -> io::Result<()> { + let mut path = path.clone(); + while path.to_str() != Some("/tmp") { + match set_permissions( + path, + Permissions::from_mode(consts::FILE_EXECUTE_PERMISSION_MODE), + ) { + Err(e) => { + info!("failed to chmod path {:?}: {}", path, e); + return Err(e); + } + _ => match path.parent() { + Some(parent) => path = parent, + None => Err("").unwrap_or_exit("should never come here"), + }, + }; + } + Ok(()) + } + + pub fn store(&self, t: &InvocationNormalTask) -> Option { + let path: &str = &self.get_task_file_path(t).display().to_string(); + info!("save task {} to {}", &t.invocation_task_id, path); + + let mut file = match self.create_file(path, true, true) { + Ok(file) => file, + Err(e) => { + error!("cannot create file {:?} error {:?}", path, e); + return None; + } + }; + + match t.decode_command() { + Ok(s) => { + let task_str = &s; + match file.write_all(task_str.as_bytes()) { + Err(why) => { + error!("couldn't write {}", why); + None + } + Ok(_) => Some(path.to_string()), + } + } + Err(e) => { + info!( + "task {} command decode failed {:?}", + &t.invocation_task_id, e + ); + None + } + } + } + + #[cfg(test)] + pub fn remove(&self, path: &str) { + match remove_file(path) { + Ok(_) => info!("remove task from {}", path), + Err(why) => panic!("couldn't remove task: {}", why), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::{read_dir, read_to_string, remove_dir, remove_file}; + + #[test] + fn test_store_task() { + logger::init_test_log(); + + #[cfg(unix)] + let workdir = format!("/root/"); + #[cfg(windows)] + let workdir = format!("C:\\Program Files\\qcloud\\tat_agent"); + let task = InvocationNormalTask { + invocation_task_id: "100001".to_string(), + command_type: format!("SHELL"), + time_out: 30, + command: format!("bHMgLWw="), + username: format!("root"), + working_directory: workdir, + }; + + let store = TaskFileStore::new(); + #[cfg(unix)] + let desired_path = format!("/tmp/tat_agent/commands/{}", Local::now().format("%Y%m")); + #[cfg(windows)] + let desired_path = format!( + "C:\\Program Files\\qcloud\\tat_agent\\tmp\\commands\\{}", + Local::now().format("%Y%m") + ); + assert_eq!( + store + .get_task_file_path(&task) + .as_path() + .parent() + .unwrap() + .display() + .to_string(), + desired_path + ); + + let path = store.store(&task).unwrap(); + let contents = read_to_string(&path).unwrap(); + assert_eq!(contents, "ls -l"); + store.remove(&path); + let paths = read_dir(Path::new(&path).parent().unwrap()).unwrap(); + for f in paths { + remove_file(f.unwrap().path()).unwrap(); + } + remove_dir(Path::new(&path).parent().unwrap()).unwrap(); + } +} diff --git a/src/http/thread.rs b/src/http/thread.rs new file mode 100644 index 0000000..65e8ac7 --- /dev/null +++ b/src/http/thread.rs @@ -0,0 +1,572 @@ +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::sync::{mpsc::Receiver, mpsc::TryRecvError}; +use std::thread; +use std::thread::JoinHandle; +use std::time::Duration; +use std::time::SystemTime; + +use async_std::task; +use log::{debug, error, info}; +use tokio::runtime::Builder; +use tokio::sync::Mutex; + +use crate::common::asserts::GracefulUnwrap; +use crate::common::consts::INVOKE_API; +use crate::common::consts::{DEFAULT_OUTPUT_BYTE, FINISH_RESULT_TERMINATED}; +use crate::executor::proc; +use crate::executor::proc::MyCommand; +use crate::http::store::TaskFileStore; +use crate::http::InvokeAPIAdapter; +use crate::types::inner_msg::KickMsg; +use crate::types::{InvocationCancelTask, InvocationNormalTask}; + +// 实现http线程的启动,内部使用异步runtime +pub fn run(msg_receiver: Receiver, running_task_num: Arc) -> JoinHandle<()> { + let adapter = InvokeAPIAdapter::build(INVOKE_API); + let worker = Arc::new(HttpWorker::new(adapter, running_task_num)); + let thread_handle = thread::spawn(move || { + let rt_res = Builder::new().basic_scheduler().enable_all().build(); + match rt_res { + Ok(mut rt) => loop { + match msg_receiver.try_recv() { + Ok(msg) => { + let worker = worker.clone(); + rt.spawn(async move { worker.process(msg).await }); + } + Err(e) => match e { + TryRecvError::Empty => { + rt.block_on(async { + task::sleep(Duration::from_millis(100)).await; + }); + debug!("http thread channel empty, async await"); + } + TryRecvError::Disconnected => { + error!("http thread channel disconnected, break"); + break; + } + }, + }; + }, + Err(e) => { + error!("http thread runtime error: {}", e); + } + } + }); + thread_handle +} + +pub struct HttpWorker { + adapter: InvokeAPIAdapter, + task_store: TaskFileStore, + running_tasks: Mutex>>>>, + running_task_num: Arc, +} + +impl HttpWorker { + pub fn new(adapter: InvokeAPIAdapter, running_task_num: Arc) -> Self { + let task_store = TaskFileStore::new(); + info!( + "http worker create success, save tasks to {}", + task_store.get_store_path().as_path().display().to_string() + ); + HttpWorker { + adapter, + task_store, + running_task_num, + running_tasks: Mutex::new(HashMap::new()), + } + } + + pub async fn process(&self, msg: KickMsg) -> Option { + info!("http thread processing message from: {}", msg.kick_source); + match self.adapter.describe_tasks().await { + Ok(resp) => { + info!("describe task success: {:?}", resp); + for task in resp.invocation_normal_task_set.iter() { + match self.task_store.store(&task) { + Some(task_file) => { + // if task is not in command cache, execute it + self.task_execute(task, &task_file).await; + } + None => { + // script file store failed, reuse this flow to report start failed + self.task_execute(task, "").await; + } + } + } + for task in resp.invocation_cancel_task_set.iter() { + self.task_cancel(&task).await; + } + Some(resp.invocation_normal_task_set.len() as u64) + } + Err(why) => { + error!("describe task failed: {:?}", why); + None + } + } + } + + async fn read_and_report( + &self, + cmd_arc: Arc>>, + task_id: &str, + ) -> u32 { + let mut stop_upload = false; + let mut final_log_index: u32 = 0; + let mut first_dropped: u64 = 0; + let mut finished = cmd_arc.lock().await.is_finished(); + loop { + // total sleep max 20 * 50 ms, i.e. 1s + for _n in 0..20 { + if finished { + break; + } else { + task::sleep(Duration::from_millis(50)).await; + finished = cmd_arc.lock().await.is_finished(); + } + } + let mut cmd = cmd_arc.lock().await; + if cmd.cur_output_len() != 0 && !stop_upload { + let (out, idx, dropped) = cmd.next_output(); + final_log_index = idx; + // print output in one line + info!( + "ready to report output length:{:?}, idx:{}, dropped:{}, output_debug:{}", + out.len(), + idx, + dropped, + String::from_utf8_lossy(&out[..]).replace("\n", "\\n"), + ); + // output report task here + self.upload_task_log(task_id, idx, out, dropped).await; + + if dropped > 0 { + stop_upload = true; + first_dropped = dropped; + } + } + if stop_upload { + // Do not report output any more in next loop, because of max report exceed + info!( + "task {}: max log size exceeds, command still running but not report output anymore, only report final dropped bytes when task finished.", + task_id + ); + } + if finished { + // report final dropped bytes of output when task is already finished and max report exceed. + // otherwise check finish flag again after a while + info!("task {} finished.", task_id); + + if stop_upload { + let (out, idx, dropped) = cmd.next_output(); + // dropped > first_dropped means more output generated after first drop occur, + // so need update final idx and drop when cmd finished. + if dropped > first_dropped { + final_log_index = idx; + info!( + "ready to report output dropped length:idx:{}, dropped:{}, output_debug:{}", + idx, + dropped, + String::from_utf8_lossy(&out[..]).replace("\n", "\\n"), + ); + // final dropped bytes report task here + self.upload_task_log(task_id, idx, out, dropped).await; + info!("report final dropped bytes of output."); + } + } + break; + } + } + final_log_index + } + + // report task start + // start Command to execute task + // upload task log + // report task finish + async fn task_execute(&self, task: &InvocationNormalTask, task_file: &str) { + info!("task execute begin: {:?}", task); + let task_id = task.invocation_task_id.clone(); + let result = self.create_proc(task_file, task).await; + if result.is_none() { + return; + } + self.running_task_num.fetch_add(1, Ordering::SeqCst); + let cmd_arc = result.unwrap(); + + let mut final_log_index = 0; + if cmd_arc.lock().await.is_started() { + final_log_index = self.read_and_report(cmd_arc.clone(), &task_id).await; + } + + let cmd = cmd_arc.lock().await; + //report finish + let finish_result = cmd.finish_result(); + let err_info = cmd.err_info(); + let exit_code = cmd.exit_code(); + let finish_time = cmd.finish_time(); + self.adapter + .report_task_finish( + &task_id, + &finish_result, + &err_info, + exit_code, + final_log_index, + finish_time, + ) + .await + .map(|_| info!("task_execute report_task_finish {} success", task_id)) + .map_err(|e| { + error!( + "report task {} finish error: {:?}", + &task.invocation_task_id, e + ); + }) + .ok(); + self.running_task_num.fetch_sub(1, Ordering::SeqCst); + // process finish ,remove + self.running_tasks + .lock() + .await + .remove(task.invocation_task_id.as_str()); + } + + async fn task_cancel(&self, task: &InvocationCancelTask) { + info!("=>task_cancel"); + let cancel_task_id = task.invocation_task_id.clone(); + //mutex with create_proc + let tasks = self.running_tasks.lock().await; + let task = tasks.get(cancel_task_id.as_str()); + if let Some(cmd_arc) = task { + let cmd = cmd_arc.lock().await; + cmd.cancel() + .map(|_| { + info!("cancel task {} success", &cancel_task_id); + }) + .map_err(|e| { + error!("task {} cancel fail, error: {}", &cancel_task_id, e); + }) + .ok(); + return; + } else { + info!( + "task {} not find, may be not start or finished", + &cancel_task_id + ); + } + + //report terminated + let finish_time = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_exit("sys time may before 1970") + .as_secs(); + self.adapter + .report_task_finish( + &cancel_task_id, + FINISH_RESULT_TERMINATED, + "", + 0, + 0, + finish_time, + ) + .await + .map(|_| info!("task_cancel report_task_finish {} success", cancel_task_id)) + .map_err(|e| { + error!("report task {} terminate error: {:?}", cancel_task_id, e); + }) + .ok(); + } + + async fn report_task_start(&self, task_id: &str, timestamp: u64) -> bool { + return match self.adapter.report_task_start(task_id, timestamp).await { + Err(e) => { + error!("report start error: {:?}", e); + false + } + Ok(_) => true, + }; + } + + async fn upload_task_log(&self, task_id: &str, idx: u32, out: Vec, dropped: u64) { + match self + .adapter + .upload_task_log(task_id, idx, out, dropped) + .await + { + Ok(_) => { + info!("success upload task {} log index: {}", task_id, idx); + } + Err(e) => { + error!("fail to upload task {} log: {:?}", task_id, e); + } + } + } + + async fn create_proc( + &self, + task_file: &str, + task: &InvocationNormalTask, + ) -> Option>>> { + let task_id = task.invocation_task_id.clone(); + //mutex with task_cancel + let mut tasks = self.running_tasks.lock().await; + if tasks.contains_key(&task_id) { + info!("fetch duplicate task,task id {}", task_id); + return None; + } + + let start_timestamp = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_exit("sys time may before 1970") + .as_secs(); + + let result = self + .report_task_start(task.invocation_task_id.as_str(), start_timestamp) + .await; + if !result { + return None; + } + + let proc_result = proc::new( + task_file, + &task.username, + &task.command_type, + &task.working_directory, + task.time_out, + DEFAULT_OUTPUT_BYTE, + ); + if proc_result.is_err() { + return None; + } + let mut proc_res = proc_result.unwrap(); + proc_res + .run() + .await + .map_err(|_| error!("start process fail")) + .ok(); + let cmd_arc = Arc::new(Mutex::new(proc_res)); + + tasks.insert(task_id, cmd_arc.clone()); + return Some(cmd_arc); + } +} + +#[cfg(test)] +mod tests { + use crate::common::consts::FINISH_RESULT_TERMINATED; + use crate::common::logger; + use crate::executor::proc; + use crate::executor::proc::MyCommand; + use crate::http::store::TaskFileStore; + use crate::http::thread::HttpWorker; + use crate::http::InvokeAPIAdapter; + use crate::types::{ + AgentError, AgentErrorCode, InvocationCancelTask, InvocationNormalTask, + ReportTaskFinishResponse, ReportTaskStartResponse, + }; + use rand::distributions::Alphanumeric; + use rand::{thread_rng, Rng}; + use std::fs; + use std::fs::File; + use std::io::Write; + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Mutex; + use tokio::time::timeout; + + fn gen_rand_str() -> String { + thread_rng().sample_iter(&Alphanumeric).take(10).collect() + } + + fn create_file(content: &str, filename: &str) { + File::create(filename) + .unwrap() + .write_all(content.as_bytes()) + .unwrap(); + #[cfg(unix)] + std::process::Command::new("chmod") + .args(&["+x", filename]) + .status() + .unwrap(); + } + + fn get_http_worker() -> HttpWorker { + HttpWorker { + adapter: InvokeAPIAdapter::faux(), + task_store: TaskFileStore::new(), + running_tasks: Default::default(), + running_task_num: Arc::new(Default::default()), + } + } + + fn build_invocation( + task_id: &str, + cmd: &str, + time_out: u64, + cmd_type: &str, + ) -> InvocationNormalTask { + InvocationNormalTask { + invocation_task_id: task_id.to_string(), + time_out, + command: cmd.to_string(), + command_type: cmd_type.to_string(), + username: "root".to_string(), + working_directory: "./".to_string(), + } + } + + fn fake_command() -> Arc>> { + #[cfg(unix)] + let cmd_type = "SHELL"; + #[cfg(windows)] + let cmd_type = "POWERSHELL"; + let result = proc::new("", "", cmd_type.as_ref(), "", 10, 1024); + Arc::new(Mutex::new(result.unwrap())) + } + + #[tokio::test] + async fn test_report_dup_task() { + let http_worker = get_http_worker(); + http_worker + .running_tasks + .lock() + .await + .insert("invt-1111".to_string(), fake_command()); + + let task = build_invocation("invt-1111", "", 5, "POWERSHELL"); + let result = http_worker.create_proc("/fake_path", &task).await; + assert_eq!(result.is_none(), true); + } + + #[tokio::test] + async fn test_report_start_fail() { + #[cfg(unix)] + let cmd_type = "SHELL"; + #[cfg(windows)] + let cmd_type = "POWERSHELL"; + + let mut http_worker = get_http_worker(); + http_worker + .running_tasks + .lock() + .await + .insert("invt-1122".to_string(), fake_command()); + let task = build_invocation("invt-1111", "", 5, cmd_type.as_ref()); + faux::when!(http_worker.adapter.report_task_start) + .then_return(Err(AgentError::new(AgentErrorCode::ResponseReadError, ""))); + let result = http_worker.create_proc("/fake_path", &task).await; + assert_eq!(result.is_none(), true); + } + + #[tokio::test] + async fn test_report_start_sucess() { + cfg_if::cfg_if! { + if #[cfg(unix)] { + let filename = format!("./.{}.sh", gen_rand_str()); + let cmd_type = "SHELL"; + create_file("sleep 1", filename.as_str()); + } else if #[cfg(windows)] { + let filename = format!("./{}.ps1", gen_rand_str()); + let cmd_type = "POWERSHELL"; + create_file( + "Start-Sleep -s 1", + filename.as_str(), + ); + } + } + let mut http_worker = get_http_worker(); + let task = build_invocation("invt-1133", "", 5, cmd_type); + faux::when!(http_worker.adapter.report_task_start) + .then_return(Ok(ReportTaskStartResponse {})); + + let result = http_worker.create_proc(filename.as_str(), &task).await; + fs::remove_file(filename.as_str()).unwrap(); + assert_eq!(result.is_some(), true); + assert_eq!( + http_worker + .running_tasks + .lock() + .await + .get("invt-1133") + .is_some(), + true + ); + } + + #[tokio::test] + async fn test_create_mutex() { + let http_worker = get_http_worker(); + let _lock = http_worker.running_tasks.lock().await; + #[cfg(unix)] + let cmd_type = "SHELL"; + #[cfg(windows)] + let cmd_type = "POWERSHELL"; + let task = build_invocation("invt-1133", "", 5, cmd_type); + let create_fut = http_worker.create_proc("", &task); + let time_out = timeout(Duration::from_secs(1), create_fut).await; + assert_eq!(time_out.is_err(), true); + } + + #[tokio::test] + async fn test_cancel_mutex() { + let http_worker = get_http_worker(); + let _lock = http_worker.running_tasks.lock().await; + let task = InvocationCancelTask { + invocation_task_id: "inv-xxxx".to_string(), + }; + let create_fut = http_worker.task_cancel(&task); + let time_out = timeout(Duration::from_secs(1), create_fut).await; + assert_eq!(time_out.is_err(), true); + } + + #[tokio::test] + async fn test_cancel() { + logger::init_test_log(); + cfg_if::cfg_if! { + if #[cfg(unix)] { + let filename = format!("./.{}.sh", gen_rand_str()); + let cmd_type = "SHELL"; + create_file("sleep 1024", filename.as_str()); + } else if #[cfg(windows)] { + let filename = format!("./{}.ps1", gen_rand_str()); + let cmd_type = "POWERSHELL"; + create_file( + "Start-Sleep -s 1024;", + filename.as_str(), + ); + } + } + let mut http_worker = get_http_worker(); + let task = build_invocation("invt-test_cancel", "", 1025, cmd_type); + faux::when!(http_worker.adapter.report_task_start) + .then_return(Ok(ReportTaskStartResponse {})); + + faux::when!(http_worker.adapter.report_task_finish) + .then_return(Ok(ReportTaskFinishResponse {})); + + let cmd = http_worker + .create_proc(filename.as_str(), &task) + .await + .unwrap(); + assert_eq!(cmd.lock().await.is_started(), true); + let http_worker1 = Arc::new(http_worker); + let http_worker2 = http_worker1.clone(); + let cmd_clone = cmd.clone(); + tokio::spawn(async move { + http_worker1 + .read_and_report(cmd_clone, &task.invocation_task_id) + .await; + }); + + let task = InvocationCancelTask { + invocation_task_id: "invt-test_cancel".to_string(), + }; + http_worker2.task_cancel(&task).await; + tokio::time::delay_for(Duration::from_secs(1)).await; + fs::remove_file(filename.as_str()).unwrap(); + let finis_result = cmd.lock().await.finish_result(); + assert_eq!(finis_result, FINISH_RESULT_TERMINATED); //need read twice + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..ef030c6 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,8 @@ +pub mod common; +pub mod http; +pub mod ontime; +pub mod types; +pub mod ws; +pub mod executor; +pub mod uname; +pub mod daemonizer; diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..0782b7a --- /dev/null +++ b/src/main.rs @@ -0,0 +1,61 @@ +use log::info; +use std::sync::atomic::AtomicU64; +use std::sync::mpsc::channel; +use std::sync::Arc; + +mod common; +mod executor; +mod http; +mod ontime; +mod types; +mod ws; +mod uname; +mod daemonizer; + +use crate::common::asserts::GracefulUnwrap; +use crate::common::consts::AGENT_VERSION; +use crate::common::logger; +use crate::common::Opts; +use crate::http::thread as http_thread; +use crate::ontime::thread as ontime_thread; +use crate::ontime::timer::Timer; +use crate::ws::thread as ws_thread; + +#[tokio::main] +async fn main() { + + let _opts = Opts::get_opts(); + daemonizer::daemonize(||{ + // log init after daemonized, so log dir will at same dir of agent + logger::init(); + info!("agent version:[{}]", AGENT_VERSION); + Timer::get_instance(); + + // code of thread communication + let (ws_kick_sender, kick_receiver) = channel(); + let ontime_kick_sender = ws_kick_sender.clone(); + + let (ping_channel_sender, + ping_channel_receiver) = channel(); + + let running_task_num = Arc::new(AtomicU64::new(0)); + + // ontime thread send ping request + let _ot_thread = ontime_thread::run( + ping_channel_receiver, + ontime_kick_sender, + running_task_num.clone(), + ); + // + // // http thread recv the notify + let _h_thread = http_thread::run(kick_receiver, running_task_num.clone()); + + loop { + info!("now spawn a new ws connection"); + let s1 = ws_kick_sender.clone(); + let s2 = ping_channel_sender.clone(); + let ws_thread = ws_thread::run(s1, s2); + ws_thread.join().or_log("ws thread joined"); + } + }); +} diff --git a/src/ontime.rs b/src/ontime.rs new file mode 100644 index 0000000..ba35f74 --- /dev/null +++ b/src/ontime.rs @@ -0,0 +1,3 @@ +pub mod self_update; +pub mod thread; +pub mod timer; diff --git a/src/ontime/self_update.rs b/src/ontime/self_update.rs new file mode 100644 index 0000000..aa37e90 --- /dev/null +++ b/src/ontime/self_update.rs @@ -0,0 +1,345 @@ +use std::fs::{create_dir_all, File}; +use std::io; +use std::io::Write; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; +#[cfg(unix)] +use std::fs::{set_permissions,Permissions}; + +#[cfg(windows)] +use crate::daemonizer::wow64_disable_exc; +use std::process::Command; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use bytes::Bytes; +use log::debug; +use log::error; +use log::info; +use log::warn; +use tokio::runtime::Builder; +use tokio::runtime::Runtime; +use unzip::{Unzipper, UnzipperStats}; + +use crate::common::consts::{ + AGENT_FILENAME, INVOKE_API, SELF_UPDATE_FILENAME, + SELF_UPDATE_PATH, SELF_UPDATE_SCRIPT, UPDATE_DOWNLOAD_TIMEOUT, UPDATE_FILE_UNZIP_DIR, +}; +#[cfg(unix)] +use crate::common::consts::{FILE_EXECUTE_PERMISSION_MODE}; +use crate::http::{HttpRequester, InvokeAPIAdapter, Requester}; +use crate::types::{AgentError, CheckUpdateResponse, HttpMethod}; + +pub fn try_update(self_updating: Arc, need_restart: Arc) { + let rt_res = Builder::new().basic_scheduler().enable_all().build(); + if let Err(e) = rt_res { + warn!( + "runtime for try update build fail:{:?}, will retry later", + e + ); + self_updating.store(false, Ordering::SeqCst); + return; + } + let mut rt = rt_res.unwrap(); + + let adapter = InvokeAPIAdapter::build(INVOKE_API); + + let check_update_rsp = check_update(&mut rt, &adapter); + if let Err(e) = check_update_rsp { + warn!("check update http request fail:{:?}", e); + self_updating.store(false, Ordering::SeqCst); + return; + } + let check_update_rsp = check_update_rsp.unwrap(); + + if check_update_rsp.need_update() == false { + info!("check update ret need_update:false, no newer version to update now"); + self_updating.store(false, Ordering::SeqCst); + return; + } + + if check_update_rsp.download_url().is_none() || check_update_rsp.md5().is_none() { + warn!( + "check update rsp invalid no url or md5:{:?}", + check_update_rsp + ); + self_updating.store(false, Ordering::SeqCst); + return; + } + + info!( + "new agent version found:{:?}, going to download", + check_update_rsp + ); + + let download_ret = download_file( + &mut rt, + check_update_rsp.download_url().clone().unwrap(), + SELF_UPDATE_PATH.to_string(), + SELF_UPDATE_FILENAME.to_string(), + ); + if let Err(e) = download_ret { + error!("download new agent fail:{}", e); + self_updating.store(false, Ordering::SeqCst); + return; + } + let download_content = download_ret.unwrap(); + + let md5_check_pass = md5_check(&download_content, check_update_rsp.md5().clone().unwrap()); + if md5_check_pass { + info!("download file md5 matched with remote"); + } else { + warn!("download file md5 mismatch with remote, ignore this update"); + self_updating.store(false, Ordering::SeqCst); + return; + } + + let unzip_ret = unzip_file( + SELF_UPDATE_PATH.to_string(), + SELF_UPDATE_FILENAME.to_string(), + UPDATE_FILE_UNZIP_DIR.to_string(), + ); + if let Err(e) = unzip_ret { + warn!("self update file unzip fail:{}, ignore this update", e); + self_updating.store(false, Ordering::SeqCst); + return; + } + info!("self update file unzip success"); + + let add_execute_ret = batch_set_execute_permission( + SELF_UPDATE_PATH.to_string(), + UPDATE_FILE_UNZIP_DIR.to_string(), + SELF_UPDATE_SCRIPT.to_string(), + AGENT_FILENAME.to_string(), + ); + if let Err(e) = add_execute_ret { + warn!( + "set execute permission for self update file fail:{}, ignore this update", + e + ); + self_updating.store(false, Ordering::SeqCst); + return; + } + info!("self update script set execute permission success"); + + let try_run_ret = try_run_agent( + SELF_UPDATE_PATH.to_string(), + UPDATE_FILE_UNZIP_DIR.to_string(), + AGENT_FILENAME.to_string(), + ); + if let Err(e) = try_run_ret { + warn!("try run agent fail:{}, ignore this update", e); + self_updating.store(false, Ordering::SeqCst); + return; + } + info!( + "try run agent --version succ ret:'{}'", + try_run_ret.unwrap() + ); + + let update_script_ret = run_self_update_script( + SELF_UPDATE_PATH.to_string(), + UPDATE_FILE_UNZIP_DIR.to_string(), + SELF_UPDATE_SCRIPT.to_string(), + ); + if let Err(e) = update_script_ret { + warn!("run self update script fail:{}", e); + self_updating.store(false, Ordering::SeqCst); + return; + } + info!("agent self update script run success, will restart later gracefully"); + need_restart.store(true, Ordering::SeqCst); +} + +fn check_update( + rt: &mut Runtime, + adapter: &InvokeAPIAdapter, +) -> Result> { + let req = adapter.check_update(); + let rsp = rt.block_on(req); + rsp +} + +fn download_file( + rt: &mut Runtime, + url: String, + path: String, + filename: String, +) -> Result { + if let Err(e) = create_dir_all(path.clone()) { + let s = format!("path:{} create fail because:{}", path, e); + return Err(s); + } + let filepath = format!("{}/{}", path, filename); + let file = File::create(filepath); + if let Err(e) = file { + let s = format!("download file create fail:{}", e); + return Err(s); + } + let mut file = file.unwrap(); + + let mut req = HttpRequester::new(); + req.with_time_out(UPDATE_DOWNLOAD_TIMEOUT); + let init_ret = req.initialize(url.as_str()); + if let None = init_ret { + let s = "http init fail, maybe url invalid".to_string(); + return Err(s); + } + + let req = req.send_request::(HttpMethod::GET, "", None); + let rsp = rt.block_on(req); + if let Err(e) = rsp { + let s = format!("self update download fail:{:?}", e); + return Err(s); + } + let rsp = rsp.unwrap(); + + let bytes = rt.block_on(rsp.bytes()); + if let Err(e) = bytes { + let s = format!("self update download bytes ret fail:{}", e); + return Err(s); + } + let bytes = bytes.unwrap(); + + let write_ret = file.write_all(bytes.as_ref()); + if let Err(e) = write_ret { + let s = format!("self update file write fail:{}", e); + return Err(s); + } + + let sync_ret = file.sync_all(); + if let Err(e) = sync_ret { + let s = format!("self update file sync disk fail:{}", e); + return Err(s); + } + + info!("self update download success"); + Ok(bytes) +} + +fn md5_check(download_content: &Bytes, md5: String) -> bool { + let digest = md5::compute(download_content); + let digest = format!("{:x}", digest); + debug!("download file md5:{}, remote md5:{}", digest, md5); + digest.eq_ignore_ascii_case(md5.as_str()) +} + +fn unzip_file( + path: String, + zip_filename: String, + unzip_dir: String, +) -> Result { + let zip_file = format!("{}/{}", path, zip_filename); + let file = File::open(zip_file)?; + let abs_unzip_dir = format!("{}/{}", path, unzip_dir); + let unzipper = Unzipper::new(file, abs_unzip_dir); + unzipper.unzip() +} + +#[cfg(unix)] +fn batch_set_execute_permission( + path: String, + unzip_dir: String, + script_filename: String, + agent_filename: String, +) -> io::Result<()> { + let script = format!("{}/{}/{}", path, unzip_dir, script_filename); + set_execute_permission(script)?; + + let agent = format!("{}/{}/{}", path, unzip_dir, agent_filename); + set_execute_permission(agent) +} + +#[cfg(windows)] +// set permission is no needed for windows. +fn batch_set_execute_permission( + _path: String, + _unzip_dir: String, + _script_filename: String, + _agent_filename: String, +) -> io::Result<()> { + Ok(()) +} + +#[cfg(unix)] +fn set_execute_permission(abs_filename: String) -> io::Result<()> { + set_permissions( + abs_filename, + Permissions::from_mode(FILE_EXECUTE_PERMISSION_MODE), + ) +} + +fn try_run_agent( + path: String, + unzip_dir: String, + agent_filename: String, +) -> Result { + let agent = format!("{}/{}/{}", path, unzip_dir, agent_filename); + let cmd = Command::new(agent).arg("--version").output(); + if let Err(e) = cmd { + return Err(format!("agent run ret: {:?}", e)); + } + let out = cmd.unwrap(); + + let out = String::from_utf8(out.stdout); + + if let Err(e) = out { + return Err(format!( + "version output contains non-utf8 char:{:?}, invalid agent", + e + )); + } + let out = out.unwrap(); + + let out_v: Vec<&str> = out.split(' ').collect(); + if out_v.len() != 2 || out_v[0] != AGENT_FILENAME { + return Err(format!("version output invalid:{}", out)); + } + Ok(out) +} + +fn run_self_update_script( + path: String, + unzip_dir: String, + script_filename: String, +) -> Result<(), String> { + let script = format!("{}/{}/{}", path, unzip_dir, script_filename); + #[cfg(unix)] + let cmd = Command::new("sh").arg("-c").arg(script).output(); + #[cfg(windows)] + let cmd = wow64_disable_exc(move ||{ + Command::new(script.clone()).output()}); + if let Err(e) = cmd { + return Err(format!("self update run ret: {:?}", e)); + } + let out = cmd.unwrap(); + let stdout = String::from_utf8_lossy(out.stdout.as_slice()); + let stderr = String::from_utf8_lossy(out.stderr.as_slice()); + debug!("stdout of self update script:[{}]", stdout); + debug!("stderr of self update script:[{}]", stderr); + + if out.status.success() { + Ok(()) + } else { + Err(format!("ret code:{:?}", out.status.code())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use crate::common::logger::init_test_log; + + #[test] + fn test_self_update() { + init_test_log(); + let updating = Arc::new(AtomicBool::new(true)); + let need_restart = Arc::new(AtomicBool::new(false)); + try_update(updating.clone(), need_restart.clone()); + let updating = updating.load(Ordering::SeqCst); + assert!(updating); + let need_restart = need_restart.load(Ordering::SeqCst); + assert!(need_restart); + } +} diff --git a/src/ontime/thread.rs b/src/ontime/thread.rs new file mode 100644 index 0000000..79324b1 --- /dev/null +++ b/src/ontime/thread.rs @@ -0,0 +1,231 @@ +use log::debug; +use log::info; +use log::warn; +use std::ops::AddAssign; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::mpsc::{Receiver, Sender}; +use std::sync::Arc; +use std::thread; +use std::time::{Duration, SystemTime}; + +use futures01::sink::{Sink, Wait}; +use futures01::sync::mpsc::UnboundedSender; +use websocket::OwnedMessage; + +use crate::common::asserts::GracefulUnwrap; +use crate::common::consts::{ + ONTIME_CHECK_TASK_NUM, ONTIME_KICK_INTERVAL, ONTIME_KICK_SOURCE, ONTIME_PING_INTERVAL, + ONTIME_THREAD_INTERVAL, ONTIME_UPDATE_INTERVAL, +}; +use crate::ontime::self_update::try_update; +use crate::ontime::timer::Timer; +use crate::types::inner_msg::KickMsg; + +pub fn run( + ping_channel_receiver: Receiver>, + ontime_kick_sender: Sender, + running_task_num: Arc, +) -> thread::JoinHandle<()> { + let ret = thread::spawn(move || { + let ping_sender = ping_channel_receiver + .recv() + .unwrap_or_exit("ping channel recv fail"); + let mut sender = ping_sender.wait(); + + let mut instant_kick = SystemTime::now(); + let mut instant_ping = SystemTime::now(); + let mut instant_update = + SystemTime::now() - Duration::from_secs(ONTIME_UPDATE_INTERVAL - 10); + let mut instant_check_tasks = SystemTime::now(); + // Will be set to true after self updating finished, + // and then exit current agent to switch to new version agent. + let need_restart = Arc::new(AtomicBool::new(false)); + let self_updating = Arc::new(AtomicBool::new(false)); + + // kick once after agent start + send_kick_msg(&ontime_kick_sender); + // ping once after agent start + send_ping_msg(&ping_channel_receiver, &mut sender); + + loop { + // inter-thread channel communication, very fast + check_ontime_kick(&mut instant_kick, &ontime_kick_sender); + // do self update in a new thread, will not block current thread + check_ontime_update(&mut instant_update, &self_updating, &need_restart); + // run the tasks whose timer is arrived, generally very quick task + schedule_timer_task(); + // check running tasks number and need_restart flag to do graceful restart when no running tasks + check_running_task_num(&mut instant_check_tasks, &need_restart, &running_task_num); + // may block thread during WebSocket reconnect, put it at end + check_ontime_ping(&mut instant_ping, &ping_channel_receiver, &mut sender); + + thread::sleep(Duration::from_secs(ONTIME_THREAD_INTERVAL)); + } + }); + ret +} + +fn schedule_timer_task() { + let tasks; + { + let timer = Timer::get_instance(); + let mut timer = timer.lock().unwrap_or_exit(""); + debug!("current status of timer:{:?}", timer); + tasks = timer.tasks_to_schedule(); + } + // release the lock and then run each task + let cnt = tasks.len(); + for task in tasks { + task.run_task(); + } + if cnt > 0 { + info!("total {} timer tasks scheduled", cnt); + } +} + +fn check_ontime_kick(instant_kick: &mut SystemTime, ontime_kick_sender: &Sender) { + let interval = Duration::from_secs(ONTIME_KICK_INTERVAL); + match instant_kick.elapsed() { + Ok(duration) => { + if duration < interval { + // interval not reach, do nothing + return; + } + } + Err(e) => { + warn!("get systemTime err: {:?}", e); + return; + } + } + // instant.add_assign() is better than *instant = now(), + // the latter may cause accumulated latency after long term running. + instant_kick.add_assign(interval); + send_kick_msg(ontime_kick_sender); +} + +fn send_kick_msg(ontime_kick_sender: &Sender) { + let msg = KickMsg { + kick_source: ONTIME_KICK_SOURCE.to_string(), + }; + ontime_kick_sender + .send(msg) + .unwrap_or_exit("ontime kick send fail"); + info!("ontime kick sent"); +} + +fn check_ontime_ping( + instant_ping: &mut SystemTime, + ping_channel_receiver: &Receiver>, + sender: &mut Wait>, +) { + let interval = Duration::from_secs(ONTIME_PING_INTERVAL); + match instant_ping.elapsed() { + Ok(duration) => { + if duration < interval { + // interval not reach, do nothing + return; + } + } + Err(e) => { + warn!("get systemTime err: {:?}", e); + return; + } + } + instant_ping.add_assign(interval); + send_ping_msg(ping_channel_receiver, sender); +} + +fn send_ping_msg( + ping_channel_receiver: &Receiver>, + sender: &mut Wait>, +) { + loop { + let ret = sender.send(OwnedMessage::Ping(Vec::new())); + if let Err(e) = ret { + info!("ping_sender ret: {:?}", e); + // may block few seconds at this line + let ping_sender = ping_channel_receiver + .recv() + .unwrap_or_exit("ping channel recv fail"); + info!("new ping_sender got, try send again"); + *sender = ping_sender.wait(); + } else { + break; + } + } + info!("ontime ping sent"); +} + +fn check_ontime_update( + instant_update: &mut SystemTime, + self_updating: &Arc, + need_restart: &Arc, +) { + if self_updating.load(Ordering::SeqCst) { + return; + } + + let interval = Duration::from_secs(ONTIME_UPDATE_INTERVAL); + match instant_update.elapsed() { + Ok(duration) => { + if duration < interval { + // interval not reach, do nothing + return; + } + } + Err(e) => { + warn!("get systemTime err: {:?}", e); + return; + } + } + + instant_update.add_assign(interval); + + self_updating.store(true, Ordering::SeqCst); + info!("start check self update"); + let self_updating_clone = self_updating.clone(); + let need_restart_clone = need_restart.clone(); + thread::Builder::new() + .spawn(move || { + try_update(self_updating_clone, need_restart_clone); + }) + .ok(); +} + +fn check_running_task_num( + instance_check: &mut SystemTime, + need_restart: &Arc, + running_task_num: &Arc, +) { + let interval = Duration::from_secs(ONTIME_CHECK_TASK_NUM); + match instance_check.elapsed() { + Ok(duration) => { + if duration < interval { + // interval not reach, do nothing + return; + } + } + Err(e) => { + warn!("get systemTime err: {:?}", e); + return; + } + } + instance_check.add_assign(interval); + let restart_flag = need_restart.load(Ordering::SeqCst); + if restart_flag { + let task_num = running_task_num.load(Ordering::SeqCst); + if task_num == 0 { + info!( + "running tasks num: {}, need_restart is {}, exit prgram", + task_num, restart_flag + ); + std::process::exit(2); + } + debug!( + "running tasks num: {}, need_restart is {}, continue running", + task_num, restart_flag + ); + } else { + debug!("need_restart is {}, continue running", restart_flag); + } +} diff --git a/src/ontime/timer.rs b/src/ontime/timer.rs new file mode 100644 index 0000000..b98f89c --- /dev/null +++ b/src/ontime/timer.rs @@ -0,0 +1,275 @@ +use std::collections::BTreeMap; +use std::fmt::{self, Debug}; +use std::ops::Fn; +use std::sync::Arc; +use std::sync::Mutex; +use std::time::Instant; + +pub struct TimerTask { + task_id: u64, + task_fn: Box ()>, +} + +impl TimerTask { + fn new(task_id: u64, task_fn: F) -> Box + where + F: 'static + Fn() -> (), + { + Box::new(TimerTask { + task_id, + task_fn: Box::new(task_fn), + }) + } + + pub fn task_id(&self) -> u64 { + self.task_id + } + + pub fn run_task(&self) { + (self.task_fn)() + } +} + +pub struct Timer { + cur_id: u64, + instant: Instant, + // Key is the elapsed nano seconds from program start. + // Value is the timer task to run. + task_map: BTreeMap>, +} + +impl Timer { + fn new() -> Timer { + Timer { + cur_id: 0, + instant: Instant::now(), + task_map: BTreeMap::new(), + } + } + + fn inc_fetch_cur_id(&mut self) -> u64 { + self.cur_id = { + if self.cur_id == std::u64::MAX { + 0 + } else { + self.cur_id + 1 + } + }; + self.cur_id + } + + // Insert the task to the map. + // Param relative_time is the key wanted, if key conflict, will find next near key to insert, + // because the key is nano second, so add 1 or some doesn't matter. + // The probability of nano second key conflict is very small. + // Return the actual inserted key + fn insert_task_map(&mut self, relative_time: u128, task: Box) -> u128 { + let mut actual_key = relative_time; + while self.task_map.contains_key(&actual_key) { + actual_key += 1; + } + self.task_map.insert(actual_key, task); + actual_key + } + + // Run task_fn() at sec_after seconds later. + // Return the actual inserted key and task_id. + pub fn add_task(&mut self, sec_after: u64, task_fn: F) -> (u128, u64) + where + F: 'static + Fn() -> (), + { + let cur_id = self.inc_fetch_cur_id(); + let task = TimerTask::new(cur_id, task_fn); + let cur_relative_time = self.instant.elapsed().as_nanos(); + let sec_after = sec_after as u128; + let actual_key = self.insert_task_map(cur_relative_time + sec_after * 1_000_000_000, task); + (actual_key, cur_id) + } + + // Delete a pre-added task, if no need to run it. + // Return whether the task existed && removed + pub fn del_task(&mut self, key: u128, task_id: u64) -> bool { + let item = self.task_map.get(&key); + if let Some(task) = item { + if task_id == task.task_id() { + self.task_map.remove(&key); + return true; + } + } + false + } + + // Move the arrived timer task out to run at somewhere else. + // Because all method of Timer is in the MutexGuard of Timer singleton, + // And run task may occupy some time, so move out to release the lock. + pub fn tasks_to_schedule(&mut self) -> Vec> { + let now = self.instant.elapsed().as_nanos(); + let mut keys = vec![]; + for (key, _value) in self.task_map.iter() { + if *key <= now { + keys.push(*key); + } else { + break; + } + } + let mut tasks = vec![]; + for key in &keys { + let task = self.task_map.remove(key).unwrap(); + tasks.push(task); + } + tasks + } + + // get the singleton, thread safe by mutex wrapped + pub fn get_instance() -> Arc> { + static mut INS: Option>> = None; + let &mut ins; + unsafe { + ins = INS.get_or_insert(Arc::new(Mutex::new(Timer::new()))); + } + ins.clone() + } +} + +impl Debug for Timer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut task_map_debug = "{".to_string(); + for (key, value) in self.task_map.iter() { + task_map_debug += + format!("(key:{}, value.task_id:{}), ", key, value.task_id()).as_str(); + } + task_map_debug += "}"; + + f.debug_struct("Timer") + .field("cur_id", &self.cur_id) + .field("instant", &self.instant) + .field("task_map_len", &self.task_map.len()) + .field("task_map_debug", &task_map_debug) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::thread; + use std::time::Duration; + + use log::info; + + use crate::common::asserts::GracefulUnwrap; + use crate::common::logger::init_test_log; + + #[test] + // because singleton timer has only one instance, + // we have to put all timer testcase in one. + // NOTICE: can NOT run together with test_shell_cmd_timeout, + // because that testcase also use the singleton + fn test_timer_in_one_case() { + init_test_log(); + test_timer_task_add_del_schedule(); + test_timer_singleton_inc_cur_id(); + test_several_task(); + } + + // usage of schedule timer tasks + fn test_several_task() { + { + let timer = Timer::get_instance(); + let mut timer = timer.lock().unwrap_or_exit(""); + timer.add_task(3, || { + info!("running task of after 3"); + }); + timer.add_task(2, || { + info!("running task of after 2"); + }); + timer.add_task(5, || { + info!("running task of after 5"); + }); + timer.add_task(2, || { + info!("running task of after 2 (another)"); + }); + } + { + let timer = Timer::get_instance(); + let mut timer = timer.lock().unwrap_or_exit(""); + info!("timer:{:?}", timer); + let mut cnt = 0; + while cnt < 4 { + let tasks = timer.tasks_to_schedule(); + cnt += tasks.len(); + for task in tasks { + task.run_task(); + } + info!("total {} tasks run", cnt); + thread::sleep(Duration::new(0, 500_000_000)); + } + info!("timer:{:?}", timer); + } + } + + fn test_timer_singleton_inc_cur_id() { + { + let timer = Timer::get_instance(); + let mut timer = timer.lock().unwrap_or_exit(""); + let cur = timer.inc_fetch_cur_id(); + assert_eq!(3, cur); + info!("timer:{:?}", timer); + } + { + let timer = Timer::get_instance(); + let mut timer = timer.lock().unwrap_or_exit(""); + let cur = timer.inc_fetch_cur_id(); + assert_eq!(4, cur); + info!("timer:{:?}", timer); + } + } + + fn test_timer_task_add_del_schedule() { + { + let timer = Timer::get_instance(); + let mut timer = timer.lock().unwrap_or_exit(""); + + timer.add_task(3, || { + info!("task after 3"); + }); + info!("timer:{:?}", timer); + assert_eq!(timer.task_map.len(), 1); + assert_eq!(timer.cur_id, 1); + + let (key, id) = timer.add_task(1, || { + info!("task after 1"); + }); + info!("timer:{:?}", timer); + assert_eq!(timer.task_map.len(), 2); + assert_eq!(timer.cur_id, 2); + assert_eq!(id, 2); + + let f = timer.del_task(key, id); + info!("timer:{:?}", timer); + assert_eq!(f, true); + assert_eq!(timer.task_map.len(), 1); + assert_eq!(timer.cur_id, 2); + } + { + let timer = Timer::get_instance(); + let mut timer = timer.lock().unwrap_or_exit(""); + + let tasks = timer.tasks_to_schedule(); + assert_eq!(0, tasks.len()); + } + thread::sleep(Duration::new(3, 0)); + { + let timer = Timer::get_instance(); + let mut timer = timer.lock().unwrap_or_exit(""); + + let tasks = timer.tasks_to_schedule(); + assert_eq!(1, tasks.len()); + assert_eq!(timer.task_map.len(), 0); + + for task in tasks { + task.run_task(); + } + } + } +} diff --git a/src/types.rs b/src/types.rs new file mode 100644 index 0000000..a86060d --- /dev/null +++ b/src/types.rs @@ -0,0 +1,20 @@ +/* 声明内部模块 */ +mod common; +mod error; +mod task; + +pub mod inner_msg; +pub mod ws_msg; + +/* 导出基础类型 */ +pub use common::HttpMethod; +pub use error::AgentError; +pub use error::AgentErrorCode; +pub use task::{ + AgentRequest, CheckUpdateRequest, CheckUpdateResponse, DescribeTasksRequest, + DescribeTasksResponse, ReportTaskFinishRequest, ReportTaskFinishResponse, + ReportTaskStartRequest, ReportTaskStartResponse, ServerRawResponse, UploadTaskLogRequest, + UploadTaskLogResponse, +}; + +pub use task::{InvocationCancelTask, InvocationNormalTask}; diff --git a/src/types/common.rs b/src/types/common.rs new file mode 100644 index 0000000..34b468b --- /dev/null +++ b/src/types/common.rs @@ -0,0 +1,4 @@ +pub enum HttpMethod { + GET, + POST, +} diff --git a/src/types/error.rs b/src/types/error.rs new file mode 100644 index 0000000..04a17c9 --- /dev/null +++ b/src/types/error.rs @@ -0,0 +1,43 @@ +use std::fmt; + +#[derive(Debug, Clone)] +pub enum AgentErrorCode { + // errors in c/s communication + ResponseReadError = 100_001, + JsonDecodeError = 100_002, + ResponseEmptyError = 100_003, + UnexpectedResponseFormat = 100_004, + RequestEmptyError = 100_005, + // errors in task executing + _StartExecutionError = 101_001, + // other error types + ClientNotInitialized = 102_001, + MaxRetryFailures = 102_002, +} + +#[derive(Debug, Clone)] +pub struct AgentError { + pub code: AgentErrorCode, + pub message: String, + pub original_error: T, +} + +impl AgentError { + pub fn wrap(code: AgentErrorCode, message: &str, original_error: T) -> Self { + AgentError { + code: code, + message: String::from(message), + original_error: original_error, + } + } +} + +impl AgentError { + pub fn new(code: AgentErrorCode, message: &str) -> Self { + AgentError { + code: code, + message: String::from(message), + original_error: String::from("None"), + } + } +} diff --git a/src/types/inner_msg.rs b/src/types/inner_msg.rs new file mode 100644 index 0000000..07746c7 --- /dev/null +++ b/src/types/inner_msg.rs @@ -0,0 +1,4 @@ +#[derive(Debug)] +pub struct KickMsg { + pub kick_source: String, +} diff --git a/src/types/task.rs b/src/types/task.rs new file mode 100644 index 0000000..e4c4b6e --- /dev/null +++ b/src/types/task.rs @@ -0,0 +1,434 @@ +use serde::{Deserialize, Serialize}; +use crate::common::consts::AGENT_VERSION; +use crate::types::AgentErrorCode; + +use crate::uname::Uname; +use crate::uname::common::UnameExt; + +//============================================================================== +// Declare standard request and response format for C/S communication +// general parameters in reqeust +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +pub struct GeneralParameters { + action: String, +} + +// standard http request format +// combined general parameters with custom parameters +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +pub struct AgentRequest { + // general parameters for all kind of request, such as Action, etc. + #[serde(flatten)] + pub general_params: GeneralParameters, + // user self defined parameters + #[serde(flatten)] + pub custom_params: T, +} + +// general parameters in response, encapsuled in ServerRawResponse +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +pub struct GeneralResponse { + // request id is always required + request_id: String, + // error is not None when some error happened + error: Option, + // content is not None in most cases + #[serde(flatten)] + content: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +pub struct ResponseError { + code: String, + message: String, +} + +// standard response format +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +pub struct ServerRawResponse { + response: GeneralResponse, +} + +impl AgentRequest { + pub fn new(action: &str, custom_params: T) -> Self { + let g = GeneralParameters { + action: action.to_string(), + }; + AgentRequest { + general_params: g, + custom_params: custom_params, + } + } +} + +impl GeneralResponse { + pub fn content(self) -> Option { + self.content + } + + pub fn error(self) -> Option { + self.error + } + + pub fn is_ok(&self) -> bool { + self.error.is_none() + } + + pub fn _request_id(&self) -> String { + self.request_id.clone() + } +} + +impl ServerRawResponse { + pub fn into_response(self) -> Result { + if self.response.is_ok() { + match self.response.content() { + Some(content) => Ok(content), + None => Err(ResponseError { + code: format!("{:?}", AgentErrorCode::UnexpectedResponseFormat), + message: format!("cannot get response content"), + }), + } + } else { + match self.response.error() { + Some(err) => Err(err), + None => Err(ResponseError { + code: format!("{:?}", AgentErrorCode::UnexpectedResponseFormat), + message: format!("cannot get response error"), + }), + } + } + } + + pub fn _into_request_id(&self) -> String { + self.response._request_id() + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "PascalCase")] +pub struct Empty {} + +//============================================================================== +// DescribeTasks API +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "PascalCase")] +pub struct InvocationNormalTask { + #[serde(default)] + pub invocation_task_id: String, + #[serde(default)] + pub time_out: u64, + #[serde(alias = "Cmd")] + #[serde(default)] + pub command: String, + #[serde(alias = "CmdType")] + #[serde(default)] + pub command_type: String, + #[serde(default)] + pub username: String, + #[serde(default)] + pub working_directory: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "PascalCase")] +pub struct InvocationCancelTask { + #[serde(default)] + pub invocation_task_id: String, +} + +// request and response +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "PascalCase")] +pub struct DescribeTasksResponse { + pub invocation_normal_task_set: Vec, + pub invocation_cancel_task_set: Vec, +} + +pub type DescribeTasksRequest = Empty; + +impl InvocationNormalTask { + pub fn decode_command(&self) -> Result { + match base64::decode(&self.command) { + Ok(command) => match std::str::from_utf8(&command) { + Ok(s) => Ok(String::from(s)), + Err(e) => Err(format!("parse error: {:?}", e)), + }, + Err(e) => Err(format!("decode error: {:?}", e)), + } + } +} +//============================================================================== +// ReportTaskStart API +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "PascalCase")] +pub struct ReportTaskStartRequest { + #[serde(default)] + pub invocation_task_id: String, + #[serde(default)] + pub time_stamp: u64, +} + +pub type ReportTaskStartResponse = Empty; + +//============================================================================== +// ReportTaskFinish API +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "PascalCase")] +pub struct ReportTaskFinishRequest { + #[serde(default)] + pub invocation_task_id: String, + #[serde(default)] + pub time_stamp: u64, + #[serde(default)] + pub result: String, + #[serde(default)] + pub error_info: String, + #[serde(default)] + pub exit_code: i32, + #[serde(default)] + pub final_log_index: u32, +} + +pub type ReportTaskFinishResponse = Empty; + +//============================================================================== +// UploadTaskLog API +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "PascalCase")] +pub struct UploadTaskLogRequest { + #[serde(default)] + invocation_task_id: String, + #[serde(default)] + index: u32, + #[serde(default)] + output: String, + #[serde(default)] + dropped: u64, +} + +impl UploadTaskLogRequest { + pub fn new(invocation_task_id: &str, index: u32, output: Vec, dropped: u64) -> Self { + UploadTaskLogRequest { + invocation_task_id: String::from(invocation_task_id), + index: index, + output: base64::encode(&output), + dropped: dropped, + } + } +} +pub type UploadTaskLogResponse = Empty; +//============================================================================== +// CheckUpdate API + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "PascalCase")] +pub struct CheckUpdateRequest { + kernel_name: String, + kernel_release: String, + kernel_version: String, + machine: String, + version: String, +} + +impl CheckUpdateRequest { + pub fn new() -> Self { + let uname = Uname::new().unwrap(); + CheckUpdateRequest { + kernel_name: uname.sys_name(), + kernel_release: uname.release(), + kernel_version: uname.version(), + machine: uname.machine(), + version: AGENT_VERSION.to_string(), + } + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "PascalCase")] +pub struct CheckUpdateResponse { + need_update: bool, + #[serde(default)] + download_url: Option, + #[serde(default)] + md5: Option, +} + +impl CheckUpdateResponse { + pub fn need_update(&self) -> bool { + self.need_update + } + + pub fn download_url(&self) -> &Option { + &self.download_url + } + + pub fn md5(&self) -> &Option { + &self.md5 + } +} + +// Unit Tests +#[cfg(test)] +mod tests { + use crate::types::{InvocationNormalTask, ServerRawResponse}; + + #[test] + fn serialize_agent_request() { + use serde::Serialize; + #[derive(Serialize)] + struct Inner { + count: u8, + } + #[derive(Serialize)] + struct Foo { + #[serde(flatten)] + inner: Inner, + bar: String, + names: Vec, + } + let i = Inner{ count: 3}; + let f = Foo{ + inner: i, + bar: String::from("woo"), + names: vec!["jack".to_string(), "john".to_string(), "ken".to_string()], + }; + let res = serde_json::to_string(&f).unwrap(); + assert_eq!(res, "{\"count\":3,\"bar\":\"woo\",\"names\":[\"jack\",\"john\",\"ken\"]}"); + } + + #[test] + fn deserialize_server_response_error() { + + let error_str = " + { + \"Response\": { + \"Error\": { + \"Code\": \"ExampleCode\", + \"Message\": \"Some message\" + }, + \"RequestId\": \"e8fc76bf-ed90-4f38-a871-4f344d35d5ff\" + } + }"; + use serde::Deserialize; + #[derive(Deserialize)] + struct MyResp { + _name: String, + } + let raw_resp = serde_json::from_str::>(&error_str).unwrap(); + let general_resp = &raw_resp.response; + assert_eq!(&general_resp.request_id, "e8fc76bf-ed90-4f38-a871-4f344d35d5ff"); + let error = &general_resp.error.as_ref().unwrap(); + assert_eq!(&error.code, "ExampleCode"); + assert_eq!(&error.message, "Some message"); + // assert_eq!(general_resp.content.as_ref().unwrap(), None); + assert_eq!(general_resp.content.is_none(), true); + } + + #[test] + fn deserialize_server_response_content() { + let resp_str = " + { + \"Response\": { + \"RequestId\": \"aee1c4f7-c782-45b7-b81b-9cea73448a31\", + \"UserSet\": [ + { + \"Name\": \"Foo\", + \"Age\": 20 + }, + { + \"Name\": \"Bar\", + \"Age\": 30 + } + ] + } + } + "; + use serde::{Deserialize, Serialize}; + #[derive(Serialize, Deserialize, Debug)] + #[serde(rename_all = "PascalCase")] + struct MyUser { + name: String, + age: u64, + } + #[derive(Serialize, Deserialize, Debug)] + #[serde(rename_all = "PascalCase")] + struct MyResp { + user_set: Vec, + } + let raw_resp = serde_json::from_str::>(&resp_str).unwrap(); + let general_resp = &raw_resp.response; + assert_eq!(&general_resp.request_id, "aee1c4f7-c782-45b7-b81b-9cea73448a31"); + let content = general_resp.content.as_ref().unwrap(); + assert_eq!(content.user_set.len(), 2); + assert_eq!(&content.user_set[0].name, "Foo"); + assert_eq!(content.user_set[1].age, 30); + } + + #[test] + fn test_decode_normal_command() { + let tasks1 = InvocationNormalTask { + invocation_task_id: format!(""), + command_type: format!("SHELL"), + time_out: 0, + command: String::from("bHMgLWw7CmVjaG8gIkhlbGxvIFdvcmxkIg=="), + username: format!("root"), + working_directory: format!("") + }; + assert_eq!( + tasks1.decode_command().unwrap(), + String::from("ls -l;\necho \"Hello World\"") + ); + } + + #[test] + fn test_decode_invalid_command() { + let tasks1 = InvocationNormalTask { + invocation_task_id: format!(""), + command_type: format!("SHELL"), + time_out: 0, + command: String::from("ls -l;\necho \"Hello World\""), + username: format!("root"), + working_directory: format!("") + }; + assert_eq!( + tasks1.decode_command().is_err(), + true + ); + } + + #[cfg(target_family = "unix")] + #[test] + fn test_encode_log() { + use std::process::Command; + use std::fs::remove_file; + use std::io::Read; + use std::fs::File; + + use crate::types::UploadTaskLogRequest; + let _cmd = Command::new("time") + .arg("dd") + .arg("if=/dev/urandom") + .arg("of=random-file") + .arg("bs=1").arg("count=1024") + .output().expect("failed to generate random binary file"); + // read binary file + let mut f = File::open("./random-file").unwrap(); + let mut buffer = Vec::new(); + // read the whole file + f.read_to_end(&mut buffer).expect("failed to read random-file"); + assert_eq!(remove_file("./random-file").is_ok(), true); + let _req = UploadTaskLogRequest::new( + "invk-123123", + 0, + buffer, + 0 + ); + // println!("{:?}", req); + } +} diff --git a/src/types/ws_msg.rs b/src/types/ws_msg.rs new file mode 100644 index 0000000..33a75fd --- /dev/null +++ b/src/types/ws_msg.rs @@ -0,0 +1,13 @@ +use serde::{Deserialize, Serialize}; +use serde_json; + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +pub struct WsMsg { + #[serde(default)] + pub r#type: String, + #[serde(default)] + pub seq: u64, + #[serde(default)] + pub data: Option, +} diff --git a/src/uname.rs b/src/uname.rs new file mode 100644 index 0000000..db0ecdf --- /dev/null +++ b/src/uname.rs @@ -0,0 +1,13 @@ +pub mod common; +#[cfg(windows)] +mod windows; +#[cfg(unix)] +mod unix; + +pub struct Uname { + sys_name: String, + node_name: String, + release: String, + version: String, + machine: String, +} diff --git a/src/uname/common.rs b/src/uname/common.rs new file mode 100644 index 0000000..1adf507 --- /dev/null +++ b/src/uname/common.rs @@ -0,0 +1,57 @@ +use crate::uname::Uname; + +pub trait UnameExt { + fn sys_name(&self) -> String; + fn node_name(&self) -> String; + fn release(&self) -> String; + fn version(&self) -> String; + fn machine(&self) -> String; +} + +impl UnameExt for Uname { + fn sys_name(&self) -> String { + format!("{}", self.sys_name) + } + + fn node_name(&self) -> String { + format!("{}", self.node_name) + } + + fn release(&self) -> String { + format!("{}", self.release) + } + + fn version(&self) -> String { + format!("{}", self.version) + } + + fn machine(&self) -> String { + format!("{}", self.machine) + } +} + +#[cfg(test)] +mod tests { + use crate::uname::Uname; + use crate::uname::common::UnameExt; + + #[test] + fn test_uname() { + let uname = Uname::new().unwrap(); + println!("machine: {}", &uname.machine()); + #[cfg(all(unix, target_arch = "x86_64"))] + assert_eq!(uname.machine(), "x86_64"); + #[cfg(all(unix, target_arch = "x86"))] + assert_eq!(uname.machine(), "i686"); + + println!("sys_name: {}", uname.sys_name()); + if cfg!(windows) { + assert_eq!(uname.sys_name(), "Windows"); + } else if !cfg!(unix) { + #[cfg(target_os = "macos")] + assert_eq!(uname.sys_name(), "Darwin"); + #[cfg(target_os = "linux")] + assert_eq!(uname.sys_name(), "Linux"); + }; + } +} diff --git a/src/uname/unix.rs b/src/uname/unix.rs new file mode 100644 index 0000000..9ad3ab5 --- /dev/null +++ b/src/uname/unix.rs @@ -0,0 +1,36 @@ +use std::io; +use std::ffi::CStr; + +use libc::{utsname, c_char}; +use crate::uname::Uname; + +impl Uname { + pub fn new() -> io::Result { + let mut n = unsafe { std::mem::zeroed() }; + let r = unsafe { libc::uname(&mut n) }; + if r == 0 { + Ok(From::from(n)) + } else { + Err(io::Error::last_os_error()) + } + } +} + +#[inline] +fn parse(buf: &[c_char]) -> String { + let s = unsafe { CStr::from_ptr(buf.as_ptr()) }; + s.to_string_lossy().into_owned().replace("\"","\\\"") +} + +impl From for Uname { + fn from(x: utsname) -> Self { + let uname = Uname { + sys_name: parse(&x.sysname), + node_name: parse(&x.nodename), + release: parse(&x.release), + version: parse(&x.version), + machine: parse(&x.machine), + }; + uname + } +} diff --git a/src/uname/windows.rs b/src/uname/windows.rs new file mode 100644 index 0000000..5b71cb9 --- /dev/null +++ b/src/uname/windows.rs @@ -0,0 +1,16 @@ +use crate::uname::Uname; +use std::io; + +impl Uname { + pub fn new() -> io::Result { + let uname = Uname { + sys_name: String::from("Windows"), + node_name: String::from("unknown"), + release: String::from("unknown"), + version: String::from("unknown"), + // TODO: need optimized with real arch. + machine: String::from("i686"), + }; + return Ok(uname); + } +} diff --git a/src/ws.rs b/src/ws.rs new file mode 100644 index 0000000..7c2b43b --- /dev/null +++ b/src/ws.rs @@ -0,0 +1 @@ +pub mod thread; diff --git a/src/ws/thread.rs b/src/ws/thread.rs new file mode 100644 index 0000000..45db934 --- /dev/null +++ b/src/ws/thread.rs @@ -0,0 +1,268 @@ +use std::{thread, time}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::mpsc::Sender; +use std::time::Duration; + +use futures01::future::Future; +use futures01::sink::{Sink, Wait}; +use futures01::stream::Stream; +use futures01::sync::mpsc; +use futures01::sync::mpsc::UnboundedSender; +use log::debug; +use log::error; +use log::info; +use serde_json; +use tokio01 as tokio; +use tokio01::prelude::FutureExt; +use websocket::{ClientBuilder, CloseData, OwnedMessage}; +use websocket::header::Headers; +use websocket::result::WebSocketError; + + +use crate::common::asserts::GracefulUnwrap; +use crate::common::consts::{ + AGENT_VERSION, MAX_PING_FROM_LAST_PONG, WS_ACTIVE_CLOSE, WS_ACTIVE_CLOSE_CODE, + WS_CONNECT_TIMEOUT, WS_KERNEL_NAME_HEADER, WS_LAST_CLOSE_INTERVAL, WS_MSG_TYPE_ACK, + WS_MSG_TYPE_KICK, WS_PASSIVE_CLOSE, WS_PASSIVE_CLOSE_CODE, WS_RECONNECT_INTERVAL, WS_URL, + WS_VERSION_HEADER,VPCID_HEADER,VIP_HEADER +}; +use crate::common::envs; +use crate::types::inner_msg::KickMsg; +use crate::types::ws_msg::WsMsg; +use crate::uname::Uname; +use crate::uname::common::UnameExt; + +pub fn run( + kick_sender: Sender, + ping_channel_sender: Sender>, +) -> thread::JoinHandle<()> { + let ret = thread::spawn(move || { + let mut runtime = tokio::runtime::current_thread::Builder::new() + .build() + .unwrap_or_exit("ws tokio runtime build fail"); + + let ping_cnt_from_last_pong = Arc::new(AtomicUsize::new(0)); + let close_sent = Arc::new(AtomicBool::new(false)); + let header = gen_ver_header(); + + loop { + // a loop of new connection, reset the flag + ping_cnt_from_last_pong.store(0, Ordering::SeqCst); + close_sent.store(false, Ordering::SeqCst); + + // new channel pair in this loop + let (ping_sender, ping_receiver) = mpsc::unbounded(); + let my_ping_sender0 = ping_sender.clone(); + let mut my_ping_sender0 = my_ping_sender0.wait(); + let my_ping_sender1 = ping_sender.clone(); + let mut my_ping_sender1 = my_ping_sender1.wait(); + // send the new sender to ontime thread + ping_channel_sender + .send(ping_sender) + .unwrap_or_exit("ping channel send fail"); + + thread::sleep(time::Duration::from_secs(WS_RECONNECT_INTERVAL)); + + let sender = kick_sender.clone(); + + let runner = ClientBuilder::new(WS_URL) + .unwrap_or_exit("ws cli builder fail") + .custom_headers(&header) + .async_connect_insecure() + .timeout(Duration::from_secs(WS_CONNECT_TIMEOUT)) + .map_err(|e| { + error!("connect fail:{:?}", e); + panic!(); + }) + .and_then(|(duplex, _)| { + info!("connection established"); + let (sink, stream) = duplex.split(); + stream + .filter_map(|msg| { + handle_server_msg( + msg, + &close_sent, + &ping_cnt_from_last_pong, + &sender, + &mut my_ping_sender0, + ) + }) + .select( + ping_receiver + .filter_map(|msg| { + handle_ping_notify_msg( + msg, + &close_sent, + &ping_cnt_from_last_pong, + &mut my_ping_sender1, + ) + }) + .map_err(|_| WebSocketError::NoDataAvailable), + ) + .forward(sink) + }) + .map(|_| { + info!("ws connection finished"); + }) + .or_else(|err| { + error!("ws connection ended with an error: {:?}", err); + Ok(()) as Result<(), ()> + }); + + info!("establishing new ws connection"); + runtime.block_on(runner).or_log("ws runtime run failed"); + } + }); + + ret +} + +fn gen_ver_header() -> Headers { + let mut headers = Headers::new(); + headers.set_raw(WS_VERSION_HEADER, vec![AGENT_VERSION.as_bytes().to_vec()]); + if envs::enable_test() { + headers.set_raw(VPCID_HEADER, vec![envs::test_vpcid().as_bytes().to_vec()]); + headers.set_raw(VIP_HEADER, vec![envs::test_vip().as_bytes().to_vec()]); + } + + if let Ok(uname) = Uname::new() { + headers.set_raw( + WS_KERNEL_NAME_HEADER, + vec![uname.sys_name().into_bytes()], + ); + } + + debug!("ws header:{:?}", headers); + headers +} + +// handle the message from WebSocket server +fn handle_server_msg( + msg: OwnedMessage, + close_sent: &Arc, + ping_cnt_from_last_pong: &Arc, + kick_sender: &Sender, + my_ping_sender: &mut Wait>, +) -> Option { + info!("ws recv msg: {:?}", msg); + if close_sent.load(Ordering::SeqCst) { + info!("ws close msg sent, ignore following msg from server"); + return None; + } + match msg { + OwnedMessage::Ping(data) => Some(OwnedMessage::Pong(data)), + OwnedMessage::Pong(_) => { + // connection ok now, just clear the cnt + ping_cnt_from_last_pong.store(0, Ordering::SeqCst); + None + } + OwnedMessage::Text(msg) => { + let ret = handle_ws_text_msg(msg, &kick_sender); + ret + } + OwnedMessage::Close(_) => { + close_sent.store(true, Ordering::SeqCst); + // notify myself to be ready to abort this task + my_ping_sender + .send(OwnedMessage::Ping("".to_string().into_bytes())) + .or_log("notify myself to abort this ws task failed"); + Some(OwnedMessage::Close(Some(CloseData { + status_code: WS_PASSIVE_CLOSE_CODE, + reason: WS_PASSIVE_CLOSE.to_string(), + }))) + } + _ => None, + } +} + +fn handle_ws_text_msg(msg: String, kick_sender: &Sender) -> Option { + let ret: Result = serde_json::from_str(msg.as_str()); + match ret { + Ok(ws_msg) => gen_ws_text_rsp(ws_msg, kick_sender), + Err(e) => { + error!("json parse fail, invalid ws text msg: {:?}", e); + // ignore it + None + } + } +} + +fn gen_ws_text_rsp(mut ws_msg: WsMsg, kick_sender: &Sender) -> Option { + if ws_msg.r#type != WS_MSG_TYPE_KICK { + error!("not kick, unknown ws msg:{:?}", ws_msg.r#type); + return None; + } + + let msg = KickMsg { + kick_source: "ws".to_string(), + }; + // notify http thread to fetch task + kick_sender.send(msg).unwrap_or_exit("ws kick send fail"); + info!("kick_sender sent to notify fetch task, kick_source ws"); + + // gen ack rsp for ws server + ws_msg.r#type = WS_MSG_TYPE_ACK.to_string(); + let ret = serde_json::to_string(&ws_msg); + match ret { + Ok(ws_rsp) => Some(OwnedMessage::Text(ws_rsp)), + Err(e) => { + error!("ws rsp json encode fail:{:?}", e); + None + } + } +} + +// msg from ontime thread, which notify ws to ping server +fn handle_ping_notify_msg( + msg: OwnedMessage, + close_sent: &Arc, + ping_cnt_from_last_pong: &Arc, + my_ping_sender: &mut Wait>, +) -> Option { + if close_sent.load(Ordering::SeqCst) { + // give an opportunity of 1s to send the last ws close msg out + thread::sleep(time::Duration::from_secs(WS_LAST_CLOSE_INTERVAL)); + panic!("in case of server never response, abort this task & reconnect"); + } + + let pre_val = ping_cnt_from_last_pong.fetch_add(1, Ordering::SeqCst); + + if pre_val >= MAX_PING_FROM_LAST_PONG { + if !close_sent.load(Ordering::SeqCst) { + close_sent.store(true, Ordering::SeqCst); + // notify myself to be ready to abort this task + my_ping_sender + .send(OwnedMessage::Ping("".to_string().into_bytes())) + .or_log("notify myself to abort this ws task failed"); + } + info!( + "pre val of ping_cnt_from_last_pong:{}, lost pong too long, now send ws close msg", + pre_val + ); + Some(OwnedMessage::Close(Some(CloseData { + status_code: WS_ACTIVE_CLOSE_CODE, + reason: WS_ACTIVE_CLOSE.to_string(), + }))) + } else { + // send ping for heartbeat + Some(msg) + } +} + +#[cfg(test)] +mod tests { + use log::info; + + use crate::common::logger::init_test_log; + + use super::*; + + // #[test] unused + fn _test_ws_cus_header() { + init_test_log(); + let header = gen_ver_header(); + assert_eq!(2, header.len()); + info!("header:{:?}", header); + } +} diff --git a/tests/http_test.rs b/tests/http_test.rs new file mode 100644 index 0000000..3d9a6a3 --- /dev/null +++ b/tests/http_test.rs @@ -0,0 +1,93 @@ +extern crate tat_agent; + +mod support; + +use support::server; +use std::thread; +use std::sync::Once; + +use tat_agent::common::consts; + +static INIT: Once = Once::new(); + +pub fn initialize() { + INIT.call_once(|| { + println!("mock server started..."); + tat_agent::common::logger::init_test_log(); + thread::spawn(move || { + server::start(8080); + }); + }) +} + +#[cfg(test)] +#[tokio::test(basic_scheduler)] +async fn test_describe_tasks() { + initialize(); + let adapter = tat_agent::http::InvokeAPIAdapter::build(consts::MOCK_INVOKE_API); + let resp = adapter.describe_tasks(); + assert_eq!(1, resp.await.unwrap().invocation_normal_task_set.len()) +} + +#[cfg(test)] +#[tokio::test(basic_scheduler)] +async fn test_report_task_start() { + use std::time::SystemTime; + use tat_agent::common::asserts::GracefulUnwrap; + initialize(); + let adapter = tat_agent::http::InvokeAPIAdapter::build(consts::MOCK_INVOKE_API); + let start_timestamp = SystemTime::now().duration_since( + SystemTime::UNIX_EPOCH + ).unwrap_or_exit("sys time may before 1970").as_secs(); + let resp = + adapter.report_task_start("invt-12345678", start_timestamp).await; + assert_eq!(true, resp.is_ok()); +} + +#[cfg(test)] +#[tokio::test(basic_scheduler)] +async fn test_upload_task_log() { + initialize(); + let adapter = tat_agent::http::InvokeAPIAdapter::build(consts::MOCK_INVOKE_API); + let resp = adapter + .upload_task_log("invk-12345678", 1, "some output info".as_bytes().to_vec(), 0) + .await; + assert_eq!(true, resp.is_ok()); +} + +#[cfg(test)] +#[tokio::test(basic_scheduler)] +async fn test_report_task_finish() { + initialize(); + let adapter = tat_agent::http::InvokeAPIAdapter::build(consts::MOCK_INVOKE_API); + let resp = adapter + .report_task_finish("invk-12345678", "some output info", "", 0, 1, 0) + .await; + assert_eq!(true, resp.is_ok()); +} + +#[cfg(test)] +#[tokio::test(basic_scheduler)] +async fn test_check_update() { + let adapter = tat_agent::http::InvokeAPIAdapter::build(consts::MOCK_INVOKE_API); + let resp = adapter.check_update(); + let resp = resp.await.unwrap(); + assert_eq!(true, resp.need_update()); + assert_eq!("http://example.com", resp.download_url().clone().unwrap()); + assert_eq!("eeb0248363b2e9b66f975abd4f092db8", resp.md5().clone().unwrap()); +} + +#[tokio::test(basic_scheduler)] +async fn test_process_msg() { + use tat_agent::http::thread::HttpWorker; + use tat_agent::types::inner_msg::KickMsg; + use std::sync::{Arc, atomic::AtomicU64}; + initialize(); + let adapter = tat_agent::http::InvokeAPIAdapter::build(consts::MOCK_INVOKE_API); + let worker = HttpWorker::new(adapter, Arc::new(AtomicU64::new(0))); + let msg = KickMsg { + kick_source: String::from("Test"), + }; + worker.process(msg).await; +} + diff --git a/tests/support/mod.rs b/tests/support/mod.rs new file mode 100644 index 0000000..ac2ea90 --- /dev/null +++ b/tests/support/mod.rs @@ -0,0 +1,2 @@ +pub mod server; +mod response; diff --git a/tests/support/response.rs b/tests/support/response.rs new file mode 100644 index 0000000..09e39a8 --- /dev/null +++ b/tests/support/response.rs @@ -0,0 +1,138 @@ +extern crate tat_agent; + +use tat_agent::types::*; +use hyper::Body; +use hyper::Response; +use serde::{Deserialize, Serialize}; +use serde_json::to_string_pretty; + +// general parameters in request +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +pub struct GeneralParameters { + pub action: String, +} + +// standard http request format +// combined general parameters with custom parameters +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +pub struct AgentRequest { + // general parameters for all kind of request, such as Action, etc. + #[serde(flatten)] + pub general_params: GeneralParameters, + // user self defined parameters + #[serde(flatten)] + pub custom_params: T, +} + +// general parameters in response, encapsulated in ServerRawResponse +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +pub struct GeneralResponse { + // request id is always required + request_id: String, + // error is not None when some error happened + error: Option, + // content is not None in most cases + #[serde(flatten)] + content: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +pub struct ResponseError { + code: String, + message: String, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +pub struct Empty {} + +pub type UploadTaskLogResponse = Empty; + +// standard response format +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +pub struct ServerRawResponse { + response: GeneralResponse, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "PascalCase")] +pub struct CheckUpdateResponse { + need_update: bool, + #[serde(default)] + download_url: Option, + #[serde(default)] + md5: Option, +} + +pub fn tasks_response() -> Response { + let task_number = 10000000; + let tasks = InvocationNormalTask { + invocation_task_id: String::from(format!("invt-{}", task_number)), + time_out: 3600, + username: String::from("root"), + command: base64::encode("ls -l".as_bytes()), + command_type: format!("SHELL"), + working_directory: String::from("/root/"), + }; + let resp = DescribeTasksResponse { + invocation_normal_task_set: vec![tasks], + invocation_cancel_task_set: vec![], + }; + let general_resp = GeneralResponse:: { + request_id: String::from("a0ed3f7a-ef51-4393-80b2-a3415c28c783"), + error: None, + content: Some(resp), + }; + let raw_resp = ServerRawResponse:: { + response: general_resp, + }; + Response::new(to_string_pretty(&raw_resp).unwrap().into()) +} + +pub fn start_response() -> Response { + let resp = ReportTaskStartResponse {}; + let general_resp = GeneralResponse:: { + request_id: String::from("a0ed3f7a-ef51-4393-80b2-a3415c28c783"), + error: None, + content: Some(resp), + }; + let raw_resp = ServerRawResponse:: { + response: general_resp, + }; + Response::new(to_string_pretty(&raw_resp).unwrap().into()) +} + +pub fn upload_response() -> Response { + let resp = UploadTaskLogResponse {}; + let general_resp = GeneralResponse:: { + request_id: String::from("a0ed3f7a-ef51-4393-80b2-a3415c28c783"), + error: None, + content: Some(resp), + }; + let raw_resp = ServerRawResponse:: { + response: general_resp, + }; + Response::new(to_string_pretty(&raw_resp).unwrap().into()) +} + +pub fn check_update_response() -> Response { + let resp = CheckUpdateResponse{ + need_update: true, + download_url: Some(String::from("http://example.com")), + md5: Some(String::from("eeb0248363b2e9b66f975abd4f092db8")) + }; + let general_resp = GeneralResponse:: { + request_id: String::from("a0ed3f7a-ef51-4393-80b2-a3415c28c783"), + error: None, + content: Some(resp), + }; + let raw_resp = ServerRawResponse:: { + response: general_resp, + }; + Response::new(to_string_pretty(&raw_resp).unwrap().into()) +} diff --git a/tests/support/server.rs b/tests/support/server.rs new file mode 100644 index 0000000..d71fcad --- /dev/null +++ b/tests/support/server.rs @@ -0,0 +1,115 @@ +extern crate tat_agent; +use futures01::{future, Future, Stream}; +use hyper::server::Server; +use hyper::service::service_fn; +use hyper::{Body, Request, Response}; +use response::AgentRequest; +use response::Empty; +use serde_json::from_str; + +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::str; + +use crate::support::response; + +type ResponseFuture = Box, Error = hyper::error::Error> + Send>; + +pub fn start(port: u16) { + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), port); + let new_service = move || { + service_fn(move |req: Request| { + println!("request: {:?}", req); + req.into_body().concat2().and_then(|whole_body| { + let body_bytes = whole_body.into_bytes().to_vec(); + let s = match str::from_utf8(&body_bytes) { + Ok(v) => v, + Err(e) => panic!("Invalid UTF-8 sequence: {}", e), + }; + let rr = from_str::>(&s).unwrap(); + let action_str = &rr.general_params.action; + println!("action: {}", action_str); + match action_str as &str { + "DescribeTasks" => { + describe_tasks(Request::new(Body::from(format!("{:?}", &body_bytes)))) + } + "ReportTaskStart" => { + report_task_start(Request::new(Body::from(format!("{:?}", &body_bytes)))) + } + "UploadTaskLog" => { + upload_task_log(Request::new(Body::from(format!("{:?}", &body_bytes)))) + } + "ReportTaskFinish" => { + report_task_finish(Request::new(Body::from(format!("{:?}", &body_bytes)))) + } + "CheckUpdate" => { + check_update(Request::new(Body::from(format!("{:?}", &body_bytes)))) + } + _ => Box::new(future::ok(Response::new(Body::from("hello")))) + } + }) + }) + }; + let server = Server::bind(&addr) + .serve(new_service) + .map_err(|e| eprintln!("Server error: {}", e)); + hyper::rt::run(server); +} + +fn describe_tasks(req: Request) -> ResponseFuture { + assert_eq!(req.uri(), "/"); + println!("describe tasks"); + let body = req.into_body(); + Box::new( + body.concat2() + .from_err() + .and_then(|_whole_body| Box::new(future::ok(response::tasks_response()))), + ) +} + +fn report_task_start(req: Request) -> ResponseFuture { + assert_eq!(req.uri(), "/"); + println!("report task start"); + let body = req.into_body(); + Box::new( + body.concat2() + .from_err() + .and_then(|_whole_body| Box::new(future::ok(response::start_response()))), + ) +} + +fn report_task_finish(req: Request) -> ResponseFuture { + assert_eq!(req.uri(), "/"); + println!("report task finish"); + let body = req.into_body(); + Box::new( + body.concat2() + .from_err() + .and_then(|_whole_body| Box::new(future::ok(response::start_response()))), + ) +} + +fn upload_task_log(req: Request) -> ResponseFuture { + assert_eq!(req.uri(), "/"); + println!("upload task log"); + let body = req.into_body(); + Box::new( + body.concat2() + .from_err() + .and_then(|whole_body| { + let str_body = String::from_utf8(whole_body.to_vec()).unwrap(); + let _words: Vec<&str> = str_body.split('=').collect(); + Box::new(future::ok(response::upload_response())) + }), + ) +} + +fn check_update(req: Request) -> ResponseFuture { + assert_eq!(req.uri(), "/"); + println!("check update"); + let body = req.into_body(); + Box::new( + body.concat2() + .from_err() + .and_then(|_whole_body| Box::new(future::ok(response::check_update_response()))), + ) +} \ No newline at end of file