From 102066c521c1ae07dd0859902c5aaa4343c8523c Mon Sep 17 00:00:00 2001 From: Vincent Zhang <118719397+vincent-dfinity@users.noreply.github.com> Date: Mon, 6 Jan 2025 18:15:07 +0800 Subject: [PATCH] feat: add pre-install hooks to canisters. (#4055) * Add pre-install hooks to canisters. * Exclude SC2317 to workaround the warnings introduced after shellcheck has been updated to '0.9.0' on the CI. --- .github/workflows/publish-manifest.yml | 3 +- CHANGELOG.md | 4 ++ docs/dfx-json-schema.json | 10 +++ .../dfx.json | 11 +++- .../main.mo | 0 .../postinstall.sh | 2 +- e2e/assets/pre_post_install/preinstall.sh | 3 + e2e/tests-dfx/install.bash | 63 +++++++++++++------ src/dfx-core/src/config/model/dfinity.rs | 6 ++ src/dfx/src/lib/canister_info.rs | 7 +++ .../operations/canister/install_canister.rs | 56 +++++++++++++---- 11 files changed, 131 insertions(+), 34 deletions(-) rename e2e/assets/{post_install => pre_post_install}/dfx.json (57%) rename e2e/assets/{post_install => pre_post_install}/main.mo (100%) rename e2e/assets/{post_install => pre_post_install}/postinstall.sh (77%) create mode 100755 e2e/assets/pre_post_install/preinstall.sh diff --git a/.github/workflows/publish-manifest.yml b/.github/workflows/publish-manifest.yml index 9f51af1611..1f3bf1dbc0 100644 --- a/.github/workflows/publish-manifest.yml +++ b/.github/workflows/publish-manifest.yml @@ -26,7 +26,8 @@ jobs: run: go install mvdan.cc/sh/v3/cmd/shfmt@latest - name: Generate run: | - shellcheck --shell=sh public/install-dfxvm.sh --exclude SC2154,SC2034,SC3003,SC3014,SC3043 + shellcheck -V + shellcheck --shell=sh public/install-dfxvm.sh --exclude SC2154,SC2034,SC3003,SC3014,SC3043,SC2317 ~/go/bin/shfmt -d -p -i 4 -ci -bn -s public/install-dfxvm.sh sed -i "s/@revision@/${GITHUB_SHA}/" public/install-dfxvm.sh mkdir _out diff --git a/CHANGELOG.md b/CHANGELOG.md index 231c12ea15..c54a65832e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,6 +86,10 @@ Your principal for ICP wallets and decentralized exchanges: ueuar-wxbnk-bdcsr-dn (run `dfx identity get-principal` to display) ``` +### feat: Add pre-install tasks + +Add pre-install tasks, which can be defined by the new `pre-install` key for canister objects in `dfx.json` with a command or list of commands. + ## Dependencies ### Frontend canister diff --git a/docs/dfx-json-schema.json b/docs/dfx-json-schema.json index b49c3a3b6a..6f6fdc505f 100644 --- a/docs/dfx-json-schema.json +++ b/docs/dfx-json-schema.json @@ -496,6 +496,16 @@ } ] }, + "pre_install": { + "title": "Pre-Install Commands", + "description": "One or more commands to run pre canister installation. These commands are executed in the root of the project.", + "default": [], + "allOf": [ + { + "$ref": "#/definitions/SerdeVec_for_String" + } + ] + }, "pullable": { "title": "Pullable", "description": "Defines required properties so that this canister is ready for `dfx deps pull` by other projects.", diff --git a/e2e/assets/post_install/dfx.json b/e2e/assets/pre_post_install/dfx.json similarity index 57% rename from e2e/assets/post_install/dfx.json rename to e2e/assets/pre_post_install/dfx.json index 13f72c635f..e414a26503 100644 --- a/e2e/assets/post_install/dfx.json +++ b/e2e/assets/pre_post_install/dfx.json @@ -1,9 +1,18 @@ { "version": 1, "canisters": { + "preinstall": { + "main": "main.mo", + "pre_install": "echo hello-pre-file" + }, + "preinstall_script": { + "main": "main.mo", + "pre_install": "preinstall.sh", + "dependencies": ["preinstall"] + }, "postinstall": { "main": "main.mo", - "post_install": "echo hello-file" + "post_install": "echo hello-post-file" }, "postinstall_script": { "main": "main.mo", diff --git a/e2e/assets/post_install/main.mo b/e2e/assets/pre_post_install/main.mo similarity index 100% rename from e2e/assets/post_install/main.mo rename to e2e/assets/pre_post_install/main.mo diff --git a/e2e/assets/post_install/postinstall.sh b/e2e/assets/pre_post_install/postinstall.sh similarity index 77% rename from e2e/assets/post_install/postinstall.sh rename to e2e/assets/pre_post_install/postinstall.sh index a20c8dc438..d18ad161fd 100755 --- a/e2e/assets/post_install/postinstall.sh +++ b/e2e/assets/pre_post_install/postinstall.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash echo "working directory of post-install script: '$(pwd)'" -echo hello-script +echo hello-post-script diff --git a/e2e/assets/pre_post_install/preinstall.sh b/e2e/assets/pre_post_install/preinstall.sh new file mode 100755 index 0000000000..386574a9a2 --- /dev/null +++ b/e2e/assets/pre_post_install/preinstall.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +echo "working directory of pre-install script: '$(pwd)'" +echo hello-pre-script diff --git a/e2e/tests-dfx/install.bash b/e2e/tests-dfx/install.bash index e947e8e22d..1286493d92 100644 --- a/e2e/tests-dfx/install.bash +++ b/e2e/tests-dfx/install.bash @@ -90,26 +90,36 @@ teardown() { assert_command_fail dfx canister install --all --wasm "${archive:?}/wallet/0.10.0/wallet.wasm" } -@test "install runs post-install tasks" { - install_asset post_install +@test "install runs pre-post-install tasks" { + install_asset pre_post_install dfx_start assert_command dfx canister create --all assert_command dfx build + assert_command dfx canister install preinstall + assert_match 'hello-pre-file' + + assert_command dfx canister install preinstall_script + assert_match 'hello-pre-script' + + echo 'return 1' >> preinstall.sh + assert_command_fail dfx canister install preinstall_script --mode upgrade + assert_match 'hello-pre-script' + assert_command dfx canister install postinstall - assert_match 'hello-file' + assert_match 'hello-post-file' assert_command dfx canister install postinstall_script - assert_match 'hello-script' + assert_match 'hello-post-script' echo 'return 1' >> postinstall.sh assert_command_fail dfx canister install postinstall_script --mode upgrade - assert_match 'hello-script' + assert_match 'hello-post-script' } -@test "post-install tasks run in project root" { - install_asset post_install +@test "pre-post-install tasks run in project root" { + install_asset pre_post_install dfx_start assert_command dfx canister create --all @@ -117,42 +127,57 @@ teardown() { cd src/e2e_project_backend + assert_command dfx canister install preinstall_script + assert_match 'hello-pre-script' + assert_match "working directory of pre-install script: '.*/working-dir/e2e_project'" + assert_command dfx canister install postinstall_script - assert_match 'hello-script' + assert_match 'hello-post-script' assert_match "working directory of post-install script: '.*/working-dir/e2e_project'" } -@test "post-install tasks receive environment variables" { - install_asset post_install +@test "pre-post-install tasks receive environment variables" { + install_asset pre_post_install dfx_start + echo "echo hello \$CANISTER_ID" >> preinstall.sh echo "echo hello \$CANISTER_ID" >> postinstall.sh assert_command dfx canister create --all assert_command dfx build - id=$(dfx canister id postinstall_script) + id_pre=$(dfx canister id preinstall_script) + id_post=$(dfx canister id postinstall_script) assert_command dfx canister install --all - assert_match "hello $id" + assert_match "hello $id_post" + assert_command dfx canister install preinstall_script --mode upgrade + assert_match "hello $id_pre" assert_command dfx canister install postinstall_script --mode upgrade - assert_match "hello $id" + assert_match "hello $id_post" assert_command dfx deploy - assert_match "hello $id" + assert_match "hello $id_post" + assert_command dfx deploy preinstall_script + assert_match "hello $id_pre" assert_command dfx deploy postinstall_script - assert_match "hello $id" + assert_match "hello $id_post" } -@test "post-install tasks discover dependencies" { - install_asset post_install +@test "pre-post-install tasks discover dependencies" { + install_asset pre_post_install dfx_start + echo "echo hello \$CANISTER_ID_PREINSTALL" >> preinstall.sh echo "echo hello \$CANISTER_ID_POSTINSTALL" >> postinstall.sh assert_command dfx canister create --all assert_command dfx build - id=$(dfx canister id postinstall) + id_pre=$(dfx canister id preinstall) + id_post=$(dfx canister id postinstall) + + assert_command dfx canister install preinstall_script + assert_match "hello $id_pre" assert_command dfx canister install postinstall_script - assert_match "hello $id" + assert_match "hello $id_post" } @test "can install gzip wasm" { diff --git a/src/dfx-core/src/config/model/dfinity.rs b/src/dfx-core/src/config/model/dfinity.rs index e05b1ecfd5..edee38d405 100644 --- a/src/dfx-core/src/config/model/dfinity.rs +++ b/src/dfx-core/src/config/model/dfinity.rs @@ -271,6 +271,12 @@ pub struct ConfigCanistersCanister { #[serde(flatten)] pub type_specific: CanisterTypeProperties, + /// # Pre-Install Commands + /// One or more commands to run pre canister installation. + /// These commands are executed in the root of the project. + #[serde(default)] + pub pre_install: SerdeVec, + /// # Post-Install Commands /// One or more commands to run post canister installation. /// These commands are executed in the root of the project. diff --git a/src/dfx/src/lib/canister_info.rs b/src/dfx/src/lib/canister_info.rs index df3dd1e027..95b4808af9 100644 --- a/src/dfx/src/lib/canister_info.rs +++ b/src/dfx/src/lib/canister_info.rs @@ -49,6 +49,7 @@ pub struct CanisterInfo { type_specific: CanisterTypeProperties, dependencies: Vec, + pre_install: Vec, post_install: Vec, main: Option, shrink: Option, @@ -171,6 +172,7 @@ impl CanisterInfo { _ => build_defaults.get_args(), }; + let pre_install = canister_config.pre_install.clone().into_vec(); let post_install = canister_config.post_install.clone().into_vec(); let metadata = CanisterMetadataConfig::new(&canister_config.metadata, &network_name); @@ -190,6 +192,7 @@ impl CanisterInfo { args, type_specific, dependencies, + pre_install, post_install, main: canister_config.main.clone(), shrink: canister_config.shrink, @@ -262,6 +265,10 @@ impl CanisterInfo { &self.packtool } + pub fn get_pre_install(&self) -> &[String] { + &self.pre_install + } + pub fn get_post_install(&self) -> &[String] { &self.post_install } diff --git a/src/dfx/src/lib/operations/canister/install_canister.rs b/src/dfx/src/lib/operations/canister/install_canister.rs index 3591021dbb..1d446fe2fb 100644 --- a/src/dfx/src/lib/operations/canister/install_canister.rs +++ b/src/dfx/src/lib/operations/canister/install_canister.rs @@ -55,6 +55,17 @@ pub async fn install_canister( let log = env.get_logger(); let agent = env.get_agent(); let network = env.get_network_descriptor(); + if !canister_info.get_pre_install().is_empty() { + let config = env.get_config()?; + run_customized_install_tasks( + env, + canister_info, + true, + network, + pool, + env_file.or_else(|| config.as_ref()?.get_config().output_env_file.as_deref()), + )?; + } if !network.is_ic && named_canister::get_ui_canister_id(canister_id_store).is_none() { named_canister::install_ui_canister(env, canister_id_store, None).await?; } @@ -282,9 +293,10 @@ The command line value will be used.", } if !canister_info.get_post_install().is_empty() { let config = env.get_config()?; - run_post_install_tasks( + run_customized_install_tasks( env, canister_info, + false, network, pool, env_file.or_else(|| config.as_ref()?.get_config().output_env_file.as_deref()), @@ -435,14 +447,16 @@ fn check_stable_compatibility( }) } -#[context("Failed to run post-install tasks")] -fn run_post_install_tasks( +#[context("Failed to run {}-install tasks", if is_pre_install { "pre" } else { "post" })] +fn run_customized_install_tasks( env: &dyn Environment, canister: &CanisterInfo, + is_pre_install: bool, network: &NetworkDescriptor, pool: Option<&CanisterPool>, env_file: Option<&Path>, ) -> DfxResult { + let pre_or_post = if is_pre_install { "pre" } else { "post" }; let tmp; let pool = match pool { Some(pool) => pool, @@ -450,8 +464,9 @@ fn run_post_install_tasks( let config = env.get_config_or_anyhow()?; let canisters_to_load = all_project_canisters_with_ids(env, &config); - tmp = CanisterPool::load(env, false, &canisters_to_load) - .context("Error collecting canisters for post-install task")?; + tmp = CanisterPool::load(env, false, &canisters_to_load).context(format!( + "Error collecting canisters for {pre_or_post}-install task" + ))?; &tmp } }; @@ -460,27 +475,43 @@ fn run_post_install_tasks( .iter() .map(|can| can.canister_id()) .collect_vec(); - for task in canister.get_post_install() { - run_post_install_task(canister, task, network, pool, &dependencies, env_file)?; + let tasks = if is_pre_install { + canister.get_pre_install() + } else { + canister.get_post_install() + }; + for task in tasks { + run_customized_install_task( + canister, + task, + is_pre_install, + network, + pool, + &dependencies, + env_file, + )?; } Ok(()) } -#[context("Failed to run post-install task {task}")] -fn run_post_install_task( +#[context("Failed to run {}-install task {}", if is_pre_install { "pre" } else { "post" }, task)] +fn run_customized_install_task( canister: &CanisterInfo, task: &str, + is_pre_install: bool, network: &NetworkDescriptor, pool: &CanisterPool, dependencies: &[Principal], env_file: Option<&Path>, ) -> DfxResult { + let pre_or_post = if is_pre_install { "pre" } else { "post" }; let cwd = canister.get_workspace_root(); let words = shell_words::split(task) - .with_context(|| format!("Error interpreting post-install task `{task}`"))?; + .with_context(|| format!("Error interpreting {pre_or_post}-install task `{task}`"))?; let canonicalized = dfx_core::fs::canonicalize(&cwd.join(&words[0])) .or_else(|_| which::which(&words[0])) .map_err(|_| anyhow!("Cannot find command or file {}", &words[0]))?; + let mut command = Command::new(canonicalized); command.args(&words[1..]); let vars = @@ -492,13 +523,14 @@ fn run_post_install_task( .current_dir(cwd) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()); + let status = command.status()?; if !status.success() { match status.code() { Some(code) => { - bail!("The post-install task `{task}` failed with exit code {code}") + bail!("The {pre_or_post}-install task `{task}` failed with exit code {code}") } - None => bail!("The post-install task `{task}` was terminated by a signal"), + None => bail!("The {pre_or_post}-install task `{task}` was terminated by a signal"), } } Ok(())