Skip to content

Commit 78ec7a7

Browse files
staticfloatstaticfloat
authored
Merge pull request #29 from staticfloat/sf/group_capable
Sf/group capable
2 parents 0d311d5 + ff817f9 commit 78ec7a7

File tree

8 files changed

+98
-68
lines changed

8 files changed

+98
-68
lines changed

‎bin/create_repo_key‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/bin/bash
1+
#!/usr/bin/env bash
22

33
## This script creates a "repository key". This key is stored, encrypted, within a special folder
44
## that the agent knows to look inside of when decrypting secrets. It also sets the repository up

‎bin/sign_treehashes‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,4 +44,4 @@ for YAML_PATH in ${YAML_PATHS[@]}; do
4444
EOD
4545
fi
4646
done
47-
done
47+
done

‎bin/verify_treehashes‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ for YAML_PATH in ${YAML_PATHS[@]}; do
3232
PIPELINE_TREEHASH_FILESOURCE="$(cut -d'&' -f4 <<<"${TRIPLET}" | tr -d '"')"
3333

3434
# Compare decrypted treehash with calculated treehash
35-
vecho "About to decrypt ${PIPELINE_ENCRYPTED_TREEHASH} with ${REPO_KEY_PATH}"
3635
PIPELINE_DECRYPTED_TREEHASH="$(base64dec <<<"${PIPELINE_ENCRYPTED_TREEHASH}" | decrypt_aes "${REPO_KEY_PATH}" 2>/dev/null || true)"
3736
if [[ "${PIPELINE_DECRYPTED_TREEHASH}" == "${PIPELINE_TREEHASH}" ]]; then
3837
echo "[${YAML_PATH}] -> ${PIPELINE_PATH}: ✔️"

‎hooks/post-command‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,13 +45,18 @@ for PIPELINE_IDX in "${!SIGNED_PIPELINES[@]}"; do
4545

4646
# Hash up the inputs
4747
readarray -d '' -t PIPELINE_INPUTS < <(collect_buildkite_array "BUILDKITE_PLUGIN_CRYPTIC_SIGNED_PIPELINES_${PIPELINE_IDX}_INPUTS")
48+
vecho " -> Performing pipeline launch:"
49+
vecho " -> ${PIPELINE_PATH}"
4850
INPUT_TREEHASHES=( "$(calc_treehash <<<"${PIPELINE_PATH}")" )
4951
for PATTERN in "${PIPELINE_INPUTS[@]}"; do
50-
INPUT_TREEHASHES+=( "$(collect_glob_pattern "${PATTERN}" | calc_treehash)" )
52+
HASH="$(collect_glob_pattern "${PATTERN}" | calc_treehash)"
53+
vecho " + ${HASH} <- ${PATTERN}"
54+
INPUT_TREEHASHES+=( "${HASH}" )
5155
done
5256

5357
# Hash all treehashes together to get full input hash
5458
FULL_TREEHASH="$(printf "%s" "${INPUT_TREEHASHES[@]}" | calc_shasum)"
59+
vecho " ∟ ${FULL_TREEHASH}"
5560

5661
# Verify this with the treehash signature
5762
SIGNATURE_VAR="BUILDKITE_PLUGIN_CRYPTIC_SIGNED_PIPELINES_${PIPELINE_IDX}_SIGNATURE"
@@ -75,7 +80,6 @@ for PIPELINE_IDX in "${!SIGNED_PIPELINES[@]}"; do
7580
fi
7681
if [[ "$(decrypt_aes "${UNENCRYPTED_REPO_KEY_PATH}" <"${SIGNATURE_FILE}")" != "${FULL_TREEHASH}" ]]; then
7782
SIGNATURE_FAIL_MSG="Pipeline '${PIPELINE_PATH}' fails treehash signature check! You may need to re-run cryptic/bin/sign_treehashes!"
78-
echo "${SIGNATURE_FAIL_MSG}" >&2
7983

8084
HASH_OVERRIDE_VAR="BUILDKITE_PLUGIN_CRYPTIC_SIGNED_PIPELINES_${PIPELINE_IDX}_ALLOW_HASH_OVERRIDE"
8185
if [[ -v "${HASH_OVERRIDE_VAR}" ]] && [[ "${!HASH_OVERRIDE_VAR}" == "true" ]]; then
@@ -103,7 +107,7 @@ for PIPELINE_IDX in "${!SIGNED_PIPELINES[@]}"; do
103107
else
104108
# Execute `die` in a subshell so that we can print out failure messages for each pipeline,
105109
# then fail out once at the end, ignoring the `exit` and failure that it creates.
106-
(die "Refusing to continue execution; pipeline '${PIPELINE_PATH}' fails treehash signature check! You may need to re-run cryptic/bin/sign_treehashes!"; ) || true
110+
(die "${SIGNATURE_FAIL_MSG}"; ) || true
107111
SHOULD_FAIL=true
108112
continue
109113
fi

‎lib/argparse.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/bin/bash
1+
#!/usr/bin/env bash
22

33
VERBOSE="${VERBOSE:-false}"
44
function verbose() {
@@ -95,4 +95,4 @@ if verbose; then
9595
fi
9696

9797
# Restore positional parameters back to `$1`, `$2`, etc...
98-
set -- "${POSITIONAL[@]}"
98+
set -- "${POSITIONAL[@]}"

‎lib/common.sh‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/bin/bash
1+
#!/usr/bin/env bash
22

33
set -eou pipefail
44
shopt -s extglob
@@ -20,7 +20,7 @@ function die() {
2020
}
2121

2222
# Returns true if verbose mode is enabled
23-
VERBOSE="${VERBOSE:-false}"
23+
VERBOSE="${BUILDKITE_PLUGIN_CRYPTIC_VERBOSE:-${VERBOSE:-false}}"
2424
function verbose() {
2525
[[ "${VERBOSE}" == "true" ]]
2626
}
@@ -439,6 +439,10 @@ function find_repository_root() {
439439
}
440440

441441
function find_repo_key() {
442+
if [[ -v "REPO_KEY_PATH" ]]; then
443+
return
444+
fi
445+
442446
# First, check to see if we have a decrypted repo key:
443447
REPO_KEY_PATH="${REPO_ROOT}/.buildkite/cryptic_repo_keys/repo_key"
444448
if [[ -f "${REPO_KEY_PATH}" ]]; then

‎lib/yaml_extraction_prologue.sh‎

Lines changed: 77 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -9,20 +9,20 @@ fi
99
# Extract the `variables:` section of a cryptic `pipeline.yml` plugin section
1010
function extract_encrypted_variables() {
1111
# Iterate over the steps in the yaml file
12-
(shyaml get-values-0 steps <"${1}" || true) |
12+
(shyaml -q get-values-0 steps <"${1}" || true) |
1313
while IFS='' read -r -d '' STEP; do
1414
# For each step, get its list of plugins
15-
(shyaml get-values-0 plugins <<<"${STEP}" 2>/dev/null || true) |
15+
(shyaml -q get-values-0 plugins <<<"${STEP}" || true) |
1616
while IFS='' read -r -d '' PLUGINS; do
1717
# Get the plugin names
18-
(shyaml keys-0 <<<"${PLUGINS}" || true) |
18+
(shyaml -q keys-0 <<<"${PLUGINS}" || true) |
1919
while IFS='' read -r -d '' PLUGIN_NAME; do
2020
# Skip plugins that are not named `cryptic`
2121
if [[ "${PLUGIN_NAME}" != staticfloat/cryptic* ]]; then
2222
continue
2323
fi
2424
# For each plugin, if its `cryptic`, extract the variables
25-
(shyaml get-values-0 "${PLUGIN_NAME}.variables" <<<"${PLUGINS}" 2>/dev/null || true) |
25+
(shyaml -q get-values-0 "${PLUGIN_NAME}.variables" <<<"${PLUGINS}" || true) |
2626
while IFS='' read -r -d '' VAR; do
2727
printf "%s\n" "${VAR}"
2828
done
@@ -34,20 +34,20 @@ function extract_encrypted_variables() {
3434
# Extract all variables that match "CRYPTIC_ADHOC_SECRET_*"
3535
function extract_adhoc_encrypted_variables() {
3636
# Iterate over any global env mappings
37-
(shyaml keys-0 env <"${1}" 2>/dev/null || true) |
37+
(shyaml -q keys-0 env <"${1}" || true) |
3838
while IFS='' read -r -d '' VARNAME; do
3939
if [[ "${VARNAME}" == CRYPTIC_ADHOC_SECRET_* ]]; then
40-
printf "%s\n" "${VARNAME:21}=$(shyaml get-value env.${VARNAME} <"${1}")"
40+
printf "%s\n" "${VARNAME:21}=$(shyaml -q get-value env.${VARNAME} <"${1}")"
4141
fi
4242
done
4343

4444
# Iterate over the steps in the yaml file
45-
(shyaml get-values-0 steps <"${1}" || true) |
45+
(shyaml -q get-values-0 steps <"${1}" || true) |
4646
while IFS='' read -r -d '' STEP; do
47-
(shyaml keys-0 env <<<"${STEP}" 2>/dev/null || true) |
47+
(shyaml -q keys-0 env <<<"${STEP}" || true) |
4848
while IFS='' read -r -d '' VARNAME; do
4949
if [[ "${VARNAME}" == CRYPTIC_ADHOC_SECRET_* ]]; then
50-
printf "%s\n" "${VARNAME:21}=$(shyaml get-value env.${VARNAME} <<<"${STEP}")"
50+
printf "%s\n" "${VARNAME:21}=$(shyaml -q get-value env.${VARNAME} <<<"${STEP}")"
5151
fi
5252
done
5353
done
@@ -56,20 +56,20 @@ function extract_adhoc_encrypted_variables() {
5656
# Extract the `files:` section of a cryptic `pipeline.yml` plugin section
5757
function extract_encrypted_files() {
5858
# Iterate over the steps in the yaml file
59-
(shyaml get-values-0 steps <"${1}" || true) |
59+
(shyaml -q get-values-0 steps <"${1}" || true) |
6060
while IFS='' read -r -d '' STEP; do
6161
# For each step, get its list of plugins
62-
(shyaml get-values-0 plugins <<<"${STEP}" 2>/dev/null || true) |
62+
(shyaml -q get-values-0 plugins <<<"${STEP}" || true) |
6363
while IFS='' read -r -d '' PLUGINS; do
6464
# Get the plugin names
65-
(shyaml keys-0 <<<"${PLUGINS}" || true) |
65+
(shyaml -q keys-0 <<<"${PLUGINS}" || true) |
6666
while IFS='' read -r -d '' PLUGIN_NAME; do
6767
# Skip plugins that are not named `cryptic`
6868
if [[ "${PLUGIN_NAME}" != staticfloat/cryptic* ]]; then
6969
continue
7070
fi
7171
# For each plugin, if its `cryptic`, extract the files
72-
(shyaml get-values-0 "${PLUGIN_NAME}.files" <<<"${PLUGINS}" 2>/dev/null || true) |
72+
(shyaml -q get-values-0 "${PLUGIN_NAME}.files" <<<"${PLUGINS}" || true) |
7373
while IFS='' read -r -d '' FILE; do
7474
FILE="$(echo ${FILE} | tr -d '"')"
7575
printf "%s\n" "${FILE}"
@@ -88,56 +88,75 @@ function extract_pipeline_treehashes() {
8888
vecho "Extracting treehashes from '${YAML_PATH}'"
8989

9090
# Iterate over the steps in the yaml file
91-
(shyaml get-values-0 steps <"${1}" || true) |
91+
(shyaml -q get-values-0 steps <"${1}" || true) |
9292
while IFS='' read -r -d '' STEP; do
93-
# For each step, get its list of plugins
94-
(shyaml get-values-0 plugins <<<"${STEP}" 2>/dev/null || true) |
95-
while IFS='' read -r -d '' PLUGINS; do
96-
# Get the plugin names
97-
(shyaml keys-0 <<<"${PLUGINS}" || true) |
98-
while IFS='' read -r -d '' PLUGIN_NAME; do
99-
# Skip plugins that are not named `cryptic`
100-
if [[ "${PLUGIN_NAME}" != staticfloat/cryptic* ]]; then
101-
continue
102-
fi
93+
# If this step is a `group` step, let's iterate over each of its steps
94+
if shyaml -q get-value 'group' >/dev/null <<<"${STEP}"; then
95+
(shyaml -q get-values-0 steps <<<"${STEP}" || true) |
96+
while IFS='' read -r -d '' INNER_STEP; do
97+
extract_plugin_treehashes "${INNER_STEP}"
98+
done
99+
else
100+
extract_plugin_treehashes "${STEP}"
101+
fi
102+
done
103103

104-
# For each plugin, if its `cryptic`, walk over the pipelines
105-
(shyaml get-values-0 "${PLUGIN_NAME}.signed_pipelines" <<<"${PLUGINS}" 2>/dev/null || true) |
106-
while IFS='' read -r -d '' PIPELINE; do
107-
# For each signed pipeline, get its pipeline path and its inputs
108-
PIPELINE_PATH="$(shyaml get-value "pipeline" <<<"${PIPELINE}" 2>/dev/null || true)"
109-
110-
# Start by calculating the treehash of the yaml file
111-
INPUT_TREEHASHES=( "$(calc_treehash <<<"${PIPELINE_PATH}")" )
112-
113-
# Next, calculate the treehash of the rest of the glob patterns
114-
for PATTERN in $(shyaml get-values "inputs" <<<"${PIPELINE}" 2>/dev/null || true); do
115-
INPUT_TREEHASHES+=( "$(collect_glob_pattern "${PATTERN}" | calc_treehash)" )
116-
done
117-
118-
# Calculate full treehash
119-
FULL_TREEHASH="$(printf "%s" "${INPUT_TREEHASHES[@]}" | calc_shasum)"
120-
121-
# If `signature_file` is defined, use it!
122-
local BASE64_ENCRYPTED_TREEHASH=""
123-
local TREEHASH_FILE_SOURCE=""
124-
if shyaml get-value "signature_file" <<<"${PIPELINE}" 2>/dev/null >/dev/null; then
125-
TREEHASH_FILE_SOURCE="$(shyaml get-value "signature_file" <<<"${PIPELINE}" 2>/dev/null)"
126-
if [[ -f "${TREEHASH_FILE_SOURCE}" ]]; then
127-
BASE64_ENCRYPTED_TREEHASH="$(base64enc <"${TREEHASH_FILE_SOURCE}")"
128-
fi
129-
else
130-
# Try to extract the signature from the yaml directly too
131-
BASE64_ENCRYPTED_TREEHASH="$(shyaml get-value "signature" <<<"${PIPELINE}" 2>/dev/null || true)"
132-
fi
104+
# Don't stay in `${REPO_ROOT}`
105+
popd >/dev/null
106+
}
107+
108+
function extract_plugin_treehashes() {
109+
# Get the list of plugins
110+
(shyaml -q get-values-0 plugins <<<"${1}" || true) |
111+
while IFS='' read -r -d '' PLUGINS; do
112+
# Get the plugin names
113+
(shyaml -q keys-0 <<<"${PLUGINS}" || true) |
114+
while IFS='' read -r -d '' PLUGIN_NAME; do
115+
# Skip plugins that are not named `cryptic`
116+
if [[ "${PLUGIN_NAME}" != staticfloat/cryptic* ]]; then
117+
continue
118+
fi
119+
120+
# For each plugin, if its `cryptic`, walk over the pipelines
121+
(shyaml -q get-values-0 "${PLUGIN_NAME}.signed_pipelines" <<<"${PLUGINS}" || true) |
122+
while IFS='' read -r -d '' PIPELINE; do
123+
# For each signed pipeline, get its pipeline path and its inputs
124+
PIPELINE_PATH="$(shyaml -q get-value "pipeline" <<<"${PIPELINE}" || true)"
133125

134-
# Print out treehash and pipeline path
135-
printf "%s&%s&%s&%s\n" "${PIPELINE_PATH}" "${FULL_TREEHASH}" "${BASE64_ENCRYPTED_TREEHASH}" "${TREEHASH_FILE_SOURCE}"
126+
vecho " -> Found pipeline launch:"
127+
vecho " -> ${PIPELINE_PATH}"
128+
129+
# Start by calculating the treehash of the yaml file
130+
INPUT_TREEHASHES=( "$(calc_treehash <<<"${PIPELINE_PATH}")" )
131+
132+
# Next, calculate the treehash of the rest of the glob patterns
133+
readarray -d '' PATTERNS -t < <(shyaml -q get-values-0 "inputs" <<<"${PIPELINE}")
134+
for PATTERN in "${PATTERNS[@]}"; do
135+
HASH="$(collect_glob_pattern "${PATTERN}" | calc_treehash)"
136+
vecho " + ${HASH} <- ${PATTERN}"
137+
INPUT_TREEHASHES+=( "${HASH}" )
136138
done
139+
140+
# Calculate full treehash
141+
FULL_TREEHASH="$(printf "%s" "${INPUT_TREEHASHES[@]}" | calc_shasum)"
142+
vecho " ∟ ${FULL_TREEHASH}"
143+
144+
# If `signature_file` is defined, use it!
145+
local BASE64_ENCRYPTED_TREEHASH=""
146+
local TREEHASH_FILE_SOURCE=""
147+
if shyaml get-value "signature_file" <<<"${PIPELINE}" >/dev/null; then
148+
TREEHASH_FILE_SOURCE="$(shyaml -q get-value "signature_file" <<<"${PIPELINE}")"
149+
if [[ -f "${TREEHASH_FILE_SOURCE}" ]]; then
150+
BASE64_ENCRYPTED_TREEHASH="$(base64enc <"${TREEHASH_FILE_SOURCE}")"
151+
fi
152+
else
153+
# Try to extract the signature from the yaml directly too
154+
BASE64_ENCRYPTED_TREEHASH="$(shyaml -q get-value "signature" <<<"${PIPELINE}" || true)"
155+
fi
156+
157+
# Print out treehash and pipeline path
158+
printf "%s&%s&%s&%s\n" "${PIPELINE_PATH}" "${FULL_TREEHASH}" "${BASE64_ENCRYPTED_TREEHASH}" "${TREEHASH_FILE_SOURCE}"
137159
done
138160
done
139161
done
140-
141-
# Don't stay in `${REPO_ROOT}`
142-
popd >/dev/null
143162
}

‎plugin.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,4 +41,8 @@ configuration:
4141
unsigned_pipelines:
4242
type: array
4343

44+
# Set this to allow for greater verbosity in the cryptic plugin's output
45+
verbose:
46+
type: bool
47+
4448
additionalProperties: false

0 commit comments

Comments
 (0)