-
Notifications
You must be signed in to change notification settings - Fork 57
/
Copy pathgarbage_collect_releases.sh
190 lines (164 loc) · 8.06 KB
/
garbage_collect_releases.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
#!/bin/bash
#
# Copyright (c) 2024 The Flatcar Maintainers.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
#
# garbage_collect_releases() should be called after sourcing.
#
# OPTIONAL INPUT
# - Number releases to keep per channel. Defaults to 10.
# - Number of LTS channels to keep. Defaults to 2 (i.e. the current and the previous (deprecated) LTS).
# - DRY_RUN (Env variable). Set to "y" to just list what would be done but not
# actually purge anything.
# Flatcar build cache releases artifacts garbage collector.
# This script removes release artifacts of past releases from the build cache.
# Note that release artifacts are copied to official mirrors upon release, so there's
# no need to keep a copy on the build cache server.
function garbage_collect_releases() {
# Run a subshell, so the traps, environment changes and global
# variables are not spilled into the caller.
(
set -euo pipefail
_garbage_collect_releases_impl "${@}"
)
}
# --
function _garbage_collect_releases_impl() {
local keep_per_chan="${1:-10}"
local keep_lts_releases="${2:-2}"
local dry_run="${DRY_RUN:-}"
echo
echo "Number of versions to keep per channel: '${keep_per_chan}'"
echo "Number of LTS major releases to keep: '${keep_lts_releases}'"
echo
source ci-automation/ci_automation_common.sh
local sshcmd="$(gen_sshcmd)"
local keep=$(
# For some reasons this is set to the empty string in some environments and it makes gawk
# behave like POSIX awk (i.e. no 'gensub').
unset POSIXLY_CORRECT
curl -s "${RELEASES_JSON_FEED}" \
| jq -r 'keys_unsorted | .[] | match("[0-9]+\\.[0-9]+\\.[0-9]+") | .string' \
| sort -Vr \
| awk -v keep="${keep_per_chan}" -v lts="${keep_lts_releases}" '
{
version = $1
chan_num = gensub("[0-9]+\\.([0-9]+)\\.[0-9]+","\\1","g", version) + 0
major = gensub("([0-9]+)\\.[0-9]+\\.[0-9]+","\\1","g", version) + 0
if (chan_num <= 2) {
if (chan_count[chan_num] < keep)
print version
chan_count[chan_num] = chan_count[chan_num] + 1
} else {
if ( (chan_count["lts"][major] < keep) \
&& (length(chan_count["lts"]) <= lts) )
print version
chan_count["lts"][major] = chan_count["lts"][major] + 1
}
} '
)
mapfile -t keep_versions <<<"${keep}"
echo
echo "######## The following version(s) will be kept ########"
if [ "$dry_run" = "y" ] ; then
echo
echo "(NOTE this is just a dry run since DRY_RUN=y)"
echo
fi
printf "%s\n" "${keep_versions[@]}"
local dir=""
for dir in "sdk/amd64" \
"containers" \
"boards/amd64-usr" \
"boards/arm64-usr" \
"images/amd64" \
"images/arm64" \
"testing" \
; do
local fullpath="${BUILDCACHE_PATH_PREFIX}/${dir}"
echo
echo "## Processing '${fullpath}'"
echo "---------------------------"
for version in $($sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
"ls -1 ${BUILDCACHE_PATH_PREFIX}/${dir} | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$'"); do
local o_fullpath="${fullpath}/${version}"
# Go through all deletion candidate versions in directory
# skip if candidate version is marked for keeping OR if it's a new release about to be published
# or if it ships an SDK used by a version in the keep list.
if printf "%s\n" "${keep_versions[@]}" \
| { unset POSIXLY_CORRECT ; awk -v candidate_version="${version}" -v path="${dir}" '
BEGIN {
# Candidate version (from build cache directory) that was passed in candidate_version variable
candidate_major = gensub("([0-9]+)\\.[0-9]+\\.[0-9]+","\\1","g", candidate_version) + 0
candidate_minor = gensub("[0-9]+\\.([0-9]+)\\.[0-9]+","\\1","g", candidate_version) + 0
candidate_patch = gensub("[0-9]+\\.[0-9]+\\.([0-9]+)","\\1","g", candidate_version) + 0
ret = 1
}
{
# The whole keep versions list is piped into AWK; match the candidate (build cache) version agains each entry.
keep_list_entry = $0
if (keep_list_entry == candidate_version) {
print ""
print "## Skipping " candidate_version " because it is in the keep list."
ret = 0
exit
}
keep_list_major = gensub("([0-9]+)\\.[0-9]+\\.[0-9]+","\\1","g", keep_list_entry) + 0
keep_list_minor = gensub("[0-9]+\\.([0-9]+)\\.[0-9]+","\\1","g", keep_list_entry) + 0
keep_list_patch = gensub("[0-9]+\\.[0-9]+\\.([0-9]+)","\\1","g", keep_list_entry) + 0
if ( (path == "sdk/amd64") && (candidate_major == keep_list_major) ) {
print ""
print "## Skipping " candidate_version " in " path " because it contains the SDK for release " keep_list_entry " in keep list."
ret = 0
exit
}
# keep list piped into AWK is sorted (descending), so the very first (i.e. highest) version on top is the most recent Alpha release
if (latest_alpha_major == "")
latest_alpha_major = keep_list_major
if (candidate_major > latest_alpha_major) {
print ""
print "## Skipping " candidate_version " because major version is higher than the latest Alpha (" latest_alpha_major ") in keep list."
print "(I.e. this is an unpublished new Alpha release)"
ret = 0
exit
}
if ((candidate_major == keep_list_major) && (candidate_minor > keep_list_minor)) {
print ""
print "## Skipping " candidate_version " because major version is in keep list and minor version is higher than the latest release."
print "(I.e. this is an unpublished channel progression " keep_list_entry " -> " candidate_version ")"
ret = 0
exit
}
if ((candidate_major == keep_list_major) && (candidate_minor == keep_list_minor) && (candidate_patch > keep_list_patch)) {
print ""
print "## Skipping " version " because major and minor versions are in keep list and patch version is higher than the latest release."
print "(I.e. this is an unpublished new patch release " keep_list_entry " -> " candidate_version ")"
ret = 0
exit
}
}
END {
exit ret
}' ; } then
continue
fi
echo
echo "## Removing version '${version}' in '${o_fullpath}'"
echo
echo "## The following files will be removed ##"
$sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
"ls -la ${o_fullpath} || true"
if [ "$dry_run" != "y" ] ; then
set -x
$sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
"rm -rf ${o_fullpath} || true"
set +x
else
echo "## (DRY_RUN=y so not doing anything) ##"
fi
done
done
}
# --