Skip to content

Commit f4efa94

Browse files
dduportaldduportal
committed
Merge pull request #777 from jenkins-infra/helpdesk-4774
feat(packaging) stop generating RPM repository on remote VM + introduces staging (cherry picked from commit 5f20e41)
1 parent eefcdd4 commit f4efa94

File tree

7 files changed

+150
-95
lines changed

7 files changed

+150
-95
lines changed

‎Jenkinsfile.d/core/package‎

Lines changed: 50 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ pipeline {
7070
}
7171

7272
// ENV JENKINS_VERSION
73-
// stable: It represents the latest stable version that satifies version pattern X.Y.Z
73+
// stable: It represents the latest stable version that satisfies version pattern X.Y.Z
7474
// weekly: It represents the latest weekly version that satisfies version pattern X.Y
7575
// <version>: where version represent any valid existing version like 2.176.2
7676

@@ -119,18 +119,17 @@ pipeline {
119119

120120
}
121121
stage('Get GPG key') {
122-
123122
steps {
124123
checkout scm
125-
dir (WORKING_DIRECTORY){
124+
dir (WORKING_DIRECTORY) {
126125
git branch: PACKAGING_GIT_BRANCH, credentialsId: 'release-key', url: PACKAGING_GIT_REPOSITORY
127126
}
128127

129128
sh '''
130129
./utils/release.bash --getGPGKeyFromAzure
131130
'''
132131

133-
dir (WORKING_DIRECTORY){
132+
dir (WORKING_DIRECTORY) {
134133
stash includes: GPG_FILE , name: 'GPG'
135134
}
136135

@@ -139,7 +138,6 @@ pipeline {
139138
}
140139

141140
stage('Get Code Signing Certificate') {
142-
143141
steps {
144142
sh '''
145143
utils/release.bash --downloadAzureKeyvaultSecret
@@ -150,52 +148,48 @@ pipeline {
150148
}
151149
}
152150

153-
stage('Download WAR archive to package'){
151+
stage('Download WAR archive to package') {
154152
steps{
155153
sh '''
156154
./utils/release.bash --downloadJenkins
157155
'''
158-
dir (WORKING_DIRECTORY){
156+
dir (WORKING_DIRECTORY) {
159157
stash includes: WAR_FILENAME, name: "WAR"
160158
archiveArtifacts artifacts: "*.war"
161159
}
162160
}
163161
}
164-
stage('Package'){
162+
stage('Package') {
165163
failFast false
166164
parallel {
167165
stage('WAR') {
168166
stages {
169-
stage('Publish'){
167+
stage('Publish') {
170168
steps {
171-
sshagent(['pkgserver']) {
172-
sh '''
173-
./utils/release.bash --packaging war.publish
174-
'''
175-
}
169+
sh '''
170+
./utils/release.bash --packaging war.publish
171+
'''
176172
}
177173
}
178174
}
179175
}
180176
stage('Debian') {
181177
stages {
182-
stage('Build'){
178+
stage('Build') {
183179
steps {
184180
sh '''
185181
./utils/release.bash --packaging deb
186182
'''
187-
dir (WORKING_DIRECTORY){
183+
dir (WORKING_DIRECTORY) {
188184
archiveArtifacts artifacts: "target/debian/*.deb"
189185
}
190186
}
191187
}
192-
stage('Publish'){
188+
stage('Publish') {
193189
steps {
194-
sshagent(['pkgserver']) {
195-
sh '''
196-
./utils/release.bash --packaging deb.publish
197-
'''
198-
}
190+
sh '''
191+
./utils/release.bash --packaging deb.publish
192+
'''
199193
}
200194
}
201195
}
@@ -214,11 +208,9 @@ pipeline {
214208
}
215209
stage('Publish'){
216210
steps {
217-
sshagent(['pkgserver']) {
218-
sh '''
219-
./utils/release.bash --packaging rpm.publish
220-
'''
221-
}
211+
sh '''
212+
./utils/release.bash --packaging rpm.publish
213+
'''
222214
}
223215
}
224216
}
@@ -262,7 +254,7 @@ pipeline {
262254
steps {
263255
container('dotnet') {
264256
checkout scm
265-
dir (WORKING_DIRECTORY){
257+
dir (WORKING_DIRECTORY) {
266258
git branch: PACKAGING_GIT_BRANCH, credentialsId: 'release-key', url: PACKAGING_GIT_REPOSITORY
267259

268260
unstash 'GPG'
@@ -283,21 +275,19 @@ pipeline {
283275
}
284276
}
285277
}
286-
stage('Publish'){
278+
stage('Publish') {
287279
steps {
288280
unarchive mapping: ['*.msi*': WORKING_DIRECTORY]
289-
sshagent(['pkgserver']) {
290-
sh '''
291-
./utils/release.bash --packaging msi.publish
292-
'''
293-
}
281+
sh '''
282+
./utils/release.bash --packaging msi.publish
283+
'''
294284
}
295285
}
296286
}
297287
}
298288
}
299289
}
300-
stage('Promote'){
290+
stage('Promote') {
301291
failFast true
302292
parallel {
303293
stage('Maven Repository') {
@@ -306,38 +296,45 @@ pipeline {
306296
}
307297

308298
steps {
309-
sshagent(['pkgserver']) {
310-
sh '''
311-
./utils/release.bash --promoteStagingMavenArtifacts
312-
'''
313-
}
299+
sh '''
300+
./utils/release.bash --promoteStagingMavenArtifacts
301+
'''
314302
}
315303
}
316304
stage('Git Repository') {
317305
when {
318306
environment name: 'GIT_STAGING_REPOSITORY_PROMOTION_ENABLED', value: 'true'
319307
}
320308
steps {
321-
sshagent(['pkgserver']) {
322-
sh '''
323-
./utils/release.bash --promoteStagingGitRepository
324-
'''
325-
}
309+
sh '''
310+
./utils/release.bash --promoteStagingGitRepository
311+
'''
326312
}
327313
}
328314
}
329315
}
330316
// Force mirror synchronization
331-
stage('Synchronize mirror'){
317+
stage('Promote Packages and sync mirrors') {
318+
environment {
319+
SSH_HOSTKEY_ARCHIVES_JENKINS_IO = credentials('ssh-hostkey-archives.jenkins.io')
320+
SSH_HOSTKEY_PKG_ORIGIN_JENKINS_IO = credentials('ssh-hostkey-pkg.origin.jenkins.io')
321+
}
332322
steps{
333-
sshagent(['pkgserver']) {
323+
sshagent(credentials: [
324+
'pkgserver',
325+
'archives.jenkins.io',
326+
]) {
327+
sh '''
328+
mkdir -m 700 -p "${HOME}/.ssh"
329+
cat "${SSH_HOSTKEY_ARCHIVES_JENKINS_IO}" "${SSH_HOSTKEY_PKG_ORIGIN_JENKINS_IO}" >> "${HOME}/.ssh/known_hosts"
330+
'''
334331
sh '''
335-
./utils/release.bash --syncMirror
332+
./utils/release.bash --promotePackages
336333
'''
337334
}
338335
}
339336
}
340-
stage('Invalidate Fastly Cache'){
337+
stage('Invalidate Fastly Cache') {
341338
environment {
342339
FASTLY_API_TOKEN = credentials('fastly-api-token')
343340
FASTLY_SERVICE_ID = credentials('fastly_pkgserver_service_id')
@@ -349,9 +346,9 @@ pipeline {
349346
}
350347
}
351348
}
352-
post {
353-
failure {
354-
input '''Can I delete the pod? '''
355-
}
349+
post {
350+
failure {
351+
input '''Can I delete the pod? '''
356352
}
353+
}
357354
}

‎Jenkinsfile.d/infra-agents-health‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ pipeline {
2626
yamlFile 'PodTemplates.d/package-linux.yaml'
2727
}
2828
}
29+
environment {
30+
SSH_HOSTKEY_ARCHIVES_JENKINS_IO = credentials('ssh-hostkey-archives.jenkins.io')
31+
SSH_HOSTKEY_PKG_ORIGIN_JENKINS_IO = credentials('ssh-hostkey-pkg.origin.jenkins.io')
32+
}
2933
steps {
3034
// Ensure we can get the secondary git repository used for packaging
3135
dir ('./release'){
@@ -40,8 +44,23 @@ pipeline {
4044
// Ensure we get the correct Java and Maven versions
4145
sh 'mvn -v'
4246
// Ensure the correct storage is mounted in expected paths
43-
sh 'ls -la /srv/releases/jenkins'
4447
sh 'ls -la /var/www/pkg.jenkins.io.staging/'
48+
sh 'ls -la /var/www/pkg.jenkins.io.production/'
49+
sh 'ls -la /var/www/get.jenkins.io.staging/'
50+
sh 'ls -la /var/www/get.jenkins.io.production/'
51+
52+
// Ensure we can SSH-connect to required remote servers
53+
sshagent(credentials: [
54+
'pkgserver',
55+
'archives.jenkins.io'
56+
]) {
57+
sh '''
58+
mkdir -m 700 -p "${HOME}/.ssh"
59+
cat "${SSH_HOSTKEY_ARCHIVES_JENKINS_IO}" "${SSH_HOSTKEY_PKG_ORIGIN_JENKINS_IO}" >> "${HOME}/.ssh/known_hosts"
60+
ssh -v [email protected] whoami
61+
ssh -v [email protected] whoami
62+
'''
63+
}
4564
}
4665
}
4766
stage('Test Linux Agent `release-linux`') {

‎PodTemplates.d/package-linux.yaml‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,6 @@ spec:
1313
image: jenkinsciinfra/packaging:8.1.21
1414
imagePullPolicy: "IfNotPresent"
1515
env:
16-
- name: "HOME"
17-
value: "/home/jenkins/agent/workspace"
1816
- name: "JENKINS_JAVA_BIN"
1917
value: "/opt/jdk-21/bin/java"
2018
- name: "JENKINS_JAVA_OPTS"
@@ -32,11 +30,17 @@ spec:
3230
runAsGroup: 1000
3331
volumeMounts:
3432
- name: data-storage-jenkins-io
35-
mountPath: /srv/releases/jenkins
33+
mountPath: /var/www/get.jenkins.io.staging
34+
subPath: ./get.jenkins.io/mirrorbits-staging/
35+
- name: data-storage-jenkins-io
36+
mountPath: /var/www/get.jenkins.io.production
3637
subPath: ./get.jenkins.io/mirrorbits/
3738
- name: data-storage-jenkins-io
3839
mountPath: /var/www/pkg.jenkins.io.staging
3940
subPath: ./pkg.jenkins.io/staging/
41+
- name: data-storage-jenkins-io
42+
mountPath: /var/www/pkg.jenkins.io.production
43+
subPath: ./pkg.jenkins.io/production/
4044
volumes:
4145
- name: data-storage-jenkins-io
4246
persistentVolumeClaim:

‎README.adoc‎

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -152,18 +152,8 @@ More information in https://github.com/jenkins-infra/release/blob/master/utils/g
152152
|
153153
| x
154154

155-
| `PKGSERVER`
156-
| Defines where the different packages will be published.
157-
|
158-
| x
159-
160-
| `PKGSERVER_SSH_OPTS`
161-
| Defines custom ssh options used to connect to `PKGSERVER`.
162-
|
163-
| x
164-
165155
| `PROMOTE_STAGING_MAVEN_ARTIFACTS_ARGS`
166-
| Defines parameters used by `promoteMavenArtifacts.py`.
156+
| Defines parameters used by `promoteMavenArtifacts.py`.
167157

168158
Default value is set to `item --mode copy --source $MAVEN_REPOSITORY_NAME --destination $MAVEN_REPOSITORY_PRODUCTION_NAME --url $MAVEN_REPOSITORY_URL --username $MAVEN_REPOSITORY_USERNAME --password $MAVEN_REPOSITORY_PASSWORD --search '/org/jenkins-ci/main' $(./utils/getJenkinsVersion.py --version)}"`
169159

@@ -435,16 +425,16 @@ Show certificate information
435425
openssl x509 -in jenkins-release.crt -text -noout
436426

437427
Convert p7b to pkcs12
438-
428+
439429
# Based from https://knowledge.digicert.com/solution/SO26449.html and https://github.com/jenkins-infra/release/blob/7a03f98eff839d4fed75ea96cf7bebbc963e3a91/README.adoc#certificate
440430
# P7B to PFX: 1/2
441431
openssl pkcs7 -print_certs -in digicert.p7b -out jenkins-release.crt
442432
## Asks for the Export password, transmitted by Digicert from another channel
443433
## Asks for the `jenkins-release.key` private key passphrase
444-
434+
445435
# Check for the intermediate certificate attributes
446436
openssl x509 -in jenkins-release.crt -text -noout
447-
437+
448438
# P7B to PFX: 2/2
449439
openssl pkcs12 -export -in jenkins-release.crt -inkey jenkins-release.key -out jenkins-release.pfx
450440
## Asks for an Export password: do not set any (type enter only)

‎env/package.mk‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#
2+
# Environment definition for the packaging process
3+
#
4+
5+
# where to put binary files
6+
export BASE_BIN_DIR=/var/www/get.jenkins.io.staging
7+
export WARDIR=${BASE_BIN_DIR}/war${RELEASELINE}
8+
export MSIDIR=${BASE_BIN_DIR}/windows${RELEASELINE}
9+
export DEBDIR=${BASE_BIN_DIR}/debian${RELEASELINE}
10+
export RPMDIR=${BASE_BIN_DIR}/rpm${RELEASELINE}
11+
12+
# where to put repository index and other web contents
13+
export BASE_PKG_DIR=/var/www/pkg.jenkins.io.staging
14+
export RPM_WEBDIR=${BASE_PKG_DIR}/rpm${RELEASELINE}
15+
export MSI_WEBDIR=${BASE_PKG_DIR}/windows${RELEASELINE}
16+
export DEB_WEBDIR=${BASE_PKG_DIR}/debian${RELEASELINE}
17+
18+
# URL to the aforementioned webdir.
19+
WEBSERVER=https://pkg.jenkins.io
20+
export RPM_URL=${WEBSERVER}/rpm${RELEASELINE}
21+
export DEB_URL=${WEBSERVER}/debian${RELEASELINE}

‎profile.d/experimental‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,3 @@ SIGN_ALIAS=jenkins
1212

1313
# PACKAGING ENV used from jenkinsci/packaging
1414
RELEASELINE="-experimental"
15-
16-
BUILDENV=env/azure.mk
17-
18-
# Define endpoint used to force mirror synchronization
19-
[email protected]
20-
PKGSERVER_SSH_OPTS="-p 22 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

0 commit comments

Comments
 (0)