Skip to content

Commit

Permalink
Change auth method in releasepy: remove shared token and use Jenkins …
Browse files Browse the repository at this point in the history
…API tokens (#1201)

Commit changes:

* Remove the existing token 
* Change the authentication model to use per-user Jekins API tokens: 41d75d8
* Implement --auth for release.py
* Use --auth from releasepy in nightly/releasepy jobs
* Add osrfbuild permission to call -debbuilder and brew_release
* Include basic python metadata files

* Corresponding developers documentation update: gazebosim/docs#538

Signed-off-by: Jose Luis Rivero <jrivero@honurobotics.com>
j-rivero authored Nov 28, 2024
1 parent 374fd4e commit bc7d636
Showing 15 changed files with 147 additions and 75 deletions.
13 changes: 9 additions & 4 deletions check_releasepy.bash
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#!/bin/bash -e

export _RELEASEPY_DEBUG=1

test_dir=$(mktemp -d)
export _RELEASEPY_TEST_RELEASE_REPO="${test_dir}/test-release"
mkdir -p ${_RELEASEPY_TEST_RELEASE_REPO}/{focal,jammy,ubuntu}/debian
@@ -25,7 +26,8 @@ exec_releasepy_test()
./release.py \
--dry-run \
--no-sanity-checks \
gz-foo 1.2.3 token ${test_params}
--auth user:fake \
gz-foo 1.2.3 ${test_params}
}

exec_ignition_releasepy_test()
@@ -35,7 +37,8 @@ exec_ignition_releasepy_test()
./release.py \
--dry-run \
--no-sanity-checks \
ign-foo 1.2.3 token ${test_params}
--auth user:fake \
ign-foo 1.2.3 ${test_params}
}

exec_ignition_gazebo_releasepy_test()
@@ -45,7 +48,8 @@ exec_ignition_gazebo_releasepy_test()
./release.py \
--dry-run \
--no-sanity-checks \
ign-gazebo 1.2.3 token ${test_params}
--auth user:fake \
ign-gazebo 1.2.3 ${test_params}
}

exec_releasepy_with_real_gz()
@@ -54,9 +58,10 @@ exec_releasepy_with_real_gz()
./release.py \
--dry-run \
--no-sanity-checks \
--auth user:fake \
--source-repo-uri http://github.com/gazebosim/gz-common \
--source-repo-existing-ref http://github.com/gazebosim/gz-common/foo-tag \
"${gz_pkg}" "${major_version}.x.y" token
"${gz_pkg}" "${major_version}.x.y"
}

expect_job_run()
34 changes: 0 additions & 34 deletions jenkins-scripts/dsl/_configs_/GenericRemoteToken.groovy

This file was deleted.

4 changes: 2 additions & 2 deletions jenkins-scripts/dsl/_configs_/OSRFCredentials.groovy
Original file line number Diff line number Diff line change
@@ -13,8 +13,8 @@ class OSRFCredentials
credentialsBinding {
crendentials_list.each { credential_keyword ->
if (credential_keyword == 'OSRFBUILD_GITHUB_TOKEN') {
usernamePassword('OSRFBUILD_USER',
'OSRFBUILD_TOKEN',
usernamePassword('OSRFBUILD_GITHUB_USER',
'OSRFBUILD_GITHUB_TOKEN',
'github-osrfbuild-apitoken')
} else if (credential_keyword == 'OSRFBUILD_JENKINS_TOKEN') {
usernamePassword('OSRFBUILD_JENKINS_USER',
2 changes: 0 additions & 2 deletions jenkins-scripts/dsl/_configs_/OSRFLinuxBackportPkg.groovy
Original file line number Diff line number Diff line change
@@ -4,7 +4,6 @@ import javaposse.jobdsl.dsl.Job

/*
-> OSRFLinuxBase
-> GenericRemoteToken
Implements:
- priorioty 300
@@ -24,7 +23,6 @@ class OSRFLinuxBackportPkg
static void create(Job job)
{
OSRFLinuxBase.create(job)
GenericRemoteToken.create(job)

job.with
{
4 changes: 2 additions & 2 deletions jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy
Original file line number Diff line number Diff line change
@@ -2,10 +2,10 @@ package _configs_

import javaposse.jobdsl.dsl.Job
import _configs_.Globals
import _configs_.OSRFCredentials

/*
-> OSRFLinuxBuildPkgBase
-> GenericRemoteToken
Implements:
- priority 100
@@ -28,7 +28,7 @@ class OSRFLinuxBuildPkg
static void create(Job job, Map default_params = [:])
{
OSRFLinuxBuildPkgBase.create(job)
GenericRemoteToken.create(job)
OSRFCredentials.allowOsrfbuildToRunTheBuild(job)

job.with
{
8 changes: 4 additions & 4 deletions jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@ class OSRFReleasepy
{
// Base class for the job
OSRFUNIXBase.create(job)
OSRFCredentials.setOSRFCrendentials(job, ['OSRFBUILD_JENKINS_TOKEN'])

job.with
{
@@ -58,8 +59,6 @@ class OSRFReleasepy

shell("""\
#!/bin/bash -xe
set +x # keep password secret
PASS=\$(cat \$HOME/build_pass)
dry_run_str=""
if \$DRY_RUN; then
@@ -72,10 +71,11 @@ class OSRFReleasepy
fi
echo "releasing \${n} (from branch \${src_branch})"
python3 ./scripts/release.py \${dry_run_str} "\${PACKAGE}" "\${VERSION}" "\${PASS}" \${extra_osrf_repo} \
python3 ./scripts/release.py \${dry_run_str} "\${PACKAGE}" "\${VERSION}" \${extra_osrf_repo} \
--auth "\${OSRFBUILD_JENKINS_USER}:\${OSRFBUILD_JENKINS_TOKEN}" \
--source-tarball-uri \${SOURCE_TARBALL_URI} \
--release-repo-branch \${RELEASE_REPO_BRANCH} \
--upload-to-repo \${UPLOAD_TO_REPO} > log
--upload-to-repo \${UPLOAD_TO_REPO}
echo " - done"
""".stripIndent())
}
1 change: 0 additions & 1 deletion jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy
Original file line number Diff line number Diff line change
@@ -57,7 +57,6 @@ class OSRFSourceCreation
static void create(Job job, Map default_params = [:], Map default_hidden_params = [:])
{
OSRFLinuxBuildPkgBase.create(job)
GenericRemoteToken.create(job)
OSRFSourceCreation.addParameters(job, default_params)

def pkg_sources_dir="pkgs"
5 changes: 1 addition & 4 deletions jenkins-scripts/dsl/brew_release.dsl
Original file line number Diff line number Diff line change
@@ -51,7 +51,7 @@ void include_common_params(Job job)
// 1. BREW pull request SHA updater
def release_job = job("generic-release-homebrew_pull_request_updater")
OSRFUNIXBase.create(release_job)
GenericRemoteToken.create(release_job)
OSRFCredentials.allowOsrfbuildToRunTheBuild(release_job)

include_common_params(release_job)
release_job.with
@@ -131,8 +131,6 @@ OSRFBrewCompilationAnyGitHub.create(bottle_job_builder,
DISABLE_TESTS,
NO_SUPPORTED_BRANCHES,
DISABLE_GITHUB_INTEGRATION)
GenericRemoteToken.create(bottle_job_builder)

bottle_job_builder.with
{
wrappers {
@@ -247,7 +245,6 @@ bottle_job_builder.with
// 4. BREW bottle hash update
def bottle_job_hash_updater = job(bottle_hash_updater_job_name)
OSRFUNIXBase.create(bottle_job_hash_updater)
GenericRemoteToken.create(bottle_job_hash_updater)

include_common_params(bottle_job_hash_updater)
bottle_job_hash_updater.with
1 change: 0 additions & 1 deletion jenkins-scripts/dsl/gazebo_ros_pkgs.dsl
Original file line number Diff line number Diff line change
@@ -260,7 +260,6 @@ bloom_debbuild_jobs.each { bloom_pkg ->

// Use the linux install as base
OSRFLinuxBuildPkgBase.create(build_pkg_job)
GenericRemoteToken.create(build_pkg_job)

build_pkg_job.with
{
9 changes: 6 additions & 3 deletions jenkins-scripts/dsl/ignition_collection.dsl
Original file line number Diff line number Diff line change
@@ -148,6 +148,7 @@ nightly_collection = gz_collections_yaml.collections

def nightly_scheduler_job = job("ignition-${gz_nightly}-nightly-scheduler")
OSRFUNIXBase.create(nightly_scheduler_job)
OSRFCredentials.setOSRFCrendentials(nightly_scheduler_job, ['OSRFBUILD_JENKINS_TOKEN'])

nightly_scheduler_job.with
{
@@ -190,8 +191,6 @@ nightly_scheduler_job.with
steps {
shell("""\
#!/bin/bash -xe
set +x # keep password secret
PASS=\$(cat \$HOME/build_pass)

dry_run_str=""
if \$DRY_RUN; then
@@ -240,7 +239,11 @@ nightly_scheduler_job.with
fi

echo "releasing \${n} (from branch \${src_branch})"
python3 ./scripts/release.py \${dry_run_str} "\${n}" nightly "\${PASS}" --release-repo-branch main --nightly-src-branch \${src_branch} --upload-to-repo nightly > log || echo "MARK_AS_UNSTABLE"
python3 ./scripts/release.py \${dry_run_str} "\${n}" nightly \
--auth "\${OSRFBUILD_JENKINS_USER}:\${OSRFBUILD_JENKINS_TOKEN}" \
--release-repo-branch main \
--nightly-src-branch \${src_branch} \
--upload-to-repo nightly
echo " - done"
done

1 change: 0 additions & 1 deletion jenkins-scripts/dsl/ros_gz_bridge.dsl
Original file line number Diff line number Diff line change
@@ -28,7 +28,6 @@ bridge_packages.each { pkg ->

// Use the linux install as base
OSRFLinuxBuildPkgBase.create(build_pkg_job)
GenericRemoteToken.create(build_pkg_job)

build_pkg_job.with
{
8 changes: 4 additions & 4 deletions jenkins-scripts/dsl/test.dsl
Original file line number Diff line number Diff line change
@@ -153,12 +153,12 @@ test_credentials_token_job.with
echo " + Testing OSRFBUILD_GITHUB_TOKEN ability to push into the fork osrfbuild/homebrew-simulation"
echo " (out of the test is the ability to create pull requests into osrf/homebrew-simulation)"
rm -fr homebrew-simulation
git clone https://github.com/\${OSRFBUILD_USER}/homebrew-simulation.git
git clone https://github.com/\${OSRFBUILD_GITHUB_USER}/homebrew-simulation.git
cd homebrew-simulation
git config user.name \${OSRFBUILD_USER} --replace-all
git config user.email "\${OSRFBUILD_USER}@openrobotics.org" --replace-all
git config user.name \${OSRFBUILD_GITHUB_USER} --replace-all
git config user.email "\${OSRFBUILD_GITHUB_USER}@openrobotics.org" --replace-all
set +x
git config url."https://osrfbuild:\${OSRFBUILD_TOKEN}@github.com/osrfbuild/homebrew-simulation.git".InsteadOf https://github.com/osrfbuild/homebrew-simulation.git
git config url."https://osrfbuild:\${OSRFBUILD_GITHUB_TOKEN}@github.com/osrfbuild/homebrew-simulation.git".InsteadOf https://github.com/osrfbuild/homebrew-simulation.git
set -x
GIT_TERMINAL_PROMPT=0 git push -u origin master --dry-run
""".stripIndent())
5 changes: 5 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[build-system]
build-backend = 'setuptools.build_meta'
requires = [
'setuptools',
]
117 changes: 104 additions & 13 deletions release.py
Original file line number Diff line number Diff line change
@@ -2,22 +2,25 @@

from __future__ import print_function
from argparse import RawTextHelpFormatter
from configparser import ConfigParser
from typing import Tuple
from urllib3.exceptions import RequestError
from urllib3.util import make_headers
import subprocess
import sys
import tempfile
import os
import urllib.parse
import urllib.request
import urllib3
import argparse
import shutil
import venv

USAGE = 'release.py <package> <version> <jenkinstoken>'
USAGE = 'release.py <package> <version>'
try:
JENKINS_URL = os.environ['JENKINS_URL']
except KeyError:
JENKINS_URL = 'http://build.osrfoundation.org'
JENKINS_URL = 'https://build.osrfoundation.org'
JOB_NAME_PATTERN = '%s-debbuilder'
GENERIC_BREW_PULLREQUEST_JOB = 'generic-release-homebrew_pull_request_updater'

@@ -111,21 +114,27 @@ def parse_args(argv):
Script to handle the release process for the Gazebo devs.
Examples:
A) Generate source: local repository tag + call source job:
$ release.py <package> <version> <jenkins_token>
$ release.py <package> <version>
(auto calculate source-repo-uri from local directory)
B) Call builders: reuse existing tarball version + call build jobs:
$ release.py --source-tarball-uri <URL> <package> <version> <jenkins_token>
$ release.py --source-tarball-uri <URL> <package> <version>
(no call to source job, directly build jobs with tarball URL)
C) Nightly builds (linux)
$ release.py --source-repo-existing-ref <git_branch> --upload-to-repo nightly <URL> <package> <version> <jenkins_token>
$ release.py --source-repo-existing-ref <git_branch> --upload-to-repo nightly <URL> <package> <version>
""")
parser.add_argument('package', help='which package to release')
parser.add_argument('version', help='which version to release')
parser.add_argument('jenkins_token', help='secret token to allow access to Jenkins to start builds')
parser.add_argument('deprecated_jenkins_token',
default=None,
nargs="?",
help=argparse.SUPPRESS)
parser.add_argument('--dry-run', dest='dry_run', action='store_true', default=False,
help='dry-run; i.e., do actually run any of the commands')
parser.add_argument('--auth', dest='auth_input_arg',
default=None,
help='Explicit jenkins user:token string overriding the jenkins.ini credentials file.')
parser.add_argument('-a', '--package-alias', dest='package_alias',
default=None,
help='different name that we are releasing under')
@@ -176,6 +185,44 @@ def parse_args(argv):

return args

#
# BEGIN: Credentials code copied from ros_buildfarm
#
def get_credentials(jenkins_url=None):
config = ConfigParser()
config_file = get_credential_path()
if not os.path.exists(config_file):
print("Could not find credential file '%s'" % config_file,
file=sys.stderr)
return None, None

config.read(config_file)
section_name = None
if jenkins_url is not None and jenkins_url in config:
section_name = jenkins_url
if section_name is None and 'DEFAULT' in config:
section_name = 'DEFAULT'

if section_name is None or 'username' not in config[section_name] or \
'password' not in config[section_name]:
print(
"Could not find credentials for '%s' in file '%s'" %
(jenkins_url, config_file), file=sys.stderr)
return None, None
return config[section_name]['username'], config[section_name]['password']


def get_credential_path():
return os.path.join(
os.path.expanduser('~'), get_relative_credential_path())


def get_relative_credential_path():
return os.path.join('.buildfarm', 'jenkins.ini')

#
# END: Credentials code copied from ros_buildfarm
#

def get_release_repository_info(package):
github_url = "https://github.com/gazebo-release/" + package + "-release"
@@ -336,6 +383,8 @@ def sanity_checks(args, repo_dir):
sanity_check_sdformat_versions(args.package, args.version)
sanity_project_package_in_stable(args.version, args.upload_to_repository)

check_credentials(args.auth_input_arg)
print_success("Jenkins credentials are good")
shutil.rmtree(repo_dir)


@@ -486,9 +535,34 @@ def generate_source_params(args):

return params


def call_jenkins_build(job_name, params, output_string,
search_description_help):
def build_credentials_header(auth_input_arg = None):
if auth_input_arg:
if len(auth_input_arg.split(':')) != 2:
error("Auth string is not in the form of 'user:token' ")
username, api_token = auth_input_arg.split(':')
else:
username, api_token = get_credentials(JENKINS_URL)
if not username:
exit(1)

return make_headers(basic_auth=f'{username}:{api_token}')

def check_credentials(auth_input_arg = None):
http = urllib3.PoolManager()
response = http.request('GET',
JENKINS_URL,
headers=build_credentials_header(auth_input_arg))
if response.status != 200:
print(f"Crendentials error: {response.status}: {response.reason}")
http.clear()
exit(1)


def call_jenkins_build(job_name,
params,
output_string,
search_description_help,
auth_input_arg = None):
# Only to help user feedback this block
help_url = f'{JENKINS_URL}/job/{job_name}'
if search_description_help:
@@ -502,9 +576,23 @@ def call_jenkins_build(job_name, params, output_string,
job_name,
params_query)
print_only_dbg(f" -- {output_string}: {url}")
if not DRY_RUN:
urllib.request.urlopen(url)

if not DRY_RUN:
http = urllib3.PoolManager()
try :
response = http.request('POST',
url ,
headers=build_credentials_header(auth_input_arg))
# 201 code is "created", it is the expected return of POST
if response.status != 201:
print(f"Error {response.status}: {response.reason}")
exit(1)
except RequestError as e:
print(f"An error occurred in the http request: {e}")
except Exception as e:
print(f"An unexpected error occurred: {e}")
finally:
http.clear()

def display_help_job_chain_for_source_calls(args):
# Encode the different ways using in the job descriptions to filter builds
@@ -686,6 +774,10 @@ def process_ros_vendor_package(args):
def go(argv):
args = parse_args(argv)

if args.deprecated_jenkins_token:
error('Build token has been removed. Please generate a user token:\n'
' - https://gazebosim.org/docs/latest/releases-instructions/#access-and-credentials')

# If only the process of ROS vendor package is set, just do it
if args.bump_ros_vendor_only:
process_ros_vendor_package(args)
@@ -707,7 +799,6 @@ def go(argv):
sanity_checks(args, repo_dir)

params = generate_source_params(args)
params['token'] = args.jenkins_token
params['PACKAGE'] = args.package
params['VERSION'] = args.version if not NIGHTLY else 'nightly'
params['RELEASE_REPO_BRANCH'] = args.release_repo_branch
10 changes: 10 additions & 0 deletions setup.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
[metadata]
name = releasepy
version = 0.0.1

[options]
install_requires =
urllib3 >= 1.26.0
argcomplete >= 1.8.0

packages = find:

0 comments on commit bc7d636

Please sign in to comment.