From 253a08442cf573a768a813ec9e601cfed89f36e1 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Thu, 23 Jan 2025 14:36:33 +0100 Subject: [PATCH 01/17] feat: use a container for creating the CA and server certificate --- .../DeveloperInstallation/stuffThatRun.rst | 8 +- tests/CI/docker-compose.yml | 21 ++- tests/Jenkins/config/ci/openssl_config_ca.cnf | 110 ----------- .../Jenkins/config/ci/openssl_config_host.cnf | 37 ---- .../Jenkins/config/ci/openssl_config_user.cnf | 20 -- tests/Jenkins/dirac_ci.sh | 38 ++-- tests/Jenkins/utilities.sh | 176 ------------------ 7 files changed, 46 insertions(+), 364 deletions(-) delete mode 100644 tests/Jenkins/config/ci/openssl_config_ca.cnf delete mode 100644 tests/Jenkins/config/ci/openssl_config_host.cnf delete mode 100644 tests/Jenkins/config/ci/openssl_config_user.cnf diff --git a/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/stuffThatRun.rst b/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/stuffThatRun.rst index 702e810ec5e..df19860b3c7 100644 --- a/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/stuffThatRun.rst +++ b/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/stuffThatRun.rst @@ -93,13 +93,9 @@ the private key. You will need two different sets certificates and the CA certif The following commands should do the trick for you, by creating a fake CA, a fake user certificate, and a fake host certificate:: cd $DEVROOT/DIRAC - git checkout release/integration - source tests/Jenkins/utilities.sh - generateCA - generateCertificates 365 - generateUserCredentials 365 + docker run ghcr.io/diracgrid/diracx/certificates-generation:latest mkdir -p ~/.globus/ - cp $DEVROOT/user/*.{pem,key} ~/.globus/ + docker cp certificates-generation:/ca/certs/client.{pem,key} ~/.globus/ mv ~/.globus/client.key ~/.globus/userkey.pem mv ~/.globus/client.pem ~/.globus/usercert.pem diff --git a/tests/CI/docker-compose.yml b/tests/CI/docker-compose.yml index 5cd1b4266ff..94b46e50d7b 100644 --- a/tests/CI/docker-compose.yml +++ b/tests/CI/docker-compose.yml @@ -1,4 +1,6 @@ volumes: + # Volume used to store the certificates of dirac + certs_data: # Volume used to store the config of diracx diracx-cs-store: # Volume used to store the pair of keys to sign the tokens @@ -101,6 +103,15 @@ services: command: /home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC/tests/CI/check_db_initialized.sh pull_policy: always + dirac-init-certificates: + image: ghcr.io/diracgrid/management/certificates-generation:latest + container_name: dirac-init-certificates + volumes: + - certs_data:/ca/certs/ + entrypoint: | + /entrypoint.sh + pull_policy: always + dirac-server: image: ${CI_REGISTRY_IMAGE}/${HOST_OS}-dirac container_name: server @@ -116,6 +127,8 @@ services: condition: service_started iam-login-service: condition: service_healthy + dirac-init-certificates: + condition: service_completed_successfully # Let the init container create the certificates diracx-init-key: condition: service_completed_successfully # Let the init container create the signing key diracx-init-cs: @@ -123,13 +136,14 @@ services: ulimits: nofile: 8192 volumes: + - certs_data:/ca/certs - diracx-cs-store:/cs_store - diracx-key-store:/signing-key environment: - DIRACX_CONFIG_BACKEND_URL=git+file:///cs_store/initialRepo - DIRACX_SERVICE_AUTH_TOKEN_KEY=file:///signing-key/rs256.key - pull_policy: always command: ["sleep", "infinity"] # This is necessary because of the issue described in https://github.com/moby/moby/issues/42275. What is added here is a hack/workaround. + pull_policy: always dirac-client: @@ -141,8 +155,10 @@ services: - dirac-server ulimits: nofile: 8192 - pull_policy: always + volumes: + - certs_data:/ca/certs command: ["sleep", "infinity"] # This is necessary because of the issue described in https://github.com/moby/moby/issues/42275. What is added here is a hack/workaround. + pull_policy: always dirac-pilot: image: ${CI_REGISTRY_IMAGE}/${HOST_OS}-dirac @@ -152,6 +168,7 @@ services: depends_on: - dirac-server volumes: + - certs_data:/ca/certs - type: bind source: ${CVMFS_DIR} target: /cvmfs diff --git a/tests/Jenkins/config/ci/openssl_config_ca.cnf b/tests/Jenkins/config/ci/openssl_config_ca.cnf deleted file mode 100644 index df7ac9909bb..00000000000 --- a/tests/Jenkins/config/ci/openssl_config_ca.cnf +++ /dev/null @@ -1,110 +0,0 @@ -# OpenSSL root CA configuration file - -[ ca ] -# `man ca` -default_ca = CA_default - -[ CA_default ] -# Directory and file locations. -# EDIT HERE -dir = #GRIDSECURITY#/ca #PUT THE RIGHT DIR HERE! -####### -certs = $dir/certs -crl_dir = $dir/crl -new_certs_dir = $dir/newcerts -database = $dir/index.txt -serial = $dir/serial -RANDFILE = $dir/private/.rand - -# The root key and root certificate. -private_key = $dir/ca.key.pem -certificate = $dir/ca.cert.pem - -# For certificate revocation lists. -crlnumber = $dir/crlnumber -crl = $dir/crl/ca.crl.pem -crl_extensions = crl_ext -default_crl_days = 30 - -# SHA-1 is deprecated, so use SHA-2 instead. -default_md = sha256 - -name_opt = ca_default -cert_opt = ca_default -default_days = 375 -preserve = no -policy = policy_loose - -# This option is dangerous, but allows to -# set subjectAlternativeName on the Request -# `man ca` is your friend -copy_extensions=copy - -[ policy_strict ] -# The root CA should only sign intermediate certificates that match. -# See the POLICY FORMAT section of `man ca`. -organizationName = match -organizationalUnitName = optional -commonName = supplied -emailAddress = optional - -[ policy_loose ] -# Allow the intermediate CA to sign a more diverse range of certificates. -# See the POLICY FORMAT section of the `ca` man page. -countryName = optional -stateOrProvinceName = optional -localityName = optional -organizationName = optional -organizationalUnitName = optional -commonName = supplied -emailAddress = optional - -[ req ] -# Options for the `req` tool (`man req`). -default_bits = 2048 -distinguished_name = req_distinguished_name -string_mask = utf8only -prompt = no - - -# SHA-1 is deprecated, so use SHA-2 instead. -default_md = sha256 - -# Extension to add when the -x509 option is used. -x509_extensions = v3_ca - -[ req_distinguished_name ] -# See . -# EDIT HERE OPTIONALLY -O = DIRAC CI -CN = DIRAC CI Signing Certification Authority -######## - -[ v3_ca ] -# Extensions for a typical CA (`man x509v3_config`). -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid:always,issuer -basicConstraints = critical, CA:true -keyUsage = critical, digitalSignature, cRLSign, keyCertSign - -[ usr_cert ] -# Extensions for client certificates (`man x509v3_config`). -basicConstraints = CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer -keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth - - -# Whatever is in the request will be copied, unless it is already defined. -# So we have some double definitions here with the other ssl config files, -# but that's allright. The basicConstraints = CA:FALSE is a protection that MUST stay -[ server_cert ] -# Extensions for server certificates (`man x509v3_config`). -basicConstraints = CA:FALSE -nsComment = "OpenSSL Generated Server Certificate" -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -# Our servers need both serverAuth and clientAuth -extendedKeyUsage = serverAuth,clientAuth diff --git a/tests/Jenkins/config/ci/openssl_config_host.cnf b/tests/Jenkins/config/ci/openssl_config_host.cnf deleted file mode 100644 index 6633552ed99..00000000000 --- a/tests/Jenkins/config/ci/openssl_config_host.cnf +++ /dev/null @@ -1,37 +0,0 @@ -# OpenSSL root CA configuration file - - -[ req ] -# Options for the `req` tool (`man req`). -default_bits = 2048 -distinguished_name = req_distinguished_name -string_mask = utf8only -prompt = no -encrypt_key = yes - -# SHA-1 is deprecated, so use SHA-2 instead. -default_md = sha256 - -# Extension to add in the request -req_extensions = v3_req - -[ req_distinguished_name ] -# CAUTION the name also needs to be in the alt_names as per RFC (don't remember the number) -# See . -C = ch -O = DIRAC -OU = DIRAC CI -CN = #hostname# - - -[ v3_req ] -# Extensions to ask for the cert (`man x509v3_config`). -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = serverAuth,clientAuth -subjectKeyIdentifier = hash -subjectAltName = @alt_names - - -[ alt_names ] -DNS.1 = #hostname# -DNS.2 = localhost diff --git a/tests/Jenkins/config/ci/openssl_config_user.cnf b/tests/Jenkins/config/ci/openssl_config_user.cnf deleted file mode 100644 index f767b58a8c5..00000000000 --- a/tests/Jenkins/config/ci/openssl_config_user.cnf +++ /dev/null @@ -1,20 +0,0 @@ -[ req ] -default_bits = 2048 -encrypt_key = yes -distinguished_name = req_dn -prompt = no -req_extensions = v3_req - -# Generates the following subject -# Subject: O=DIRAC CI, O=CERN, CN=ciuser -[ req_dn ] -C = ch -O = DIRAC -OU = DIRAC CI -CN = ciuser - -[ v3_req ] -# Extensions for client certificates (`man x509v3_config`). -nsComment = "OpenSSL Generated Client Certificate" -keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth diff --git a/tests/Jenkins/dirac_ci.sh b/tests/Jenkins/dirac_ci.sh index b57ddb93fd5..424f9d020e2 100644 --- a/tests/Jenkins/dirac_ci.sh +++ b/tests/Jenkins/dirac_ci.sh @@ -90,10 +90,7 @@ source "${TESTCODE}/DIRAC/tests/Jenkins/utilities.sh" installSite() { echo "==> [installSite]" - generateCA - generateCertificates - - echo -n > "${SERVERINSTALLDIR}/dirac-ci-install.cfg" + # echo -n > "${SERVERINSTALLDIR}/dirac-ci-install.cfg" getCFGFile echo "==> Fixing install.cfg file" @@ -127,7 +124,30 @@ installSite() { bash "installer.sh" rm "installer.sh" echo "source \"$PWD/diracos/diracosrc\"" > "$PWD/bashrc" - mv "${SERVERINSTALLDIR}/etc/grid-security/"* "${SERVERINSTALLDIR}/diracos/etc/grid-security/" + + mkdir -p "${SERVERINSTALLDIR}/diracos/etc/grid-security/certificates/" + mkdir -p "${SERVERINSTALLDIR}/user/" + + echo "==> CAs and certificates" + + # Copy the CA to the list of trusted CA + cp "/ca/certs/ca.cert.pem" "${SERVERINSTALLDIR}/diracos/etc/grid-security/certificates/" + + # Copy the cert and host key to the certificates directory + cp /ca/certs/hostcert.pem "${SERVERINSTALLDIR}/diracos/etc/grid-security/" + cp /ca/certs/hostkey.pem "${SERVERINSTALLDIR}/diracos/etc/grid-security/" + + # Generate the hash link file required by openSSL to index CA certificates + caHash=$(openssl x509 -in "${SERVERINSTALLDIR}/diracos/etc/grid-security/certificates/ca.cert.pem" -noout -hash) + # We make a relative symlink on purpose (i.e. not the full path to ca.cert.pem) + # because otherwise the BundleDeliveryClient will send the full path, which + # will be wrong on the client + ln -s "${SERVERINSTALLDIR}/diracos/etc/grid-security/certificates/ca.cert.pem" "${SERVERINSTALLDIR}/diracos/etc/grid-security/certificates/$caHash.0" + + # Copy the user cert and key to the correct directory + cp /ca/certs/client.pem "${SERVERINSTALLDIR}/user/" + cp /ca/certs/client.key "${SERVERINSTALLDIR}/user/" + rm -rf "${SERVERINSTALLDIR}/etc" ln -s "${SERVERINSTALLDIR}/diracos/etc" "${SERVERINSTALLDIR}/etc" source diracos/diracosrc @@ -136,7 +156,6 @@ installSite() { done cd - - echo "==> Sourcing bashrc" source "${SERVERINSTALLDIR}/bashrc" @@ -212,13 +231,6 @@ fullInstallDIRAC() { cat "${SERVERINSTALLDIR}/diracos/etc/Production.cfg" fi - # Dealing with security stuff - # generateCertificates - if ! generateUserCredentials; then - echo "ERROR: generateUserCredentials failed" >&2 - exit 1 - fi - if ! diracCredentials; then echo "ERROR: diracCredentials failed" >&2 exit 1 diff --git a/tests/Jenkins/utilities.sh b/tests/Jenkins/utilities.sh index d920556dd75..89faf966f09 100644 --- a/tests/Jenkins/utilities.sh +++ b/tests/Jenkins/utilities.sh @@ -370,182 +370,6 @@ getUserProxy() { echo '==> Done getUserProxy' } - -#------------------------------------------------------------------------------- -# OPEN SSL... let's create a fake CA and certificates -#------------------------------------------------------------------------------- - - -# function generateCA() -# -# This generates the CA that will be used to sign the server and client certificates - -generateCA() { - echo '==> [generateCA]' - - mkdir -p "${SERVERINSTALLDIR}/etc/grid-security/certificates" - mkdir -p "${SERVERINSTALLDIR}/etc/grid-security/ca/" - if ! cd "${SERVERINSTALLDIR}/etc/grid-security/ca"; then - echo "ERROR: cannot change to ${SERVERINSTALLDIR}/etc/grid-security/ca" >&2 - exit 1 - fi - - # Initialize the ca - mkdir -p newcerts certs crl - touch index.txt - echo 1000 > serial - echo 1000 > crlnumber - - # Create the CA key - openssl genrsa -out ca.key.pem 2048 # for unencrypted key - chmod 400 ca.key.pem - - - # Prepare OpenSSL config file, it contains extensions to put into place, - # DN configuration, etc.. - cp "${CI_CONFIG}/openssl_config_ca.cnf" "openssl_config_ca.cnf" - sed -i "s|#GRIDSECURITY#|${SERVERINSTALLDIR}/etc/grid-security|g" openssl_config_ca.cnf - - - # Generate the CA certificate - openssl req -config openssl_config_ca.cnf \ - -key ca.key.pem \ - -new -x509 \ - -days 7300 \ - -sha256 \ - -extensions v3_ca \ - -out ca.cert.pem - - # Copy the CA to the list of trusted CA - cp ca.cert.pem "${SERVERINSTALLDIR}/etc/grid-security/certificates/" - - # Generate the hash link file required by openSSL to index CA certificates - caHash=$(openssl x509 -in ca.cert.pem -noout -hash) - # We make a relative symlink on purpose (i.e. not the full path to ca.cert.pem) - # because otherwsie the BundleDeliveryClient will send the full path, which - # will be wrong on the client - ln -s "ca.cert.pem" "${SERVERINSTALLDIR}/etc/grid-security/certificates/$caHash.0" -} - -#............................................................................. -# -# function generateCertificates -# -# This function generates a random host certificate ( certificate and key ), -# which will be stored on etc/grid-security. -# We use the self signed CA created by generateCA function -# In real, we'd copy them from -# CVMFS: -# /cvmfs/grid.cern.ch/etc/grid-security/certificates -# -# Additional info: -# http://www.openssl.org/docs/apps/req.html -# -#............................................................................. - -generateCertificates() { - echo '==> [generateCertificates]' - nDays=${1:-7} - - mkdir -p "${SERVERINSTALLDIR}/etc/grid-security/" - if ! cd "${SERVERINSTALLDIR}/etc/grid-security/"; then - echo "ERROR: cannot change to ${SERVERINSTALLDIR}/etc/grid-security/" >&2 - exit 1 - fi - - # Generate private RSA key - openssl genrsa -out hostkey.pem 2048 &> /dev/null - chmod 400 hostkey.pem - - # Prepare OpenSSL config file, it contains extensions to put into place, - # DN configuration, etc.. - cp "${CI_CONFIG}/openssl_config_host.cnf" "openssl_config_host.cnf" - - # man hostname to see why we use --all-fqdns - # Note: if there's no dns entry for the localhost, the fqdns will be empty - # so we append to it the local hostname, and we take the first one in the list - fqdn=$( (hostname --all-fqdn; hostname ) | paste -sd ' ' | awk '{print $1}') - sed -i "s/#hostname#/$fqdn/g" openssl_config_host.cnf - - # Generate X509 Certificate request based on the private key and the OpenSSL configuration - # file, valid for nDays days (default 1). - openssl req -config openssl_config_host.cnf \ - -key hostkey.pem \ - -new \ - -sha256 \ - -out request.csr.pem - - # Sign it using the self generated CA - openssl ca -config "${SERVERINSTALLDIR}/etc/grid-security/ca/openssl_config_ca.cnf" \ - -days "$nDays" \ - -extensions server_cert \ - -batch \ - -in request.csr.pem \ - -out hostcert.pem - - cd - -} - - -#............................................................................. -# -# generateUserCredentials: -# -# Given we know the "CA" certificates, we can use them to sign a randomly -# generated key / host certificate. This function is very similar to -# generateCertificates. User credentials will be stored at: -# ${SERVERINSTALLDIR}/user -# The user will be called "ciuser". Do not confuse with the admin user, -# which is "ci". -# The argument that can be passed is the validity of the certificate -# -# Additional info: -# http://acs.lbl.gov/~boverhof/openssl_certs.html -# -#............................................................................. - -generateUserCredentials() { - echo '==> [generateUserCredentials]' - - # validity of the certificate - nDays=${1:-7} - - USERCERTDIR=${SERVERINSTALLDIR}/user - # Generate directory where to store credentials - mkdir -p "${USERCERTDIR}" - if ! cd "${USERCERTDIR}"; then - echo "ERROR: cannot change to ${USERCERTDIR}" >&2 - exit 1 - fi - - # What is that ? - save=$- - if [[ $save =~ e ]]; then - set +e - fi - - cp "${CI_CONFIG}/openssl_config_user.cnf" "${USERCERTDIR}/openssl_config_user.cnf" - if [[ $save =~ e ]]; then - set -e - fi - - openssl genrsa -out client.key 2048 &> /dev/null - chmod 400 client.key - - openssl req -config "${USERCERTDIR}/openssl_config_user.cnf" \ - -key "${USERCERTDIR}/client.key" \ - -new \ - -out "$USERCERTDIR/client.req" - - openssl ca -config "${SERVERINSTALLDIR}/etc/grid-security/ca/openssl_config_ca.cnf" \ - -extensions usr_cert \ - -batch \ - -days "$nDays" \ - -in "$USERCERTDIR/client.req" \ - -out "$USERCERTDIR/client.pem" -} - - #............................................................................. # # diracCredentials: From 190ddcea02801b107d59f0bc04d9f94db0ce01e0 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Fri, 21 Feb 2025 17:11:45 +0100 Subject: [PATCH 02/17] fix: no need for docker cp: certificates from the volume --- integration_tests.py | 58 --------------------------------------- tests/Jenkins/dirac_ci.sh | 2 +- 2 files changed, 1 insertion(+), 59 deletions(-) diff --git a/integration_tests.py b/integration_tests.py index 42fc4c2b5bb..afca857305c 100755 --- a/integration_tests.py +++ b/integration_tests.py @@ -409,35 +409,6 @@ def install_server(): ], check=True, ) - for path in [ - "etc/grid-security", - "user/client.pem", - "user/client.key", - f"/tmp/x509up_u{os.getuid()}", - ]: - source = os.path.join("/home/dirac/ServerInstallDIR", path) - ret = subprocess.run( - ["docker", "cp", f"server:{source}", "-"], - check=True, - text=False, - stdout=subprocess.PIPE, - ) - if path.startswith("user/"): - dest = f"client:/home/dirac/ServerInstallDIR/{os.path.dirname(path)}" - elif path.startswith("/"): - dest = f"client:{os.path.dirname(path)}" - else: - dest = f"client:/home/dirac/ClientInstallDIR/{os.path.dirname(path)}" - subprocess.run(["docker", "cp", "-", dest], check=True, text=False, input=ret.stdout) - subprocess.run( - base_cmd - + [ - "bash", - "-c", - "cp /home/dirac/ServerInstallDIR/user/client.* /home/dirac/.globus/", - ], - check=True, - ) base_cmd = _build_docker_cmd("pilot", tty=False) subprocess.run( @@ -451,35 +422,6 @@ def install_server(): ], check=True, ) - for path in [ - "etc/grid-security", - "user/client.pem", - "user/client.key", - f"/tmp/x509up_u{os.getuid()}", - ]: - source = os.path.join("/home/dirac/ServerInstallDIR", path) - ret = subprocess.run( - ["docker", "cp", f"server:{source}", "-"], - check=True, - text=False, - stdout=subprocess.PIPE, - ) - if path.startswith("user/"): - dest = f"pilot:/home/dirac/ServerInstallDIR/{os.path.dirname(path)}" - elif path.startswith("/"): - dest = f"pilot:{os.path.dirname(path)}" - else: - dest = f"pilot:/home/dirac/PilotInstallDIR/{os.path.dirname(path)}" - subprocess.run(["docker", "cp", "-", dest], check=True, text=False, input=ret.stdout) - subprocess.run( - base_cmd - + [ - "bash", - "-c", - "cp /home/dirac/ServerInstallDIR/user/client.* /home/dirac/.globus/", - ], - check=True, - ) @app.command() diff --git a/tests/Jenkins/dirac_ci.sh b/tests/Jenkins/dirac_ci.sh index 424f9d020e2..f8119340fdb 100644 --- a/tests/Jenkins/dirac_ci.sh +++ b/tests/Jenkins/dirac_ci.sh @@ -83,7 +83,7 @@ source "${TESTCODE}/DIRAC/tests/Jenkins/utilities.sh" # # installSite: # -# This function will install DIRAC +# This function will install DIRAC server # #............................................................................... From 22206264f5e57e508c9d97fed94a6964284d90f7 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Fri, 21 Feb 2025 17:12:22 +0100 Subject: [PATCH 03/17] fix: use dirac-proxy-init instead of dirac-login --- integration_tests.py | 2 +- .../scripts/dirac_proxy_init.py | 13 ++++++------ tests/CI/install_client.sh | 2 +- tests/Jenkins/utilities.sh | 21 ++++++++++++------- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/integration_tests.py b/integration_tests.py index afca857305c..78069fa8aa5 100755 --- a/integration_tests.py +++ b/integration_tests.py @@ -378,7 +378,7 @@ def install_server(): # for the diracx container to use # It needs to be started and running before the DIRAC server installation # because after installing the databases, the install server script - # calls dirac-login. + # calls dirac-proxy-init. # At this point we need the new CS to have been updated # already else the token exchange fails. diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py index 193cf19db71..8d2274973e4 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py @@ -6,21 +6,20 @@ $ dirac-proxy-init -g dirac_user Enter Certificate password: ************** """ +import datetime +import glob import os import sys -import glob import time -import datetime import DIRAC - -from DIRAC import gLogger, gConfig, S_OK, S_ERROR +from DIRAC import S_ERROR, S_OK, gLogger +from DIRAC.ConfigurationSystem.Client.Helpers import Registry from DIRAC.Core.Base.Script import Script -from DIRAC.FrameworkSystem.Client import ProxyGeneration, ProxyUpload -from DIRAC.Core.Security import X509Chain, ProxyInfo, VOMS +from DIRAC.Core.Security import VOMS, ProxyInfo, X509Chain from DIRAC.Core.Security.DiracX import addTokenToPEM from DIRAC.Core.Security.Locations import getCAsLocation, getDefaultProxyLocation -from DIRAC.ConfigurationSystem.Client.Helpers import Registry +from DIRAC.FrameworkSystem.Client import ProxyGeneration, ProxyUpload from DIRAC.FrameworkSystem.Client.BundleDeliveryClient import BundleDeliveryClient diff --git a/tests/CI/install_client.sh b/tests/CI/install_client.sh index 11c52883e6d..fa21f03453f 100755 --- a/tests/CI/install_client.sh +++ b/tests/CI/install_client.sh @@ -61,7 +61,7 @@ echo -e "*** $(date -u) **** Client INSTALLATION START ****\n" installDIRAC echo -e "*** $(date -u) Getting a non privileged user\n" |& tee -a clientTestOutputs.txt -dirac-login -C "${SERVERINSTALLDIR}/user/client.pem" -K "${SERVERINSTALLDIR}/user/client.key" "${DEBUG}" |& tee -a clientTestOutputs.txt +dirac-proxy-init -C "${SERVERINSTALLDIR}/user/client.pem" -K "${SERVERINSTALLDIR}/user/client.key" "${DEBUG}" |& tee -a clientTestOutputs.txt #-------------------------------------------------------------------------------# echo -e "*** $(date -u) **** Submit a job ****\n" diff --git a/tests/Jenkins/utilities.sh b/tests/Jenkins/utilities.sh index 89faf966f09..b6bced37bd7 100644 --- a/tests/Jenkins/utilities.sh +++ b/tests/Jenkins/utilities.sh @@ -247,6 +247,13 @@ installDIRAC() { echo "source \"$PWD/diracos/diracosrc\"" > "$PWD/bashrc" echo "export X509_CERT_DIR=\"$PWD/diracos/etc/grid-security/certificates\"" >> "$PWD/bashrc" source diracos/diracosrc + + # Copy the user cert and key to the correct directory + cp /ca/certs/client.pem "${SERVERINSTALLDIR}/user/" + cp /ca/certs/client.key "${SERVERINSTALLDIR}/user/" + cp /ca/certs/client.pem /home/dirac/.globus/usercert.pem + cp /ca/certs/client.key /home/dirac/.globus/userkey.pem + if [[ -n "${DIRAC_RELEASE+x}" ]]; then if [[ -z "${ALTERNATIVE_MODULES}" ]]; then pip install DIRAC "${DIRAC_RELEASE}" @@ -383,8 +390,8 @@ diracCredentials() { echo '==> [diracCredentials]' sed -i 's/commitNewData = CSAdministrator/commitNewData = authenticated/g' "${SERVERINSTALLDIR}/etc/Configuration_Server.cfg" - if ! dirac-login dirac_admin --nocs -C "${SERVERINSTALLDIR}/user/client.pem" -K "${SERVERINSTALLDIR}/user/client.key" -T 72 "${DEBUG}"; then - echo 'ERROR: dirac-login failed' >&2 + if ! dirac-proxy-init dirac_admin --nocs -C "${SERVERINSTALLDIR}/user/client.pem" -K "${SERVERINSTALLDIR}/user/client.key" "${DEBUG}"; then + echo 'ERROR: dirac-proxy-init failed' >&2 exit 1 fi sed -i 's/commitNewData = authenticated/commitNewData = CSAdministrator/g' "${SERVERINSTALLDIR}/etc/Configuration_Server.cfg" @@ -466,7 +473,7 @@ diracProxies() { if [[ -n $TEST_DIRACX ]]; then echo "Waiting for for DiracX to be available" >&2 for i in {1..10}; do - if dirac-login -C "${SERVERINSTALLDIR}/user/client.pem" -K "${SERVERINSTALLDIR}/user/client.key" -T 72 "${DEBUG}"; then + if dirac-proxy-init -C "${SERVERINSTALLDIR}/user/client.pem" -K "${SERVERINSTALLDIR}/user/client.key" "${DEBUG}"; then break fi sleep 5 @@ -474,13 +481,13 @@ diracProxies() { fi # User proxy - if ! dirac-login -C "${SERVERINSTALLDIR}/user/client.pem" -K "${SERVERINSTALLDIR}/user/client.key" -T 72 "${DEBUG}"; then - echo 'ERROR: dirac-login failed' >&2 + if ! dirac-proxy-init -C "${SERVERINSTALLDIR}/user/client.pem" -K "${SERVERINSTALLDIR}/user/client.key" "${DEBUG}"; then + echo 'ERROR: dirac-init failed' >&2 exit 1 fi # group proxy - if ! dirac-login prod -C "${SERVERINSTALLDIR}/user/client.pem" -K "${SERVERINSTALLDIR}/user/client.key" -T 72 "${DEBUG}"; then - echo 'ERROR: dirac-login failed' >&2 + if ! dirac-proxy-init prod -C "${SERVERINSTALLDIR}/user/client.pem" -K "${SERVERINSTALLDIR}/user/client.key" "${DEBUG}"; then + echo 'ERROR: dirac-init failed' >&2 exit 1 fi } From 6b7f9ac5f441ddfe7164c73abee2a974f6693d03 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Mon, 24 Feb 2025 17:55:13 +0100 Subject: [PATCH 04/17] fix: monitoring the installation takes a user --- src/DIRAC/Core/scripts/dirac_install_db.py | 31 +++++++++++++++---- .../Client/SystemAdministratorClientCLI.py | 17 ++++++---- .../Utilities/MonitoringUtilities.py | 21 ++----------- 3 files changed, 38 insertions(+), 31 deletions(-) diff --git a/src/DIRAC/Core/scripts/dirac_install_db.py b/src/DIRAC/Core/scripts/dirac_install_db.py index ed61e8226d2..dbf16c095dd 100755 --- a/src/DIRAC/Core/scripts/dirac_install_db.py +++ b/src/DIRAC/Core/scripts/dirac_install_db.py @@ -2,6 +2,8 @@ """ Create a new DB in the MySQL server """ +from DIRAC import exit as DIRACExit +from DIRAC import gConfig, gLogger from DIRAC.Core.Base.Script import Script @@ -12,7 +14,8 @@ def main(): _, args = Script.parseCommandLine() # Script imports - from DIRAC import gConfig + from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import useServerCertificate + from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities @@ -21,15 +24,31 @@ def main(): for db in args: result = gComponentInstaller.installDatabase(db) if not result["OK"]: - print(f"ERROR: failed to correctly install {db}", result["Message"]) - continue + gLogger.error(f"ERROR: failed to correctly install {db}", result["Message"]) + DIRACExit(1) extension, system = result["Value"] - gComponentInstaller.addDatabaseOptionsToCS(gConfig, system, db, overwrite=True) + result = gComponentInstaller.addDatabaseOptionsToCS(gConfig, system, db, overwrite=True) + if not result["OK"]: + gLogger.error(f"ERROR: failed to add database options to CS: {result['Message']}") + DIRACExit(1) if db != "InstalledComponentsDB": - result = MonitoringUtilities.monitorInstallation("DB", system, db) + + # get the user that installed the DB + if useServerCertificate(): + user = "DIRAC" + else: + result = getProxyInfo() + if not result["OK"]: + return result + proxyInfo = result["Value"] + if "username" in proxyInfo: + user = proxyInfo["username"] + + result = MonitoringUtilities.monitorInstallation("DB", system, db, user=user) if not result["OK"]: - print(f"ERROR: failed to register installation in database: {result['Message']}") + gLogger.error(f"ERROR: failed to register installation in database: {result['Message']}") + DIRACExit(1) if __name__ == "__main__": diff --git a/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py b/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py index e1303de123c..9b65a237518 100644 --- a/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py +++ b/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py @@ -11,7 +11,6 @@ import time from DIRAC import gConfig, gLogger -from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals from DIRAC.Core.Base.CLI import CLI, colorize from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.Core.Utilities import List @@ -623,6 +622,11 @@ def do_install(self, args): install agent [-m ] [-p