Skip to content

Commit

Permalink
Merge branch 'master' into undx/TCOMP-2339_remove_static_modifier
Browse files Browse the repository at this point in the history
  • Loading branch information
yyin-talend committed Oct 7, 2023
2 parents d59f8b3 + 1ceafce commit b93c9ee
Show file tree
Hide file tree
Showing 98 changed files with 712 additions and 241 deletions.
42 changes: 42 additions & 0 deletions .jenkins/scripts/mvn_sonar_branch.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
#!/usr/bin/env bash
#
# Copyright (C) 2006-2023 Talend Inc. - www.talend.com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

set -xe

# Execute sonar analysis
# $1: Sonar analyzed branch
# $@: the extra parameters to be used in the maven commands
main() (
branch="${1?Missing branch name}"; shift
extraBuildParams=("$@")

declare -a LIST_FILE_ARRAY=( $(find $(pwd) -type f -name 'jacoco.xml') )
LIST_FILE=$(IFS=, ; echo "${LIST_FILE_ARRAY[*]}")

# Why sonar plugin is not declared in pom.xml: https://blog.sonarsource.com/we-had-a-dream-mvn-sonarsonar
# TODO https://jira.talendforge.org/browse/TDI-48980 (CI: Reactivate Sonar cache)
mvn sonar:sonar \
--define 'sonar.host.url=https://sonar-eks.datapwn.com' \
--define "sonar.login=${SONAR_LOGIN}" \
--define "sonar.password=${SONAR_PASSWORD}" \
--define "sonar.branch.name=${branch}" \
--define "sonar.coverage.jacoco.xmlReportPaths='${LIST_FILE}'" \
--define "sonar.analysisCache.enabled=false" \
"${extraBuildParams[@]}"
)

main "$@"
53 changes: 53 additions & 0 deletions .jenkins/scripts/mvn_sonar_pr.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
#!/usr/bin/env bash
#
# Copyright (C) 2006-2023 Talend Inc. - www.talend.com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

set -xe

# Execute sonar analysis
# $1: Sonar analyzed branch
# $2: Sonar target branch for compare
# $3: PR id
# $@: the extra parameters to be used in the maven commands
main() (
branch="${1?Missing branch name}"; shift
target_branch="${1?Missing target branch name}"; shift
pull_request_id="${1?Missing pull request id}"; shift
extraBuildParams=("$@")

declare -a LIST_FILE_ARRAY=( $(find $(pwd) -type f -name 'jacoco.xml') )
LIST_FILE=$(IFS=, ; echo "${LIST_FILE_ARRAY[*]}")

# Fetch target branch for Sonar diff purpose
git fetch origin "${target_branch}":"${target_branch}"

# Why sonar plugin is not declared in pom.xml: https://blog.sonarsource.com/we-had-a-dream-mvn-sonarsonar
# TODO https://jira.talendforge.org/browse/TDI-48980 (CI: Reactivate Sonar cache)
mvn sonar:sonar \
--define 'sonar.host.url=https://sonar-eks.datapwn.com' \
--define "sonar.login=${SONAR_LOGIN}" \
--define "sonar.password=${SONAR_PASSWORD}" \
--define "sonar.coverage.jacoco.xmlReportPaths='${LIST_FILE}'" \
--define "sonar.analysisCache.enabled=false" \
--define "sonar.pullrequest.branch=${branch}" \
--define "sonar.pullrequest.base=${target_branch}" \
--define "sonar.pullrequest.key=${pull_request_id}" \
"${extraBuildParams[@]}"


)

main "$@"
140 changes: 84 additions & 56 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,49 @@ import java.time.LocalDateTime
import java.util.regex.Matcher

// Credentials
final def ossrhCredentials = usernamePassword(credentialsId: 'ossrh-credentials', usernameVariable: 'OSSRH_USER', passwordVariable: 'OSSRH_PASS')
final def nexusCredentials = usernamePassword(credentialsId: 'nexus-artifact-zl-credentials', usernameVariable: 'NEXUS_USER', passwordVariable: 'NEXUS_PASS')
final def jetbrainsCredentials = usernamePassword(credentialsId: 'jetbrains-credentials', usernameVariable: 'JETBRAINS_USER', passwordVariable: 'JETBRAINS_PASS')
final def jiraCredentials = usernamePassword(credentialsId: 'jira-credentials', usernameVariable: 'JIRA_USER', passwordVariable: 'JIRA_PASS')
final def gitCredentials = usernamePassword(credentialsId: 'github-credentials', usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_PASS')
final def dockerCredentials = usernamePassword(credentialsId: 'artifactory-datapwn-credentials', usernameVariable: 'DOCKER_USER', passwordVariable: 'DOCKER_PASS')
final def sonarCredentials = usernamePassword( credentialsId: 'sonar-credentials', usernameVariable: 'SONAR_USER', passwordVariable: 'SONAR_PASS')
final def keyImportCredentials = usernamePassword(credentialsId: 'component-runtime-import-key-credentials', usernameVariable: 'KEY_USER', passwordVariable: 'KEY_PASS')
final def gpgCredentials = usernamePassword(credentialsId: 'component-runtime-gpg-credentials', usernameVariable: 'GPG_KEYNAME', passwordVariable: 'GPG_PASSPHRASE')
final def ossrhCredentials = usernamePassword(
credentialsId: 'ossrh-credentials',
usernameVariable: 'OSSRH_USER',
passwordVariable: 'OSSRH_PASS')
final def nexusCredentials = usernamePassword(
credentialsId: 'nexus-artifact-zl-credentials',
usernameVariable: 'NEXUS_USER', passwordVariable: 'NEXUS_PASS')
final def jetbrainsCredentials = usernamePassword(
credentialsId: 'jetbrains-credentials',
usernameVariable: 'JETBRAINS_USER',
passwordVariable: 'JETBRAINS_PASS')
final def jiraCredentials = usernamePassword(
credentialsId: 'jira-credentials',
usernameVariable: 'JIRA_USER',
passwordVariable: 'JIRA_PASS')
final def gitCredentials = usernamePassword(
credentialsId: 'github-credentials',
usernameVariable: 'GITHUB_USER',
passwordVariable: 'GITHUB_PASS')
final def dockerCredentials = usernamePassword(
credentialsId: 'artifactory-datapwn-credentials',
usernameVariable: 'DOCKER_USER',
passwordVariable: 'DOCKER_PASS')
final def sonarCredentials = usernamePassword(
credentialsId: 'sonar-credentials',
usernameVariable: 'SONAR_LOGIN',
passwordVariable: 'SONAR_PASSWORD')
final def keyImportCredentials = usernamePassword(
credentialsId: 'component-runtime-import-key-credentials',
usernameVariable: 'KEY_USER',
passwordVariable: 'KEY_PASS')
final def gpgCredentials = usernamePassword(
credentialsId: 'component-runtime-gpg-credentials',
usernameVariable: 'GPG_KEYNAME',
passwordVariable: 'GPG_PASSPHRASE')

// In PR environment, the branch name is not valid and should be swap with pr name.
final String pull_request_id = env.CHANGE_ID
final String branch_name = pull_request_id != null ? env.CHANGE_BRANCH : env.BRANCH_NAME

// Job config
final String slackChannel = 'components-ci'
final Boolean isMasterBranch = env.BRANCH_NAME == "master"
final Boolean isStdBranch = (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("maintenance/"))
final Boolean isMasterBranch = branch_name == "master"
final Boolean isStdBranch = (branch_name == "master" || branch_name.startsWith("maintenance/"))
final Boolean hasPostLoginScript = params.POST_LOGIN_SCRIPT != ""
final String extraBuildParams = ""
final String buildTimestamp = String.format('-%tY%<tm%<td%<tH%<tM%<tS', LocalDateTime.now())
Expand Down Expand Up @@ -70,13 +99,13 @@ pipeline {
}

options {
buildDiscarder(logRotator(artifactNumToKeepStr: '10', numToKeepStr: env.BRANCH_NAME == 'master' ? '15' : '10'))
buildDiscarder(logRotator(artifactNumToKeepStr: '10', numToKeepStr: branch_name == 'master' ? '15' : '10'))
timeout(time: 180, unit: 'MINUTES')
skipStagesAfterUnstable()
}

triggers {
cron(env.BRANCH_NAME == "master" ? "0 12 * * *" : "")
cron(branch_name == "master" ? "0 12 * * *" : "")
}

parameters {
Expand Down Expand Up @@ -113,9 +142,9 @@ pipeline {
defaultValue: '',
description: 'Execute a shell command after login. Useful for maintenance.')
booleanParam(
name: 'FORCE_SONAR',
name: 'DISABLE_SONAR',
defaultValue: false,
description: 'Force Sonar analysis')
description: 'Cancel the Sonar analysis stage execution')
booleanParam(
name: 'FORCE_DOC',
defaultValue: false,
Expand All @@ -129,6 +158,7 @@ pipeline {
stages {
stage('Preliminary steps') {
steps {

///////////////////////////////////////////
// Login tasks
///////////////////////////////////////////
Expand Down Expand Up @@ -198,10 +228,11 @@ pipeline {
""".stripIndent()
}
else {
echo "Validate the branch name"
println "Validate the branch name"

(branch_user,
branch_ticket,
branch_description) = extract_branch_info("$env.BRANCH_NAME")
branch_description) = extract_branch_info(branch_name)

// Check only branch_user, because if there is an error all three params are empty.
if (branch_user == ("")) {
Expand All @@ -225,7 +256,7 @@ pipeline {
"$params.VERSION_QUALIFIER" as String)

echo """
Configure the version qualifier for the curent branche: $env.BRANCH_NAME
Configure the version qualifier for the curent branche: $branch_name
requested qualifier: $params.VERSION_QUALIFIER
with User = $branch_user, Ticket = $branch_ticket, Description = $branch_description
Qualified Version = $finalVersion"""
Expand Down Expand Up @@ -264,7 +295,7 @@ pipeline {
// updating build description
String description = """
Version = $finalVersion - $params.Action Build
Sonar forced: $params.FORCE_SONAR - Script: $hasPostLoginScript
Disable Sonar: $params.DISABLE_SONAR - Script: $hasPostLoginScript
Debug: $params.JENKINS_DEBUG
Extra build args: $extraBuildParams""".stripIndent()
job_description_append(description)
Expand Down Expand Up @@ -316,7 +347,7 @@ pipeline {
post {
always {
recordIssues(
enabledForFailure: true,
enabledForFailure: false,
tools: [
junitParser(
id: 'unit-test',
Expand Down Expand Up @@ -466,24 +497,13 @@ pipeline {
sh """\
#!/usr/bin/env bash
set -xe
mvn versions:dependency-updates-report versions:plugin-updates-report \
versions:property-updates-report \
-pl '!bom'
mvn versions:dependency-updates-report versions:plugin-updates-report -pl '!bom'
""".stripIndent()
}
}
}
post {
always {
publishHTML(
target: [
allowMissing : true,
alwaysLinkToLastBuild: false,
keepAll : true,
reportDir : 'target/site/',
reportFiles : 'property-updates-report.html',
reportName : "outdated::property"
])
publishHTML(
target: [
allowMissing : true,
Expand All @@ -507,24 +527,32 @@ pipeline {
}
stage('Sonar') {
when {
expression { params.Action != 'RELEASE' }
branch 'master'
expression { (params.Action != 'RELEASE') && !params.DISABLE_SONAR}
}
steps {
withCredentials([sonarCredentials]) {
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
// TODO https://jira.talendforge.org/browse/TDI-48980 (CI: Reactivate Sonar cache)
sh """\
#!/usr/bin/env bash
set -xe
_JAVA_OPTIONS='--add-opens=java.base/java.lang=ALL-UNNAMED'
mvn sonar:sonar \
--define sonar.host.url=https://sonar-eks.datapwn.com \
--define sonar.login='$SONAR_USER' \
--define sonar.password='$SONAR_PASS' \
--define sonar.branch.name=${env.BRANCH_NAME} \
--define sonar.analysisCache.enabled=false
""".stripIndent()
script {
withCredentials([nexusCredentials,
sonarCredentials,
gitCredentials]) {

if (pull_request_id != null) {

println 'Run analysis for PR'
sh """\
bash .jenkins/scripts/mvn_sonar_pr.sh \
'${branch_name}' \
'${env.CHANGE_TARGET}' \
'${pull_request_id}' \
${extraBuildParams}
""".stripIndent()
} else {
echo 'Run analysis for branch'
sh """\
bash .jenkins/scripts/mvn_sonar_branch.sh \
'${branch_name}' \
${extraBuildParams}
""".stripIndent()
}
}
}
}
Expand All @@ -542,7 +570,7 @@ pipeline {
configFileProvider([configFile(fileId: 'maven-settings-nexus-zl', variable: 'MAVEN_SETTINGS')]) {
sh """\
#!/usr/bin/env bash
bash .jenkins/scripts/release.sh ${env.BRANCH_NAME} ${finalVersion}
bash .jenkins/scripts/release.sh $branch_name $finalVersion
""".stripIndent()
}
}
Expand All @@ -558,7 +586,7 @@ pipeline {
slackSend(
color: '#00FF00',
message: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})",
channel: "${slackChannel}"
channel: "${env.SLACK_CI_CHANNEL}"
)
}
}
Expand Down Expand Up @@ -588,22 +616,22 @@ pipeline {
slackSend(
color: '#FF0000',
message: "@here : NEW FAILURE: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})",
channel: "${slackChannel}"
channel: "${env.SLACK_CI_CHANNEL}"
)
} else {
//else send notification without pinging channel
slackSend(
color: '#FF0000',
message: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})",
channel: "${slackChannel}"
channel: "${env.SLACK_CI_CHANNEL}"
)
}
}
}
}
always {
recordIssues(
enabledForFailure: true,
enabledForFailure: false,
tools: [
taskScanner(
id: 'disabled',
Expand Down Expand Up @@ -744,7 +772,7 @@ private static String add_qualifier_to_version(String version, String ticket, St
* @return A list containing the extracted: [user, ticket, description]
* The method also raise an assert exception in case of wrong branch name
*/
private static ArrayList<String> extract_branch_info(GString branch_name) {
private static ArrayList<String> extract_branch_info(String branch_name) {

String branchRegex = /^(?<user>.*)\/(?<ticket>[A-Z]{2,8}-\d{1,6})[_-](?<description>.*)/
Matcher branchMatcher = branch_name =~ branchRegex
Expand All @@ -761,4 +789,4 @@ private static ArrayList<String> extract_branch_info(GString branch_name) {
String description = branchMatcher.group("description")

return [user, ticket, description]
}
}
4 changes: 2 additions & 2 deletions bom/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

<groupId>org.talend.sdk.component</groupId>
<artifactId>component-bom</artifactId>
<version>1.61.0-SNAPSHOT</version>
<version>1.62.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Component BOM</name>
<description>Talend Component Kit</description>
Expand Down Expand Up @@ -82,7 +82,7 @@
<log4j2.version>2.20.0</log4j2.version>

<commons-codec.version>1.15</commons-codec.version>
<commons-compress.version>1.23.0</commons-compress.version>
<commons-compress.version>1.24.0</commons-compress.version>
<commons-lang3.version>3.11</commons-lang3.version>

<jackson.version>2.15.2</jackson.version>
Expand Down
2 changes: 1 addition & 1 deletion component-api/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
<parent>
<groupId>org.talend.sdk.component</groupId>
<artifactId>component-runtime</artifactId>
<version>1.61.0-SNAPSHOT</version>
<version>1.62.0-SNAPSHOT</version>
</parent>

<artifactId>component-api</artifactId>
Expand Down
Loading

0 comments on commit b93c9ee

Please sign in to comment.