Add release notes generation action

Signed-off-by: Sayali Gaikawad <[email protected]>

jenkins/release-notes-check/release-notes-check.jenkinsfile

@@ -16,13 +16,25 @@ pipeline {
     options {
             timeout(time: 2, unit: 'HOURS')
     }
-    agent none
+    agent {
+        docker {
+            label 'Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host'
+            image 'opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3'
+            registryUrl 'https://public.ecr.aws/'
+            alwaysPull true
+        }
+    }
     parameters {
         string(
-            name: 'INPUT_MANIFEST',
-            description: 'Input manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0.yml.',
+            name: 'RELEASE_VERSION',
+            description: 'Release version number to fetch input manifest path',
             trim: true
         )
+        choice(
+                choices: ['check', 'compile'],
+                name: 'ACTION',
+                description: 'check - Checks the release notes for all components. compile - Consolidates all release notes', 
+        )
         string(
             name: 'GIT_LOG_DATE',
             description: 'in format yyyy-mm-dd, example 2022-07-26.',
@@ -44,87 +56,99 @@ pipeline {
             trim: true
         )
     }
-    environment {
-        AGENT_X64 = 'Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host'
-    }
     stages {
-        stage('detect docker image + args') {
-            agent {
-                docker {
-                    label AGENT_X64
-                    image 'docker/library/alpine:3'
-                    registryUrl 'https://public.ecr.aws/'
-                    alwaysPull true
-                }
-            }
-            steps {
-                script {
-                    dockerAgent = detectDockerAgent()
-                    currentBuild.description = INPUT_MANIFEST
-                }
-            }
-            post {
-                always {
-                    postCleanup()
-                }
-            }
-        }
         stage('Parameters Check') {
             steps {
                 script {
-                    currentBuild.description = "Comment:${COMMENT}, Manifest:${INPUT_MANIFEST}"
-                    if (GIT_LOG_DATE.isEmpty() || INPUT_MANIFEST.isEmpty()) {
-                                currentBuild.result = 'ABORTED'
-                                error('Make sure all the parameters are passed in.')
+                    if (ACTION.isEmpty() || RELEASE_VERSION.isEmpty()) {
+                        currentBuild.result = 'ABORTED'
+                        error('ACTION or RELEASE_VERSION cannot be empty!')
                     }
-                    if (COMMENT == "ADD" && GIT_ISSUE_NUMBER.isEmpty()) {
-                                currentBuild.result = 'ABORTED'
-                                error('Make sure all the parameters are passed in.')
+                    if (ACTION == "check" && (COMMENT == "ADD" || GIT_ISSUE_NUMBER.isEmpty())) {
+                        currentBuild.result = 'ABORTED'
+                        error('ACTION or COMMENT or GIT_ISSUE_NUMBER cannot be empty!')
                     }
-                    if (COMMENT == "UPDATE" && COMMENT_UNIQUE_ID.isEmpty()) {
-                                currentBuild.result = 'ABORTED'
-                                error('Make sure all the parameters are passed in.')
+                    if (ACTION == "check" && (COMMENT == "UPDATE" || COMMENT_UNIQUE_ID.isEmpty())) {
+                        currentBuild.result = 'ABORTED'
+                        error('ACTION or COMMENT or COMMENT_UNIQUE_ID cannot be empty!')
                     }
                 }
             }
         }
-        stage('Generate MarkDown table') {
-            agent {
-                docker {
-                    label AGENT_X64
-                    image dockerAgent.image
-                    registryUrl 'https://public.ecr.aws/'
-                    alwaysPull true
-                }
+        stage('Check release notes') {
+            when {
+                expression { params.ACTION == 'check' }
             }
             steps {
                 script {
                     withCredentials([usernamePassword(credentialsId: "jenkins-github-bot-token", usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN')]) {
                         if (params.COMMENT == "ADD") {
-                            sh '''
+                            sh """
                                 #!/bin/bash
                                 set +e
-                                ./release_notes.sh check manifests/${INPUT_MANIFEST} --date ${GIT_LOG_DATE} --output table.md
+                                ./release_notes.sh check manifests/${RELEASE_VERSION}/opensearch-${RELEASE_VERSION}.yml manifests/${RELEASE_VERSION}/opensearch-dashboards-${RELEASE_VERSION}.yml --date ${GIT_LOG_DATE} --output table.md
                                 echo "Adding Comment on issue $GIT_ISSUE_NUMBER"
                                 gh repo clone https://github.com/opensearch-project/opensearch-build.git; cd opensearch-build
                                 gh issue comment ${GIT_ISSUE_NUMBER} --body-file ../table.md --repo opensearch-project/opensearch-build
-                            '''
+                            """
                         }
                         if (params.COMMENT == "UPDATE") {
-                            sh '''
+                            sh """
                                 #!/bin/bash
                                 set +e
-                                ./release_notes.sh check manifests/${INPUT_MANIFEST} --date ${GIT_LOG_DATE} --output table.md
+                                ./release_notes.sh check manifests/${RELEASE_VERSION}/opensearch-${RELEASE_VERSION}.yml manifests/${RELEASE_VERSION}/opensearch-dashboards-${RELEASE_VERSION}.yml --date ${GIT_LOG_DATE} --output table.md
                                 echo "Updating Comment on issue $GIT_ISSUE_NUMBER"
                                 gh repo clone https://github.com/opensearch-project/opensearch-build.git; cd opensearch-build
                                 IFS=
-                                gh api --method PATCH -H "Accept: application/vnd.github+json" /repos/opensearch-project/opensearch-build/issues/comments/${COMMENT_UNIQUE_ID} -f body=$(cat ../table.md)
-                            '''
+                                gh api --method PATCH -H "Accept: application/vnd.github+json" /repos/opensearch-project/opensearch-build/issues/comments/${COMMENT_UNIQUE_ID} -f body=\$(cat ../table.md)
+                            """
                         }
                         if (params.COMMENT == "NO_COMMENT") {
-                            sh '''
-                            ./release_notes.sh check manifests/${INPUT_MANIFEST} --date ${GIT_LOG_DATE}
-                            '''
+                            sh """
+                            ./release_notes.sh check manifests/${RELEASE_VERSION}/opensearch-${RELEASE_VERSION}.yml manifests/${RELEASE_VERSION}/opensearch-dashboards-${RELEASE_VERSION}.yml --date ${GIT_LOG_DATE}
+                            """
+                        }
+                    }
+                }
+            }
+            post {
+                always {
+                    postCleanup()
+                }
+            }
+        }
+        stage('Generate consolidated release notes') {
+            when {
+                expression { params.ACTION == 'compile' }
+            }
+            steps {
+                script {
+                    sh """
+                        #!/bin/bash
+                        set +e
+                        ./release_notes.sh compile manifests/${RELEASE_VERSION}/opensearch-${RELEASE_VERSION}.yml manifests/${RELEASE_VERSION}/opensearch-dashboards-${RELEASE_VERSION}.yml --output table.md
+                    """
+                    withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) {
+                        try {
+                            sh """
+                                git remote set-url origin "https://opensearch-ci:${GITHUB_TOKEN}@github.com/opensearch-project/opensearch-build"
+                                git config user.email "[email protected]"
+                                git config user.name "opensearch-ci"
+                                git checkout -b release-notes
+                            """
+                            def status = sh(returnStdout: true, script: 'git status --porcelain')
+                            if (status) {
+                                sh """
+                                    git status --porcelain | grep '^ M' | cut -d " " -f3 | xargs git add
+                                    git commit -sm "Add consolidated release notes for ${params.RELEASE_VERSION}"
+                                    git push origin release-notes --force
+                                    gh pr create --title 'Add consolidated release notes for ${params.RELEASE_VERSION}' --body 'Add consolidated release notes for ${params.RELEASE_VERSION}' -H release-notes -B main
+                                """
+                            } else {
+                                println 'Nothing to commit!'
+                            }
+                        } catch (e) {
+                            error 'An error occured while creating pull request for consolidated release notes' + e.toString()
                         }
                     }
                 }

tests/jenkins/TestReleaseNotesCheck.groovy

@@ -13,14 +13,15 @@ import static com.lesfurets.jenkins.unit.global.lib.LibraryConfiguration.library
 import static com.lesfurets.jenkins.unit.global.lib.GitSource.gitSource
 import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString
 import static org.assertj.core.api.Assertions.assertThat
+import static org.hamcrest.CoreMatchers.hasItem
 
 class TestReleaseNotesCheck extends BuildPipelineTest {
 
     String gitLogDate = '2022-10-10'
     String comment = 'NO_COMMENT'
     String gitIssueNumber = '123456'
     String commentUniqueID = '123456'
-    String inputManifest = '3.0.0/opensearch-3.0.0.yml'
+    String releaseVersion = '3.0.0'
 
     @Override
     @Before
@@ -37,32 +38,48 @@ class TestReleaseNotesCheck extends BuildPipelineTest {
             )
         super.setUp()
 
-        binding.setVariable('INPUT_MANIFEST', inputManifest)
-        binding.setVariable('GIT_LOG_DATE', gitLogDate)
-        binding.setVariable('COMMENT', comment)
-        binding.setVariable('GIT_ISSUE_NUMBER', gitIssueNumber)
-        binding.setVariable('COMMENT_UNIQUE_ID', commentUniqueID)
-        binding.setVariable('AGENT_X64','Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host')
-        binding.setVariable('dockerAgent', [image:'opensearchstaging/ci-runner:ci-runner-centos7-v1', args:'-e JAVA_HOME=/opt/java/openjdk-11'])
-
+        addParam('RELEASE_VERSION', releaseVersion)
     }
 
     @Test
-    public void testReleaseNoteCheckPipeline() {
+    public void releaseNotesCheck() {
+        addParam('ACTION', 'check')
+        addParam('GIT_LOG_DATE', gitLogDate)
+        addParam('COMMENT', comment)
+        addParam('GIT_ISSUE_NUMBER', gitIssueNumber)
+        addParam('COMMENT_UNIQUE_ID', commentUniqueID)
+
         super.testPipeline("jenkins/release-notes-check/release-notes-check.jenkinsfile",
                 "tests/jenkins/jenkinsjob-regression-files/release-notes-check/release-notes-check.jenkinsfile")
+        assertJobStatusSuccess()
+        def callStack = helper.getCallStack()
+        assertCallStack().contains('Check release notes, groovy.lang.Closure')
+        assertCallStack().contains('Skipping stage Generate consolidated release notes')
+        assertThat(getShellCommands('release_notes'), hasItem('./release_notes.sh check manifests/3.0.0/opensearch-3.50.yml manifests/3.0.0/opensearch-dashboards-3.0.0.yml --date 2022-10-10'))
     }
 
     @Test
-    public void releaseNoteExecuteWithoutErrors() {
-        runScript("jenkins/release-notes-check/release-notes-check.jenkinsfile")
-
+    public void releaseNoteCompile() {
+        addParam('ACTION', 'compile')
+        super.testPipeline("jenkins/release-notes-check/release-notes-check.jenkinsfile",
+                "tests/jenkins/jenkinsjob-regression-files/release-notes-check/release-notes-compile.jenkinsfile")
         assertJobStatusSuccess()
+        def callStack = helper.getCallStack()
+        assertCallStack().contains('Skipping stage Check release notes')
+        assertCallStack().contains('Generate consolidated release notes, groovy.lang.Closure')
+        print(getShellCommands('release_notes'))
+        assertThat(getShellCommands('release_notes'), hasItem('./release_notes.sh compile manifests/3.0.0/opensearch-3.0.0.yml manifests/3.0.0/opensearch-dashboards-3.0.0.yml --output table.md'))
+    }
+
 
-        assertThat(helper.callStack.findAll { call ->
+    def getShellCommands(String searchtext) {
+        def shCommands = helper.callStack.findAll { call ->
             call.methodName == 'sh'
-        }.any { call ->
-            callArgsToString(call).contains('release_notes.sh')
-        }).isTrue()
+        }.collect { call ->
+            callArgsToString(call)
+        }.findAll { command ->
+            command.contains(searchtext)
+        }
+        return shCommands
     }
 }

tests/jenkins/jenkinsjob-regression-files/release-notes-check/release-notes-check.jenkinsfile.txt

@@ -3,33 +3,16 @@
       release-notes-check.library({[email protected], retriever=null})
       release-notes-check.pipeline(groovy.lang.Closure)
          release-notes-check.timeout({time=2, unit=HOURS})
-         release-notes-check.echo(Executing on agent [label:none])
-         release-notes-check.stage(detect docker image + args, groovy.lang.Closure)
-            release-notes-check.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, image:docker/library/alpine:3, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
-            release-notes-check.script(groovy.lang.Closure)
-               release-notes-check.detectDockerAgent()
-                  detectDockerAgent.legacySCM(groovy.lang.Closure)
-                  detectDockerAgent.library({[email protected], retriever=null})
-                  detectDockerAgent.readYaml({file=manifests/3.0.0/opensearch-3.0.0.yml})
-                  InputManifest.asBoolean()
-                  detectDockerAgent.echo(Using Docker image opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3 (-e JAVA_HOME=/opt/java/openjdk-21))
-                  detectDockerAgent.echo(Using java version openjdk-21)
-         release-notes-check.postCleanup()
-            postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})
+         release-notes-check.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
          release-notes-check.stage(Parameters Check, groovy.lang.Closure)
             release-notes-check.script(groovy.lang.Closure)
-         release-notes-check.stage(Generate MarkDown table, groovy.lang.Closure)
-            release-notes-check.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
+         release-notes-check.stage(Check release notes, groovy.lang.Closure)
             release-notes-check.script(groovy.lang.Closure)
                release-notes-check.usernamePassword({credentialsId=jenkins-github-bot-token, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
                release-notes-check.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
                   release-notes-check.sh(
-                                #!/bin/bash
-                                set +e
-                                ./release_notes.sh check manifests/${INPUT_MANIFEST} --date ${GIT_LOG_DATE} --output table.md
-                                echo "Adding Comment on issue $GIT_ISSUE_NUMBER"
-                                gh repo clone https://github.com/opensearch-project/opensearch-build.git; cd opensearch-build
-                                gh issue comment ${GIT_ISSUE_NUMBER} --body-file ../table.md --repo opensearch-project/opensearch-build
+                            ./release_notes.sh check manifests/3.0.0/opensearch-3.0.0.yml manifests/3.0.0/opensearch-dashboards-3.0.0.yml --date 2022-10-10
                             )
          release-notes-check.postCleanup()
             postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})
+         release-notes-check.echo(Skipping stage Generate consolidated release notes)

tests/jenkins/jenkinsjob-regression-files/release-notes-check/release-notes-compile.jenkinsfile.txt

@@ -0,0 +1,33 @@
+   release-notes-check.run()
+      release-notes-check.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git})
+      release-notes-check.library({[email protected], retriever=null})
+      release-notes-check.pipeline(groovy.lang.Closure)
+         release-notes-check.timeout({time=2, unit=HOURS})
+         release-notes-check.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
+         release-notes-check.stage(Parameters Check, groovy.lang.Closure)
+            release-notes-check.script(groovy.lang.Closure)
+         release-notes-check.echo(Skipping stage Check release notes)
+         release-notes-check.stage(Generate consolidated release notes, groovy.lang.Closure)
+            release-notes-check.script(groovy.lang.Closure)
+               release-notes-check.sh(
+                        #!/bin/bash
+                        set +e
+                        ./release_notes.sh compile manifests/3.0.0/opensearch-3.0.0.yml manifests/3.0.0/opensearch-dashboards-3.0.0.yml --output table.md
+                    )
+               release-notes-check.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER})
+               release-notes-check.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
+                  release-notes-check.sh(
+                                git remote set-url origin "https://opensearch-ci:[email protected]/opensearch-project/opensearch-build"
+                                git config user.email "[email protected]"
+                                git config user.name "opensearch-ci"
+                                git checkout -b release-notes
+                            )
+                  release-notes-check.sh({returnStdout=true, script=git status --porcelain})
+                  release-notes-check.sh(
+                                    git status --porcelain | grep '^ M' | cut -d " " -f3 | xargs git add
+                                    git commit -sm "Add consolidated release notes for 3.0.0"
+                                    git push origin release-notes --force
+                                    gh pr create --title 'Add consolidated release notes for 3.0.0' --body 'Add consolidated release notes for 3.0.0' -H release-notes -B main
+                                )
+         release-notes-check.postCleanup()
+            postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})