Merge branch 'master' into dependabot/maven/org.apache.spark-spark-core_2.12-3.4.0

Author: yruslan

.fossa.yml

@@ -1,22 +1,30 @@
-# Generated by FOSSA CLI (https://github.com/fossas/fossa-cli)
 # Visit https://fossa.com to learn more
 
-version: 2
-cli:
-  server: https://app.fossa.com
-  fetcher: custom
-  project: cobrix
-analyze:
-  modules:
-  - name: cobol-parser_2.11
-    type: mvn
-    target: cobol-parser/pom.xml
-    path: .
-    options:
-      cmd: "mvn dependency:tree -Dscope=compile | grep -v provided"
-  - name: spark-cobol_2.11
-    type: mvn
-    target: spark-cobol/pom.xml
-    path: .
-    options:
-      cmd: "mvn dependency:tree -Dscope=compile | grep -v provided"
+version: 3
+
+server: https://app.fossa.com
+
+project:
+  id: github.com/AbsaOSS/cobrix
+  name: cobrix
+  policy: custom-cli-policy
+  link: fossa.com
+  url: github.com/AbsaOSS/cobrix
+
+targets:
+  only:
+    - type: scala
+      path: cobrix
+
+paths:
+  only:
+    - cobol-parser
+    - cobol-converters
+    - spark-cobol
+    - target
+  exclude:
+    - ./examples
+    - ./project
+
+telemetry:
+  scope: "off"

.github/workflows/build.yml

@@ -5,28 +5,52 @@ on:
     branches: [ master ]
   pull_request:
     branches: [ master ]
+    paths:
+      - "cobol-parser/**"
+      - "cobol-converters/**"
+      - "spark-cobol/**"
+      - "project/**"
+      - "build.sbt"
+      - ".github/workflows/build.yml"
 
 jobs:
   build-sbt:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        scala: [ 2.11.12, 2.12.17, 2.13.10 ]
-        spark: [ 2.4.8, 3.3.2 ]
+        scala: [ 2.11.12, 2.12.20, 2.13.15 ]
+        spark: [ 2.4.8, 3.4.3, 3.5.3 ]
         exclude:
           - scala: 2.11.12
-            spark: 3.3.2
-          - scala: 2.13.10
+            spark: 3.4.3
+          - scala: 2.11.12
+            spark: 3.5.3
+          - scala: 2.12.20
+            spark: 2.4.8
+          - scala: 2.13.15
             spark: 2.4.8
     name: Spark ${{matrix.spark}} on Scala ${{matrix.scala}}
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - uses: coursier/cache-action@v5
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@v10
+      - name: Setup JDK
+        uses: actions/setup-java@v4
         with:
-          java-version: "[email protected]"
+          distribution: temurin
+          java-version: 8
+          cache: sbt
+      - name: Install sbt
+        run: | 
+          sudo apt-get update
+          sudo apt-get install apt-transport-https curl gnupg -yqq
+          echo "deb https://repo.scala-sbt.org/scalasbt/debian all main" | sudo tee /etc/apt/sources.list.d/sbt.list
+          echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | sudo tee /etc/apt/sources.list.d/sbt_old.list
+          curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x2EE0EA64E40A89B84B2DF73499E82A75642AC823" | sudo -H gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --import
+          sudo chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
+          sudo apt-get update
+          sudo apt-get install sbt
+
       - name: Build and run tests
         run: sbt ++${{matrix.scala}} test -DSPARK_VERSION=${{matrix.spark}}

.github/workflows/fossa.yml

@@ -11,14 +11,14 @@ on:
       - '**.svg'
 
 jobs:
-  build:
+  fossa:
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Run FOSSA scan and upload build data
-        uses: fossa-contrib/fossa-action@v1
+        uses: fossa-contrib/fossa-action@v2
         with:
-          fossa-api-key: ab616dfa5101a6039259372543d60ccb
+          fossa-api-key: ${{secrets.FOSSA_API_KEY}}

.github/workflows/jacoco_check.yml

@@ -19,57 +19,84 @@ name: JaCoCo report
 on:
   pull_request:
     branches: [ master ]
-    types: [ opened, edited, synchronize, reopened ]
+    paths:
+      - "cobol-parser/**"
+      - "cobol-converters/**"
+      - "spark-cobol/**"
+      - "project/**"
+      - "build.sbt"
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
+        # The project supports Scala 2.11, 2.12, 2.13
+        # The CI runs all tests suites for all supported Scala versions at build.yml
+        # The codebase for all Scala versions is the same, so the coverage is calculated only once
+        # Scala 2.12 is chosen since it is supported by the most wide range of Spark versions and
+        # vendor distributions.
         include:
-          - scala: 2.11.12
-            scalaShort: "2.11"
-            spark: 2.4.8
-            overall: 0.0
-            changed: 80.0
-          - scala: 2.12.17
+          - scala: 2.12.20
             scalaShort: "2.12"
-            spark: 3.2.3
-            overall: 0.0
-            changed: 80.0
-          - scala: 2.13.10
-            scalaShort: "2.13"
-            spark: 3.3.2
+            spark: 3.3.4
             overall: 0.0
             changed: 80.0
+
+            # Historically, 'cobol-parser' module was part of 'spark-cobol' and the coverage is combined.
+            # Jacoco can't combine coverage across modules, so until 'cobol-parser' has unit tests
+            # that covers the same codepade from the same module, the coverage requirement is lowered to 20%.
+            changedCobolParserOverride: 20
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@v10
+        uses: actions/checkout@v4
+      - name: Setup JDK
+        uses: actions/setup-java@v4
         with:
-          java-version: "[email protected]"
+          distribution: temurin
+          java-version: 8
+          cache: sbt
       - name: Build and run tests
         run: sbt ++${{matrix.scala}} jacoco -DSPARK_VERSION=${{matrix.spark}}
-      - name: Add coverage to PR
+      - name: Add coverage of 'cobol-parser' to PR
+        id: jacocoParser
+        uses: madrapps/[email protected]
+        with:
+          paths: >
+            ${{ github.workspace }}/cobol-parser/target/scala-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml
+          token: ${{ secrets.GITHUB_TOKEN }}
+          min-coverage-overall: ${{ matrix.overall }}
+          min-coverage-changed-files: ${{ matrix.changedCobolParserOverride }}
+          title: JaCoCo code coverage report - 'cobol-parser'
+          update-comment: true
+      - name: Get the Coverage info
+        run: |
+          echo "Total coverage ${{ steps.jacoco.outputs.coverage-overall }}"
+          echo "Changed Files coverage ${{ steps.jacoco.outputs.coverage-changed-files }}"
+      - name: Add coverage of 'spark-cobol' to PR
         id: jacoco
         uses: madrapps/[email protected]
         with:
           paths: >
-            ${{ github.workspace }}/cobol-parser/target/scala-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml,
             ${{ github.workspace }}/spark-cobol/target/scala-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml
           token: ${{ secrets.GITHUB_TOKEN }}
           min-coverage-overall: ${{ matrix.overall }}
           min-coverage-changed-files: ${{ matrix.changed }}
-          title: JaCoCo code coverage report - scala ${{ matrix.scala }} - spark ${{ matrix.spark }}
+          title: JaCoCo code coverage report - 'spark-cobol'
           update-comment: true
       - name: Get the Coverage info
         run: |
           echo "Total coverage ${{ steps.jacoco.outputs.coverage-overall }}"
           echo "Changed Files coverage ${{ steps.jacoco.outputs.coverage-changed-files }}"
-      - name: Fail PR if changed files coverage is less than ${{ matrix.changed }}%
-        if: ${{ steps.jacoco.outputs.coverage-changed-files < 80.0 }}
+      - name: Fail PR if changed files coverage for 'cobol-parser' is less than ${{ matrix.changedCobolParserOverride }}%
+        if: ${{ steps.jacocoParser.outputs.coverage-changed-files < matrix.changedCobolParserOverride }}
+        uses: actions/github-script@v6
+        with:
+          script: |
+            core.setFailed('Changed files coverage is less than ${{ matrix.changedCobolParserOverride }}% for 'cobol-parser'!')
+      - name: Fail PR if changed files coverage for 'spark-cobol' is less than ${{ matrix.changed }}%
+        if: ${{ steps.jacoco.outputs.coverage-changed-files < matrix.changed }}
         uses: actions/github-script@v6
         with:
           script: |
-            core.setFailed('Changed files coverage is less than ${{ matrix.changed }}%!')
+            core.setFailed('Changed files coverage is less than ${{ matrix.changed }}% for 'spark-cobol!')

.github/workflows/release.yaml

@@ -0,0 +1,107 @@
+#
+# Copyright 2022 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Release
+
+on:
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  release-sbt:
+    runs-on: ubuntu-latest
+    name: Release Scala artifacts
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref || github.ref_name }}
+
+      - name: Prepare the release branch
+        run: |
+          VERSION=$(grep "ThisBuild / version" version.sbt | cut -d\" -f2 | sed 's/-SNAPSHOT//')
+          git config --global user.email "[email protected]"
+          git config --global user.name "CI/CD bot"
+          git checkout -b release/$VERSION
+          git push --set-upstream origin release/$VERSION
+
+      - name: Setup JDK and sbt
+        uses: actions/[email protected]
+        with:
+          distribution: temurin
+          java-version: 8
+          cache: sbt
+
+      - name: Install sbt
+        run: |
+          sudo apt-get update
+          sudo apt-get install apt-transport-https curl gnupg -yqq
+          echo "deb https://repo.scala-sbt.org/scalasbt/debian all main" | sudo tee /etc/apt/sources.list.d/sbt.list
+          echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | sudo tee /etc/apt/sources.list.d/sbt_old.list
+          curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x2EE0EA64E40A89B84B2DF73499E82A75642AC823" | sudo -H gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --import
+          sudo chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
+          sudo apt-get update
+          sudo apt-get install sbt
+
+      - name: Import GPG Key
+        run: |
+          echo "${{ secrets.ABSA_OSS_CI_CD_BOT_GPG_KEY }}" > gpg-secret-key.asc
+          gpg --import --batch gpg-secret-key.asc && rm -rf gpg-secret-key.asc
+          mkdir -p ~/.gnupg
+
+      - name: Setup SonaType config
+        run: |
+          mkdir -p ~/.sbt/1.0
+          echo "${{ secrets.SONATYPE_CONFIG }}" | base64 --decode > ~/.sbt/1.0/sonatype.sbt
+
+      - name: Checkout the release branch
+        run: |
+          VERSION=$(grep "ThisBuild / version" version.sbt | cut -d\" -f2 | sed 's/-SNAPSHOT//')
+          git fetch origin release/$VERSION 
+          git checkout release/$VERSION
+
+      - name: Run the release plugin
+        run: sbt releaseNow
+
+  create-pr:
+    needs: [ "release-sbt" ]
+    runs-on: ubuntu-latest
+    name: Create Pull Request
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref || github.ref_name }}
+
+      - name: Checkout the release branch
+        id: release_branch3
+        run: |
+          VERSION=$(grep "ThisBuild / version" version.sbt | cut -d\" -f2 | sed 's/-SNAPSHOT//')
+          echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+          git config --global user.email "[email protected]"
+          git config --global user.name "CI/CD bot"
+          git fetch origin release/$VERSION
+          git checkout release/$VERSION
+
+      - name: Create Pull Request
+        run: gh pr create -B master -H "release/$VERSION" --title "Release Cobrix v$VERSION" --body 'Created by Github action'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          VERSION: ${{ steps.release_branch3.outputs.VERSION }}