ARROW-13051: [Release][Java] Use artifacts built by Crossbow

Closes apache#11696 from kou/release-java-nexus

Authored-by: Sutou Kouhei <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>

Author: kou

Diff for: dev/release/05-binary-upload.sh

@@ -68,7 +68,6 @@ fi
 : ${UPLOAD_AMAZON_LINUX:=${UPLOAD_DEFAULT}}
 : ${UPLOAD_CENTOS:=${UPLOAD_DEFAULT}}
 : ${UPLOAD_DEBIAN:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_JAVA:=${UPLOAD_DEFAULT}}
 : ${UPLOAD_NUGET:=${UPLOAD_DEFAULT}}
 : ${UPLOAD_PYTHON:=${UPLOAD_DEFAULT}}
 : ${UPLOAD_UBUNTU:=${UPLOAD_DEFAULT}}
@@ -92,9 +91,6 @@ if [ ${UPLOAD_DEBIAN} -gt 0 ]; then
   rake_tasks+=(apt:rc)
   apt_targets+=(debian)
 fi
-if [ ${UPLOAD_JAVA} -gt 0 ]; then
-  rake_tasks+=(java:rc)
-fi
 if [ ${UPLOAD_NUGET} -gt 0 ]; then
   rake_tasks+=(nuget:rc)
 fi

Diff for: dev/release/06-java-upload.sh

@@ -0,0 +1,140 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+set -u
+set -o pipefail
+
+SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+if [ "$#" -ne 2 ]; then
+  echo "Usage: $0 <version> <rc-num>"
+  exit
+fi
+
+version=$1
+rc=$2
+
+pushd "${SOURCE_DIR}"
+if [ ! -f .env ]; then
+  echo "You must create $(pwd)/.env"
+  echo "You can use $(pwd)/.env.example as template"
+  exit 1
+fi
+. .env
+popd
+
+version_with_rc="${version}-rc${rc}"
+crossbow_job_prefix="release-${version_with_rc}"
+crossbow_package_dir="${SOURCE_DIR}/../../packages"
+
+: ${CROSSBOW_JOB_NUMBER:="0"}
+: ${CROSSBOW_JOB_ID:="${crossbow_job_prefix}-${CROSSBOW_JOB_NUMBER}"}
+artifact_dir="${crossbow_package_dir}/${CROSSBOW_JOB_ID}"
+
+if [ ! -e "${artifact_dir}" ]; then
+  echo "${artifact_dir} does not exist"
+  exit 1
+fi
+
+if [ ! -d "${artifact_dir}" ]; then
+  echo "${artifact_dir} is not a directory"
+  exit 1
+fi
+
+cd "${artifact_dir}/java-jars"
+
+files=
+types=
+classifiers=
+
+sign() {
+  local path="$1"
+  local classifier="$2"
+  local type=$(echo "${path}" | grep -o "[^.]*$")
+
+  local asc_path="${path}.asc"
+  rm -f "${asc_path}"
+  gpg \
+    --detach-sig \
+    --local-user "${GPG_KEY_ID}" \
+    --output "${asc_path}" \
+    "${path}"
+  if [ -n "${files}" ]; then
+    files="${files},"
+    types="${types},"
+    classifiers="${classifiers},"
+  fi
+  files="${files}${asc_path}"
+  types="${types}${type}.asc"
+  classifiers="${classifiers}${classifier}"
+
+  # .md5 and .sha1 are generated automatically on repository side.
+  # local sha512_path="${path}.sha512"
+  # shasum --algorithm 512 "${path}" > "${sha512_path}"
+  # files="${files},${sha512_path}"
+  # types="${types},${type}.sha512"
+  # classifiers="${classifiers},${classifier}"
+}
+
+for pom in *.pom; do
+  base=$(basename ${pom} .pom)
+  args=()
+  args+=(deploy:deploy-file)
+  args+=(-Durl=https://repository.apache.org/service/local/staging/deploy/maven2)
+  args+=(-DrepositoryId=apache.releases.https)
+  pom="${PWD}/${pom}"
+  args+=(-DpomFile="${pom}")
+  if [ -f "${base}.jar" ]; then
+    jar="${PWD}/${base}.jar"
+    args+=(-Dfile="${jar}")
+    sign "${jar}" ""
+  else
+    args+=(-Dfile="${pom}")
+  fi
+  sign "${pom}" ""
+  if [ "$(echo ${base}-*)" != "${base}-*" ]; then
+    for other_file in ${base}-*; do
+      file="${PWD}/${other_file}"
+      type=$(echo "${other_file}" | grep -o "[^.]*$")
+      case "${type}" in
+        asc|sha256|sha512)
+          continue
+          ;;
+      esac
+      classifier=$(basename "${other_file}" ".${type}" | sed -e "s/${base}-//g")
+      files="${files},${file}"
+      types="${types},${type}"
+      classifiers="${classifiers},${classifier}"
+      sign "${file}" "${classifier}"
+    done
+  fi
+  args+=(-Dfiles="${files}")
+  args+=(-Dtypes="${types}")
+  args+=(-Dclassifiers="${classifiers}")
+  pushd "${SOURCE_DIR}"
+  mvn deploy:deploy-file "${args[@]}"
+  popd
+done
+
+echo "Success!"
+echo "Press the 'Close' button manually by Web interface:"
+echo "    https://repository.apache.org/#stagingRepositories"
+echo "It publishes the artifacts to the staging repository:"
+echo "    https://repository.apache.org/content/repositories/staging/org/apache/arrow/"

Diff for: dev/release/binary-task.rb

@@ -739,7 +739,6 @@ def upload
   def define
     define_apt_tasks
     define_yum_tasks
-    define_java_tasks
     define_nuget_tasks
     define_python_tasks
     define_summary_tasks
@@ -1857,14 +1856,6 @@ def define_generic_data_tasks(label,
     define_generic_data_release_tasks(label, id, release_dir)
   end
 
-  def define_java_tasks
-    define_generic_data_tasks("Java",
-                              :java,
-                              "#{rc_dir}/java/#{full_version}",
-                              "#{release_dir}/java/#{full_version}",
-                              "java-jars/**/*")
-  end
-
   def define_nuget_tasks
     define_generic_data_tasks("NuGet",
                               :nuget,
@@ -1893,7 +1884,6 @@ def define_summary_tasks
   https://apache.jfrog.io/artifactory/arrow/amazon-linux#{suffix}-rc/
   https://apache.jfrog.io/artifactory/arrow/centos#{suffix}-rc/
   https://apache.jfrog.io/artifactory/arrow/debian#{suffix}-rc/
-  https://apache.jfrog.io/artifactory/arrow/java#{suffix}-rc/#{version}
   https://apache.jfrog.io/artifactory/arrow/nuget#{suffix}-rc/#{full_version}
   https://apache.jfrog.io/artifactory/arrow/python#{suffix}-rc/#{full_version}
   https://apache.jfrog.io/artifactory/arrow/ubuntu#{suffix}-rc/
@@ -1910,7 +1900,6 @@ def define_summary_tasks
   https://apache.jfrog.io/artifactory/arrow/amazon-linux#{suffix}/
   https://apache.jfrog.io/artifactory/arrow/centos#{suffix}/
   https://apache.jfrog.io/artifactory/arrow/debian#{suffix}/
-  https://apache.jfrog.io/artifactory/arrow/java#{suffix}/#{version}
   https://apache.jfrog.io/artifactory/arrow/nuget#{suffix}/#{version}
   https://apache.jfrog.io/artifactory/arrow/python#{suffix}/#{version}
   https://apache.jfrog.io/artifactory/arrow/ubuntu#{suffix}/

Diff for: dev/release/download_rc_binaries.py

@@ -25,27 +25,31 @@
 import urllib.request
 
 
-ARTIFACTORY_ROOT = "https://apache.jfrog.io/artifactory/arrow"
 DEFAULT_PARALLEL_DOWNLOADS = 8
 
 
-class Artifactory:
+class Downloader:
 
-    def get_file_list(self, prefix):
+    def get_file_list(self, prefix, filter=None):
         def traverse(directory, files, directories):
-            url = f'{ARTIFACTORY_ROOT}/{directory}'
+            url = f'{self.URL_ROOT}/{directory}'
             response = urllib.request.urlopen(url).read().decode()
             paths = re.findall('<a href="(.+?)"', response)
             for path in paths:
+                path = re.sub(f'^{re.escape(url)}',
+                              '',
+                              path)
                 if path == '../':
                     continue
                 resolved_path = f'{directory}{path}'
+                if filter and not filter(path):
+                    continue
                 if path.endswith('/'):
                     directories.append(resolved_path)
                 else:
                     files.append(resolved_path)
         files = []
-        if not prefix.endswith('/'):
+        if prefix != '' and not prefix.endswith('/'):
             prefix += '/'
         directories = [prefix]
         while len(directories) > 0:
@@ -98,7 +102,7 @@ def _download_file(self, dest, path):
 
         print("Downloading {} to {}".format(path, dest_path))
 
-        url = f'{ARTIFACTORY_ROOT}/{path}'
+        url = f'{self.URL_ROOT}/{path}'
 
         cmd = [
             'curl', '--fail', '--location', '--retry', '5',
@@ -112,6 +116,15 @@ def _download_file(self, dest, path):
                             .format(path, stdout, stderr))
 
 
+class Artifactory(Downloader):
+    URL_ROOT = "https://apache.jfrog.io/artifactory/arrow"
+
+
+class Maven(Downloader):
+    URL_ROOT = "https://repository.apache.org" + \
+        "/content/repositories/staging/org/apache/arrow"
+
+
 def parallel_map_terminate_early(f, iterable, num_parallel):
     tasks = []
     with cf.ProcessPoolExecutor(num_parallel) as pool:
@@ -133,38 +146,41 @@ def parallel_map_terminate_early(f, iterable, num_parallel):
     'debian',
     'ubuntu',
 ]
-ARROW_STANDALONE_PACKAGE_TYPES = ['java', 'nuget', 'python']
+ARROW_STANDALONE_PACKAGE_TYPES = ['nuget', 'python']
 ARROW_PACKAGE_TYPES = \
     ARROW_REPOSITORY_PACKAGE_TYPES + \
     ARROW_STANDALONE_PACKAGE_TYPES
 
 
 def download_rc_binaries(version, rc_number, re_match=None, dest=None,
                          num_parallel=None, target_package_type=None):
-    artifactory = Artifactory()
-
     version_string = '{}-rc{}'.format(version, rc_number)
+    version_pattern = re.compile(r'\d+\.\d+\.\d+')
     if target_package_type:
         package_types = [target_package_type]
     else:
         package_types = ARROW_PACKAGE_TYPES
     for package_type in package_types:
-        if package_type in ARROW_REPOSITORY_PACKAGE_TYPES:
+        def is_target(path):
+            match = version_pattern.search(path)
+            if not match:
+                return True
+            return match[0] == version
+        filter = is_target
+
+        if package_type == 'jars':
+            downloader = Maven()
+            prefix = ''
+        elif package_type in ARROW_REPOSITORY_PACKAGE_TYPES:
+            downloader = Artifactory()
             prefix = f'{package_type}-rc'
         else:
+            downloader = Artifactory()
             prefix = f'{package_type}-rc/{version_string}'
-        files = artifactory.get_file_list(prefix)
-        if package_type in ARROW_REPOSITORY_PACKAGE_TYPES:
-            version_pattern = re.compile(r'\d+\.\d+\.\d+')
-
-            def is_old_release(path):
-                match = version_pattern.search(path)
-                if not match:
-                    return False
-                return match[0] != version
-            files = [x for x in files if not is_old_release(x)]
-        artifactory.download_files(files, re_match=re_match, dest=dest,
-                                   num_parallel=num_parallel)
+            filter = None
+        files = downloader.get_file_list(prefix, filter=filter)
+        downloader.download_files(files, re_match=re_match, dest=dest,
+                                  num_parallel=num_parallel)
 
 
 if __name__ == '__main__':

Diff for: dev/release/post-03-binary.sh

@@ -49,7 +49,6 @@ fi
 : ${DEPLOY_AMAZON_LINUX:=${DEPLOY_DEFAULT}}
 : ${DEPLOY_CENTOS:=${DEPLOY_DEFAULT}}
 : ${DEPLOY_DEBIAN:=${DEPLOY_DEFAULT}}
-: ${DEPLOY_JAVA:=${DEPLOY_DEFAULT}}
 : ${DEPLOY_NUGET:=${DEPLOY_DEFAULT}}
 : ${DEPLOY_PYTHON:=${DEPLOY_DEFAULT}}
 : ${DEPLOY_UBUNTU:=${DEPLOY_DEFAULT}}
@@ -73,9 +72,6 @@ if [ ${DEPLOY_DEBIAN} -gt 0 ]; then
   rake_tasks+=(apt:release)
   apt_targets+=(debian)
 fi
-if [ ${DEPLOY_JAVA} -gt 0 ]; then
-  rake_tasks+=(java:release)
-fi
 if [ ${DEPLOY_NUGET} -gt 0 ]; then
   rake_tasks+=(nuget:release)
 fi