diff --git a/hadoop-3.4.yaml b/hadoop-3.4.yaml new file mode 100644 index 0000000000..b7929be29e --- /dev/null +++ b/hadoop-3.4.yaml @@ -0,0 +1,116 @@ +package: + name: hadoop-3.4 + version: 3.4.0 + epoch: 0 + description: Open Source Identity and Access Management For Modern Applications and Services + copyright: + - license: Apache-2.0 + dependencies: + runtime: + - bash + - curl + - findutils + - gawk + - hadoop-runner-3.4 + - openjdk-8-default-jvm + +environment: + contents: + packages: + - abseil-cpp-dev + - autoconf + - automake + - bash + - boost + - boost-dev + - build-base + - busybox + - bzip2-dev + - ca-certificates-bundle + - cmake + - cyrus-sasl-dev + - fuse2-dev + - fuse2-static + - gcc + - gnutar + - libcrypto3 + - libtirpc-dev + - libtool + - maven + - openjdk-8 + - openssl-dev + - protobuf-dev~3.21 + - protobuf~3.21 + - protoc~3.21 + - snappy-dev + - snappy-static + - zlib-dev + - zstd-dev + environment: + LANG: en_US.UTF-8 + JAVA_HOME: /usr/lib/jvm/java-1.8-openjdk + +pipeline: + - uses: git-checkout + with: + repository: https://github.com/apache/hadoop + tag: rel/release-${{package.version}} + expected-commit: bd8b77f398f626bb7791783192ee7a5dfaeec760 + + # Fix build when using OpenSSL > 1.1.1 + - uses: patch + with: + patches: openssl.patch + + - uses: maven/pombump + with: + patch-file: pombump-deps.yaml + properties-file: pombump-properties.yaml + pom: hadoop-project/pom.xml + + - runs: | + mkdir -p ${{targets.destdir}}/usr/share/java/hadoop + mkdir -p ${{targets.destdir}}/var/log/hadoop + + - if: ${{build.arch}} == "aarch64" + runs: | + mvn -Pdist,native,src -DskipTests -Denforcer.skip=true -Dtar -Dmaven.javadoc.skip=true -Drequire.openssl -Drequire.fuse -Drequire.snappy clean package -e -X + tar -xf hadoop-dist/target/hadoop-${{package.version}}.tar.gz --strip-components=1 -C ${{targets.destdir}}/usr/share/java/hadoop + + - if: ${{build.arch}} == "x86_64" + runs: | + mvn -Pdist,yarn-ui,native,src -DskipTests -Denforcer.skip=true -Dtar -Dmaven.javadoc.skip=true -Drequire.openssl -Drequire.fuse -Drequire.snappy clean package -e -X + tar -xf hadoop-dist/target/hadoop-${{package.version}}.tar.gz --strip-components=1 -C ${{targets.destdir}}/usr/share/java/hadoop + +subpackages: + - name: ${{package.name}}-compat + pipeline: + - runs: | + mkdir -p ${{targets.contextdir}}/opt + ln -sf /usr/share/java/hadoop ${{targets.contextdir}}/opt/hadoop + +update: + enabled: true + ignore-regex-patterns: + - -RC + github: + identifier: apache/hadoop + use-tag: true + strip-prefix: rel/release- + tag-filter: rel/release- + +test: + environment: + contents: + packages: + - bash + - openjdk-8-jre + environment: + JAVA_HOME: /usr/lib/jvm/java-1.8-openjdk + pipeline: + - name: version check + runs: | + /usr/share/java/hadoop/bin/hadoop version + - name: help message + runs: | + /usr/share/java/hadoop/bin/hadoop --help diff --git a/hadoop-3.4/openssl.patch b/hadoop-3.4/openssl.patch new file mode 100644 index 0000000000..d5fb4c083f --- /dev/null +++ b/hadoop-3.4/openssl.patch @@ -0,0 +1,143 @@ +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +index 3f141be05b5..cd0b55848b0 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +@@ -42,13 +42,13 @@ + #ifdef UNIX + static void * (*dlsym_CRYPTO_malloc) (int, const char *, int); + static void (*dlsym_CRYPTO_free) (void *); +-#if OPENSSL_VERSION_NUMBER < 0x10100000L ++//#if OPENSSL_VERSION_NUMBER < 0x10100000L + static int (*dlsym_CRYPTO_num_locks) (void); + static void (*dlsym_CRYPTO_set_locking_callback) (void (*)()); + static void (*dlsym_CRYPTO_set_id_callback) (unsigned long (*)()); + static void (*dlsym_ENGINE_load_rdrand) (void); + static void (*dlsym_ENGINE_cleanup) (void); +-#endif ++//#endif + static ENGINE * (*dlsym_ENGINE_by_id) (const char *); + static int (*dlsym_ENGINE_init) (ENGINE *); + static int (*dlsym_ENGINE_set_default) (ENGINE *, unsigned int); +diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/content_summary.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/content_summary.h +index 6762a65ca90..d9809bab592 100644 +--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/content_summary.h ++++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/content_summary.h +@@ -19,6 +19,7 @@ + #define HDFSPP_CONTENT_SUMMARY_H_ + + #include ++#include + + namespace hdfs { + +diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/fsinfo.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/fsinfo.h +index 103cf133bec..156ed445181 100644 +--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/fsinfo.h ++++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/fsinfo.h +@@ -19,6 +19,7 @@ + #define HDFSPP_FSINFO_H_ + + #include ++#include + + namespace hdfs { + +diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h +index 1da1f18a592..a73556c4588 100644 +--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h ++++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h +@@ -19,6 +19,8 @@ + #define HDFSPP_STATINFO_H_ + + #include ++#include ++ + + namespace hdfs { + +diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h +index bc3d8b96d3f..09045d7a5af 100644 +--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h ++++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h +@@ -19,6 +19,7 @@ + #ifndef COMMON_HDFS_URI_H_ + #define COMMON_HDFS_URI_H_ + ++#include + #include + #include + #include +diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/sasl_digest_md5.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/sasl_digest_md5.cc +index cdb25d7e1a0..b4397efae7d 100644 +--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/sasl_digest_md5.cc ++++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/sasl_digest_md5.cc +@@ -20,7 +20,7 @@ + #include "common/util.h" + + #include +-#include ++#include + #include + + #include +@@ -30,7 +30,7 @@ + namespace hdfs { + + static std::string QuoteString(const std::string &src); +-static std::string GetMD5Digest(const std::string &src); ++static std::string GetSHA256Digest(const std::string &src); + static std::string BinaryToHex(const std::string &src); + + static const char kDigestUri[] = "hdfs/0"; +@@ -146,7 +146,7 @@ Status DigestMD5Authenticator::ParseFirstChallenge(const std::string &payload) { + } + } + +- if (props["algorithm"] != "md5-sess" || props["charset"] != "utf-8" || ++ if (props["algorithm"] != "sha256-sess" || props["charset"] != "utf-8" || + props.find("nonce") == props.end()) { + return Status::Error("Invalid challenge"); + } +@@ -204,15 +204,15 @@ DigestMD5Authenticator::GenerateResponseValue(std::string *response_value) { + } + + begin_a1 << username_ << ":" << realm_ << ":" << password_; +- a1_ss << GetMD5Digest(begin_a1.str()) << ":" << nonce_ << ":" << cnonce_ ++ a1_ss << GetSHA256Digest(begin_a1.str()) << ":" << nonce_ << ":" << cnonce_ + << ":" << username_; + + std::stringstream combine_ss; +- combine_ss << BinaryToHex(GetMD5Digest(a1_ss.str())) << ":" << nonce_ << ":" ++ combine_ss << BinaryToHex(GetSHA256Digest(a1_ss.str())) << ":" << nonce_ << ":" + << std::hex << std::setw(8) << std::setfill('0') << nonce_count_ + << ":" << cnonce_ << ":" << qop_ << ":" +- << BinaryToHex(GetMD5Digest(a2)); +- *response_value = BinaryToHex(GetMD5Digest(combine_ss.str())); ++ << BinaryToHex(GetSHA256Digest(a2)); ++ *response_value = BinaryToHex(GetSHA256Digest(combine_ss.str())); + return Status::OK(); + } + +@@ -230,13 +230,10 @@ static std::string QuoteString(const std::string &src) { + return dst; + } + +-static std::string GetMD5Digest(const std::string &src) { +- MD5_CTX ctx; +- unsigned long long res[2]; +- MD5_Init(&ctx); +- MD5_Update(&ctx, src.c_str(), src.size()); +- MD5_Final(reinterpret_cast(res), &ctx); +- return std::string(reinterpret_cast(res), sizeof(res)); ++static std::string GetSHA256Digest(const std::string &src) { ++ unsigned char hash[SHA256_DIGEST_LENGTH]; ++ SHA256(reinterpret_cast(src.c_str()), src.size(), hash); ++ return std::string(reinterpret_cast(hash), SHA256_DIGEST_LENGTH); + } + + static std::string BinaryToHex(const std::string &src) { +@@ -249,3 +246,4 @@ static std::string BinaryToHex(const std::string &src) { + return ss.str(); + } + } ++ diff --git a/hadoop-3.4/pombump-properties.yaml b/hadoop-3.4/pombump-properties.yaml new file mode 100644 index 0000000000..383b2e29df --- /dev/null +++ b/hadoop-3.4/pombump-properties.yaml @@ -0,0 +1,13 @@ +properties: + - property: protobuf.version + value: "3.25.5" + - property: zookeeper.version + value: "3.8.4" + - property: guava.version + value: "32.0.0-android" + - property: solr.version + value: "8.11.3" + - property: netty4.version + value: "4.1.108.Final" + - property: nimbus-jose-jwt.version + value: "9.37.2"