diff --git a/scripts/setup_package.sh b/scripts/setup_package.sh index 2ac832ed..72256b4d 100755 --- a/scripts/setup_package.sh +++ b/scripts/setup_package.sh @@ -24,6 +24,7 @@ PROGNAME=`basename "$0"` SLE_VERSION_REGEX="[0-9]{6}" VERSION_REGEX="([0-9\.a-zA-Z]+-$SLE_VERSION_REGEX\.[0-9\.]+[0-9])" PLATFORM= +PRODUCT= URL= PACKAGE= NO_CLEANUP=0 @@ -37,10 +38,29 @@ NO_IPA_CLONES_DOWNLOAD=0 # If this flag is enabled, then download of debuginfo packages will be blocked. NO_DEBUGINFO_DOWNLOAD=0 +# Pushd and popd are not silent. Silence them. +pushd () +{ + command pushd "$@" > /dev/null +} +popd () +{ + command popd "$@" > /dev/null +} + set_url_platform() { PLATFORM=$1 - URL="https://download.suse.de/updates/SUSE/Updates/SLE-Module-Basesystem/$PLATFORM/x86_64/update/x86_64" + PRODUCT=$2 + local element=$3 + + URL="https://download.suse.de/download/ibs/SUSE:/SLE-$PLATFORM:/$PRODUCT/standard" + + if [ "$element" == "src" ]; then + URL="$URL/src" + elif [ "$element" != "ipa-clones" ]; then + URL="$URL/x86_64" + fi } web_get() @@ -89,6 +109,7 @@ get_sle_version_from_package_name() local version=$(echo "$1" | grep -Eo "($SLE_VERSION_REGEX)") + local sle_version=${sle_hash[$version]} if [ "x$sle_version" = "x" ]; then @@ -120,6 +141,7 @@ download_package_list() local url="$URL" local list_path=$1 + echo downloading package list: "$url" web_get "$url" "$list_path" } @@ -211,11 +233,6 @@ get_list_of_debuginfo_packages() local package_name=$(get_name_from_package_name $package) local version=$(get_version_from_package_name $package) - # libopenssl1_1 src comes from openssl. - if [ "$package_name" = "libopenssl1_1" ]; then - package_name="openssl-1_1" - fi - src_package_list="$src_package_list $package_name-debuginfo-$version.x86_64.rpm" done @@ -229,10 +246,8 @@ download_debuginfo_packages() echo $packages - URL="https://download.suse.de/updates/SUSE/Updates/SLE-Module-Basesystem/$PLATFORM/x86_64/update_debug/x86_64/" + set_url_platform $PLATFORM $PRODUCT "debuginfo" parallel_download_packages "$packages" - - URL=$old_url } download_src_packages() @@ -240,10 +255,8 @@ download_src_packages() local packages=$(get_list_of_src_packages "$*") local old_url=$URL - URL="https://download.suse.de/updates/SUSE/Updates/SLE-Module-Basesystem/$PLATFORM/x86_64/update/src/" + set_url_platform $PLATFORM $PRODUCT "src" parallel_download_packages "$packages" - - URL=$old_url } download_ipa_clones() @@ -254,10 +267,8 @@ download_ipa_clones() local sle_ver=$(get_sle_version_from_package_name $1) - URL="https://download.suse.de/download/ibs/SUSE:/SLE-$sle_ver:/Update/standard/" + set_url_platform $PLATFORM $PRODUCT "ipa-clones" parallel_download_packages "$ipa_clones_list" - - URL=$old_url } extract_libs_from_package() @@ -272,22 +283,47 @@ extract_libs_from_package() mkdir -p $PLATFORM/$name/$version cp $package $PLATFORM/$name/$version/$package - cp $src_package $PLATFORM/$name/$version/$src_package - cp $ipa_clones $PLATFORM/$name/$version/$ipa_clones - cp $debuginfo_package $PLATFORM/$name/$version/$debuginfo_package + if [ $? -ne 0 ]; then + echo "error: $package not downloaded." + exit 1 + fi + + if [ $NO_SRC_DOWNLOAD -eq 0 ]; then + cp $src_package $PLATFORM/$name/$version/$src_package + if [ $? -ne 0 ]; then + echo "error: $src_package not downloaded." + exit 1 + fi + fi + + if [ $NO_IPA_CLONES_DOWNLOAD -eq 0 ]; then + cp $ipa_clones $PLATFORM/$name/$version/$ipa_clones + if [ $? -ne 0 ]; then + echo "error: $ipa_clones not downloaded." + exit 1 + fi + fi + + if [ $NO_DEBUGINFO_DOWNLOAD -eq 0 ]; then + cp $debuginfo_package $PLATFORM/$name/$version/$debuginfo_package + if [ $? -ne 0 ]; then + echo "error: $debuginfo not downloaded." + exit 1 + fi + fi cd $PLATFORM/$name/$version mkdir -p binaries cd binaries if [ -f ../$package ]; then - rpm2cpio ../$package | cpio -idmv --quiet + rpm2cpio ../$package | cpio -idm --quiet fi cd .. mkdir -p src cd src if [ -f ../$src_package ]; then - rpm2cpio ../$src_package | cpio -idmv --quiet + rpm2cpio ../$src_package | cpio -idm --quiet tar xf $(ls | grep -E "(\.tar\.xz$|\.tar\.gz$)") fi cd .. @@ -296,7 +332,7 @@ extract_libs_from_package() cd debuginfo if [ -f ../$debuginfo_package ]; then echo "Extracting $debuginfo_package" - rpm2cpio ../$debuginfo_package | cpio -idmv --quiet + rpm2cpio ../$debuginfo_package | cpio -idm --quiet fi cd .. @@ -314,26 +350,98 @@ extract_libs_from_package() cd ../../../ } +# List of .debug files in folder. Stored here for cache reasons. +_LIST_OF_DEBUG="" + +match_so_to_debuginfo() +{ + local so=$1 + local base_so=$(basename $so) + + local list_of_debug=$(echo $_LIST_OF_DEBUG | xargs -n1 | grep -E "$base_so.*\.debug$") + + let num=0 + + # Count how many files we got. + for dbg in $list_of_debug; do + let "num=num+1" + done + + # Assert that we got only one file + if [ $num -ne 1 ]; then + echo "Expected only 1 file matching $base_so, got $num: $list_of_debug" > /dev/stderr + exit 1 + fi + + # Return the file we got. + echo $list_of_debug +} + dump_interesting_info_from_elfs() { - local list_of_sos=$(find $PLATFORM | grep -E "*.so[\.0-9]*$") + pushd $1 + local list_of_sos=$(find . | grep -E ".*\.so[\.0-9]*$") + + # Populate cache of list of .debug + _LIST_OF_DEBUG=$(find . -name "*.debug") - # Extract the relevant so information needed for livepatching. + # Iterate on every so in the folder. for so in $list_of_sos; do - ulp extract $so -o $so.json - done + if [ $NO_DEBUGINFO_DOWNLOAD -eq 0 ]; then + # Get the debuginfo that matches this library. + local debug=$(match_so_to_debuginfo $so) - echo $list_of_sos + if [ "$debug" == "" ]; then + exit 1 + fi + + # Run the ulp extract command on both the library and debuginfo. + echo ulp extract $so -d $debug -o $so.json + ulp extract $so -d $debug -o $so.json + else + # Run the ulp extract command only on the library. + echo ulp extract $so -o $so.json + ulp extract $so -o $so.json + fi + done # Delete all .so we don't need. for so in $list_of_sos; do rm -f $so done + + # Delete all .debug we we don't need. + for debug in $_LIST_OF_DEBUG; do + rm -f $debug + done + + # Delete empty directories left. + find . -type d -empty -delete + + # Invalidate the cache. + _LIST_OF_DEBUG="" + + popd +} + +dump_interesting_info_from_elfs_in_lib() +{ + local platform=$1 + + # Enter in platform folder + pushd $platform + + # Iterate on every version + for dir in $(ls); do + dump_interesting_info_from_elfs $dir + done + popd + } sanitize_platform() { - local platforms="15-SP3 15-SP4" + local platforms="15-SP3 15-SP4 15-SP5" for platform in ${platforms}; do if [ "$PLATFORM" = "$platform" ]; then @@ -440,36 +548,49 @@ parse_program_argv() sanitize_package # Set platform globally - set_url_platform "$PLATFORM" + set_url_platform "$PLATFORM" "GA" } main() { - # Set default URL platform to "15-SP4". - set_url_platform "15-SP4" parse_program_argv $* + # Clean the directory + rm -rf $PLATFORM - download_package_list "/tmp/suse_package_list.html" - local names=$(extract_lib_package_names "/tmp/suse_package_list.html" $PACKAGE) + local all_names="" + local products="GA Update" - parallel_download_packages "$names" + for product in GA Update; do + # Set platform globally + set_url_platform "$PLATFORM" $product - if [ $NO_SRC_DOWNLOAD -eq 0 ]; then - download_src_packages "$names" - fi - if [ $NO_IPA_CLONES_DOWNLOAD -eq 0 ]; then - download_ipa_clones "$names" - fi - if [ $NO_DEBUGINFO_DOWNLOAD -eq 0 ]; then - download_debuginfo_packages "$names" - fi + download_package_list "/tmp/suse_package_list.html" + local names=$(extract_lib_package_names "/tmp/suse_package_list.html" $PACKAGE) + + # Clean the directory + rm -rf $PLATFORM + + parallel_download_packages "$names" + + if [ $NO_SRC_DOWNLOAD -eq 0 ]; then + download_src_packages "$names" + fi + if [ $NO_IPA_CLONES_DOWNLOAD -eq 0 ]; then + download_ipa_clones "$names" + fi + if [ $NO_DEBUGINFO_DOWNLOAD -eq 0 ]; then + download_debuginfo_packages "$names" + fi + + all_names="$all_names $names" + done - for package in $names; do + for package in $all_names; do extract_libs_from_package "$package" done - dump_interesting_info_from_elfs + dump_interesting_info_from_elfs_in_lib $PLATFORM/$PACKAGE # Delete all packages to cleanup. if [ $NO_CLEANUP -ne 1 ]; then diff --git a/tools/arguments.h b/tools/arguments.h index b94ebccc..58cf5f13 100644 --- a/tools/arguments.h +++ b/tools/arguments.h @@ -50,6 +50,7 @@ struct arguments const char *process_wildcard; const char *user_wildcard; const char *prefix; + const char *with_debuginfo; command_t command; int retries; int quiet; diff --git a/tools/extract.c b/tools/extract.c index c97b6950..b33d545f 100644 --- a/tools/extract.c +++ b/tools/extract.c @@ -252,6 +252,10 @@ write_ulp_so_info_json(FILE *stream, const struct ulp_so_info *info) static struct symbol * get_list_of_symbols_in_section(Elf *elf, Elf_Scn *s) { + if (elf == NULL || s == NULL) { + return NULL; + } + struct symbol *symbol_head = NULL; int nsyms, i; @@ -638,12 +642,57 @@ get_symbol_with_name(struct ulp_so_info *info, const char *sym) return symbol; } +struct ulp_so_info *merge_ulp_so_infos(struct ulp_so_info *a, + struct ulp_so_info *b) +{ + /* Sanity check. */ + if (memcmp(a->buildid, b->buildid, sizeof(a->buildid)) != 0) { + char a_buildid[2 * BUILDID_LEN + 1]; + char b_buildid[2 * BUILDID_LEN + 1]; + memcpy(a_buildid, buildid_to_string(a->buildid), sizeof(a_buildid)); + memcpy(b_buildid, buildid_to_string(b->buildid), sizeof(b_buildid)); + + WARN("Attempt to merge symbol lists with distinct buildids\n" + " lib: %s\n" + " debuginfo: %s", a_buildid, b_buildid); + } + + /* Merge both lists. */ + struct symbol **it = &(a->symbols); + for (; *it != NULL; it = &(*it)->next) + ; + + /* Duplicate the list for the sake of avoiding silly memory errors. */ + struct symbol *it2 = b->symbols; + for (; it2 != NULL; it2 = it2->next) { + struct symbol *new = calloc(1, sizeof(struct symbol)); + assert(new && "Error allocating a new symbol element."); + + new->name = strdup(it2->name); + new->offset = it2->offset; + new->size = it2->size; + new->st_info = it2->st_info; + new->st_other = it2->st_other; + new->next = NULL; + + /* Insert into the list. */ + *it = new; + it = &(new->next); + } + + return a; +} + int run_extract(struct arguments *arguments) { /* Path to input livepatch library target. */ const char *input_file = arguments->args[0]; + /* Path to the debuginfo file, which can be used to extract more + information. */ + const char *debuginfo_file = arguments->with_debuginfo; + /* arguments->metadata is what is captured by the -o option. */ const char *output_file = arguments->metadata; @@ -652,6 +701,18 @@ run_extract(struct arguments *arguments) output_file = "out.json"; struct ulp_so_info *info = parse_so_elf(input_file); + struct ulp_so_info *debuginfo = NULL; + + if (info == NULL) { + WARN("error reading elf file %s", input_file); + return 1; + } + + if (debuginfo_file) { + debuginfo = parse_so_elf(debuginfo_file); + info = merge_ulp_so_infos(info, debuginfo); + } + FILE *out; if (!strcmp(output_file, "-")) @@ -666,5 +727,6 @@ run_extract(struct arguments *arguments) fclose(out); release_so_info(info); + release_so_info(debuginfo); return 0; } diff --git a/tools/ulp.c b/tools/ulp.c index 16b7a445..e20a6495 100644 --- a/tools/ulp.c +++ b/tools/ulp.c @@ -171,8 +171,10 @@ static struct argp_option options[] = { "Use this livepatch file\nDefaults to the one described in ARG1", 0 }, { "target", 't', "LIBRARY", 0, "Use this target library\nDefaults to the one described in ARG1", 0 }, - { "color", ULP_OP_COLOR, "yes/no/auto", 0, "Enable/disable colored messages", - 0 }, + { "color", ULP_OP_COLOR, "yes/no/auto", 0, "Enable/disable colored messages", 0 }, + { 0, 0, 0, 0, "extract command only:", 0 }, + { "with-debuginfo", 'd', "DEBUGINFO", 0, + "Use debuginfo information for symbolextraction", 0 }, { 0 } }; @@ -344,6 +346,9 @@ parser(int key, char *arg, struct argp_state *state) case 'R': arguments->prefix = arg; break; + case 'd': + arguments->with_debuginfo = arg; + break; case ULP_OP_REVERT_ALL: arguments->library = get_basename(arg); break;