s390x: vectorize crc32

Use vector extensions when compiling for s390x and binutils knows
about them. At runtime, check whether kernel supports vector
extensions (it has to be not just the CPU, but also the kernel) and
choose between the regular and the vectorized implementations.

Co-authored-by: Eduard Stefes <[email protected]>

Author: iii-i

CMakeLists.txt

@@ -76,6 +76,10 @@ if(NOT ZLIB_CONF_WRITTEN)
     mark_as_advanced(ZLIB_CONF_WRITTEN)
 endif(NOT ZLIB_CONF_WRITTEN)
 
+#
+# Add contrib code
+#
+add_subdirectory(contrib/s390x)
 #
 # Check to see if we have large file support
 #
@@ -201,6 +205,7 @@ if(ZLIB_BUILD_SHARED)
         UNIX
         AND NOT APPLE
         AND NOT (CMAKE_SYSTEM_NAME STREQUAL AIX))
+    target_link_libraries(zlib PRIVATE $<TARGET_NAME_IF_EXISTS:zlib_crc32_vx>)
 endif(ZLIB_BUILD_SHARED)
 
 if(ZLIB_BUILD_STATIC)
@@ -223,6 +228,7 @@ if(ZLIB_BUILD_STATIC)
     set_target_properties(
         zlibstatic PROPERTIES EXPORT_NAME ZLIBSTATIC OUTPUT_NAME
                                                      z${zlib_static_suffix})
+    target_link_libraries(zlibstatic PRIVATE $<TARGET_NAME_IF_EXISTS:zlib_crc32_vx>)
 endif(ZLIB_BUILD_STATIC)
 
 if(ZLIB_INSTALL)

Makefile.in

@@ -27,6 +27,7 @@ LDFLAGS=
 TEST_LIBS=-L. libz.a
 LDSHARED=$(CC)
 CPP=$(CC) -E
+VGFMAFLAG=
 
 STATICLIB=libz.a
 SHAREDLIB=libz.so
@@ -164,6 +165,9 @@ adler32.o: $(SRCDIR)adler32.c
 crc32.o: $(SRCDIR)crc32.c
 	$(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)crc32.c
 
+crc32_vx.o: $(SRCDIR)contrib/s390x/crc32_vx.c
+	$(CC) $(CFLAGS) $(VGFMAFLAG) $(ZINC) -c -o $@ $(SRCDIR)contrib/s390x/crc32_vx.c
+
 deflate.o: $(SRCDIR)deflate.c
 	$(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)deflate.c
 
@@ -214,6 +218,11 @@ crc32.lo: $(SRCDIR)crc32.c
 	$(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/crc32.o $(SRCDIR)crc32.c
 	-@mv objs/crc32.o $@
 
+crc32_vx.lo: $(SRCDIR)contrib/s390x/crc32_vx.c
+	-@mkdir objs 2>/dev/null || test -d objs
+	$(CC) $(SFLAGS) $(VGFMAFLAG) $(ZINC) -DPIC -c -o objs/crc32_vx.o $(SRCDIR)contrib/s390x/crc32_vx.c
+	-@mv objs/crc32_vx.o $@
+
 deflate.lo: $(SRCDIR)deflate.c
 	-@mkdir objs 2>/dev/null || test -d objs
 	$(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/deflate.o $(SRCDIR)deflate.c
@@ -406,6 +415,7 @@ infback.o inflate.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.
 inffast.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h $(SRCDIR)inflate.h $(SRCDIR)inffast.h
 inftrees.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h
 trees.o: $(SRCDIR)deflate.h $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)trees.h
+crc32_vx.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)contrib/s390x/crc32_vx_hooks.h
 
 adler32.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h
 zutil.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)gzguts.h
@@ -417,3 +427,4 @@ infback.lo inflate.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftree
 inffast.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h $(SRCDIR)inflate.h $(SRCDIR)inffast.h
 inftrees.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h
 trees.lo: $(SRCDIR)deflate.h $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)trees.h
+crc32_vx.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)contrib/s390x/crc32_vx_hooks.h

configure

@@ -93,6 +93,7 @@ debug=0
 address=0
 memory=0
 unknown=0
+enable_crcvx=1
 old_cc="$CC"
 old_cflags="$CFLAGS"
 OBJC='$(OBJZ) $(OBJG)'
@@ -120,6 +121,7 @@ case "$1" in
       echo '  configure [--const] [--zprefix] [--prefix=PREFIX]  [--eprefix=EXPREFIX]' | tee -a configure.log
       echo '    [--static] [--64] [--libdir=LIBDIR] [--sharedlibdir=LIBDIR]' | tee -a configure.log
       echo '    [--includedir=INCLUDEDIR] [--archs="-arch i386 -arch x86_64"]' | tee -a configure.log
+      echo '    [--disable-crcvx]' | tee -a configure.log
         exit 0 ;;
     -p*=* | --prefix=*) prefix=`echo $1 | sed 's/.*=//'`; shift ;;
     -e*=* | --eprefix=*) exec_prefix=`echo $1 | sed 's/.*=//'`; shift ;;
@@ -146,6 +148,7 @@ case "$1" in
     --sanitize) address=1; shift ;;
     --address) address=1; shift ;;
     --memory) memory=1; shift ;;
+    --disable-crcvx) enable_crcvx=0; shift ;;
     *) unknown=1; echo "unknown option ignored: $1" | tee -a configure.log; shift;;
     esac
 done
@@ -870,6 +873,70 @@ EOF
   fi
 fi
 
+# check for ibm s390x build
+HAVE_S390X=0
+cat > $test.c << EOF
+#ifndef __s390x__
+    #error
+#endif
+EOF
+if try $CC -c $CFLAGS $test.c; then 
+    echo "Checking for s390x build ... Yes." | tee -a configure.log
+    HAVE_S390X=1
+else
+    echo "Checking for s390x build ... No." | tee -a configure.log
+fi
+
+# check for ibm s390x vx vector extensions
+HAVE_S390X_VX=0
+if test $HAVE_S390X -eq 1 && test $enable_crcvx -eq 1 ; then
+    # preset the compiler specific flags
+    if test $clang -eq 1; then
+        VGFMAFLAG=-fzvector
+    else
+        VGFMAFLAG=-mzarch
+    fi
+
+    cat > $test.c <<EOF
+#ifndef __s390x__
+#error
+#endif
+#include <vecintrin.h>
+int main(void) {
+    unsigned long long a __attribute__((vector_size(16))) = { 0 };
+    unsigned long long b __attribute__((vector_size(16))) = { 0 };
+    unsigned char c __attribute__((vector_size(16))) = { 0 };
+    c = vec_gfmsum_accum_128(a, b, c);
+    return c[0];
+}
+EOF
+
+    # cflags already contains a valid march
+    if try $CC -c $CFLAGS $VGFMAFLAG $test.c; then
+        echo "Checking for s390x vx vector extension ... Yes." | tee -a configure.log
+        HAVE_S390X_VX=1
+    # or set march for our compile units
+    elif try $CC -c $CFLAGS $VGFMAFLAG -march=z13 $test.c; then
+        echo "Checking for s390x vx vector extension (march=z13) ... Yes." | tee -a configure.log
+        HAVE_S390X_VX=1
+        VGFMAFLAG="$VGFMAFLAG -march=z13"
+    # else we are not on s390x
+    else 
+        echo "Checking for s390x vx vector extension ... No." | tee -a configure.log
+    fi
+
+    # prepare compiling for s390x
+    if test $HAVE_S390X_VX -eq 1; then
+        CFLAGS="$CFLAGS -DHAVE_S390X_VX"
+        SFLAGS="$SFLAGS -DHAVE_S390X_VX"
+        OBJC="$OBJC crc32_vx.o"
+        PIC_OBJC="$PIC_OBJC crc32_vx.lo"
+    else
+        # target has no vx extension
+        VGFMAFLAG=""
+    fi
+fi
+
 # show the results in the log
 echo >> configure.log
 echo ALL = $ALL >> configure.log
@@ -901,6 +968,9 @@ echo mandir = $mandir >> configure.log
 echo prefix = $prefix >> configure.log
 echo sharedlibdir = $sharedlibdir >> configure.log
 echo uname = $uname >> configure.log
+echo HAVE_S390X = $HAVE_S390X >> configure.log
+echo HAVE_S390X_VX = $HAVE_S390X_VX >> configure.log
+echo VGFMAFLAG = $VGFMAFLAG >> configure.log
 
 # update Makefile with the configure results
 sed < ${SRCDIR}Makefile.in "

contrib/README.contrib

@@ -46,6 +46,9 @@ puff/       by Mark Adler <[email protected]>
         Small, low memory usage inflate.  Also serves to provide an
         unambiguous description of the deflate format.
 
+s390x/      by Ilya Leoshkevich <[email protected]>
+        Hardware-accelerated CRC32 on IBM Z with Z13 VX extension.
+
 testzlib/   by Gilles Vollant <[email protected]>
         Example of the use of zlib
 

contrib/hooks.h

@@ -0,0 +1,8 @@
+#ifndef Z_HOOKS_H__
+#define Z_HOOKS_H__
+
+#ifdef HAVE_S390X_VX
+#include "s390x/crc32_vx_hooks.h"
+#endif
+
+#endif

contrib/s390x/CMakeLists.txt

@@ -0,0 +1,61 @@
+option(ZLIB_CRC32VX "Enable building S390-CRC32VX implementation" ON)
+
+set(CMAKE_TRY_COMPILE_TARGET_TYPE "STATIC_LIBRARY")
+
+#
+# check if we compile for IBM s390x
+#
+CHECK_C_SOURCE_COMPILES("
+#ifndef __s390x__
+    #error
+#endif
+" HAS_S390X_SUPPORT)
+
+#
+# Check for IBM S390X - VX extensions
+#
+if(ZLIB_CRC32VX AND HAS_S390X_SUPPORT)
+    # preset the compiler specific flags
+    if (CMAKE_C_COMPILER_ID STREQUAL "Clang")
+        set(VGFMAFLAG "-fzvector")
+    else()
+        set(VGFMAFLAG "-mzarch")
+    endif(CMAKE_C_COMPILER_ID STREQUAL "Clang")
+
+    set(S390X_VX_TEST
+        "#ifndef __s390x__ \n\
+        #error \n\
+        #endif \n\
+        #include <vecintrin.h> \n\
+        int main(void) { \
+           unsigned long long a __attribute__((vector_size(16))) = { 0 }; \
+           unsigned long long b __attribute__((vector_size(16))) = { 0 }; \
+           unsigned char c __attribute__((vector_size(16))) = { 0 }; \
+           c = vec_gfmsum_accum_128(a, b, c); \
+           return c[0]; \
+        }")
+
+    # cflags already contains a valid march
+    set(CMAKE_REQUIRED_FLAGS "${VGFMAFLAG}")
+    check_c_source_compiles("${S390X_VX_TEST}" HAS_S390X_VX_SUPPORT)
+    unset(CMAKE_REQUIRED_FLAGS)
+
+    # or set march for our compile units
+    if(NOT HAS_S390X_VX_SUPPORT)
+        set(CMAKE_REQUIRED_FLAGS "${VGFMAFLAG} -march=z13")
+        check_c_source_compiles("${S390X_VX_TEST}" HAS_Z13_S390X_VX_SUPPORT)
+        unset(CMAKE_REQUIRED_FLAGS )
+        list(APPEND VGFMAFLAG "-march=z13")
+    endif(NOT HAS_S390X_VX_SUPPORT)
+
+    # prepare compiling for s390x
+    if(HAS_S390X_VX_SUPPORT OR HAS_Z13_S390X_VX_SUPPORT)
+        add_library(zlib_crc32_vx OBJECT 
+            crc32_vx.c
+            crc32_vx_hooks.h)
+        set_source_files_properties(
+            crc32_vx.c
+            PROPERTIES COMPILE_OPTIONS "${VGFMAFLAG}")
+        target_compile_definitions(zlib_crc32_vx PUBLIC HAVE_S390X_VX=1)
+    endif(HAS_S390X_VX_SUPPORT OR HAS_Z13_S390X_VX_SUPPORT)
+endif(ZLIB_CRC32VX AND HAS_S390X_SUPPORT)

contrib/s390x/README

@@ -0,0 +1,9 @@
+IBM Z mainframes starting from version z13 provide vector instructions, which 
+allows vectorization of crc32. This extension is build by default when targeting
+ibm s390x. However this extension can disabled if desired:
+
+    # for configure build
+    $ ./configure --disable-crcvx
+
+    # for cmake build
+    $ cmake .. -DZLIB_CRC32VX=off