Merge pull request #51 from Intel-HLS/pv_libz

Pv libz
IntelLabs · Apr 4, 2017 · dd638ad · dd638ad
2 parents 7c27959 + 2471588
commit dd638ad
Show file tree

Hide file tree

Showing 5 changed files with 253 additions and 2 deletions.
diff --git a/src/main/native/compression/IntelDeflater.cc b/src/main/native/compression/IntelDeflater.cc
@@ -313,9 +313,14 @@ JNIEXPORT void JNICALL
 Java_com_intel_gkl_compression_IntelDeflater_endNative(JNIEnv *env, jobject obj)
 {
   jint level = env->GetIntField(obj, FID_level);
-  z_stream* lz_stream = (z_stream*)env->GetLongField(obj, FID_lz_stream);
+
   if (level != 1) {
+    z_stream* lz_stream = (z_stream*)env->GetLongField(obj, FID_lz_stream);
     deflateEnd(lz_stream);
   }
-  else free(lz_stream);
+  else {
+   isal_zstream* lz_stream = (isal_zstream*)env->GetLongField(obj, FID_lz_stream);
+   free(lz_stream->hufftables);
+   free(lz_stream);
+  }
 }
diff --git a/src/main/native/compression/libz/CMakeLists.txt b/src/main/native/compression/libz/CMakeLists.txt
@@ -0,0 +1,70 @@
+project(GKL)
+cmake_minimum_required(VERSION 2.8)
+
+
+# common compiler and linker settings
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O3 -mavx")
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+if(APPLE)
+  set(CMAKE_SHARED_LINKER_FLAGS "-static-libstdc++")
+elseif(UNIX)
+  set(CMAKE_SHARED_LINKER_FLAGS "-static-libstdc++ -static-libgcc")
+endif()
+
+
+include(ExternalProject)
+
+#---------------------------------------------------------------------
+# isa-l
+#---------------------------------------------------------------------
+
+ExternalProject_Add(
+   isalExternal
+   URL ${CMAKE_CURRENT_SOURCE_DIR}/../isa-l-master
+   UPDATE_COMMAND <SOURCE_DIR>/autogen.sh
+#PATCH_COMMAND patch < ${CMAKE_CURRENT_SOURCE_DIR}/isa-l.patch
+   CONFIGURE_COMMAND <SOURCE_DIR>/configure --prefix=<INSTALL_DIR> 
+   BUILD_COMMAND make
+   INSTALL_COMMAND make install
+)
+add_library(isal STATIC IMPORTED GLOBAL)
+set_target_properties(isal PROPERTIES IMPORTED_LOCATION
+  ${CMAKE_CURRENT_BINARY_DIR}/isalExternal-prefix/lib/libisal.a
+)
+
+ADD_CUSTOM_COMMAND(
+      TARGET isalExternal
+      PRE_BUILD
+      COMMAND cp
+      ARGS ${CMAKE_CURRENT_BINARY_DIR}/../isa-l-master/include/igzip_lib.h  ${CMAKE_CURRENT_BINARY_DIR}/../otc_zlib/igzip_lib.h)
+
+ADD_CUSTOM_COMMAND(
+      TARGET isalExternal
+      PRE_BUILD
+      COMMAND cp
+      ARGS ${CMAKE_CURRENT_BINARY_DIR}/../isa-l-master/include/types.h  ${CMAKE_CURRENT_BINARY_DIR}/../otc_zlib/types.h)
+
+ADD_CUSTOM_COMMAND(
+      TARGET isalExternal
+      PRE_BUILD
+      COMMAND cp
+      ARGS ${CMAKE_CURRENT_BINARY_DIR}/isalExternal-prefix/lib/libisal.a  ${CMAKE_CURRENT_BINARY_DIR}/zlib-1.2.8devn-jtkv5External-prefix/src/zlib-1.2.8devn-jtkv5External-build/libisal.a)
+
+#---------------------------------------------------------------------
+# otc_zlib
+#---------------------------------------------------------------------
+#add_subdirectory(otc_zlib EXCLUDE_FROM_ALL)
+#add_subdirectory(zlib-1.2.8devn-jtkv5 EXCLUDE_FROM_ALL)
+
+ExternalProject_Add(
+   zlib-1.2.8devn-jtkv5External
+   URL ${CMAKE_CURRENT_SOURCE_DIR}/../otc_zlib 
+   PATCH_COMMAND patch < ${CMAKE_CURRENT_SOURCE_DIR}/zlib.patch
+#   PATCH_COMMAND patch < ${CMAKE_CURRENT_SOURCE_DIR}/zlib_deflate.patch
+   CONFIGURE_COMMAND <SOURCE_DIR>/configure --static --prefix=<INSTALL_DIR>
+   BUILD_COMMAND make
+   INSTALL_COMMAND make install
+)
+
+
+
diff --git a/src/main/native/compression/libz/README b/src/main/native/compression/libz/README
@@ -0,0 +1,11 @@
+BUILDING optimized version of compression library for C/C++ using GKL
+
+1. In the libz directory run cmake .
+
+2. Apply the zlib_deflate.patch and zlib.patch to files in /otc_zlib
+	$cd ../otc_zlib
+        $patch < ../libz/zlib_deflate.patch
+
+3. Run make
+
+4. libz.so.1.2.8.1-motley with integrated optimized isal for level 1 is generated. 
diff --git a/src/main/native/compression/libz/zlib.patch b/src/main/native/compression/libz/zlib.patch
@@ -0,0 +1,20 @@
+--- Makefile.in	2017-03-14 10:53:14.850526414 -0400
++++ Makefile.in.new	2017-03-14 10:52:48.009526529 -0400
+@@ -26,7 +26,7 @@
+
+ SFLAGS=-O
+ LDFLAGS=
+-TEST_LDFLAGS=-L. libz.a
++TEST_LDFLAGS=-L. libz.a libisal.a
+ LDSHARED=$(CC)
+ CPP=$(CC) -E
+
+@@ -318,7 +318,7 @@
+
+
+ placebo $(SHAREDLIBV): $(PIC_OBJS) libz.a
+-	$(LDSHARED) $(SFLAGS) -o $@ $(PIC_OBJS) $(LDSHAREDLIBC) $(LDFLAGS)
++	$(LDSHARED) $(SFLAGS) -o $@ $(PIC_OBJS) $(LDSHAREDLIBC) $(TEST_LDFLAGS)
+ 	rm -f $(SHAREDLIB) $(SHAREDLIBM)
+ 	ln -s $@ $(SHAREDLIB)
+ 	ln -s $@ $(SHAREDLIBM)
diff --git a/src/main/native/compression/libz/zlib_deflate.patch b/src/main/native/compression/libz/zlib_deflate.patch
@@ -0,0 +1,145 @@
+--- deflate.c	2016-12-08 10:07:15.000000000 -0500
++++ deflate_new.c	2017-04-04 11:11:11.433780075 -0400
+@@ -50,11 +50,13 @@
+ /* @(#) $Id$ */
+
+ #include "deflate.h"
+-
++#define USE_IGZIP
+ #if defined(CHECK_SSE2) || defined(USE_SSE4_2_CRC_HASH) || defined(USE_QUICK)
+ #include "x86.h"
+ #endif
+-
++#ifdef USE_IGZIP
++#include "igzip_lib.h"
++#endif
+ const char deflate_copyright[] =
+    " deflate 1.2.8.1 Copyright 1995-2013 Jean-loup Gailly and Mark Adler ";
+ /*
+@@ -382,7 +384,30 @@
+     s->strategy = strategy;
+     s->method = (Byte)method;
+
+-    return deflateReset(strm);
++    int ret =  deflateReset(strm);
++        #ifdef USE_IGZIP
++            if(level ==1){
++                if(s->wrap == 0)
++    	        {
++                    strm->state->strm = (z_streamp) ZALLOC(strm, 1, sizeof(struct isal_zstream));
++                    isal_deflate_init((struct isal_zstream*)strm->state->strm);
++                    ((struct isal_zstream*)strm->state->strm)->end_of_stream = 0;
++                    ((struct isal_zstream*)strm->state->strm)->hufftables = 0x0;
++                    ((struct isal_zstream*)strm->state->strm)->gzip_flag = 0;
++
++                }
++                else if(s->wrap == 2)
++                {
++                     strm->state->strm = (z_streamp) ZALLOC(strm, 1, sizeof(struct isal_zstream));
++                     isal_deflate_init((struct isal_zstream*)strm->state->strm);
++                     ((struct isal_zstream*)strm->state->strm)->end_of_stream = 0;
++                     ((struct isal_zstream*)strm->state->strm)->hufftables = 0x0;
++                     ((struct isal_zstream*)strm->state->strm)->gzip_flag = 1;
++                }
++            }
++
++        #endif
++        return ret;
+ }
+
+ /* =========================================================================
+@@ -755,6 +780,72 @@
+     int old_flush; /* value of flush param for previous deflate call */
+     deflate_state *s;
+
++    #ifdef USE_IGZIP
++             s = strm->state;
++
++
++            if( (s->level == 1) && (s->wrap ==0 || s->wrap ==2) )
++            {
++
++
++            int original_flush = flush;
++
++            if(flush == Z_FINISH || flush == Z_FULL_FLUSH) {
++            flush = FULL_FLUSH;
++            }
++            else if(flush == Z_SYNC_FLUSH ) {
++            flush = SYNC_FLUSH;
++            }
++            else if(flush != Z_NO_FLUSH ){
++            ERR_RETURN(strm, Z_STREAM_ERROR);
++            }
++
++
++
++            if(((struct isal_zstream*)s->strm)->hufftables == 0x0)
++                      {
++                        struct isal_huff_histogram histogram;
++                        struct isal_hufftables *hufftables_custom;
++                        int sixtyfourK = 64*1024;
++                        int usable_buffer= (strm->avail_in < sixtyfourK) ? strm->avail_in : sixtyfourK;
++
++                         hufftables_custom = (struct isal_hufftables*) malloc(sizeof(struct isal_hufftables));
++
++                         memset(&histogram, 0, sizeof(histogram));
++                         isal_update_histogram((unsigned char*)strm->next_in,usable_buffer, &histogram);
++                         isal_create_hufftables(hufftables_custom, &histogram);
++                         ((struct isal_zstream*)s->strm)->hufftables = hufftables_custom;
++                 }
++
++
++                ((struct isal_zstream *)s->strm)->next_in = strm->next_in;
++                ((struct isal_zstream *)s->strm)->avail_in = strm->avail_in;
++                ((struct isal_zstream *)s->strm)->avail_out = strm->avail_out;
++                ((struct isal_zstream *)s->strm)->next_out = strm->next_out;
++                ((struct isal_zstream *)s->strm)->flush = flush;
++                ((struct isal_zstream *)s->strm)->end_of_stream = 1;
++
++                //printf("Next_in =  %p, Next_out = %p, Avail_in = %d, Avail_out = %d\n",strm->next_in, strm->next_out, strm->avail_in, strm->avail_out);
++
++                 int ret = isal_deflate_stateless((struct isal_zstream*)s->strm);
++
++
++                 strm->avail_in = ((struct isal_zstream *)s->strm)->avail_in;
++                 strm->next_in = ((struct isal_zstream *)s->strm)->next_in;
++                 strm->avail_out = ((struct isal_zstream *)s->strm)->avail_out;
++                 strm->next_out = ((struct isal_zstream *)s->strm)->next_out;
++                 strm->total_out =((struct isal_zstream *)s->strm)->total_out;
++                 strm->total_in =((struct isal_zstream *)s->strm)->total_in;
++
++                 if(original_flush == Z_FINISH && ret == COMP_OK) ret = Z_STREAM_END;
++
++                 return ret;
++
++
++             }
++
++        #endif
++
+     if (deflateStateCheck(strm) || flush > Z_BLOCK || flush < 0) {
+         return Z_STREAM_ERROR;
+     }
+@@ -1076,6 +1167,21 @@
+ {
+     int status;
+
++     #ifdef USE_IGZIP
++            if(strm->state->level ==1 &&  ( (strm->state->wrap == 0 ) || (strm->state->wrap == 2 )) )
++            {
++              /* Deallocate in reverse order of allocations: */
++              TRY_FREE(strm, strm->state->pending_buf);
++              TRY_FREE(strm, strm->state->head);
++              TRY_FREE(strm, strm->state->prev);
++              TRY_FREE(strm, strm->state->window);
++              ZFREE(strm, strm->state->strm);
++              ZFREE(strm, strm->state);
++              strm->state = Z_NULL;
++              return Z_OK;
++            }
++        #endif
++
+     if (deflateStateCheck(strm)) return Z_STREAM_ERROR;
+
+     status = strm->state->status;