Merge pull request #41 from Intel-HLS/merge_pv_inflater_newrelease_gp…

…_native_loader_and_comments

Update compression libraries

.travis.yml

@@ -6,3 +6,7 @@ jdk:
 before_install:
   - sudo apt-get -qq update
   - sudo apt-get install -y yasm
+
+
+
+

build.gradle

@@ -23,7 +23,7 @@ dependencies {
     compile 'org.broadinstitute:gatk-native-bindings:0.0.3'
     compile 'org.apache.logging.log4j:log4j-api:2.5'
     compile 'org.apache.logging.log4j:log4j-core:2.5'
-    compile 'com.github.samtools:htsjdk:2.4.1-36-g313bf83-SNAPSHOT'
+    compile 'com.github.samtools:htsjdk:2.7.0-18-g4d0070b-SNAPSHOT'
 
     testCompile 'org.testng:testng:6.9.9'
 }

src/main/java/com/intel/gkl/IntelGKLUtils.java

@@ -28,37 +28,25 @@
 
 package com.intel.gkl;
 
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.FilenameUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import java.io.File;
 import java.io.IOException;
-import java.net.URL;
 
 /**
- *This class provides support for Utilities for compression libraries support.
- *The shared library libIntelGKL is packaged as a jar files and its path is retireved
- *if the  GKL_USE_LIB_PATH variable is set. This utility class implements the load function
- *to load the library path for the packaged jar file.
+ * Provides utilities used by the GKL library.
  */
-
-
 public final class IntelGKLUtils {
     private final static Logger logger = LogManager.getLogger(IntelGKLUtils.class);
-    private final static String GKL_USE_LIB_PATH = "GKL_USE_LIB_PATH";
-    private final static String GKL_LIB_NAME = "IntelGKL";
-    private final static String GKL_OMP_LIB_NAME = "IntelGKL_omp";
-    private final static Boolean runningOnMac =
-            System.getProperty("os.name", "unknown").toLowerCase().startsWith("mac");
 
     /**
-     * Check if AVX is supported and enabled on the CPU
+     * Check if AVX is supported on the platform.
      *
-     * @return {@code true} if AVX is supported and enabled on the CPU, {@code false} otherwise
+     * @return  true if AVX is supported and enabled on the CPU, false otherwise
      */
-    public static Boolean isAvxSupported() {
+    public static boolean isAvxSupported() {
+        final boolean runningOnMac = System.getProperty("os.name", "unknown").toLowerCase().startsWith("mac");
         // use a grep command to check for AVX support
         // grep exit code = 0 if a match was found
         final String command = runningOnMac ? "sysctl -a | grep machdep.cpu.features | grep -i avx" :

src/main/java/com/intel/gkl/NativeLibraryLoader.java

@@ -11,81 +11,62 @@
 import java.util.Set;
 
 /**
- *
+ * Loads native libraries from the classpath, usually from a jar file.
  */
 public final class NativeLibraryLoader {
     private static final Logger logger = LogManager.getLogger(NativeLibraryLoader.class);
-    private static final String USE_LIB_PATH = "USE_LIB_PATH";
+    private static final String USE_LIBRARY_PATH = "USE_LIBRARY_PATH";
     private static final Set<String> loadedLibraries = new HashSet<String>();
 
-    private final String libraryName;
-
-    public NativeLibraryLoader(String libraryName) {
-        this.libraryName = libraryName;
-    }
-
-    public boolean isLoaded() {
-        return loadedLibraries.contains(libraryName);
-    }
-
     /**
-     * Tries to load the GKL shared library. If AVX is not supported, it returns {@code false} without
-     * trying to load the library.
+     * Tries to load the native library from the classpath, usually from a jar file. <p>
      *
-     * If GKL is loaded from a jar file, the shared library file is extracted to the
-     * {@code tempDir} directory first.
+     * If the USE_LIBRARY_PATH environment variable is defined, the native library will be loaded from the
+     * java.library.path instead of the classpath.
      *
-     * @param tempDir directory where the shared library file is extracted
-     * @param libFileName name of the library file to be loaded
-     * @return {@code true} if GKL loaded successfully, {@code false} otherwise
+     * @param tempDir  directory where the native library is extracted or null to use the system temp directory
+     * @param libraryName  name of the shared library without system dependent modifications
+     * @return true if the library was loaded successfully, false otherwise
      */
-    /**
-     *
-     * @param tempDir
-     * @return {@code true} if GKL loaded successfully, {@code false} otherwise
-     */
-    public synchronized boolean load(File tempDir) {
-        if (isLoaded()) {
+    public static synchronized boolean load(File tempDir, String libraryName) {
+        if (loadedLibraries.contains(libraryName)) {
             return true;
         }
 
-        // try to load from Java library path if USE_LIB_PATH env var is defined
-        if (System.getenv(USE_LIB_PATH) != null) {
+        final String systemLibraryName = System.mapLibraryName(libraryName);
+
+        // load from the java.library.path
+        if (System.getenv(USE_LIBRARY_PATH) != null) {
+            final String javaLibraryPath = System.getProperty("java.library.path");
             try {
-                String javaLibraryPath = System.getProperty("java.library.path");
-                logger.info(String.format("Trying to load native library from: \n\t%s",
-                        javaLibraryPath.replaceAll(":", "\n\t")));
+                logger.warn(String.format("OVERRIDE DEFAULT: Loading %s from %s", systemLibraryName, javaLibraryPath));
+                logger.warn(String.format("LD_LIBRARY_PATH = %s", System.getenv("LD_LIBRARY_PATH")));
                 System.loadLibrary(libraryName);
-                logger.info("Native library loaded from Java library path.");
-
                 return true;
-            } catch (UnsatisfiedLinkError ule) {
-                logger.warn("Unable to load library.");
+            } catch (Exception|Error e) {
+                logger.warn(String.format("OVERRIDE DEFAULT: Unable to load %s from %s", systemLibraryName, javaLibraryPath));
                 return false;
             }
         }
 
-        // try to extract from classpath
-        try {
-            String resourcePath = "native/" +  System.mapLibraryName(libraryName);
-            URL inputUrl = NativeLibraryLoader.class.getResource(resourcePath);
-            if (inputUrl == null) {
-                logger.warn("Unable to find native library: " + resourcePath);
-                return false;
-            }
-
-            logger.info(String.format("Trying to load native library from:\n\t%s", inputUrl.toString()));
+        // load from the java classpath
+        final String resourcePath = "native/" +  systemLibraryName;
+        final URL inputUrl = NativeLibraryLoader.class.getResource(resourcePath);
+        if (inputUrl == null) {
+            logger.warn("Unable to find native library: " + resourcePath);
+            return false;
+        }
+        logger.info(String.format("Loading %s from %s", systemLibraryName, inputUrl.toString()));
 
-            File temp = File.createTempFile(FilenameUtils.getBaseName(resourcePath),
+        try {
+            final File temp = File.createTempFile(FilenameUtils.getBaseName(resourcePath),
                     "." + FilenameUtils.getExtension(resourcePath), tempDir);
             FileUtils.copyURLToFile(inputUrl, temp);
             temp.deleteOnExit();
-            logger.debug(String.format("Extracted native to %s\n", temp.getAbsolutePath()));
-
+            logger.debug(String.format("Extracting %s to %s", systemLibraryName, temp.getAbsolutePath()));
             System.load(temp.getAbsolutePath());
-            logger.info("Native library loaded from classpath.");
-        } catch (Exception e) {
-            logger.warn("Unable to load native library.");
+        } catch (Exception|Error e) {
+            logger.warn(String.format("Unable to load %s from %s", systemLibraryName, resourcePath));
             return false;
         }
 

src/main/java/com/intel/gkl/compression/IntelDeflater.java

@@ -29,27 +29,34 @@
 package com.intel.gkl.compression;
 
 import com.intel.gkl.IntelGKLUtils;
+import com.intel.gkl.NativeLibraryLoader;
+import org.broadinstitute.gatk.nativebindings.NativeLibrary;
 
 import java.io.File;
 import java.util.zip.Deflater;
 
-import com.intel.gkl.NativeLibraryLoader;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.broadinstitute.gatk.nativebindings.NativeLibrary;
-
+/**
+ * Provides a native Deflater implementation accelerated for the Intel Architecture.
+ */
 public final class IntelDeflater extends Deflater implements NativeLibrary {
-    private static final Logger logger = LogManager.getLogger(IntelDeflater.class);
-
-    private static final NativeLibraryLoader libraryLoader = new NativeLibraryLoader("gkl_compression");
+    private static final String NATIVE_LIBRARY_NAME = "gkl_compression";
     private static boolean initialized = false;
 
+    /**
+     * Loads the native library, if it is supported on this platform. <p>
+     * Returns false if AVX is not supported. <br>
+     * Returns false if the native library cannot be loaded for any reason. <br>
+     * Initializes the native library after the first load. <br>
+     *
+     * @param tempDir  directory where the native library is extracted or null to use the system temp directory
+     * @return  true if the native library is supported and loaded, false otherwise
+     */
     @Override
     public synchronized boolean load(File tempDir) {
         if (!IntelGKLUtils.isAvxSupported()) {
             return false;
         }
-        if (!libraryLoader.load(tempDir)) {
+        if (!NativeLibraryLoader.load(tempDir, NATIVE_LIBRARY_NAME)) {
             return false;
         }
         if (!initialized) {
@@ -109,7 +116,6 @@ public IntelDeflater() {
 
     @Override
     public void reset() {
-        logger.debug("Reset deflater");
         resetNative(nowrap);
         inputBuffer = null;
         inputBufferLength = 0;

src/main/java/com/intel/gkl/compression/IntelDeflaterFactory.java

@@ -7,8 +7,10 @@
 import java.io.File;
 import java.util.zip.Deflater;
 
+/**
+ * Provides an IntelDeflater object using the DeflaterFactory API defined in HTSJDK
+ */
 public class IntelDeflaterFactory extends DeflaterFactory {
-
     private final static Logger logger = LogManager.getLogger(IntelDeflaterFactory.class);
     private boolean intelDeflaterSupported;
 
@@ -20,14 +22,21 @@ public IntelDeflaterFactory() {
         this(null);
     }
 
-    public Deflater makeDeflater(final int compressionLevel, final boolean nowrap) {
+    /**
+     * Returns an IntelDeflater if supported on the platform, otherwise returns a Java Deflater
+     *
+     * @param compressionLevel  the compression level (0-9)
+     * @param gzipCompatible  if true the use GZIP compatible compression
+     * @return a Deflater object
+     */
+    public Deflater makeDeflater(final int compressionLevel, final boolean gzipCompatible) {
         if (intelDeflaterSupported) {
-            if ((compressionLevel == 1 && nowrap) || compressionLevel != 1) {
-                return new IntelDeflater(compressionLevel, nowrap);
+            if ((compressionLevel == 1 && gzipCompatible) || compressionLevel != 1) {
+                return new IntelDeflater(compressionLevel, gzipCompatible);
             }
         }
         logger.warn("IntelDeflater is not supported, using Java.util.zip.Deflater");
-        return new Deflater(compressionLevel, nowrap);
+        return new Deflater(compressionLevel, gzipCompatible);
     }
 
     public boolean usingIntelDeflater() {

src/main/java/com/intel/gkl/compression/IntelInflater.java

@@ -29,32 +29,34 @@
 package com.intel.gkl.compression;
 
 import com.intel.gkl.IntelGKLUtils;
+import com.intel.gkl.NativeLibraryLoader;
+import org.broadinstitute.gatk.nativebindings.NativeLibrary;
 
 import java.io.File;
 import java.util.zip.Inflater;
 
-import com.intel.gkl.NativeLibraryLoader;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.broadinstitute.gatk.nativebindings.NativeLibrary;
-
 /**
- * Created by pnvaidya on 8/30/16.
+ * Provides a native Inflater implementation accelerated for the Intel Architecture.
  */
-
-
 public final class IntelInflater extends Inflater implements NativeLibrary {
-    private final static Logger logger = LogManager.getLogger(IntelInflater.class);
-
-    private static NativeLibraryLoader libraryLoader = new NativeLibraryLoader("gkl_compression");
+    private static final String NATIVE_LIBRARY_NAME = "gkl_compression";
     private static boolean initialized = false;
 
+    /**
+     * Loads the native library, if it is supported on this platform. <p>
+     * Returns false if AVX is not supported. <br>
+     * Returns false if the native library cannot be loaded for any reason. <br>
+     * Initializes the native library after the first load. <br>
+     *
+     * @param tempDir  directory where the native library is extracted or null to use the system temp directory
+     * @return  true if the native library is supported and loaded, false otherwise
+     */
     @Override
     public synchronized boolean load(File tempDir) {
         if (!IntelGKLUtils.isAvxSupported()) {
             return false;
         }
-        if (!libraryLoader.load(tempDir)) {
+        if (!NativeLibraryLoader.load(tempDir, NATIVE_LIBRARY_NAME)) {
             return false;
         }
         if (!initialized) {