Merge branch 'branch-0.7' into fea-ext-cython-exceptions

Author: kkraus14

CHANGELOG.md

@@ -22,6 +22,7 @@
 - PR #1445 Parquet Reader: Add selective reading of rows and row group
 - PR #1532 Parquet Reader: Add support for INT96 timestamps
 - PR #1516 Add Series and DataFrame.ndim
+- PR #1466 Add GPU-accelerated ORC Reader
 
 ## Improvements
 
@@ -59,12 +60,11 @@
 - PR #1463 Allow and default melt keyword argument var_name to be None
 - PR #1486 Parquet Reader: Use device_buffer rather than device_ptr
 - PR #1520 Renamed `src/dataframe` to `src/table` and moved `table.hpp`. Made `types.hpp` to be type declarations only.
-- PR #1492 Convert transpose CFFI to Cython
-- PR #1495 Convert binary and unary ops CFFI to Cython
-- PR #1503 Convert sorting and hashing ops CFFI to Cython
+- PR #1521 Added `row_bitmask` to compute bitmask for rows of a table. Merged `valids_ops.cu` and `bitmask_ops.cu`
 - PR #1553 Overload `hash_row` to avoid using intial hash values. Updated `gdf_hash` to select between overloads
 - PR #1559 Add `except +` to all Cython function definitions to catch C++ exceptions properly
 
+
 ## Bug Fixes
 
 - PR #1233 Fix dtypes issue while adding the column to `str` dataframe.
@@ -106,6 +106,7 @@
 - PR #1535 Fix doc issue to ensure correct labelling of cudf.series
 - PR #1537 Fix `undefined reference` link error in HashPartitionTest
 - PR #1548 Fix ci/local/build.sh README from using an incorrect image example
+- PR #1551 CSV Reader: Fix integer column name indexing
 
 
 # cuDF 0.6.1 (25 Mar 2019)

cpp/CMakeLists.txt

@@ -91,16 +91,16 @@ include(FeatureSummary)
 include(CheckIncludeFiles)
 include(CheckLibraryExists)
 
-include(ConfigureArrow)
-
 ###################################################################################################
 # - find arrow ------------------------------------------------------------------------------------
 
+include(ConfigureArrow)
+
 if (ARROW_FOUND)
     message(STATUS "Apache Arrow found in ${ARROW_INCLUDE_DIR}")
 else()
     message(FATAL_ERROR "Apache Arrow not found, please check your settings.")
-endif()
+endif(ARROW_FOUND)
 
 ###################################################################################################
 # - find zlib -------------------------------------------------------------------------------------
@@ -241,7 +241,6 @@ add_library(cudf SHARED
             src/binary/jit/util/operator.cpp
             src/binary/jit/util/type.cpp
             src/bitmask/bitmask_ops.cu
-            src/bitmask/valid_ops.cu
             src/compaction/stream_compaction_ops.cu
             src/datetime/datetime_ops.cu
             src/hash/hashing.cu
@@ -259,6 +258,11 @@ add_library(cudf SHARED
             src/io/convert/dlpack/cudf_dlpack.cpp
             src/io/csv/csv_reader.cu
             src/io/csv/csv_writer.cu
+            src/io/orc/orc_reader.cu
+            src/io/orc/orc.cpp
+            src/io/orc/timezone.cpp
+            src/io/orc/stripe_data.cu
+            src/io/orc/stripe_init.cu
             src/io/parquet/page_data.cu
             src/io/parquet/page_hdr.cu
             src/io/parquet/parquet_reader.cu

cpp/include/bitmask.hpp

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef BITMASK_HPP
+#define BITMASK_HPP
+
+#include <cudf.h>
+#include <types.hpp>
+
+/**
+ * @brief  Counts the number of valid bits for the specified number of rows
+ * in a validity bitmask.
+ *
+ * If the bitmask is null, returns a count equal to the number of rows.
+ *
+ * @param[in] masks The validity bitmask buffer in device memory
+ * @param[in] num_rows The number of bits to count
+ * @param[out] count The number of valid bits in the buffer from [0, num_rows)
+ *
+ * @returns  GDF_SUCCESS upon successful completion
+ *
+ */
+gdf_error gdf_count_nonzero_mask(gdf_valid_type const* masks,
+                                 gdf_size_type num_rows, gdf_size_type* count);
+
+/** ---------------------------------------------------------------------------*
+ * @brief Concatenate the validity bitmasks of multiple columns
+ *
+ * Accounts for the differences between lengths of columns and their bitmasks
+ * (e.g. because gdf_valid_type is larger than one bit).
+ *
+ * @param[out] output_mask The concatenated mask
+ * @param[in] output_column_length The total length (in data elements) of the
+ *                                 concatenated column
+ * @param[in] masks_to_concat The array of device pointers to validity bitmasks
+ *                            for the columns to concatenate
+ * @param[in] column_lengths An array of lengths of the columns to concatenate
+ * @param[in] num_columns The number of columns to concatenate
+ * @return gdf_error GDF_SUCCESS or GDF_CUDA_ERROR if there is a runtime CUDA
+           error
+ *
+ ---------------------------------------------------------------------------**/
+gdf_error gdf_mask_concat(gdf_valid_type* output_mask,
+                          gdf_size_type output_column_length,
+                          gdf_valid_type* masks_to_concat[],
+                          gdf_size_type* column_lengths,
+                          gdf_size_type num_columns);
+
+
+#endif

cpp/include/cudf/functions.h

@@ -41,21 +41,6 @@ gdf_error gdf_nvtx_range_push_hex(char const * const name, unsigned int color );
  */
 gdf_error gdf_nvtx_range_pop();
 
-/**
- * @brief  Counts the number of valid bits for the specified number of rows
- * in a validity bitmask.
- *
- * If the bitmask is null, returns a count equal to the number of rows.
- *
- * @param[in] masks The validity bitmask buffer in device memory
- * @param[in] num_rows The number of bits to count
- * @param[out] count The number of valid bits in the buffer from [0, num_rows)
- *
- * @returns  GDF_SUCCESS upon successful completion
- *
- */
-gdf_error gdf_count_nonzero_mask(gdf_valid_type const *masks,
-                                 gdf_size_type num_rows, gdf_size_type *count);
 
 /**
  * Calculates the number of bytes to allocate for a column's validity bitmask

cpp/include/cudf/io_functions.h

@@ -34,6 +34,18 @@ gdf_error read_csv(csv_read_arg *args);
  */
 gdf_error write_csv(csv_write_arg* args);
 
+/*
+ * @brief Interface to parse ORC data to GDF columns
+ *
+ * This function accepts an input source for an Apache ORC dataset and outputs
+ * an array of gdf_columns.
+ *
+ * @param[in,out] args Structure containing input and output args
+ *
+ * @return gdf_error GDF_SUCCESS if successful
+ **/
+gdf_error read_orc(orc_read_arg *args);
+
 /*
  * @brief Interface to parse Parquet data to GDF columns
  */

cpp/include/cudf/io_types.h

@@ -158,6 +158,33 @@ typedef struct
 
 } csv_write_arg;
 
+/**---------------------------------------------------------------------------*
+ * @brief Input and output arguments to the read_orc interface.
+ *---------------------------------------------------------------------------**/
+typedef struct {
+
+  /*
+   * Output arguments
+   */
+  int           num_cols_out;               ///< Out: Number of columns returned
+  int           num_rows_out;               ///< Out: Number of rows returned
+  gdf_column    **data;                     ///< Out: Array of gdf_columns*
+
+  /*
+   * Input arguments
+   */
+  gdf_input_type source_type;               ///< In: Type of data source
+  const char    *source;                    ///< In: If source_type is FILE_PATH, contains the filepath. If input_data_type is HOST_BUFFER, points to the host memory buffer
+  size_t        buffer_size;                ///< In: If source_type is HOST_BUFFER, represents the size of the buffer in bytes. Unused otherwise.
+
+  const char    **use_cols;                 ///< In: Columns of interest; only these columns will be parsed and returned.
+  int           use_cols_len;               ///< In: Number of columns
+
+  int           skip_rows;                  ///< In: Number of rows to skip from the start
+  int           num_rows;                   ///< In: Number of rows to read. Actual number of returned rows may be less
+
+} orc_read_arg;
+
 /**---------------------------------------------------------------------------*
  * @brief Input and output arguments to the read_parquet interface.
  *---------------------------------------------------------------------------**/

cpp/include/types.hpp

@@ -19,8 +19,12 @@
 /**---------------------------------------------------------------------------*
  * @file types.hpp
  * @brief Type declarations for libcudf.
- * 
-*---------------------------------------------------------------------------**/
+ *
+ *---------------------------------------------------------------------------**/
+
+namespace bit_mask {
+using bit_mask_t = uint32_t;
+}
 
 // Forward declaration
 namespace cudf {

cpp/src/binary/jit/core/binop.cpp

@@ -19,7 +19,7 @@
 
 #include "binary/jit/core/launcher.h"
 #include "binary/jit/util/operator.h"
-#include "bitmask/bitmask_ops.h"
+#include <bitmask/bitmask_ops.hpp>
 #include "utilities/error_utils.hpp"
 #include "utilities/cudf_utils.h"
 #include "cudf.h"