Merge pull request #159 from duckdb/bump-v1.3.0

Bump bundled DuckDB to v1.3.0

Author: hannes

.github/workflows/Electron.yml

@@ -19,7 +19,7 @@ jobs:
   set-up-npm:
     if: false
     name: Set up NPM
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       DUCKDB_NODE_BUILD_CACHE: 0
     steps:
@@ -41,7 +41,7 @@ jobs:
   # From `npm show electron time --json` and https://www.electronjs.org/docs/latest/tutorial/electron-timelines
   set-up-electron-versions:
     name: Set up Electron version
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     outputs:
       matrix: ${{ steps.setup.outputs.matrix }}
     steps:
@@ -59,7 +59,7 @@ jobs:
 
   linux-electron:
     name: Electron Linux
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     needs: [set-up-npm, set-up-electron-versions]
     env:
       TARGET_ARCH: ${{ matrix.target_arch }}
@@ -271,7 +271,7 @@ jobs:
       - win-electron
     strategy:
       matrix:
-        os: [windows-latest, ubuntu-latest, ubuntu-22.04, ubuntu-20.04, windows-2019, macos-12, macos-13, macos-14]
+        os: [windows-latest, ubuntu-latest, ubuntu-22.04, windows-2019, macos-12, macos-13, macos-14]
         version: [20]
     runs-on: ${{ matrix.os }}
     steps:

.github/workflows/NodeJS.yml

@@ -18,7 +18,7 @@ env:
 jobs:
   set-up-npm:
     name: Set up NPM
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       DUCKDB_NODE_BUILD_CACHE: 0
     steps:
@@ -48,7 +48,7 @@ jobs:
     strategy:
       matrix:
         # node.js current support policy to be found at https://github.com/duckdb/duckdb-node/tree/main/#Supported-Node-versions
-        node: [ '18', '20', '22', '23']
+        node: [ '18', '20', '22', '23', '24']
         target_arch: [ x64, arm64 ]
         isRelease:
           - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
@@ -113,7 +113,7 @@ jobs:
     strategy:
       matrix:
         target_arch: [ arm64 ]
-        node: [ '18', '20', '22', '23']
+        node: [ '18', '20', '22', '23', '24']
         isRelease:
           - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
         exclude:
@@ -164,7 +164,7 @@ jobs:
     strategy:
       matrix:
         target_arch: [ x64 ]
-        node: [ '18', '20', '22', '23']
+        node: [ '18', '20', '22', '23', '24']
         isRelease:
           - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
     env:
@@ -212,7 +212,7 @@ jobs:
 
     strategy:
       matrix:
-        node: [ '18', '20', '22', '23']
+        node: [ '18', '20', '22', '23', '24']
         isRelease:
           - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
         exclude:
@@ -273,7 +273,7 @@ jobs:
       - win-nodejs
     strategy:
       matrix:
-        os: [windows-latest, ubuntu-latest, ubuntu-22.04, ubuntu-20.04, windows-2019, macos-12, macos-13, macos-14]
+        os: [windows-latest, ubuntu-latest, ubuntu-22.04, windows-2019, macos-12, macos-13, macos-14]
         version: [20]
     runs-on: ${{ matrix.os }}
     steps:

binding.gyp

@@ -21,6 +21,7 @@
                 "src/duckdb/ub_src_common_crypto.cpp", 
                 "src/duckdb/ub_src_common_enums.cpp", 
                 "src/duckdb/ub_src_common_exception.cpp", 
+                "src/duckdb/ub_src_common_multi_file.cpp", 
                 "src/duckdb/ub_src_common_operator.cpp", 
                 "src/duckdb/ub_src_common_progress_bar.cpp", 
                 "src/duckdb/ub_src_common_row_operations.cpp", 
@@ -94,7 +95,12 @@
                 "src/duckdb/ub_src_main_capi.cpp", 
                 "src/duckdb/ub_src_main_capi_cast.cpp", 
                 "src/duckdb/ub_src_main_chunk_scan_state.cpp", 
-                "src/duckdb/ub_src_main_extension.cpp", 
+                "src/duckdb/src/main/extension/extension_alias.cpp", 
+                "src/duckdb/src/main/extension/extension_helper.cpp", 
+                "src/duckdb/src/main/extension/extension_install.cpp", 
+                "src/duckdb/src/main/extension/extension_load.cpp", 
+                "src/duckdb/src/main/extension/extension_util.cpp", 
+                "src/duckdb/ub_src_main_http.cpp", 
                 "src/duckdb/ub_src_main_relation.cpp", 
                 "src/duckdb/ub_src_main_secret.cpp", 
                 "src/duckdb/ub_src_main_settings.cpp", 
@@ -136,6 +142,7 @@
                 "src/duckdb/ub_src_storage_compression_alp.cpp", 
                 "src/duckdb/ub_src_storage_compression.cpp", 
                 "src/duckdb/ub_src_storage_compression_chimp.cpp", 
+                "src/duckdb/ub_src_storage_compression_dict_fsst.cpp", 
                 "src/duckdb/ub_src_storage_compression_dictionary.cpp", 
                 "src/duckdb/ub_src_storage_compression_roaring.cpp", 
                 "src/duckdb/ub_src_storage_metadata.cpp", 
@@ -195,29 +202,27 @@
                 "src/duckdb/third_party/libpg_query/src_backend_parser_scansup.cpp", 
                 "src/duckdb/third_party/libpg_query/src_common_keywords.cpp", 
                 "src/duckdb/third_party/mbedtls/library/aes.cpp", 
-                "src/duckdb/third_party/mbedtls/library/aria.cpp", 
                 "src/duckdb/third_party/mbedtls/library/asn1parse.cpp", 
+                "src/duckdb/third_party/mbedtls/library/asn1write.cpp", 
                 "src/duckdb/third_party/mbedtls/library/base64.cpp", 
                 "src/duckdb/third_party/mbedtls/library/bignum.cpp", 
-                "src/duckdb/third_party/mbedtls/library/camellia.cpp", 
+                "src/duckdb/third_party/mbedtls/library/bignum_core.cpp", 
                 "src/duckdb/third_party/mbedtls/library/cipher.cpp", 
                 "src/duckdb/third_party/mbedtls/library/cipher_wrap.cpp", 
                 "src/duckdb/third_party/mbedtls/library/constant_time.cpp", 
-                "src/duckdb/third_party/mbedtls/library/entropy.cpp", 
-                "src/duckdb/third_party/mbedtls/library/entropy_poll.cpp", 
                 "src/duckdb/third_party/mbedtls/library/gcm.cpp", 
                 "src/duckdb/third_party/mbedtls/library/md.cpp", 
                 "src/duckdb/third_party/mbedtls/library/oid.cpp", 
                 "src/duckdb/third_party/mbedtls/library/pem.cpp", 
                 "src/duckdb/third_party/mbedtls/library/pk.cpp", 
                 "src/duckdb/third_party/mbedtls/library/pk_wrap.cpp", 
                 "src/duckdb/third_party/mbedtls/library/pkparse.cpp", 
+                "src/duckdb/third_party/mbedtls/library/platform.cpp", 
                 "src/duckdb/third_party/mbedtls/library/platform_util.cpp", 
                 "src/duckdb/third_party/mbedtls/library/rsa.cpp", 
                 "src/duckdb/third_party/mbedtls/library/rsa_alt_helpers.cpp", 
                 "src/duckdb/third_party/mbedtls/library/sha1.cpp", 
                 "src/duckdb/third_party/mbedtls/library/sha256.cpp", 
-                "src/duckdb/third_party/mbedtls/library/sha512.cpp", 
                 "src/duckdb/third_party/mbedtls/mbedtls_wrapper.cpp", 
                 "src/duckdb/third_party/yyjson/yyjson.cpp", 
                 "src/duckdb/third_party/zstd/common/debug.cpp", 
@@ -252,18 +257,24 @@
                 "src/duckdb/third_party/zstd/dict/divsufsort.cpp", 
                 "src/duckdb/third_party/zstd/dict/fastcover.cpp", 
                 "src/duckdb/third_party/zstd/dict/zdict.cpp", 
-                "src/duckdb/extension/parquet/column_reader.cpp", 
-                "src/duckdb/extension/parquet/column_writer.cpp", 
-                "src/duckdb/extension/parquet/parquet_crypto.cpp", 
+                "src/duckdb/extension/parquet/serialize_parquet.cpp", 
+                "src/duckdb/extension/parquet/parquet_multi_file_info.cpp", 
+                "src/duckdb/extension/parquet/geo_parquet.cpp", 
                 "src/duckdb/extension/parquet/parquet_extension.cpp", 
-                "src/duckdb/extension/parquet/parquet_metadata.cpp", 
+                "src/duckdb/extension/parquet/zstd_file_system.cpp", 
+                "src/duckdb/extension/parquet/parquet_file_metadata_cache.cpp", 
+                "src/duckdb/extension/parquet/parquet_timestamp.cpp", 
                 "src/duckdb/extension/parquet/parquet_reader.cpp", 
+                "src/duckdb/extension/parquet/column_writer.cpp", 
                 "src/duckdb/extension/parquet/parquet_statistics.cpp", 
-                "src/duckdb/extension/parquet/parquet_timestamp.cpp", 
+                "src/duckdb/extension/parquet/parquet_crypto.cpp", 
+                "src/duckdb/extension/parquet/parquet_metadata.cpp", 
+                "src/duckdb/extension/parquet/parquet_float16.cpp", 
+                "src/duckdb/extension/parquet/column_reader.cpp", 
                 "src/duckdb/extension/parquet/parquet_writer.cpp", 
-                "src/duckdb/extension/parquet/serialize_parquet.cpp", 
-                "src/duckdb/extension/parquet/zstd_file_system.cpp", 
-                "src/duckdb/extension/parquet/geo_parquet.cpp", 
+                "src/duckdb/ub_extension_parquet_reader.cpp", 
+                "src/duckdb/ub_extension_parquet_decoder.cpp", 
+                "src/duckdb/ub_extension_parquet_writer.cpp", 
                 "src/duckdb/third_party/parquet/parquet_types.cpp", 
                 "src/duckdb/third_party/thrift/thrift/protocol/TProtocol.cpp", 
                 "src/duckdb/third_party/thrift/thrift/transport/TTransportException.cpp", 
@@ -318,15 +329,16 @@
                 "src/duckdb/ub_extension_icu_third_party_icu_common.cpp", 
                 "src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp", 
                 "src/duckdb/extension/icu/third_party/icu/stubdata/stubdata.cpp", 
-                "src/duckdb/extension/json/buffered_json_reader.cpp", 
-                "src/duckdb/extension/json/json_enums.cpp", 
+                "src/duckdb/extension/json/json_deserializer.cpp", 
+                "src/duckdb/extension/json/json_reader.cpp", 
+                "src/duckdb/extension/json/serialize_json.cpp", 
                 "src/duckdb/extension/json/json_extension.cpp", 
-                "src/duckdb/extension/json/json_common.cpp", 
-                "src/duckdb/extension/json/json_functions.cpp", 
                 "src/duckdb/extension/json/json_scan.cpp", 
+                "src/duckdb/extension/json/json_functions.cpp", 
                 "src/duckdb/extension/json/json_serializer.cpp", 
-                "src/duckdb/extension/json/json_deserializer.cpp", 
-                "src/duckdb/extension/json/serialize_json.cpp", 
+                "src/duckdb/extension/json/json_multi_file_info.cpp", 
+                "src/duckdb/extension/json/json_enums.cpp", 
+                "src/duckdb/extension/json/json_common.cpp", 
                 "src/duckdb/ub_extension_json_json_functions.cpp", 
                 "src/duckdb/extension/core_functions/function_list.cpp", 
                 "src/duckdb/extension/core_functions/core_functions_extension.cpp", 
@@ -371,7 +383,6 @@
                 "src/duckdb/third_party/brotli/common", 
                 "src/duckdb/third_party/brotli/dec", 
                 "src/duckdb/third_party/brotli/enc", 
-                "src/duckdb/third_party/mbedtls", 
                 "src/duckdb/third_party/mbedtls/include", 
                 "src/duckdb/third_party/mbedtls/library", 
                 "src/duckdb/third_party/miniz", 

src/duckdb/extension/core_functions/aggregate/algebraic/avg.cpp

@@ -1,6 +1,7 @@
 #include "core_functions/aggregate/algebraic_functions.hpp"
 #include "core_functions/aggregate/sum_helpers.hpp"
 #include "duckdb/common/types/hugeint.hpp"
+#include "duckdb/common/types/time.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/function/function_set.hpp"
 #include "duckdb/planner/expression.hpp"
@@ -22,6 +23,21 @@ struct AvgState {
 	}
 };
 
+struct IntervalAvgState {
+	int64_t count;
+	interval_t value;
+
+	void Initialize() {
+		this->count = 0;
+		this->value = interval_t();
+	}
+
+	void Combine(const IntervalAvgState &other) {
+		this->count += other.count;
+		this->value = AddOperator::Operation<interval_t, interval_t, interval_t>(this->value, other.value);
+	}
+};
+
 struct KahanAvgState {
 	uint64_t count;
 	double value;
@@ -105,6 +121,20 @@ struct IntegerAverageOperationHugeint : public BaseSumOperation<AverageSetOperat
 	}
 };
 
+struct DiscreteAverageOperation : public BaseSumOperation<AverageSetOperation, AddToHugeint> {
+	template <class T, class STATE>
+	static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) {
+		if (state.count == 0) {
+			finalize_data.ReturnNull();
+		} else {
+			hugeint_t remainder;
+			target = Hugeint::Cast<T>(Hugeint::DivMod(state.value, state.count, remainder));
+			// Round the result
+			target += (remainder > (state.count / 2));
+		}
+	}
+};
+
 struct HugeintAverageOperation : public BaseSumOperation<AverageSetOperation, HugeintAdd> {
 	template <class T, class STATE>
 	static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) {
@@ -139,6 +169,74 @@ struct KahanAverageOperation : public BaseSumOperation<AverageSetOperation, Kaha
 	}
 };
 
+struct IntervalAverageOperation : public BaseSumOperation<AverageSetOperation, IntervalAdd> {
+	// Override BaseSumOperation::Initialize because
+	// IntervalAvgState does not have an assignment constructor from 0
+	static void Initialize(IntervalAvgState &state) {
+		AverageSetOperation::Initialize<IntervalAvgState>(state);
+	}
+
+	template <class RESULT_TYPE, class STATE>
+	static void Finalize(STATE &state, RESULT_TYPE &target, AggregateFinalizeData &finalize_data) {
+		if (state.count == 0) {
+			finalize_data.ReturnNull();
+		} else {
+			// DivideOperator does not borrow fractions right,
+			// TODO: Maybe it should?
+			// Copy PG implementation.
+			const auto &value = state.value;
+			const auto count = UnsafeNumericCast<int64_t>(state.count);
+
+			target.months = value.months / count;
+			auto months_remainder = value.months % count;
+
+			target.days = value.days / count;
+			auto days_remainder = value.days % count;
+
+			target.micros = value.micros / count;
+			auto micros_remainder = value.micros % count;
+
+			//	Shift the remainders right
+			months_remainder *= Interval::DAYS_PER_MONTH;
+			target.days += months_remainder / count;
+			days_remainder += months_remainder % count;
+
+			days_remainder *= Interval::MICROS_PER_DAY;
+			micros_remainder += days_remainder / count;
+			target.micros += micros_remainder;
+		}
+	}
+};
+
+struct TimeTZAverageOperation : public BaseSumOperation<AverageSetOperation, AddToHugeint> {
+	template <class INPUT_TYPE, class STATE, class OP>
+	static void Operation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &aggr_unary) {
+		const auto micros = Time::NormalizeTimeTZ(input).micros;
+		AverageSetOperation::template AddValues<STATE>(state, 1);
+		AddToHugeint::template AddNumber<STATE, int64_t>(state, micros);
+	}
+
+	template <class INPUT_TYPE, class STATE, class OP>
+	static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &aggr_unary, idx_t count) {
+		const auto micros = Time::NormalizeTimeTZ(input).micros;
+		AverageSetOperation::template AddValues<STATE>(state, count);
+		AddToHugeint::template AddConstant<STATE, int64_t>(state, micros, count);
+	}
+
+	template <class T, class STATE>
+	static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) {
+		if (state.count == 0) {
+			finalize_data.ReturnNull();
+		} else {
+			uint64_t remainder;
+			auto micros = Hugeint::Cast<int64_t>(Hugeint::DivModPositive(state.value, state.count, remainder));
+			// Round the result
+			micros += (remainder > (state.count / 2));
+			target = dtime_tz_t(dtime_t(micros), 0);
+		}
+	}
+};
+
 AggregateFunction GetAverageAggregate(PhysicalType type) {
 	switch (type) {
 	case PhysicalType::INT16: {
@@ -157,6 +255,10 @@ AggregateFunction GetAverageAggregate(PhysicalType type) {
 		return AggregateFunction::UnaryAggregate<AvgState<hugeint_t>, hugeint_t, double, HugeintAverageOperation>(
 		    LogicalType::HUGEINT, LogicalType::DOUBLE);
 	}
+	case PhysicalType::INTERVAL: {
+		return AggregateFunction::UnaryAggregate<IntervalAvgState, interval_t, interval_t, IntervalAverageOperation>(
+		    LogicalType::INTERVAL, LogicalType::INTERVAL);
+	}
 	default:
 		throw InternalException("Unimplemented average aggregate");
 	}
@@ -183,8 +285,20 @@ AggregateFunctionSet AvgFun::GetFunctions() {
 	avg.AddFunction(GetAverageAggregate(PhysicalType::INT32));
 	avg.AddFunction(GetAverageAggregate(PhysicalType::INT64));
 	avg.AddFunction(GetAverageAggregate(PhysicalType::INT128));
+	avg.AddFunction(GetAverageAggregate(PhysicalType::INTERVAL));
 	avg.AddFunction(AggregateFunction::UnaryAggregate<AvgState<double>, double, double, NumericAverageOperation>(
 	    LogicalType::DOUBLE, LogicalType::DOUBLE));
+
+	avg.AddFunction(AggregateFunction::UnaryAggregate<AvgState<hugeint_t>, int64_t, int64_t, DiscreteAverageOperation>(
+	    LogicalType::TIMESTAMP, LogicalType::TIMESTAMP));
+	avg.AddFunction(AggregateFunction::UnaryAggregate<AvgState<hugeint_t>, int64_t, int64_t, DiscreteAverageOperation>(
+	    LogicalType::TIMESTAMP_TZ, LogicalType::TIMESTAMP_TZ));
+	avg.AddFunction(AggregateFunction::UnaryAggregate<AvgState<hugeint_t>, int64_t, int64_t, DiscreteAverageOperation>(
+	    LogicalType::TIME, LogicalType::TIME));
+	avg.AddFunction(
+	    AggregateFunction::UnaryAggregate<AvgState<hugeint_t>, dtime_tz_t, dtime_tz_t, TimeTZAverageOperation>(
+	        LogicalType::TIME_TZ, LogicalType::TIME_TZ));
+
 	return avg;
 }
 

src/duckdb/extension/core_functions/aggregate/distributive/arg_min_max.cpp

@@ -545,8 +545,8 @@ class ArgMinMaxNState {
 	BinaryAggregateHeap<K, V, COMPARATOR> heap;
 
 	bool is_initialized = false;
-	void Initialize(idx_t nval) {
-		heap.Initialize(nval);
+	void Initialize(ArenaAllocator &allocator, idx_t nval) {
+		heap.Initialize(allocator, nval);
 		is_initialized = true;
 	}
 };
@@ -601,7 +601,7 @@ static void ArgMinMaxNUpdate(Vector inputs[], AggregateInputData &aggr_input, id
 			if (nval >= MAX_N) {
 				throw InvalidInputException("Invalid input for arg_min/arg_max: n value must be < %d", MAX_N);
 			}
-			state.Initialize(UnsafeNumericCast<idx_t>(nval));
+			state.Initialize(aggr_input.allocator, UnsafeNumericCast<idx_t>(nval));
 		}
 
 		// Now add the input to the heap