Update vendored DuckDB sources to e78a49891b

CMakeLists.txt

@@ -343,44 +343,44 @@ set(DUCKDB_SRC_FILES
   src/duckdb/extension/core_functions/core_functions_extension.cpp
   src/duckdb/extension/core_functions/function_list.cpp
   src/duckdb/extension/core_functions/lambda_functions.cpp
-  src/duckdb/ub_extension_core_functions_scalar_generic.cpp
-  src/duckdb/ub_extension_core_functions_scalar_union.cpp
-  src/duckdb/ub_extension_core_functions_scalar_list.cpp
-  src/duckdb/ub_extension_core_functions_scalar_random.cpp
-  src/duckdb/ub_extension_core_functions_scalar_operators.cpp
-  src/duckdb/ub_extension_core_functions_scalar_blob.cpp
   src/duckdb/ub_extension_core_functions_scalar_map.cpp
+  src/duckdb/ub_extension_core_functions_scalar_operators.cpp
+  src/duckdb/ub_extension_core_functions_scalar_struct.cpp
+  src/duckdb/ub_extension_core_functions_scalar_random.cpp
   src/duckdb/ub_extension_core_functions_scalar_string.cpp
+  src/duckdb/ub_extension_core_functions_scalar_blob.cpp
+  src/duckdb/ub_extension_core_functions_scalar_union.cpp
+  src/duckdb/ub_extension_core_functions_scalar_bit.cpp
+  src/duckdb/ub_extension_core_functions_scalar_array.cpp
+  src/duckdb/ub_extension_core_functions_scalar_math.cpp
   src/duckdb/ub_extension_core_functions_scalar_debug.cpp
   src/duckdb/ub_extension_core_functions_scalar_enum.cpp
-  src/duckdb/ub_extension_core_functions_scalar_array.cpp
+  src/duckdb/ub_extension_core_functions_scalar_generic.cpp
   src/duckdb/ub_extension_core_functions_scalar_date.cpp
-  src/duckdb/ub_extension_core_functions_scalar_struct.cpp
-  src/duckdb/ub_extension_core_functions_scalar_bit.cpp
-  src/duckdb/ub_extension_core_functions_scalar_math.cpp
+  src/duckdb/ub_extension_core_functions_scalar_list.cpp
   src/duckdb/ub_extension_core_functions_aggregate_nested.cpp
-  src/duckdb/ub_extension_core_functions_aggregate_holistic.cpp
   src/duckdb/ub_extension_core_functions_aggregate_regression.cpp
-  src/duckdb/ub_extension_core_functions_aggregate_distributive.cpp
   src/duckdb/ub_extension_core_functions_aggregate_algebraic.cpp
-  src/duckdb/extension/parquet/serialize_parquet.cpp
-  src/duckdb/extension/parquet/parquet_reader.cpp
-  src/duckdb/extension/parquet/parquet_crypto.cpp
+  src/duckdb/ub_extension_core_functions_aggregate_distributive.cpp
+  src/duckdb/ub_extension_core_functions_aggregate_holistic.cpp
   src/duckdb/extension/parquet/parquet_writer.cpp
-  src/duckdb/extension/parquet/zstd_file_system.cpp
   src/duckdb/extension/parquet/geo_parquet.cpp
+  src/duckdb/extension/parquet/column_reader.cpp
   src/duckdb/extension/parquet/parquet_float16.cpp
+  src/duckdb/extension/parquet/parquet_metadata.cpp
+  src/duckdb/extension/parquet/parquet_reader.cpp
+  src/duckdb/extension/parquet/zstd_file_system.cpp
   src/duckdb/extension/parquet/parquet_timestamp.cpp
-  src/duckdb/extension/parquet/column_reader.cpp
   src/duckdb/extension/parquet/parquet_multi_file_info.cpp
-  src/duckdb/extension/parquet/parquet_file_metadata_cache.cpp
-  src/duckdb/extension/parquet/parquet_metadata.cpp
   src/duckdb/extension/parquet/column_writer.cpp
   src/duckdb/extension/parquet/parquet_extension.cpp
+  src/duckdb/extension/parquet/parquet_crypto.cpp
+  src/duckdb/extension/parquet/parquet_file_metadata_cache.cpp
+  src/duckdb/extension/parquet/serialize_parquet.cpp
   src/duckdb/extension/parquet/parquet_statistics.cpp
   src/duckdb/ub_extension_parquet_decoder.cpp
-  src/duckdb/ub_extension_parquet_writer.cpp
   src/duckdb/ub_extension_parquet_reader.cpp
+  src/duckdb/ub_extension_parquet_writer.cpp
   src/duckdb/third_party/parquet/parquet_types.cpp
   src/duckdb/third_party/thrift/thrift/protocol/TProtocol.cpp
   src/duckdb/third_party/thrift/thrift/transport/TTransportException.cpp
@@ -419,32 +419,32 @@ set(DUCKDB_SRC_FILES
   src/duckdb/third_party/brotli/enc/metablock.cpp
   src/duckdb/third_party/brotli/enc/static_dict.cpp
   src/duckdb/third_party/brotli/enc/utf8_util.cpp
-  src/duckdb/extension/icu/./icu-datetrunc.cpp
-  src/duckdb/extension/icu/./icu-timezone.cpp
   src/duckdb/extension/icu/./icu-list-range.cpp
+  src/duckdb/extension/icu/./icu_extension.cpp
+  src/duckdb/extension/icu/./icu-strptime.cpp
+  src/duckdb/extension/icu/./icu-current.cpp
+  src/duckdb/extension/icu/./icu-makedate.cpp
   src/duckdb/extension/icu/./icu-datesub.cpp
-  src/duckdb/extension/icu/./icu-datefunc.cpp
+  src/duckdb/extension/icu/./icu-timezone.cpp
   src/duckdb/extension/icu/./icu-timebucket.cpp
-  src/duckdb/extension/icu/./icu-current.cpp
-  src/duckdb/extension/icu/./icu-dateadd.cpp
-  src/duckdb/extension/icu/./icu-datepart.cpp
   src/duckdb/extension/icu/./icu-table-range.cpp
-  src/duckdb/extension/icu/./icu-strptime.cpp
-  src/duckdb/extension/icu/./icu_extension.cpp
-  src/duckdb/extension/icu/./icu-makedate.cpp
+  src/duckdb/extension/icu/./icu-datefunc.cpp
+  src/duckdb/extension/icu/./icu-datepart.cpp
+  src/duckdb/extension/icu/./icu-datetrunc.cpp
+  src/duckdb/extension/icu/./icu-dateadd.cpp
   src/duckdb/ub_extension_icu_third_party_icu_common.cpp
   src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp
   src/duckdb/extension/icu/third_party/icu/stubdata/stubdata.cpp
-  src/duckdb/extension/json/json_enums.cpp
-  src/duckdb/extension/json/json_deserializer.cpp
-  src/duckdb/extension/json/json_reader.cpp
   src/duckdb/extension/json/json_common.cpp
-  src/duckdb/extension/json/json_serializer.cpp
-  src/duckdb/extension/json/serialize_json.cpp
   src/duckdb/extension/json/json_extension.cpp
-  src/duckdb/extension/json/json_functions.cpp
-  src/duckdb/extension/json/json_scan.cpp
   src/duckdb/extension/json/json_multi_file_info.cpp
+  src/duckdb/extension/json/json_scan.cpp
+  src/duckdb/extension/json/json_enums.cpp
+  src/duckdb/extension/json/json_functions.cpp
+  src/duckdb/extension/json/json_reader.cpp
+  src/duckdb/extension/json/json_deserializer.cpp
+  src/duckdb/extension/json/serialize_json.cpp
+  src/duckdb/extension/json/json_serializer.cpp
   src/duckdb/ub_extension_json_json_functions.cpp)
 
 set(JEMALLOC_SRC_FILES 

src/duckdb/extension/core_functions/scalar/generic/current_setting.cpp

@@ -50,9 +50,11 @@ unique_ptr<FunctionData> CurrentSettingBind(ClientContext &context, ScalarFuncti
 	auto key = StringUtil::Lower(StringValue::Get(key_val));
 	Value val;
 	if (!context.TryGetCurrentSetting(key, val)) {
-		Catalog::AutoloadExtensionByConfigName(context, key);
+		auto extension_name = Catalog::AutoloadExtensionByConfigName(context, key);
 		// If autoloader didn't throw, the config is now available
-		context.TryGetCurrentSetting(key, val);
+		if (!context.TryGetCurrentSetting(key, val)) {
+			throw InternalException("Extension %s did not provide the '%s' config setting", extension_name, key);
+		}
 	}
 
 	bound_function.return_type = val.type();

src/duckdb/extension/parquet/include/writer/templated_column_writer.hpp

@@ -20,8 +20,9 @@ namespace duckdb {
 template <class SRC, class TGT, class OP = ParquetCastOperator, bool ALL_VALID>
 static void TemplatedWritePlain(Vector &col, ColumnWriterStatistics *stats, const idx_t chunk_start,
                                 const idx_t chunk_end, const ValidityMask &mask, WriteStream &ser) {
-	static constexpr bool COPY_DIRECTLY_FROM_VECTOR =
-	    ALL_VALID && std::is_same<SRC, TGT>::value && std::is_arithmetic<TGT>::value;
+	static constexpr bool COPY_DIRECTLY_FROM_VECTOR = ALL_VALID && std::is_same<SRC, TGT>::value &&
+	                                                  std::is_arithmetic<TGT>::value &&
+	                                                  std::is_same<OP, ParquetCastOperator>::value;
 
 	const auto *const ptr = FlatVector::GetData<SRC>(col);
 

src/duckdb/extension/parquet/parquet_reader.cpp

@@ -54,12 +54,7 @@ static unique_ptr<duckdb_apache::thrift::protocol::TProtocol> CreateThriftFilePr
 
 static bool ShouldAndCanPrefetch(ClientContext &context, CachingFileHandle &file_handle) {
 	Value disable_prefetch = false;
-	Value prefetch_all_files =
-#ifdef DUCKDB_PREFETCH_ALL_PARQUET_FILES
-	    true; // For debugging purposes we can toggle this to always enable
-#else
-	    false; // Defaults to false
-#endif
+	Value prefetch_all_files = false;
 	context.TryGetCurrentSetting("disable_parquet_prefetching", disable_prefetch);
 	context.TryGetCurrentSetting("prefetch_all_parquet_files", prefetch_all_files);
 	bool should_prefetch = !file_handle.OnDiskFile() || prefetch_all_files.GetValue<bool>();

src/duckdb/src/catalog/catalog.cpp

@@ -534,14 +534,14 @@ bool Catalog::TryAutoLoad(ClientContext &context, const string &original_name) n
 	return false;
 }
 
-void Catalog::AutoloadExtensionByConfigName(ClientContext &context, const string &configuration_name) {
+string Catalog::AutoloadExtensionByConfigName(ClientContext &context, const string &configuration_name) {
 #ifndef DUCKDB_DISABLE_EXTENSION_LOAD
 	auto &dbconfig = DBConfig::GetConfig(context);
 	if (dbconfig.options.autoload_known_extensions) {
 		auto extension_name = ExtensionHelper::FindExtensionInEntries(configuration_name, EXTENSION_SETTINGS);
 		if (ExtensionHelper::CanAutoloadExtension(extension_name)) {
 			ExtensionHelper::AutoLoadExtension(context, extension_name);
-			return;
+			return extension_name;
 		}
 	}
 #endif

src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp

@@ -858,12 +858,12 @@ unique_ptr<CatalogEntry> DuckTableEntry::RenameField(ClientContext &context, Ren
 	if (!ColumnExists(info.column_path[0])) {
 		throw CatalogException("Cannot rename field from column \"%s\" - it does not exist", info.column_path[0]);
 	}
+
 	// follow the path
 	auto &col = GetColumn(info.column_path[0]);
 	auto res = RenameFieldFromStruct(col.Type(), info.column_path, info.new_name, 1);
 	if (res.error.HasError()) {
 		res.error.Throw();
-		return nullptr;
 	}
 
 	// construct the struct remapping expression
@@ -874,7 +874,6 @@ unique_ptr<CatalogEntry> DuckTableEntry::RenameField(ClientContext &context, Ren
 	children.push_back(make_uniq<ConstantExpression>(Value()));
 
 	auto function = make_uniq<FunctionExpression>("remap_struct", std::move(children));
-
 	ChangeColumnTypeInfo change_column_type(info.GetAlterEntryData(), info.column_path[0], std::move(res.new_type),
 	                                        std::move(function));
 	return ChangeColumnType(context, change_column_type);

src/duckdb/src/common/arrow/appender/append_data.cpp

@@ -0,0 +1,29 @@
+#include "duckdb/common/arrow/appender/append_data.hpp"
+
+namespace duckdb {
+
+void ArrowAppendData::AppendValidity(UnifiedVectorFormat &format, idx_t from, idx_t to) {
+	// resize the buffer, filling the validity buffer with all valid values
+	idx_t size = to - from;
+	ResizeValidity(GetValidityBuffer(), row_count + size);
+	if (format.validity.AllValid()) {
+		// if all values are valid we don't need to do anything else
+		return;
+	}
+
+	// otherwise we iterate through the validity mask
+	auto validity_data = (uint8_t *)GetValidityBuffer().data();
+	uint8_t current_bit;
+	idx_t current_byte;
+	GetBitPosition(row_count, current_byte, current_bit);
+	for (idx_t i = from; i < to; i++) {
+		auto source_idx = format.sel->get_index(i);
+		// append the validity mask
+		if (!format.validity.RowIsValid(source_idx)) {
+			SetNull(validity_data, current_byte, current_bit);
+		}
+		NextBit(current_byte, current_bit);
+	}
+}
+
+} // namespace duckdb

src/duckdb/src/common/arrow/appender/bool_data.cpp

@@ -6,7 +6,6 @@ namespace duckdb {
 void ArrowBoolData::Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
 	auto byte_count = (capacity + 7) / 8;
 	result.GetMainBuffer().reserve(byte_count);
-	(void)AppendValidity; // silence a compiler warning about unused static function
 }
 
 void ArrowBoolData::Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
@@ -16,24 +15,24 @@ void ArrowBoolData::Append(ArrowAppendData &append_data, Vector &input, idx_t fr
 	auto &main_buffer = append_data.GetMainBuffer();
 	auto &validity_buffer = append_data.GetValidityBuffer();
 	// we initialize both the validity and the bit set to 1's
-	ResizeValidity(validity_buffer, append_data.row_count + size);
-	ResizeValidity(main_buffer, append_data.row_count + size);
+	ArrowAppendData::ResizeValidity(validity_buffer, append_data.row_count + size);
+	ArrowAppendData::ResizeValidity(main_buffer, append_data.row_count + size);
 	auto data = UnifiedVectorFormat::GetData<bool>(format);
 
 	auto result_data = main_buffer.GetData<uint8_t>();
 	auto validity_data = validity_buffer.GetData<uint8_t>();
 	uint8_t current_bit;
 	idx_t current_byte;
-	GetBitPosition(append_data.row_count, current_byte, current_bit);
+	ArrowAppendData::GetBitPosition(append_data.row_count, current_byte, current_bit);
 	for (idx_t i = from; i < to; i++) {
 		auto source_idx = format.sel->get_index(i);
 		// append the validity mask
 		if (!format.validity.RowIsValid(source_idx)) {
-			SetNull(append_data, validity_data, current_byte, current_bit);
+			append_data.SetNull(validity_data, current_byte, current_bit);
 		} else if (!data[source_idx]) {
-			UnsetBit(result_data, current_byte, current_bit);
+			ArrowAppendData::UnsetBit(result_data, current_byte, current_bit);
 		}
-		NextBit(current_byte, current_bit);
+		ArrowAppendData::NextBit(current_byte, current_bit);
 	}
 	append_data.row_count += size;
 }

src/duckdb/src/common/arrow/appender/fixed_size_list_data.cpp

@@ -18,7 +18,7 @@ void ArrowFixedSizeListData::Append(ArrowAppendData &append_data, Vector &input,
 	UnifiedVectorFormat format;
 	input.ToUnifiedFormat(input_size, format);
 	idx_t size = to - from;
-	AppendValidity(append_data, format, from, to);
+	append_data.AppendValidity(format, from, to);
 	input.Flatten(input_size);
 	auto array_size = ArrayType::GetSize(input.GetType());
 	auto &child_vector = ArrayVector::GetEntry(input);

src/duckdb/src/common/arrow/appender/struct_data.cpp

@@ -18,7 +18,7 @@ void ArrowStructData::Append(ArrowAppendData &append_data, Vector &input, idx_t
 	UnifiedVectorFormat format;
 	input.ToUnifiedFormat(input_size, format);
 	idx_t size = to - from;
-	AppendValidity(append_data, format, from, to);
+	append_data.AppendValidity(format, from, to);
 	// append the children of the struct
 	auto &children = StructVector::GetEntries(input);
 	for (idx_t child_idx = 0; child_idx < children.size(); child_idx++) {

src/duckdb/src/common/arrow/appender/union_data.cpp

@@ -14,7 +14,6 @@ void ArrowUnionData::Initialize(ArrowAppendData &result, const LogicalType &type
 		auto child_buffer = ArrowAppender::InitializeChild(child.second, capacity, result.options);
 		result.child_data.push_back(std::move(child_buffer));
 	}
-	(void)AppendValidity; // silence a compiler warning about unused static functiondep
 }
 
 void ArrowUnionData::Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {

src/duckdb/src/common/arrow/schema_metadata.cpp

@@ -66,8 +66,8 @@ ArrowSchemaMetadata ArrowSchemaMetadata::NonCanonicalType(const string &type_nam
 	ArrowSchemaMetadata metadata;
 	metadata.AddOption(ARROW_EXTENSION_NAME, ArrowExtensionMetadata::ARROW_EXTENSION_NON_CANONICAL);
 	// We have to set the metadata key with type_name and vendor_name.
-	metadata.extension_metadata_map->AddObject("vendor_name", make_uniq<ComplexJSON>(vendor_name));
-	metadata.extension_metadata_map->AddObject("type_name", make_uniq<ComplexJSON>(type_name));
+	metadata.extension_metadata_map->AddObjectEntry("vendor_name", make_uniq<ComplexJSON>(vendor_name));
+	metadata.extension_metadata_map->AddObjectEntry("type_name", make_uniq<ComplexJSON>(type_name));
 	metadata.AddOption(ARROW_METADATA_KEY, StringUtil::ToComplexJSONMap(*metadata.extension_metadata_map));
 	return metadata;
 }

src/duckdb/src/common/complex_json.cpp

@@ -1,19 +1,24 @@
 #include "duckdb/common/complex_json.hpp"
 
 namespace duckdb {
-ComplexJSON::ComplexJSON(const string &str) : str_value(str), is_object(false) {
+ComplexJSON::ComplexJSON(const string &str) : str_value(str), type(ComplexJSONType::VALUE) {
 }
 
-ComplexJSON::ComplexJSON() : is_object(false) {
+ComplexJSON::ComplexJSON() : type(ComplexJSONType::VALUE) {
 }
 
-void ComplexJSON::AddObject(const string &key, unique_ptr<ComplexJSON> object) {
-	is_object = true;
+void ComplexJSON::AddObjectEntry(const string &key, unique_ptr<ComplexJSON> object) {
+	type = ComplexJSONType::OBJECT;
 	obj_value[key] = std::move(object);
 }
 
+void ComplexJSON::AddArrayElement(unique_ptr<ComplexJSON> object) {
+	type = ComplexJSONType::ARRAY;
+	arr_value.push_back(std::move(object));
+}
+
 ComplexJSON &ComplexJSON::GetObject(const string &key) {
-	if (is_object) {
+	if (type == ComplexJSONType::OBJECT) {
 		if (obj_value.find(key) == obj_value.end()) {
 			throw InvalidInputException("Complex JSON Key not found");
 		}
@@ -22,6 +27,16 @@ ComplexJSON &ComplexJSON::GetObject(const string &key) {
 	throw InvalidInputException("ComplexJson is not an object");
 }
 
+ComplexJSON &ComplexJSON::GetArrayElement(const idx_t &index) {
+	if (type == ComplexJSONType::ARRAY) {
+		if (index >= arr_value.size()) {
+			throw InvalidInputException("Complex JSON array element out of bounds");
+		}
+		return *arr_value[index];
+	}
+	throw InvalidInputException("ComplexJson is not an array");
+}
+
 unordered_map<string, string> ComplexJSON::Flatten() const {
 	unordered_map<string, string> result;
 	for (auto &obj : obj_value) {