Update vendored DuckDB sources to 56852a4

Author: duckdblabs-bot

src/duckdb/extension/jemalloc/jemalloc/include/jemalloc/jemalloc.h

@@ -5,8 +5,8 @@
 extern "C" {
 #endif
 
-// DuckDB uses a 5s decay
-#define DUCKDB_JEMALLOC_DECAY 5
+// DuckDB uses a 1s decay
+#define DUCKDB_JEMALLOC_DECAY 1
 
 /* Defined if __attribute__((...)) syntax is supported. */
 #define JEMALLOC_HAVE_ATTR

src/duckdb/extension/json/json_multi_file_info.cpp

@@ -474,11 +474,12 @@ void ReadJSONFunction(ClientContext &context, JSONReader &json_reader, JSONScanG
 	const auto count = lstate.Read();
 	yyjson_val **values = scan_state.values;
 
+	auto &column_ids = json_reader.reader_data.column_ids;
 	if (!gstate.names.empty()) {
 		vector<Vector *> result_vectors;
-		result_vectors.reserve(gstate.column_ids.size());
-		for (const auto &col_idx : gstate.column_ids) {
-			result_vectors.emplace_back(&output.data[col_idx]);
+		result_vectors.reserve(column_ids.size());
+		for (idx_t i = 0; i < column_ids.size(); i++) {
+			result_vectors.emplace_back(&output.data[i]);
 		}
 
 		D_ASSERT(gstate.json_data.options.record_type != JSONRecordType::AUTO_DETECT);
@@ -521,9 +522,8 @@ void ReadJSONObjectsFunction(ClientContext &context, JSONReader &json_reader, JS
 
 	if (!gstate.names.empty()) {
 		// Create the strings without copying them
-		const auto col_idx = gstate.column_ids[0];
-		auto strings = FlatVector::GetData<string_t>(output.data[col_idx]);
-		auto &validity = FlatVector::Validity(output.data[col_idx]);
+		auto strings = FlatVector::GetData<string_t>(output.data[0]);
+		auto &validity = FlatVector::Validity(output.data[0]);
 		for (idx_t i = 0; i < count; i++) {
 			if (objects[i]) {
 				strings[i] = string_t(units[i].pointer, units[i].size);

src/duckdb/extension/parquet/parquet_reader.cpp

@@ -1120,9 +1120,8 @@ bool ParquetReader::ScanInternal(ClientContext &context, ParquetReaderScanState
 			} else {
 				auto local_idx = filter_entry.GetLocalIndex();
 				auto column_id = reader_data.column_ids[local_idx];
-				auto result_idx = reader_data.column_mapping[local_idx];
 
-				auto &result_vector = result.data[result_idx];
+				auto &result_vector = result.data[local_idx];
 				auto &child_reader = root_reader.GetChildReader(column_id);
 				child_reader.Filter(scan_count, define_ptr, repeat_ptr, result_vector, scan_filter.filter,
 				                    *scan_filter.filter_state, state.sel, filter_count, i == 0);
@@ -1142,7 +1141,7 @@ bool ParquetReader::ScanInternal(ClientContext &context, ParquetReaderScanState
 				root_reader.GetChildReader(file_col_idx).Skip(result.size());
 				continue;
 			}
-			auto &result_vector = result.data[reader_data.column_mapping[col_idx].GetIndex()];
+			auto &result_vector = result.data[i];
 			auto &child_reader = root_reader.GetChildReader(file_col_idx);
 			child_reader.Select(result.size(), define_ptr, repeat_ptr, result_vector, state.sel, filter_count);
 		}
@@ -1153,8 +1152,7 @@ bool ParquetReader::ScanInternal(ClientContext &context, ParquetReaderScanState
 		for (idx_t i = 0; i < reader_data.column_ids.size(); i++) {
 			auto col_idx = MultiFileLocalIndex(i);
 			auto file_col_idx = reader_data.column_ids[col_idx];
-			auto global_col_idx = reader_data.column_mapping[col_idx];
-			auto &result_vector = result.data[global_col_idx];
+			auto &result_vector = result.data[i];
 			auto &child_reader = root_reader.GetChildReader(file_col_idx);
 			auto rows_read = child_reader.Read(scan_count, define_ptr, repeat_ptr, result_vector);
 			if (rows_read != scan_count) {

src/duckdb/src/common/adbc/adbc.cpp

@@ -17,7 +17,7 @@
 #include "duckdb/common/adbc/options.h"
 #include "duckdb/common/adbc/single_batch_array_stream.hpp"
 #include "duckdb/function/table/arrow.hpp"
-
+#include "duckdb/common/adbc/wrappers.hpp"
 #include <stdlib.h>
 #include <string.h>
 
@@ -249,7 +249,9 @@ AdbcStatusCode ConnectionNew(struct AdbcConnection *connection, struct AdbcError
 		return ADBC_STATUS_INVALID_ARGUMENT;
 	}
 
-	connection->private_data = nullptr;
+	auto connection_wrapper = new duckdb::DuckDBAdbcConnectionWrapper();
+	connection_wrapper->connection = nullptr;
+	connection->private_data = connection_wrapper;
 	return ADBC_STATUS_OK;
 }
 
@@ -263,45 +265,65 @@ AdbcStatusCode ExecuteQuery(duckdb::Connection *conn, const char *query, struct
 	return ADBC_STATUS_OK;
 }
 
+AdbcStatusCode InternalSetOption(duckdb::Connection &conn, std::unordered_map<std::string, std::string> &options,
+                                 struct AdbcError *error) {
+	// If we got here, the options have already been validated and are acceptable
+	for (auto &option : options) {
+		if (strcmp(option.first.c_str(), ADBC_CONNECTION_OPTION_AUTOCOMMIT) == 0) {
+			if (strcmp(option.second.c_str(), ADBC_OPTION_VALUE_ENABLED) == 0) {
+				if (conn.HasActiveTransaction()) {
+					AdbcStatusCode status = ExecuteQuery(&conn, "COMMIT", error);
+					if (status != ADBC_STATUS_OK) {
+						options.clear();
+						return status;
+					}
+				}
+			} else if (strcmp(option.second.c_str(), ADBC_OPTION_VALUE_DISABLED) == 0) {
+				if (!conn.HasActiveTransaction()) {
+					AdbcStatusCode status = ExecuteQuery(&conn, "START TRANSACTION", error);
+					if (status != ADBC_STATUS_OK) {
+						options.clear();
+						return status;
+					}
+				}
+			}
+		}
+	}
+	options.clear();
+	return ADBC_STATUS_OK;
+}
 AdbcStatusCode ConnectionSetOption(struct AdbcConnection *connection, const char *key, const char *value,
                                    struct AdbcError *error) {
 	if (!connection) {
 		SetError(error, "Connection is not set");
 		return ADBC_STATUS_INVALID_ARGUMENT;
 	}
-
-	auto conn = static_cast<duckdb::Connection *>(connection->private_data);
+	std::string key_string = std::string(key);
+	std::string key_value = std::string(value);
+	auto conn_wrapper = static_cast<duckdb::DuckDBAdbcConnectionWrapper *>(connection->private_data);
 	if (strcmp(key, ADBC_CONNECTION_OPTION_AUTOCOMMIT) == 0) {
 		if (strcmp(value, ADBC_OPTION_VALUE_ENABLED) == 0) {
-			if (conn->HasActiveTransaction()) {
-				AdbcStatusCode status = ExecuteQuery(conn, "COMMIT", error);
-				if (status != ADBC_STATUS_OK) {
-					return status;
-				}
-			} else {
-				// no-op
-			}
+			conn_wrapper->options[key_string] = key_value;
 		} else if (strcmp(value, ADBC_OPTION_VALUE_DISABLED) == 0) {
-			if (conn->HasActiveTransaction()) {
-				// no-op
-			} else {
-				// begin
-				AdbcStatusCode status = ExecuteQuery(conn, "START TRANSACTION", error);
-				if (status != ADBC_STATUS_OK) {
-					return status;
-				}
-			}
+			conn_wrapper->options[key_string] = key_value;
 		} else {
 			auto error_message = "Invalid connection option value " + std::string(key) + "=" + std::string(value);
 			SetError(error, error_message);
 			return ADBC_STATUS_INVALID_ARGUMENT;
 		}
+	} else {
+		// This is an unknown option to the DuckDB driver
+		auto error_message =
+		    "Unknown connection option " + std::string(key) + "=" + (value ? std::string(value) : "(NULL)");
+		SetError(error, error_message);
+		return ADBC_STATUS_NOT_IMPLEMENTED;
+	}
+	if (!conn_wrapper->connection) {
+		// If the connection has not yet been initialized, we just return here.
 		return ADBC_STATUS_OK;
 	}
-	auto error_message =
-	    "Unknown connection option " + std::string(key) + "=" + (value ? std::string(value) : "(NULL)");
-	SetError(error, error_message);
-	return ADBC_STATUS_NOT_IMPLEMENTED;
+	auto conn = reinterpret_cast<duckdb::Connection *>(conn_wrapper->connection);
+	return InternalSetOption(*conn, conn_wrapper->options, error);
 }
 
 AdbcStatusCode ConnectionReadPartition(struct AdbcConnection *connection, const uint8_t *serialized_partition,
@@ -323,7 +345,8 @@ AdbcStatusCode ConnectionCommit(struct AdbcConnection *connection, struct AdbcEr
 		SetError(error, "Connection is not set");
 		return ADBC_STATUS_INVALID_ARGUMENT;
 	}
-	auto conn = static_cast<duckdb::Connection *>(connection->private_data);
+	auto conn_wrapper = static_cast<duckdb::DuckDBAdbcConnectionWrapper *>(connection->private_data);
+	auto conn = reinterpret_cast<duckdb::Connection *>(conn_wrapper->connection);
 	if (!conn->HasActiveTransaction()) {
 		SetError(error, "No active transaction, cannot commit");
 		return ADBC_STATUS_INVALID_STATE;
@@ -341,7 +364,8 @@ AdbcStatusCode ConnectionRollback(struct AdbcConnection *connection, struct Adbc
 		SetError(error, "Connection is not set");
 		return ADBC_STATUS_INVALID_ARGUMENT;
 	}
-	auto conn = static_cast<duckdb::Connection *>(connection->private_data);
+	auto conn_wrapper = static_cast<duckdb::DuckDBAdbcConnectionWrapper *>(connection->private_data);
+	auto conn = reinterpret_cast<duckdb::Connection *>(conn_wrapper->connection);
 	if (!conn->HasActiveTransaction()) {
 		SetError(error, "No active transaction, cannot rollback");
 		return ADBC_STATUS_INVALID_STATE;
@@ -479,16 +503,25 @@ AdbcStatusCode ConnectionInit(struct AdbcConnection *connection, struct AdbcData
 		return ADBC_STATUS_INVALID_ARGUMENT;
 	}
 	auto database_wrapper = static_cast<DuckDBAdbcDatabaseWrapper *>(database->private_data);
+	auto conn_wrapper = static_cast<duckdb::DuckDBAdbcConnectionWrapper *>(connection->private_data);
+	conn_wrapper->connection = nullptr;
 
-	connection->private_data = nullptr;
-	auto res =
-	    duckdb_connect(database_wrapper->database, reinterpret_cast<duckdb_connection *>(&connection->private_data));
-	return CheckResult(res, error, "Failed to connect to Database");
+	auto res = duckdb_connect(database_wrapper->database, &conn_wrapper->connection);
+	auto adbc_status = CheckResult(res, error, "Failed to connect to Database");
+	if (adbc_status != ADBC_STATUS_OK) {
+		return adbc_status;
+	}
+	// We might have options to set
+	auto conn = reinterpret_cast<duckdb::Connection *>(conn_wrapper->connection);
+	return InternalSetOption(*conn, conn_wrapper->options, error);
 }
 
 AdbcStatusCode ConnectionRelease(struct AdbcConnection *connection, struct AdbcError *error) {
 	if (connection && connection->private_data) {
-		duckdb_disconnect(reinterpret_cast<duckdb_connection *>(&connection->private_data));
+		auto conn_wrapper = static_cast<duckdb::DuckDBAdbcConnectionWrapper *>(connection->private_data);
+		auto conn = reinterpret_cast<duckdb::Connection *>(conn_wrapper->connection);
+		duckdb_disconnect(reinterpret_cast<duckdb_connection *>(&conn));
+		delete conn_wrapper;
 		connection->private_data = nullptr;
 	}
 	return ADBC_STATUS_OK;
@@ -638,7 +671,9 @@ AdbcStatusCode StatementNew(struct AdbcConnection *connection, struct AdbcStatem
 	}
 
 	statement->private_data = statement_wrapper;
-	statement_wrapper->connection = static_cast<duckdb_connection>(connection->private_data);
+	auto conn_wrapper = static_cast<duckdb::DuckDBAdbcConnectionWrapper *>(connection->private_data);
+
+	statement_wrapper->connection = conn_wrapper->connection;
 	statement_wrapper->statement = nullptr;
 	statement_wrapper->result = nullptr;
 	statement_wrapper->ingestion_stream.release = nullptr;

src/duckdb/src/common/adbc/driver_manager.cpp

@@ -1080,13 +1080,15 @@ AdbcStatusCode AdbcConnectionGetTableTypes(struct AdbcConnection *connection, st
 
 AdbcStatusCode AdbcConnectionInit(struct AdbcConnection *connection, struct AdbcDatabase *database,
                                   struct AdbcError *error) {
+
 	if (!connection->private_data) {
 		SetError(error, "Must call AdbcConnectionNew first");
 		return ADBC_STATUS_INVALID_STATE;
 	} else if (!database->private_driver) {
 		SetError(error, "Database is not initialized");
 		return ADBC_STATUS_INVALID_ARGUMENT;
 	}
+
 	TempConnection *args = reinterpret_cast<TempConnection *>(connection->private_data);
 	connection->private_data = nullptr;
 	std::unordered_map<std::string, std::string> options = std::move(args->options);
@@ -1176,7 +1178,7 @@ AdbcStatusCode AdbcConnectionRollback(struct AdbcConnection *connection, struct
 
 AdbcStatusCode AdbcConnectionSetOption(struct AdbcConnection *connection, const char *key, const char *value,
                                        struct AdbcError *error) {
-	if (!connection->private_data) {
+	if (!connection || !connection->private_data) {
 		SetError(error, "AdbcConnectionSetOption: must AdbcConnectionNew first");
 		return ADBC_STATUS_INVALID_STATE;
 	}

src/duckdb/src/common/arrow/appender/null_data.cpp

@@ -0,0 +1,19 @@
+#include "duckdb/common/arrow/arrow_appender.hpp"
+#include "duckdb/common/arrow/appender/null_data.hpp"
+
+namespace duckdb {
+
+void ArrowNullData::Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
+	// nop
+}
+
+void ArrowNullData::Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
+	idx_t size = to - from;
+	append_data.row_count += size;
+}
+
+void ArrowNullData::Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result) {
+	result->n_buffers = 0;
+}
+
+} // namespace duckdb

src/duckdb/src/common/arrow/arrow_appender.cpp

@@ -144,6 +144,9 @@ static void InitializeAppenderForType(ArrowAppendData &append_data) {
 static void InitializeFunctionPointers(ArrowAppendData &append_data, const LogicalType &type) {
 	// handle special logical types
 	switch (type.id()) {
+	case LogicalTypeId::SQLNULL:
+		InitializeAppenderForType<ArrowNullData>(append_data);
+		break;
 	case LogicalTypeId::BOOLEAN:
 		InitializeAppenderForType<ArrowBoolData>(append_data);
 		break;

src/duckdb/src/common/arrow/arrow_converter.cpp

@@ -61,6 +61,30 @@ void InitializeChild(ArrowSchema &child, DuckDBArrowSchemaHolder &root_holder, c
 void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type,
                     ClientProperties &options, ClientContext &context);
 
+void SetArrowStructFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type,
+                          ClientProperties &options, ClientContext &context, bool map_is_parent = false) {
+	child.format = "+s";
+	auto &child_types = StructType::GetChildTypes(type);
+	child.n_children = NumericCast<int64_t>(child_types.size());
+	root_holder.nested_children.emplace_back();
+	root_holder.nested_children.back().resize(child_types.size());
+	root_holder.nested_children_ptr.emplace_back();
+	root_holder.nested_children_ptr.back().resize(child_types.size());
+	for (idx_t type_idx = 0; type_idx < child_types.size(); type_idx++) {
+		root_holder.nested_children_ptr.back()[type_idx] = &root_holder.nested_children.back()[type_idx];
+	}
+	child.children = &root_holder.nested_children_ptr.back()[0];
+	for (size_t type_idx = 0; type_idx < child_types.size(); type_idx++) {
+		InitializeChild(*child.children[type_idx], root_holder);
+		root_holder.owned_type_names.push_back(AddName(child_types[type_idx].first));
+		child.children[type_idx]->name = root_holder.owned_type_names.back().get();
+		SetArrowFormat(root_holder, *child.children[type_idx], child_types[type_idx].second, options, context);
+	}
+	if (map_is_parent) {
+		child.children[0]->flags = 0; // Set the 'keys' field to non-nullable
+	}
+}
+
 void SetArrowMapFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type,
                        ClientProperties &options, ClientContext &context) {
 	child.format = "+m";
@@ -74,7 +98,7 @@ void SetArrowMapFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child,
 	child.children = &root_holder.nested_children_ptr.back()[0];
 	child.children[0]->name = "entries";
 	child.children[0]->flags = 0; // Set the 'entries' field to non-nullable
-	SetArrowFormat(root_holder, **child.children, ListType::GetChildType(type), options, context);
+	SetArrowStructFormat(root_holder, **child.children, ListType::GetChildType(type), options, context, true);
 }
 
 bool SetArrowExtension(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type,
@@ -265,26 +289,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		break;
 	}
 	case LogicalTypeId::STRUCT: {
-		child.format = "+s";
-		auto &child_types = StructType::GetChildTypes(type);
-		child.n_children = NumericCast<int64_t>(child_types.size());
-		root_holder.nested_children.emplace_back();
-		root_holder.nested_children.back().resize(child_types.size());
-		root_holder.nested_children_ptr.emplace_back();
-		root_holder.nested_children_ptr.back().resize(child_types.size());
-		for (idx_t type_idx = 0; type_idx < child_types.size(); type_idx++) {
-			root_holder.nested_children_ptr.back()[type_idx] = &root_holder.nested_children.back()[type_idx];
-		}
-		child.children = &root_holder.nested_children_ptr.back()[0];
-		for (size_t type_idx = 0; type_idx < child_types.size(); type_idx++) {
-
-			InitializeChild(*child.children[type_idx], root_holder);
-
-			root_holder.owned_type_names.push_back(AddName(child_types[type_idx].first));
-
-			child.children[type_idx]->name = root_holder.owned_type_names.back().get();
-			SetArrowFormat(root_holder, *child.children[type_idx], child_types[type_idx].second, options, context);
-		}
+		SetArrowStructFormat(root_holder, child, type, options, context);
 		break;
 	}
 	case LogicalTypeId::ARRAY: {