Update vendored DuckDB sources to 24c295a9dd

src/duckdb/src/catalog/catalog_set.cpp

@@ -401,8 +401,6 @@ bool CatalogSet::DropEntryInternal(CatalogTransaction transaction, const string
 		throw CatalogException("Cannot drop entry \"%s\" because it is an internal system entry", entry->name);
 	}
 
-	entry->OnDrop();
-
 	// create a new tombstone entry and replace the currently stored one
 	// set the timestamp to the timestamp of the current transaction
 	// and point it at the tombstone node
@@ -454,6 +452,7 @@ void CatalogSet::VerifyExistenceOfDependency(transaction_t commit_id, CatalogEnt
 void CatalogSet::CommitDrop(transaction_t commit_id, transaction_t start_time, CatalogEntry &entry) {
 	auto &duck_catalog = GetCatalog();
 
+	entry.OnDrop();
 	// Make sure that we don't see any uncommitted changes
 	auto transaction_id = MAX_TRANSACTION_ID;
 	// This will allow us to see all committed changes made before this COMMIT happened

src/duckdb/src/common/file_system.cpp

@@ -628,39 +628,9 @@ bool FileSystem::CanHandleFile(const string &fpath) {
 	throw NotImplementedException("%s: CanHandleFile is not implemented!", GetName());
 }
 
-static string LookupExtensionForPattern(const string &pattern) {
-	for (const auto &entry : EXTENSION_FILE_PREFIXES) {
-		if (StringUtil::StartsWith(pattern, entry.name)) {
-			return entry.extension;
-		}
-	}
-	return "";
-}
-
 vector<OpenFileInfo> FileSystem::GlobFiles(const string &pattern, ClientContext &context, const FileGlobInput &input) {
 	auto result = Glob(pattern);
 	if (result.empty()) {
-		string required_extension = LookupExtensionForPattern(pattern);
-		if (!required_extension.empty() && !context.db->ExtensionIsLoaded(required_extension)) {
-			auto &dbconfig = DBConfig::GetConfig(context);
-			if (!ExtensionHelper::CanAutoloadExtension(required_extension) ||
-			    !dbconfig.options.autoload_known_extensions) {
-				auto error_message =
-				    "File " + pattern + " requires the extension " + required_extension + " to be loaded";
-				error_message =
-				    ExtensionHelper::AddExtensionInstallHintToErrorMsg(context, error_message, required_extension);
-				throw MissingExtensionException(error_message);
-			}
-			// an extension is required to read this file, but it is not loaded - try to load it
-			ExtensionHelper::AutoLoadExtension(context, required_extension);
-			// success! glob again
-			// check the extension is loaded just in case to prevent an infinite loop here
-			if (!context.db->ExtensionIsLoaded(required_extension)) {
-				throw InternalException("Extension load \"%s\" did not throw but somehow the extension was not loaded",
-				                        required_extension);
-			}
-			return GlobFiles(pattern, context, input);
-		}
 		if (input.behavior == FileGlobOptions::FALLBACK_GLOB && !HasGlob(pattern)) {
 			// if we have no glob in the pattern and we have an extension, we try to glob
 			if (!HasGlob(pattern)) {

src/duckdb/src/common/sorting/sort.cpp

@@ -378,6 +378,15 @@ class SortGlobalSourceState : public GlobalSourceState {
 		return merger_global_state ? merger_global_state->MaxThreads() : 1;
 	}
 
+	void Destroy() {
+		if (!merger_global_state) {
+			return;
+		}
+		auto guard = merger_global_state->Lock();
+		merger.sorted_runs.clear();
+		sink.temporary_memory_state.reset();
+	}
+
 public:
 	//! The global sink state
 	SortGlobalSinkState &sink;
@@ -477,16 +486,26 @@ SourceResultType Sort::MaterializeColumnData(ExecutionContext &context, Operator
 	}
 
 	// Merge into global output collection
-	auto guard = gstate.Lock();
-	if (!gstate.column_data) {
-		gstate.column_data = std::move(local_column_data);
-	} else {
-		gstate.column_data->Merge(*local_column_data);
+	{
+		auto guard = gstate.Lock();
+		if (!gstate.column_data) {
+			gstate.column_data = std::move(local_column_data);
+		} else {
+			gstate.column_data->Merge(*local_column_data);
+		}
 	}
 
+	// Destroy local state before returning
+	input.local_state.Cast<SortLocalSourceState>().merger_local_state.reset();
+
 	// Return type indicates whether materialization is done
 	const auto progress_data = GetProgress(context.client, input.global_state);
-	return progress_data.done == progress_data.total ? SourceResultType::FINISHED : SourceResultType::HAVE_MORE_OUTPUT;
+	if (progress_data.done == progress_data.total) {
+		// Destroy global state before returning
+		gstate.Destroy();
+		return SourceResultType::FINISHED;
+	}
+	return SourceResultType::HAVE_MORE_OUTPUT;
 }
 
 unique_ptr<ColumnDataCollection> Sort::GetColumnData(OperatorSourceInput &input) const {

src/duckdb/src/common/sorting/sorted_run_merger.cpp

@@ -888,6 +888,7 @@ SortedRunMerger::SortedRunMerger(const Expression &decode_sort_key_p, shared_ptr
 unique_ptr<LocalSourceState> SortedRunMerger::GetLocalSourceState(ExecutionContext &,
                                                                   GlobalSourceState &gstate_p) const {
 	auto &gstate = gstate_p.Cast<SortedRunMergerGlobalState>();
+	auto guard = gstate.Lock();
 	return make_uniq<SortedRunMergerLocalState>(gstate);
 }
 

src/duckdb/src/common/virtual_file_system.cpp

@@ -17,6 +17,7 @@ VirtualFileSystem::VirtualFileSystem(unique_ptr<FileSystem> &&inner) : default_f
 
 unique_ptr<FileHandle> VirtualFileSystem::OpenFileExtended(const OpenFileInfo &file, FileOpenFlags flags,
                                                            optional_ptr<FileOpener> opener) {
+
 	auto compression = flags.Compression();
 	if (compression == FileCompressionType::AUTO_DETECT) {
 		// auto-detect compression settings based on file name
@@ -34,8 +35,9 @@ unique_ptr<FileHandle> VirtualFileSystem::OpenFileExtended(const OpenFileInfo &f
 		}
 	}
 	// open the base file handle in UNCOMPRESSED mode
+
 	flags.SetCompression(FileCompressionType::UNCOMPRESSED);
-	auto file_handle = FindFileSystem(file.path).OpenFile(file, flags, opener);
+	auto file_handle = FindFileSystem(file.path, opener).OpenFile(file, flags, opener);
 	if (!file_handle) {
 		return nullptr;
 	}
@@ -111,15 +113,15 @@ void VirtualFileSystem::RemoveDirectory(const string &directory, optional_ptr<Fi
 bool VirtualFileSystem::ListFilesExtended(const string &directory,
                                           const std::function<void(OpenFileInfo &info)> &callback,
                                           optional_ptr<FileOpener> opener) {
-	return FindFileSystem(directory).ListFiles(directory, callback, opener);
+	return FindFileSystem(directory, opener).ListFiles(directory, callback, opener);
 }
 
 void VirtualFileSystem::MoveFile(const string &source, const string &target, optional_ptr<FileOpener> opener) {
 	FindFileSystem(source).MoveFile(source, target, opener);
 }
 
 bool VirtualFileSystem::FileExists(const string &filename, optional_ptr<FileOpener> opener) {
-	return FindFileSystem(filename).FileExists(filename, opener);
+	return FindFileSystem(filename, opener).FileExists(filename, opener);
 }
 
 bool VirtualFileSystem::IsPipe(const string &filename, optional_ptr<FileOpener> opener) {
@@ -139,7 +141,7 @@ string VirtualFileSystem::PathSeparator(const string &path) {
 }
 
 vector<OpenFileInfo> VirtualFileSystem::Glob(const string &path, FileOpener *opener) {
-	return FindFileSystem(path).Glob(path, opener);
+	return FindFileSystem(path, opener).Glob(path, opener);
 }
 
 void VirtualFileSystem::RegisterSubSystem(unique_ptr<FileSystem> fs) {
@@ -216,16 +218,61 @@ bool VirtualFileSystem::SubSystemIsDisabled(const string &name) {
 	return disabled_file_systems.find(name) != disabled_file_systems.end();
 }
 
+FileSystem &VirtualFileSystem::FindFileSystem(const string &path, optional_ptr<FileOpener> opener) {
+	return FindFileSystem(path, FileOpener::TryGetDatabase(opener));
+}
+
+FileSystem &VirtualFileSystem::FindFileSystem(const string &path, optional_ptr<DatabaseInstance> db_instance) {
+	auto fs = FindFileSystemInternal(path);
+
+	if (!fs && db_instance) {
+		string required_extension;
+
+		for (const auto &entry : EXTENSION_FILE_PREFIXES) {
+			if (StringUtil::StartsWith(path, entry.name)) {
+				required_extension = entry.extension;
+			}
+		}
+		if (!required_extension.empty() && db_instance && !db_instance->ExtensionIsLoaded(required_extension)) {
+			auto &dbconfig = DBConfig::GetConfig(*db_instance);
+			if (!ExtensionHelper::CanAutoloadExtension(required_extension) ||
+			    !dbconfig.options.autoload_known_extensions) {
+				auto error_message = "File " + path + " requires the extension " + required_extension + " to be loaded";
+				error_message =
+				    ExtensionHelper::AddExtensionInstallHintToErrorMsg(*db_instance, error_message, required_extension);
+				throw MissingExtensionException(error_message);
+			}
+			// an extension is required to read this file, but it is not loaded - try to load it
+			ExtensionHelper::AutoLoadExtension(*db_instance, required_extension);
+		}
+
+		// Retry after having autoloaded
+		fs = FindFileSystem(path);
+	}
+
+	if (!fs) {
+		fs = default_fs;
+	}
+	if (!disabled_file_systems.empty() && disabled_file_systems.find(fs->GetName()) != disabled_file_systems.end()) {
+		throw PermissionException("File system %s has been disabled by configuration", fs->GetName());
+	}
+	return *fs;
+}
+
 FileSystem &VirtualFileSystem::FindFileSystem(const string &path) {
-	auto &fs = FindFileSystemInternal(path);
-	if (!disabled_file_systems.empty() && disabled_file_systems.find(fs.GetName()) != disabled_file_systems.end()) {
-		throw PermissionException("File system %s has been disabled by configuration", fs.GetName());
+	auto fs = FindFileSystemInternal(path);
+	if (!fs) {
+		fs = default_fs;
+	}
+	if (!disabled_file_systems.empty() && disabled_file_systems.find(fs->GetName()) != disabled_file_systems.end()) {
+		throw PermissionException("File system %s has been disabled by configuration", fs->GetName());
 	}
-	return fs;
+	return *fs;
 }
 
-FileSystem &VirtualFileSystem::FindFileSystemInternal(const string &path) {
+optional_ptr<FileSystem> VirtualFileSystem::FindFileSystemInternal(const string &path) {
 	FileSystem *fs = nullptr;
+
 	for (auto &sub_system : sub_systems) {
 		if (sub_system->CanHandleFile(path)) {
 			if (sub_system->IsManuallySet()) {
@@ -237,7 +284,9 @@ FileSystem &VirtualFileSystem::FindFileSystemInternal(const string &path) {
 	if (fs) {
 		return *fs;
 	}
-	return *default_fs;
+
+	// We could use default_fs, that's on the caller
+	return nullptr;
 }
 
 } // namespace duckdb

src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,5 +1,5 @@
 #ifndef DUCKDB_PATCH_VERSION
-#define DUCKDB_PATCH_VERSION "1-dev136"
+#define DUCKDB_PATCH_VERSION "1-dev178"
 #endif
 #ifndef DUCKDB_MINOR_VERSION
 #define DUCKDB_MINOR_VERSION 4
@@ -8,10 +8,10 @@
 #define DUCKDB_MAJOR_VERSION 1
 #endif
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "v1.4.1-dev136"
+#define DUCKDB_VERSION "v1.4.1-dev178"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "76a3383248"
+#define DUCKDB_SOURCE_ID "24c295a9dd"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

src/duckdb/src/include/duckdb/common/virtual_file_system.hpp

@@ -11,6 +11,7 @@
 #include "duckdb/common/file_system.hpp"
 #include "duckdb/common/map.hpp"
 #include "duckdb/common/unordered_set.hpp"
+#include "duckdb/main/extension_helper.hpp"
 
 namespace duckdb {
 
@@ -82,8 +83,10 @@ class VirtualFileSystem : public FileSystem {
 	}
 
 private:
+	FileSystem &FindFileSystem(const string &path, optional_ptr<FileOpener> file_opener);
+	FileSystem &FindFileSystem(const string &path, optional_ptr<DatabaseInstance> database_instance);
 	FileSystem &FindFileSystem(const string &path);
-	FileSystem &FindFileSystemInternal(const string &path);
+	optional_ptr<FileSystem> FindFileSystemInternal(const string &path);
 
 private:
 	vector<unique_ptr<FileSystem>> sub_systems;

src/duckdb/src/include/duckdb/main/secret/secret.hpp

@@ -296,7 +296,9 @@ class KeyValueSecretReader {
 		Value result;
 		auto lookup_result = TryGetSecretKeyOrSetting(secret_key, setting_name, result);
 		if (lookup_result) {
-			value_out = result.GetValue<TYPE>();
+			if (!result.IsNull()) {
+				value_out = result.GetValue<TYPE>();
+			}
 		}
 		return lookup_result;
 	}

src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp

@@ -102,8 +102,9 @@ class FilterPushdown {
 	void ExtractFilterBindings(const Expression &expr, vector<ColumnBinding> &bindings);
 	//! Generate filters from the current set of filters stored in the FilterCombiner
 	void GenerateFilters();
-	//! if there are filters in this FilterPushdown node, push them into the combiner
-	void PushFilters();
+	//! if there are filters in this FilterPushdown node, push them into the combiner. Returns
+	//! FilterResult::UNSATISFIABLE if the subtree should be stripped, or FilterResult::SUCCESS otherwise
+	FilterResult PushFilters();
 };
 
 } // namespace duckdb

src/duckdb/src/include/duckdb/storage/table/array_column_data.hpp

@@ -48,10 +48,10 @@ class ArrayColumnData : public ColumnData {
 	idx_t Fetch(ColumnScanState &state, row_t row_id, Vector &result) override;
 	void FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result,
 	              idx_t result_idx) override;
-	void Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
-	            idx_t update_count) override;
-	void UpdateColumn(TransactionData transaction, const vector<column_t> &column_path, Vector &update_vector,
-	                  row_t *row_ids, idx_t update_count, idx_t depth) override;
+	void Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector,
+	            row_t *row_ids, idx_t update_count) override;
+	void UpdateColumn(TransactionData transaction, DataTable &data_table, const vector<column_t> &column_path,
+	                  Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) override;
 	unique_ptr<BaseStatistics> GetUpdateStatistics() override;
 
 	void CommitDropColumn() override;

src/duckdb/src/include/duckdb/storage/table/column_data.hpp

@@ -154,10 +154,10 @@ class ColumnData {
 	virtual void FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result,
 	                      idx_t result_idx);
 
-	virtual void Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
-	                    idx_t update_count);
-	virtual void UpdateColumn(TransactionData transaction, const vector<column_t> &column_path, Vector &update_vector,
-	                          row_t *row_ids, idx_t update_count, idx_t depth);
+	virtual void Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector,
+	                    row_t *row_ids, idx_t update_count);
+	virtual void UpdateColumn(TransactionData transaction, DataTable &data_table, const vector<column_t> &column_path,
+	                          Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth);
 	virtual unique_ptr<BaseStatistics> GetUpdateStatistics();
 
 	virtual void CommitDropColumn();
@@ -217,8 +217,8 @@ class ColumnData {
 	void FetchUpdates(TransactionData transaction, idx_t vector_index, Vector &result, idx_t scan_count,
 	                  bool allow_updates, bool scan_committed);
 	void FetchUpdateRow(TransactionData transaction, row_t row_id, Vector &result, idx_t result_idx);
-	void UpdateInternal(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
-	                    idx_t update_count, Vector &base_vector);
+	void UpdateInternal(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector,
+	                    row_t *row_ids, idx_t update_count, Vector &base_vector);
 	idx_t FetchUpdateData(ColumnScanState &state, row_t *row_ids, Vector &base_vector);
 
 	idx_t GetVectorCount(idx_t vector_index) const;

src/duckdb/src/include/duckdb/storage/table/list_column_data.hpp

@@ -46,10 +46,10 @@ class ListColumnData : public ColumnData {
 	idx_t Fetch(ColumnScanState &state, row_t row_id, Vector &result) override;
 	void FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result,
 	              idx_t result_idx) override;
-	void Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
-	            idx_t update_count) override;
-	void UpdateColumn(TransactionData transaction, const vector<column_t> &column_path, Vector &update_vector,
-	                  row_t *row_ids, idx_t update_count, idx_t depth) override;
+	void Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector,
+	            row_t *row_ids, idx_t update_count) override;
+	void UpdateColumn(TransactionData transaction, DataTable &data_table, const vector<column_t> &column_path,
+	                  Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) override;
 	unique_ptr<BaseStatistics> GetUpdateStatistics() override;
 
 	void CommitDropColumn() override;

src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -160,12 +160,12 @@ class RowGroup : public SegmentBase<RowGroup> {
 	void InitializeAppend(RowGroupAppendState &append_state);
 	void Append(RowGroupAppendState &append_state, DataChunk &chunk, idx_t append_count);
 
-	void Update(TransactionData transaction, DataChunk &updates, row_t *ids, idx_t offset, idx_t count,
-	            const vector<PhysicalIndex> &column_ids);
+	void Update(TransactionData transaction, DataTable &data_table, DataChunk &updates, row_t *ids, idx_t offset,
+	            idx_t count, const vector<PhysicalIndex> &column_ids);
 	//! Update a single column; corresponds to DataTable::UpdateColumn
 	//! This method should only be called from the WAL
-	void UpdateColumn(TransactionData transaction, DataChunk &updates, Vector &row_ids, idx_t offset, idx_t count,
-	                  const vector<column_t> &column_path);
+	void UpdateColumn(TransactionData transaction, DataTable &data_table, DataChunk &updates, Vector &row_ids,
+	                  idx_t offset, idx_t count, const vector<column_t> &column_path);
 
 	void MergeStatistics(idx_t column_idx, const BaseStatistics &other);
 	void MergeIntoStatistics(idx_t column_idx, BaseStatistics &other);

src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp

@@ -36,6 +36,7 @@ struct CollectionCheckpointState;
 struct PersistentCollectionData;
 class CheckpointTask;
 class TableIOManager;
+class DataTable;
 
 class RowGroupCollection {
 public:
@@ -99,9 +100,10 @@ class RowGroupCollection {
 	void RemoveFromIndexes(TableIndexList &indexes, Vector &row_identifiers, idx_t count);
 
 	idx_t Delete(TransactionData transaction, DataTable &table, row_t *ids, idx_t count);
-	void Update(TransactionData transaction, row_t *ids, const vector<PhysicalIndex> &column_ids, DataChunk &updates);
-	void UpdateColumn(TransactionData transaction, Vector &row_ids, const vector<column_t> &column_path,
-	                  DataChunk &updates);
+	void Update(TransactionData transaction, DataTable &table, row_t *ids, const vector<PhysicalIndex> &column_ids,
+	            DataChunk &updates);
+	void UpdateColumn(TransactionData transaction, DataTable &table, Vector &row_ids,
+	                  const vector<column_t> &column_path, DataChunk &updates);
 
 	void Checkpoint(TableDataWriter &writer, TableStatistics &global_stats);