Update vendored DuckDB sources to 6ddac802ff

src/duckdb/src/common/file_system.cpp

@@ -63,6 +63,7 @@ constexpr FileOpenFlags FileFlags::FILE_FLAGS_EXCLUSIVE_CREATE;
 constexpr FileOpenFlags FileFlags::FILE_FLAGS_NULL_IF_EXISTS;
 constexpr FileOpenFlags FileFlags::FILE_FLAGS_MULTI_CLIENT_ACCESS;
 constexpr FileOpenFlags FileFlags::FILE_FLAGS_DISABLE_LOGGING;
+constexpr FileOpenFlags FileFlags::FILE_FLAGS_ENABLE_EXTENSION_INSTALL;
 
 void FileOpenFlags::Verify() {
 #ifdef DEBUG

src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,5 +1,5 @@
 #ifndef DUCKDB_PATCH_VERSION
-#define DUCKDB_PATCH_VERSION "4-dev256"
+#define DUCKDB_PATCH_VERSION "4-dev272"
 #endif
 #ifndef DUCKDB_MINOR_VERSION
 #define DUCKDB_MINOR_VERSION 4
@@ -8,10 +8,10 @@
 #define DUCKDB_MAJOR_VERSION 1
 #endif
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "v1.4.4-dev256"
+#define DUCKDB_VERSION "v1.4.4-dev272"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "e3b14d4d3b"
+#define DUCKDB_SOURCE_ID "6ddac802ff"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

src/duckdb/src/include/duckdb/common/file_open_flags.hpp

@@ -30,6 +30,7 @@ class FileOpenFlags {
 	static constexpr idx_t FILE_FLAGS_NULL_IF_EXISTS = idx_t(1 << 10);
 	static constexpr idx_t FILE_FLAGS_MULTI_CLIENT_ACCESS = idx_t(1 << 11);
 	static constexpr idx_t FILE_FLAGS_DISABLE_LOGGING = idx_t(1 << 12);
+	static constexpr idx_t FILE_FLAGS_ENABLE_EXTENSION_INSTALL = idx_t(1 << 13);
 
 public:
 	FileOpenFlags() = default;
@@ -115,6 +116,9 @@ class FileOpenFlags {
 	inline bool DisableLogging() const {
 		return flags & FILE_FLAGS_DISABLE_LOGGING;
 	}
+	inline bool EnableExtensionInstall() const {
+		return flags & FILE_FLAGS_ENABLE_EXTENSION_INSTALL;
+	}
 	inline idx_t GetFlagsInternal() const {
 		return flags;
 	}
@@ -159,6 +163,9 @@ class FileFlags {
 	//! Disables logging to avoid infinite loops when using FileHandle-backed log storage
 	static constexpr FileOpenFlags FILE_FLAGS_DISABLE_LOGGING =
 	    FileOpenFlags(FileOpenFlags::FILE_FLAGS_DISABLE_LOGGING);
+	//! Opened file is allowed to be a duckdb_extension
+	static constexpr FileOpenFlags FILE_FLAGS_ENABLE_EXTENSION_INSTALL =
+	    FileOpenFlags(FileOpenFlags::FILE_FLAGS_ENABLE_EXTENSION_INSTALL);
 };
 
 } // namespace duckdb

src/duckdb/src/include/duckdb/common/opener_file_system.hpp

@@ -21,6 +21,18 @@ class OpenerFileSystem : public FileSystem {
 	void VerifyNoOpener(optional_ptr<FileOpener> opener);
 	void VerifyCanAccessDirectory(const string &path);
 	void VerifyCanAccessFile(const string &path);
+	void VerifyCanAccessExtension(const string &path, const FileOpenFlags &flags) {
+		if (flags.OpenForWriting() && !flags.EnableExtensionInstall()) {
+			throw PermissionException(
+			    "File '%s' cannot be opened for writing since files ending with '.duckdb_extension' are reserved for "
+			    "DuckDB extensions, and these can only be installed through the INSTALL command",
+			    path);
+		}
+	}
+
+	bool IsDuckDBExtensionName(const string &path) {
+		return StringUtil::EndsWith(path, ".duckdb_extension");
+	}
 
 	void Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override {
 		GetFileSystem().Read(handle, buffer, nr_bytes, location);
@@ -80,6 +92,13 @@ class OpenerFileSystem : public FileSystem {
 		VerifyNoOpener(opener);
 		VerifyCanAccessFile(source);
 		VerifyCanAccessFile(target);
+		if (IsDuckDBExtensionName(target) && !IsDuckDBExtensionName(source)) {
+			throw PermissionException(
+			    "File '%s' cannot be moved to '%s', files ending with '.duckdb_extension' are reserved for DuckDB "
+			    "extensions, and these can only be installed through the INSTALL command, or moved if both are "
+			    "extensions'",
+			    source, target);
+		}
 		GetFileSystem().MoveFile(source, target, GetOpener());
 	}
 
@@ -156,6 +175,9 @@ class OpenerFileSystem : public FileSystem {
 	                                        optional_ptr<FileOpener> opener = nullptr) override {
 		VerifyNoOpener(opener);
 		VerifyCanAccessFile(file.path);
+		if (IsDuckDBExtensionName(file.path)) {
+			VerifyCanAccessExtension(file.path, flags);
+		}
 		return GetFileSystem().OpenFile(file, flags, GetOpener());
 	}
 

src/duckdb/src/include/duckdb/main/extension_helper.hpp

@@ -126,8 +126,15 @@ class ExtensionHelper {
 	static string GetExtensionDirectoryPath(ClientContext &context);
 	static string GetExtensionDirectoryPath(DatabaseInstance &db, FileSystem &fs);
 
+	// Check signature of an Extension stored as FileHandle
 	static bool CheckExtensionSignature(FileHandle &handle, ParsedExtensionMetaData &parsed_metadata,
 	                                    const bool allow_community_extensions);
+	// Check signature of an Extension, represented by a buffer and total_buffer_length, and a signature to be added
+	static bool CheckExtensionBufferSignature(const char *buffer, idx_t buffer_length, const string &signature,
+	                                          const bool allow_community_extensions);
+	// Check signature of an Extension, represented by a buffer and total_buffer_length
+	static bool CheckExtensionBufferSignature(const char *buffer, idx_t total_buffer_length,
+	                                          const bool allow_community_extensions);
 	static ParsedExtensionMetaData ParseExtensionMetaData(const char *metadata) noexcept;
 	static ParsedExtensionMetaData ParseExtensionMetaData(FileHandle &handle);
 

src/duckdb/src/main/extension/extension_install.cpp

@@ -175,10 +175,25 @@ static unsafe_unique_array<data_t> ReadExtensionFileFromDisk(FileSystem &fs, con
 }
 
 static void WriteExtensionFileToDisk(QueryContext &query_context, FileSystem &fs, const string &path, void *data,
-                                     idx_t data_size) {
-	auto target_file = fs.OpenFile(path, FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_APPEND |
-	                                         FileFlags::FILE_FLAGS_FILE_CREATE_NEW);
+                                     idx_t data_size, DBConfig &config) {
+	if (!config.options.allow_unsigned_extensions) {
+		const bool signature_valid = ExtensionHelper::CheckExtensionBufferSignature(
+		    static_cast<char *>(data), data_size, config.options.allow_community_extensions);
+		if (!signature_valid) {
+			throw IOException("Attempting to install an extension file that doesn't have a valid signature, see "
+			                  "https://duckdb.org/docs/stable/operations_manual/securing_duckdb/securing_extensions");
+		}
+	}
+
+	// Now signature has been checked (if signature checking is enabled)
+
+	// Open target_file, at this points ending with '.duckdb_extension'
+	auto target_file =
+	    fs.OpenFile(path, FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_APPEND |
+	                          FileFlags::FILE_FLAGS_FILE_CREATE_NEW | FileFlags::FILE_FLAGS_ENABLE_EXTENSION_INSTALL);
+	// Write content to the file
 	target_file->Write(query_context, data, data_size);
+
 	target_file->Close();
 	target_file.reset();
 }
@@ -239,9 +254,23 @@ static void CheckExtensionMetadataOnInstall(DatabaseInstance &db, void *in_buffe
 //   3. Crash after extension move: extension is now uninstalled, new metadata file present
 static void WriteExtensionFiles(QueryContext &query_context, FileSystem &fs, const string &temp_path,
                                 const string &local_extension_path, void *in_buffer, idx_t file_size,
-                                ExtensionInstallInfo &info) {
+                                ExtensionInstallInfo &info, DBConfig &config) {
+	// temp_path ends with '.duckdb_extension'
+	if (!StringUtil::EndsWith(temp_path, ".duckdb_extension")) {
+		throw InternalException("Extension install temp_path of '%s' is not valid, should end in '.duckdb_extension'",
+		                        temp_path);
+	}
+	// local_extension_path ends with '.duckdb_extension', and given it will be written only after signature checks,
+	// it's now loadable
+	if (!StringUtil::EndsWith(local_extension_path, ".duckdb_extension")) {
+		throw InternalException("Extension install local_extension_path of '%s' is not valid, should end in "
+		                        "'.duckdb_extension'",
+		                        temp_path);
+	}
+
 	// Write extension to tmp file
-	WriteExtensionFileToDisk(query_context, fs, temp_path, in_buffer, file_size);
+	WriteExtensionFileToDisk(query_context, fs, temp_path, in_buffer, file_size, config);
+	// When this exit, signature has already being checked (if enabled by config)
 
 	// Write metadata to tmp file
 	auto metadata_tmp_path = temp_path + ".info";
@@ -331,7 +360,7 @@ static unique_ptr<ExtensionInstallInfo> DirectInstallExtension(DatabaseInstance
 
 	QueryContext query_context(context);
 	WriteExtensionFiles(query_context, fs, temp_path, local_extension_path, extension_decompressed,
-	                    extension_decompressed_size, info);
+	                    extension_decompressed_size, info, db.config);
 
 	return make_uniq<ExtensionInstallInfo>(info);
 }
@@ -423,7 +452,7 @@ static unique_ptr<ExtensionInstallInfo> InstallFromHttpUrl(DatabaseInstance &db,
 	QueryContext query_context(context);
 	auto fs = FileSystem::CreateLocal();
 	WriteExtensionFiles(query_context, *fs, temp_path, local_extension_path, (void *)decompressed_body.data(),
-	                    decompressed_body.size(), info);
+	                    decompressed_body.size(), info, db.config);
 
 	return make_uniq<ExtensionInstallInfo>(info);
 }
@@ -489,7 +518,8 @@ unique_ptr<ExtensionInstallInfo> ExtensionHelper::InstallExtensionInternal(Datab
 
 	auto extension_name = ApplyExtensionAlias(fs.ExtractBaseName(extension));
 	string local_extension_path = fs.JoinPath(local_path, extension_name + ".duckdb_extension");
-	string temp_path = local_extension_path + ".tmp-" + UUID::ToString(UUID::GenerateRandomUUID());
+	string temp_path =
+	    local_extension_path + ".tmp-" + UUID::ToString(UUID::GenerateRandomUUID()) + ".duckdb_extension";
 
 	if (fs.FileExists(local_extension_path) && !options.force_install) {
 		// File exists: throw error if origin mismatches

src/duckdb/src/main/extension/extension_load.cpp

@@ -34,6 +34,7 @@ struct DuckDBExtensionLoadState {
 	//! Create a DuckDBExtensionLoadState reference from a C API opaque pointer
 	static DuckDBExtensionLoadState &Get(duckdb_extension_info info) {
 		D_ASSERT(info);
+
 		return *reinterpret_cast<duckdb::DuckDBExtensionLoadState *>(info);
 	}
 
@@ -164,9 +165,41 @@ static T TryLoadFunctionFromDLL(void *dll, const string &function_name, const st
 	return (T)function;
 }
 
-static void ComputeSHA256String(const string &to_hash, string *res) {
+static void ComputeSHA256Buffer(const char *buffer, const idx_t start, const idx_t end, string *res) {
 	// Invoke MbedTls function to actually compute sha256
-	*res = duckdb_mbedtls::MbedTlsWrapper::ComputeSha256Hash(to_hash);
+	char hash[duckdb_mbedtls::MbedTlsWrapper::SHA256_HASH_LENGTH_BYTES];
+	duckdb_mbedtls::MbedTlsWrapper::ComputeSha256Hash(buffer + start, end - start, hash);
+	*res = std::string(hash, duckdb_mbedtls::MbedTlsWrapper::SHA256_HASH_LENGTH_BYTES);
+}
+
+static void ComputeSHA256String(const string &to_hash, string *res) {
+	ComputeSHA256Buffer(to_hash.data(), 0, to_hash.length(), res);
+}
+
+static string ComputeFinalHash(const vector<string> &chunks) {
+	string hash_concatenation;
+	hash_concatenation.reserve(32 * chunks.size()); // 256 bits -> 32 bytes per chunk
+
+	for (auto &chunk : chunks) {
+		hash_concatenation += chunk;
+	}
+
+	string two_level_hash;
+	ComputeSHA256String(hash_concatenation, &two_level_hash);
+
+	return two_level_hash;
+}
+
+static void IntializeAncillaryData(vector<string> &hash_chunks, vector<idx_t> &splits, idx_t length) {
+	const idx_t maxLenChunks = 1024ULL * 1024ULL;
+	const idx_t numChunks = (length + maxLenChunks - 1) / maxLenChunks;
+	hash_chunks.resize(numChunks);
+	splits.resize(numChunks + 1);
+
+	for (idx_t i = 0; i < numChunks; i++) {
+		splits[i] = maxLenChunks * i;
+	}
+	splits.back() = length;
 }
 
 static void ComputeSHA256FileSegment(FileHandle *handle, const idx_t start, const idx_t end, string *res) {
@@ -189,6 +222,26 @@ static void ComputeSHA256FileSegment(FileHandle *handle, const idx_t start, cons
 	*res = state.Finalize();
 }
 
+template <typename T, typename F>
+static void ComputeHashesOnSegments(F ComputeHashFun, T handle, const vector<idx_t> &splits,
+                                    vector<string> &hash_chunks) {
+#ifndef DUCKDB_NO_THREADS
+	vector<std::thread> threads;
+	threads.reserve(hash_chunks.size());
+	for (idx_t i = 0; i < hash_chunks.size(); i++) {
+		threads.emplace_back(ComputeHashFun, handle, splits[i], splits[i + 1], &hash_chunks[i]);
+	}
+
+	for (auto &thread : threads) {
+		thread.join();
+	}
+#else
+	for (idx_t i = 0; i < hash_chunks.size(); i++) {
+		ComputeHashFun(handle, splits[i], splits[i + 1], &hash_chunks[i]);
+	}
+#endif // DUCKDB_NO_THREADS
+}
+
 static string FilterZeroAtEnd(string s) {
 	while (!s.empty() && s.back() == '\0') {
 		s.pop_back();
@@ -257,57 +310,54 @@ ParsedExtensionMetaData ExtensionHelper::ParseExtensionMetaData(FileHandle &hand
 	return ParseExtensionMetaData(metadata_segment.data());
 }
 
+static bool CheckKnownSignatures(const string &two_level_hash, const string &signature,
+                                 const bool allow_community_extensions) {
+	for (auto &key : ExtensionHelper::GetPublicKeys(allow_community_extensions)) {
+		if (duckdb_mbedtls::MbedTlsWrapper::IsValidSha256Signature(key, signature, two_level_hash)) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
 bool ExtensionHelper::CheckExtensionSignature(FileHandle &handle, ParsedExtensionMetaData &parsed_metadata,
                                               const bool allow_community_extensions) {
 	auto signature_offset = handle.GetFileSize() - ParsedExtensionMetaData::SIGNATURE_SIZE;
 
-	const idx_t maxLenChunks = 1024ULL * 1024ULL;
-	const idx_t numChunks = (signature_offset + maxLenChunks - 1) / maxLenChunks;
-	vector<string> hash_chunks(numChunks);
-	vector<idx_t> splits(numChunks + 1);
+	vector<string> hash_chunks;
+	vector<idx_t> splits;
+	IntializeAncillaryData(hash_chunks, splits, signature_offset);
 
-	for (idx_t i = 0; i < numChunks; i++) {
-		splits[i] = maxLenChunks * i;
-	}
-	splits.back() = signature_offset;
+	ComputeHashesOnSegments(ComputeSHA256FileSegment, &handle, splits, hash_chunks);
 
-#ifndef DUCKDB_NO_THREADS
-	vector<std::thread> threads;
-	threads.reserve(numChunks);
-	for (idx_t i = 0; i < numChunks; i++) {
-		threads.emplace_back(ComputeSHA256FileSegment, &handle, splits[i], splits[i + 1], &hash_chunks[i]);
-	}
+	const string resulting_hash = ComputeFinalHash(hash_chunks);
 
-	for (auto &thread : threads) {
-		thread.join();
-	}
-#else
-	for (idx_t i = 0; i < numChunks; i++) {
-		ComputeSHA256FileSegment(&handle, splits[i], splits[i + 1], &hash_chunks[i]);
-	}
-#endif // DUCKDB_NO_THREADS
+	// TODO maybe we should do a stream read / hash update here
+	handle.Read((void *)parsed_metadata.signature.data(), parsed_metadata.signature.size(), signature_offset);
 
-	string hash_concatenation;
-	hash_concatenation.reserve(32 * numChunks); // 256 bits -> 32 bytes per chunk
+	return CheckKnownSignatures(resulting_hash, parsed_metadata.signature, allow_community_extensions);
+}
 
-	for (auto &hash_chunk : hash_chunks) {
-		hash_concatenation += hash_chunk;
-	}
+bool ExtensionHelper::CheckExtensionBufferSignature(const char *buffer, idx_t buffer_length, const string &signature,
+                                                    const bool allow_community_extensions) {
+	vector<string> hash_chunks;
+	vector<idx_t> splits;
+	IntializeAncillaryData(hash_chunks, splits, buffer_length);
 
-	string two_level_hash;
-	ComputeSHA256String(hash_concatenation, &two_level_hash);
+	ComputeHashesOnSegments(ComputeSHA256Buffer, buffer, splits, hash_chunks);
 
-	// TODO maybe we should do a stream read / hash update here
-	handle.Read((void *)parsed_metadata.signature.data(), parsed_metadata.signature.size(), signature_offset);
+	const string resulting_hash = ComputeFinalHash(hash_chunks);
 
-	for (auto &key : ExtensionHelper::GetPublicKeys(allow_community_extensions)) {
-		if (duckdb_mbedtls::MbedTlsWrapper::IsValidSha256Signature(key, parsed_metadata.signature, two_level_hash)) {
-			return true;
-			break;
-		}
-	}
+	return CheckKnownSignatures(resulting_hash, signature, allow_community_extensions);
+}
 
-	return false;
+bool ExtensionHelper::CheckExtensionBufferSignature(const char *buffer, idx_t total_buffer_length,
+                                                    const bool allow_community_extensions) {
+	auto signature_offset = total_buffer_length - ParsedExtensionMetaData::SIGNATURE_SIZE;
+	string signature = std::string(buffer + signature_offset, ParsedExtensionMetaData::SIGNATURE_SIZE);
+
+	return CheckExtensionBufferSignature(buffer, signature_offset, signature, allow_community_extensions);
 }
 
 bool ExtensionHelper::TryInitialLoad(DatabaseInstance &db, FileSystem &fs, const string &extension,
@@ -368,6 +418,12 @@ bool ExtensionHelper::TryInitialLoad(DatabaseInstance &db, FileSystem &fs, const
 		direct_load = true;
 		filename = fs.ExpandPath(filename);
 	}
+	if (!StringUtil::EndsWith(filename, ".duckdb_extension")) {
+		throw PermissionException(
+		    "DuckDB extensions are files ending with '.duckdb_extension', loading different "
+		    "files is not possible, error while loading from '%s', consider 'INSTALL <path>; LOAD <name>;'",
+		    filename);
+	}
 	if (!fs.FileExists(filename)) {
 		string message;
 		bool exact_match = ExtensionHelper::CreateSuggestions(extension, message);