Update vendored DuckDB sources to 9cb1e3d3ca

src/duckdb/extension/json/json_functions/copy_json.cpp

@@ -18,6 +18,13 @@ static void ThrowJSONCopyParameterException(const string &loption) {
 }
 
 static BoundStatement CopyToJSONPlan(Binder &binder, CopyStatement &stmt) {
+	static const unordered_set<string> SUPPORTED_BASE_OPTIONS {
+	    "compression", "encoding", "use_tmp_file", "overwrite_or_ignore", "overwrite", "append", "filename_pattern",
+	    "file_extension", "per_thread_output", "file_size_bytes",
+	    // "partition_by", unsupported
+	    "return_files", "preserve_order", "return_stats", "write_partition_columns", "write_empty_file",
+	    "hive_file_pattern"};
+
 	auto stmt_copy = stmt.Copy();
 	auto &copy = stmt_copy->Cast<CopyStatement>();
 	auto &copied_info = *copy.info;
@@ -48,9 +55,7 @@ static BoundStatement CopyToJSONPlan(Binder &binder, CopyStatement &stmt) {
 				csv_copy_options["suffix"] = {"\n]\n"};
 				csv_copy_options["new_line"] = {",\n\t"};
 			}
-		} else if (loption == "compression" || loption == "encoding" || loption == "per_thread_output" ||
-		           loption == "file_size_bytes" || loption == "use_tmp_file" || loption == "overwrite_or_ignore" ||
-		           loption == "filename_pattern" || loption == "file_extension") {
+		} else if (SUPPORTED_BASE_OPTIONS.find(loption) != SUPPORTED_BASE_OPTIONS.end()) {
 			// We support these base options
 			csv_copy_options.insert(kv);
 		} else {

src/duckdb/extension/parquet/parquet_crypto.cpp

@@ -90,8 +90,7 @@ class EncryptionTransport : public TTransport {
 public:
 	EncryptionTransport(TProtocol &prot_p, const string &key, const EncryptionUtil &encryption_util_p)
 	    : prot(prot_p), trans(*prot.getTransport()),
-	      aes(encryption_util_p.CreateEncryptionState(EncryptionTypes::GCM,
-	                                                  reinterpret_cast<const_data_ptr_t>(key.data()), key.size())),
+	      aes(encryption_util_p.CreateEncryptionState(EncryptionTypes::GCM, key.size())),
 	      allocator(Allocator::DefaultAllocator(), ParquetCrypto::CRYPTO_BLOCK_SIZE) {
 		Initialize(key);
 	}
@@ -174,9 +173,8 @@ class DecryptionTransport : public TTransport {
 public:
 	DecryptionTransport(TProtocol &prot_p, const string &key, const EncryptionUtil &encryption_util_p)
 	    : prot(prot_p), trans(*prot.getTransport()),
-	      aes(encryption_util_p.CreateEncryptionState(EncryptionTypes::GCM,
-	                                                  reinterpret_cast<const_data_ptr_t>(key.data()), key.size())),
-	      read_buffer_size(0), read_buffer_offset(0) {
+	      aes(encryption_util_p.CreateEncryptionState(EncryptionTypes::GCM, key.size())), read_buffer_size(0),
+	      read_buffer_offset(0) {
 		Initialize(key);
 	}
 	uint32_t read_virt(uint8_t *buf, uint32_t len) override {

src/duckdb/src/common/encryption_functions.cpp

@@ -73,7 +73,7 @@ void EncryptionEngine::AddTempKeyToCache(DatabaseInstance &db) {
 	data_t temp_key[length];
 
 	auto encryption_state = db.GetEncryptionUtil()->CreateEncryptionState(
-	    /* only for random generator */ EncryptionTypes::GCM, temp_key, length);
+	    /* only for random generator */ EncryptionTypes::GCM, length);
 	encryption_state->GenerateRandomData(temp_key, length);
 
 	string key_id = "temp_key";
@@ -85,8 +85,8 @@ void EncryptionEngine::EncryptBlock(AttachedDatabase &attached_db, const string
 	auto &db = attached_db.GetDatabase();
 	data_ptr_t block_offset_internal = temp_buffer_manager.InternalBuffer();
 	auto encrypt_key = GetKeyFromCache(db, key_id);
-	auto encryption_state = db.GetEncryptionUtil()->CreateEncryptionState(
-	    attached_db.GetStorageManager().GetCipher(), encrypt_key, MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH);
+	auto encryption_state = db.GetEncryptionUtil()->CreateEncryptionState(attached_db.GetStorageManager().GetCipher(),
+	                                                                      MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH);
 
 	EncryptionTag tag;
 	EncryptionNonce nonce;
@@ -121,8 +121,8 @@ void EncryptionEngine::DecryptBlock(AttachedDatabase &attached_db, const string
 	auto &db = attached_db.GetDatabase();
 
 	auto decrypt_key = GetKeyFromCache(db, key_id);
-	auto encryption_state = db.GetEncryptionUtil()->CreateEncryptionState(
-	    attached_db.GetStorageManager().GetCipher(), decrypt_key, MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH);
+	auto encryption_state = db.GetEncryptionUtil()->CreateEncryptionState(attached_db.GetStorageManager().GetCipher(),
+	                                                                      MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH);
 
 	//! load the stored nonce and tag
 	EncryptionTag tag;
@@ -158,8 +158,8 @@ void EncryptionEngine::EncryptTemporaryBuffer(DatabaseInstance &db, data_ptr_t b
 
 	auto encryption_util = db.GetEncryptionUtil();
 	// we hard-code GCM here for now, it's the safest and we don't know what is configured here
-	auto encryption_state = encryption_util->CreateEncryptionState(EncryptionTypes::GCM, temp_key,
-	                                                               MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH);
+	auto encryption_state =
+	    encryption_util->CreateEncryptionState(EncryptionTypes::GCM, MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH);
 
 	// zero-out the metadata buffer
 	memset(metadata, 0, DEFAULT_ENCRYPTED_BUFFER_HEADER_SIZE);
@@ -219,8 +219,8 @@ void EncryptionEngine::DecryptTemporaryBuffer(DatabaseInstance &db, data_ptr_t b
 	//! initialize encryption state
 	auto encryption_util = db.GetEncryptionUtil();
 	auto temp_key = GetKeyFromCache(db, "temp_key");
-	auto encryption_state = encryption_util->CreateEncryptionState(EncryptionTypes::GCM, temp_key,
-	                                                               MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH);
+	auto encryption_state =
+	    encryption_util->CreateEncryptionState(EncryptionTypes::GCM, MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH);
 
 	DecryptBuffer(*encryption_state, temp_key, buffer, buffer_size, metadata);
 }

src/duckdb/src/common/encryption_state.cpp

@@ -2,8 +2,8 @@
 
 namespace duckdb {
 
-EncryptionState::EncryptionState(EncryptionTypes::CipherType, const_data_ptr_t, idx_t) {
-	// abstract class, no implementation needed
+EncryptionState::EncryptionState(EncryptionTypes::CipherType cipher_p, idx_t key_len_p)
+    : cipher(cipher_p), key_len(key_len_p) {
 }
 
 EncryptionState::~EncryptionState() {
@@ -51,7 +51,7 @@ EncryptionTypes::CipherType EncryptionTypes::StringToCipher(const string &encryp
 		return CTR;
 	}
 	if (encryption_cipher == "CBC") {
-		return CBC;
+		throw NotImplementedException("CBC encryption is disabled");
 	}
 	return INVALID;
 }

src/duckdb/src/common/sorting/sorted_run.cpp

@@ -304,6 +304,9 @@ void SortedRun::Finalize(bool external) {
 		const auto sort_key_type = key_data->GetLayout().GetSortKeyType();
 		if (!SortKeyUtils::IsConstantSize(sort_key_type) || SortKeyUtils::HasPayload(sort_key_type)) {
 			Reorder(context, key_data, payload_data);
+		} else {
+			// This ensures keys are unpinned even if they are constant size and have no payload
+			key_data->Unpin();
 		}
 	}
 

src/duckdb/src/common/types.cpp

@@ -624,6 +624,10 @@ LogicalType GetUserTypeRecursive(const LogicalType &type, ClientContext &context
 	if (type.id() == LogicalTypeId::LIST) {
 		return LogicalType::LIST(GetUserTypeRecursive(ListType::GetChildType(type), context));
 	}
+	if (type.id() == LogicalTypeId::ARRAY) {
+		return LogicalType::ARRAY(GetUserTypeRecursive(ArrayType::GetChildType(type), context),
+		                          ArrayType::GetSize(type));
+	}
 	if (type.id() == LogicalTypeId::MAP) {
 		return LogicalType::MAP(GetUserTypeRecursive(MapType::KeyType(type), context),
 		                        GetUserTypeRecursive(MapType::ValueType(type), context));

src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,5 +1,5 @@
 #ifndef DUCKDB_PATCH_VERSION
-#define DUCKDB_PATCH_VERSION "0-dev4049"
+#define DUCKDB_PATCH_VERSION "0-dev4076"
 #endif
 #ifndef DUCKDB_MINOR_VERSION
 #define DUCKDB_MINOR_VERSION 4
@@ -8,10 +8,10 @@
 #define DUCKDB_MAJOR_VERSION 1
 #endif
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "v1.4.0-dev4049"
+#define DUCKDB_VERSION "v1.4.0-dev4076"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "a92d9fc987"
+#define DUCKDB_SOURCE_ID "9cb1e3d3ca"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

src/duckdb/src/include/duckdb/common/encryption_state.hpp

@@ -29,8 +29,7 @@ class EncryptionTypes {
 class EncryptionState {
 
 public:
-	DUCKDB_API explicit EncryptionState(EncryptionTypes::CipherType cipher_p, const_data_ptr_t key = nullptr,
-	                                    idx_t key_len = 0);
+	DUCKDB_API explicit EncryptionState(EncryptionTypes::CipherType cipher_p, idx_t key_len);
 	DUCKDB_API virtual ~EncryptionState();
 
 public:
@@ -41,6 +40,10 @@ class EncryptionState {
 	DUCKDB_API virtual size_t Process(const_data_ptr_t in, idx_t in_len, data_ptr_t out, idx_t out_len);
 	DUCKDB_API virtual size_t Finalize(data_ptr_t out, idx_t out_len, data_ptr_t tag, idx_t tag_len);
 	DUCKDB_API virtual void GenerateRandomData(data_ptr_t data, idx_t len);
+
+protected:
+	EncryptionTypes::CipherType cipher;
+	idx_t key_len;
 };
 
 class EncryptionUtil {
@@ -50,8 +53,8 @@ class EncryptionUtil {
 
 public:
 	virtual shared_ptr<EncryptionState> CreateEncryptionState(EncryptionTypes::CipherType cipher_p,
-	                                                          const_data_ptr_t key = nullptr, idx_t key_len = 0) const {
-		return make_shared_ptr<EncryptionState>(cipher_p, key, key_len);
+	                                                          idx_t key_len = 0) const {
+		return make_shared_ptr<EncryptionState>(cipher_p, key_len);
 	}
 
 	virtual ~EncryptionUtil() {

src/duckdb/src/include/duckdb/storage/storage_manager.hpp

@@ -129,6 +129,9 @@ class StorageManager {
 	}
 	void SetCipher(EncryptionTypes::CipherType cipher_p) {
 		D_ASSERT(cipher_p != EncryptionTypes::INVALID);
+		if (cipher_p == EncryptionTypes::CBC) {
+			throw InvalidInputException("CBC cipher is disabled");
+		}
 		storage_options.encryption_cipher = cipher_p;
 	}
 	bool IsEncrypted() const {

src/duckdb/src/main/http/http_util.cpp

@@ -379,8 +379,12 @@ HTTPUtil::RunRequestWithRetry(const std::function<unique_ptr<HTTPResponse>(void)
 		// Note: request errors will always be retried
 		bool should_retry = !response || response->ShouldRetry();
 		if (!should_retry) {
+			auto response_code = static_cast<uint16_t>(response->status);
+			if (response_code >= 200 && response_code < 300) {
+				response->success = true;
+				return response;
+			}
 			switch (response->status) {
-			case HTTPStatusCode::OK_200:
 			case HTTPStatusCode::NotModified_304:
 				response->success = true;
 				break;

src/duckdb/src/main/query_profiler.cpp

@@ -38,10 +38,12 @@ bool QueryProfiler::IsDetailedEnabled() const {
 
 ProfilerPrintFormat QueryProfiler::GetPrintFormat(ExplainFormat format) const {
 	auto print_format = ClientConfig::GetConfig(context).profiler_print_format;
-	if (format == ExplainFormat::DEFAULT) {
-		return print_format;
-	}
 	switch (format) {
+	case ExplainFormat::DEFAULT:
+		if (print_format != ProfilerPrintFormat::NO_OUTPUT) {
+			return print_format;
+		}
+		DUCKDB_EXPLICIT_FALLTHROUGH;
 	case ExplainFormat::TEXT:
 		return ProfilerPrintFormat::QUERY_TREE;
 	case ExplainFormat::JSON:

src/duckdb/src/storage/single_file_block_manager.cpp

@@ -76,11 +76,11 @@ void EncryptCanary(MainHeader &main_header, const shared_ptr<EncryptionState> &e
 	uint8_t canary_buffer[MainHeader::CANARY_BYTE_SIZE];
 
 	// we zero-out the iv and the (not yet) encrypted canary
-	uint8_t iv[16];
-	memset(iv, 0, sizeof(iv));
+	uint8_t iv[MainHeader::AES_IV_LEN];
+	memset(iv, 0, MainHeader::AES_IV_LEN);
 	memset(canary_buffer, 0, MainHeader::CANARY_BYTE_SIZE);
 
-	encryption_state->InitializeEncryption(iv, MainHeader::AES_NONCE_LEN, derived_key,
+	encryption_state->InitializeEncryption(iv, MainHeader::AES_IV_LEN, derived_key,
 	                                       MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH);
 	encryption_state->Process(reinterpret_cast<const_data_ptr_t>(MainHeader::CANARY), MainHeader::CANARY_BYTE_SIZE,
 	                          canary_buffer, MainHeader::CANARY_BYTE_SIZE);
@@ -91,15 +91,15 @@ void EncryptCanary(MainHeader &main_header, const shared_ptr<EncryptionState> &e
 bool DecryptCanary(MainHeader &main_header, const shared_ptr<EncryptionState> &encryption_state,
                    data_ptr_t derived_key) {
 	// just zero-out the iv
-	uint8_t iv[16];
-	memset(iv, 0, sizeof(iv));
+	uint8_t iv[MainHeader::AES_IV_LEN];
+	memset(iv, 0, MainHeader::AES_IV_LEN);
 
 	//! allocate a buffer for the decrypted canary
 	data_t decrypted_canary[MainHeader::CANARY_BYTE_SIZE];
 	memset(decrypted_canary, 0, MainHeader::CANARY_BYTE_SIZE);
 
 	//! Decrypt the canary
-	encryption_state->InitializeDecryption(iv, MainHeader::AES_NONCE_LEN, derived_key,
+	encryption_state->InitializeDecryption(iv, MainHeader::AES_IV_LEN, derived_key,
 	                                       MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH);
 	encryption_state->Process(main_header.GetEncryptedCanary(), MainHeader::CANARY_BYTE_SIZE, decrypted_canary,
 	                          MainHeader::CANARY_BYTE_SIZE);
@@ -299,7 +299,7 @@ void SingleFileBlockManager::StoreEncryptedCanary(AttachedDatabase &db, MainHead
 	const_data_ptr_t key = EncryptionEngine::GetKeyFromCache(db.GetDatabase(), key_id);
 	// Encrypt canary with the derived key
 	auto encryption_state = db.GetDatabase().GetEncryptionUtil()->CreateEncryptionState(
-	    main_header.GetEncryptionCipher(), key, MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH);
+	    main_header.GetEncryptionCipher(), MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH);
 	EncryptCanary(main_header, encryption_state, key);
 }
 
@@ -341,7 +341,7 @@ void SingleFileBlockManager::CheckAndAddEncryptionKey(MainHeader &main_header, s
 	EncryptionKeyManager::DeriveKey(user_key, db_identifier, derived_key);
 
 	auto encryption_state = db.GetDatabase().GetEncryptionUtil()->CreateEncryptionState(
-	    main_header.GetEncryptionCipher(), derived_key, MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH);
+	    main_header.GetEncryptionCipher(), MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH);
 	if (!DecryptCanary(main_header, encryption_state, derived_key)) {
 		throw IOException("Wrong encryption key used to open the database file");
 	}

src/duckdb/src/storage/storage_manager.cpp

@@ -40,9 +40,9 @@ void StorageOptions::Initialize(const unordered_map<string, Value> &options) {
 			encryption = true;
 		} else if (entry.first == "encryption_cipher") {
 			auto parsed_cipher = EncryptionTypes::StringToCipher(entry.second.ToString());
-			if (parsed_cipher == EncryptionTypes::CipherType::INVALID) {
-				throw BinderException("\"%s\" is not a valid cipher. Try 'GCM', 'CTR', or 'CBC'.",
-				                      entry.second.ToString());
+			if (parsed_cipher != EncryptionTypes::CipherType::GCM &&
+			    parsed_cipher != EncryptionTypes::CipherType::CTR) {
+				throw BinderException("\"%s\" is not a valid cipher. Try 'GCM' or 'CTR'.", entry.second.ToString());
 			}
 			encryption_cipher = parsed_cipher;
 		} else if (entry.first == "row_group_size") {

src/duckdb/src/storage/wal_replay.cpp

@@ -145,7 +145,7 @@ class WriteAheadLogDeserializer {
 
 			//! initialize the decryption
 			auto encryption_state = database.GetEncryptionUtil()->CreateEncryptionState(
-			    state_p.db.GetStorageManager().GetCipher(), derived_key, MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH);
+			    state_p.db.GetStorageManager().GetCipher(), MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH);
 			encryption_state->InitializeDecryption(nonce.data(), nonce.size(), derived_key,
 			                                       MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH);
 

src/duckdb/src/storage/write_ahead_log.cpp

@@ -156,8 +156,7 @@ class ChecksumWriter : public WriteStream {
 		auto &keys = EncryptionKeyManager::Get(db.GetDatabase());
 
 		auto encryption_state = db.GetDatabase().GetEncryptionUtil()->CreateEncryptionState(
-		    db.GetStorageManager().GetCipher(), keys.GetKey(encryption_key_id),
-		    MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH);
+		    db.GetStorageManager().GetCipher(), MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH);
 
 		// temp buffer
 		const idx_t ciphertext_size = size + sizeof(uint64_t);

src/duckdb/third_party/mbedtls/include/mbedtls/mbedtls_config.h

@@ -23,4 +23,7 @@
 #define MBEDTLS_RSA_C
 #define MBEDTLS_SHA1_C
 #define MBEDTLS_SHA256_C
-#define MBEDTLS_CIPHER_MODE_CTR
+#define MBEDTLS_CIPHER_MODE_CTR
+#define MBEDTLS_CIPHER_MODE_CBC
+#define MBEDTLS_CIPHER_MODE_WITH_PADDING
+#define MBEDTLS_CIPHER_PADDING_PKCS7

src/duckdb/third_party/mbedtls/include/mbedtls_wrapper.hpp

@@ -66,7 +66,7 @@ class MbedTlsWrapper {
 
 class AESStateMBEDTLS : public duckdb::EncryptionState {
 	public:
-		DUCKDB_API explicit AESStateMBEDTLS(duckdb::EncryptionTypes::CipherType cipher_p, duckdb::const_data_ptr_t key = nullptr, duckdb::idx_t key_len = 0);
+		DUCKDB_API explicit AESStateMBEDTLS(duckdb::EncryptionTypes::CipherType cipher_p, duckdb::idx_t key_len);
 		DUCKDB_API ~AESStateMBEDTLS() override;
 
 	public:
@@ -87,15 +87,14 @@ class AESStateMBEDTLS : public duckdb::EncryptionState {
 
 	private:
 		duckdb::EncryptionTypes::Mode mode;
-		duckdb::EncryptionTypes::CipherType cipher;
 		duckdb::unique_ptr<mbedtls_cipher_context_t> context;
 	};
 
 	class AESStateMBEDTLSFactory : public duckdb::EncryptionUtil {
 
 	public:
-		duckdb::shared_ptr<duckdb::EncryptionState> CreateEncryptionState(duckdb::EncryptionTypes::CipherType cipher_p, duckdb::const_data_ptr_t key = nullptr, duckdb::idx_t key_len = 0) const override {
-			return duckdb::make_shared_ptr<MbedTlsWrapper::AESStateMBEDTLS>(cipher_p, key, key_len);
+		duckdb::shared_ptr<duckdb::EncryptionState> CreateEncryptionState(duckdb::EncryptionTypes::CipherType cipher_p, duckdb::idx_t key_len = 0) const override {
+			return duckdb::make_shared_ptr<MbedTlsWrapper::AESStateMBEDTLS>(cipher_p, key_len);
 		}
 
 		~AESStateMBEDTLSFactory() override {} //