Update vendored DuckDB sources to 2259ad7316

CMakeLists.txt

@@ -347,48 +347,48 @@ set(DUCKDB_SRC_FILES
   src/duckdb/third_party/zstd/dict/divsufsort.cpp
   src/duckdb/third_party/zstd/dict/fastcover.cpp
   src/duckdb/third_party/zstd/dict/zdict.cpp
-  src/duckdb/extension/core_functions/core_functions_extension.cpp
-  src/duckdb/extension/core_functions/function_list.cpp
   src/duckdb/extension/core_functions/lambda_functions.cpp
-  src/duckdb/ub_extension_core_functions_aggregate_algebraic.cpp
+  src/duckdb/extension/core_functions/function_list.cpp
+  src/duckdb/extension/core_functions/core_functions_extension.cpp
+  src/duckdb/ub_extension_core_functions_aggregate_distributive.cpp
   src/duckdb/ub_extension_core_functions_aggregate_nested.cpp
-  src/duckdb/ub_extension_core_functions_aggregate_holistic.cpp
   src/duckdb/ub_extension_core_functions_aggregate_regression.cpp
-  src/duckdb/ub_extension_core_functions_aggregate_distributive.cpp
-  src/duckdb/ub_extension_core_functions_scalar_generic.cpp
-  src/duckdb/ub_extension_core_functions_scalar_array.cpp
-  src/duckdb/ub_extension_core_functions_scalar_random.cpp
-  src/duckdb/ub_extension_core_functions_scalar_list.cpp
-  src/duckdb/ub_extension_core_functions_scalar_struct.cpp
-  src/duckdb/ub_extension_core_functions_scalar_date.cpp
-  src/duckdb/ub_extension_core_functions_scalar_enum.cpp
+  src/duckdb/ub_extension_core_functions_aggregate_algebraic.cpp
+  src/duckdb/ub_extension_core_functions_aggregate_holistic.cpp
+  src/duckdb/ub_extension_core_functions_scalar_string.cpp
+  src/duckdb/ub_extension_core_functions_scalar_bit.cpp
   src/duckdb/ub_extension_core_functions_scalar_operators.cpp
+  src/duckdb/ub_extension_core_functions_scalar_enum.cpp
+  src/duckdb/ub_extension_core_functions_scalar_map.cpp
+  src/duckdb/ub_extension_core_functions_scalar_random.cpp
   src/duckdb/ub_extension_core_functions_scalar_math.cpp
-  src/duckdb/ub_extension_core_functions_scalar_string.cpp
+  src/duckdb/ub_extension_core_functions_scalar_union.cpp
+  src/duckdb/ub_extension_core_functions_scalar_generic.cpp
+  src/duckdb/ub_extension_core_functions_scalar_struct.cpp
+  src/duckdb/ub_extension_core_functions_scalar_list.cpp
+  src/duckdb/ub_extension_core_functions_scalar_array.cpp
   src/duckdb/ub_extension_core_functions_scalar_debug.cpp
-  src/duckdb/ub_extension_core_functions_scalar_bit.cpp
   src/duckdb/ub_extension_core_functions_scalar_blob.cpp
-  src/duckdb/ub_extension_core_functions_scalar_union.cpp
-  src/duckdb/ub_extension_core_functions_scalar_map.cpp
-  src/duckdb/extension/parquet/parquet_crypto.cpp
-  src/duckdb/extension/parquet/parquet_reader.cpp
-  src/duckdb/extension/parquet/parquet_metadata.cpp
+  src/duckdb/ub_extension_core_functions_scalar_date.cpp
   src/duckdb/extension/parquet/parquet_writer.cpp
   src/duckdb/extension/parquet/zstd_file_system.cpp
+  src/duckdb/extension/parquet/parquet_crypto.cpp
+  src/duckdb/extension/parquet/parquet_reader.cpp
   src/duckdb/extension/parquet/parquet_timestamp.cpp
+  src/duckdb/extension/parquet/parquet_float16.cpp
+  src/duckdb/extension/parquet/parquet_statistics.cpp
   src/duckdb/extension/parquet/parquet_multi_file_info.cpp
-  src/duckdb/extension/parquet/column_writer.cpp
+  src/duckdb/extension/parquet/column_reader.cpp
   src/duckdb/extension/parquet/geo_parquet.cpp
-  src/duckdb/extension/parquet/parquet_file_metadata_cache.cpp
-  src/duckdb/extension/parquet/parquet_statistics.cpp
   src/duckdb/extension/parquet/parquet_extension.cpp
-  src/duckdb/extension/parquet/parquet_float16.cpp
+  src/duckdb/extension/parquet/column_writer.cpp
+  src/duckdb/extension/parquet/parquet_file_metadata_cache.cpp
   src/duckdb/extension/parquet/serialize_parquet.cpp
-  src/duckdb/extension/parquet/column_reader.cpp
+  src/duckdb/extension/parquet/parquet_metadata.cpp
   src/duckdb/ub_extension_parquet_decoder.cpp
-  src/duckdb/ub_extension_parquet_writer.cpp
   src/duckdb/ub_extension_parquet_reader.cpp
   src/duckdb/ub_extension_parquet_reader_variant.cpp
+  src/duckdb/ub_extension_parquet_writer.cpp
   src/duckdb/third_party/parquet/parquet_types.cpp
   src/duckdb/third_party/thrift/thrift/protocol/TProtocol.cpp
   src/duckdb/third_party/thrift/thrift/transport/TTransportException.cpp
@@ -427,32 +427,32 @@ set(DUCKDB_SRC_FILES
   src/duckdb/third_party/brotli/enc/metablock.cpp
   src/duckdb/third_party/brotli/enc/static_dict.cpp
   src/duckdb/third_party/brotli/enc/utf8_util.cpp
-  src/duckdb/extension/icu/./icu-datetrunc.cpp
-  src/duckdb/extension/icu/./icu-timezone.cpp
-  src/duckdb/extension/icu/./icu-current.cpp
-  src/duckdb/extension/icu/./icu-list-range.cpp
-  src/duckdb/extension/icu/./icu-datefunc.cpp
-  src/duckdb/extension/icu/./icu-strptime.cpp
-  src/duckdb/extension/icu/./icu-dateadd.cpp
   src/duckdb/extension/icu/./icu_extension.cpp
-  src/duckdb/extension/icu/./icu-timebucket.cpp
-  src/duckdb/extension/icu/./icu-table-range.cpp
+  src/duckdb/extension/icu/./icu-strptime.cpp
   src/duckdb/extension/icu/./icu-datepart.cpp
   src/duckdb/extension/icu/./icu-makedate.cpp
+  src/duckdb/extension/icu/./icu-datefunc.cpp
+  src/duckdb/extension/icu/./icu-current.cpp
+  src/duckdb/extension/icu/./icu-timezone.cpp
+  src/duckdb/extension/icu/./icu-table-range.cpp
+  src/duckdb/extension/icu/./icu-dateadd.cpp
+  src/duckdb/extension/icu/./icu-timebucket.cpp
   src/duckdb/extension/icu/./icu-datesub.cpp
+  src/duckdb/extension/icu/./icu-datetrunc.cpp
+  src/duckdb/extension/icu/./icu-list-range.cpp
   src/duckdb/ub_extension_icu_third_party_icu_common.cpp
   src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp
   src/duckdb/extension/icu/third_party/icu/stubdata/stubdata.cpp
+  src/duckdb/extension/json/json_reader.cpp
   src/duckdb/extension/json/json_functions.cpp
+  src/duckdb/extension/json/json_extension.cpp
+  src/duckdb/extension/json/json_scan.cpp
   src/duckdb/extension/json/json_multi_file_info.cpp
+  src/duckdb/extension/json/json_enums.cpp
   src/duckdb/extension/json/serialize_json.cpp
   src/duckdb/extension/json/json_common.cpp
-  src/duckdb/extension/json/json_scan.cpp
-  src/duckdb/extension/json/json_enums.cpp
-  src/duckdb/extension/json/json_reader.cpp
-  src/duckdb/extension/json/json_serializer.cpp
-  src/duckdb/extension/json/json_extension.cpp
   src/duckdb/extension/json/json_deserializer.cpp
+  src/duckdb/extension/json/json_serializer.cpp
   src/duckdb/ub_extension_json_json_functions.cpp)
 
 set(JEMALLOC_SRC_FILES 

src/duckdb/extension/parquet/column_writer.cpp

@@ -187,9 +187,12 @@ void ColumnWriter::HandleRepeatLevels(ColumnWriterState &state, ColumnWriterStat
 		// no repeat levels without a parent node
 		return;
 	}
-	while (state.repetition_levels.size() < parent->repetition_levels.size()) {
-		state.repetition_levels.push_back(parent->repetition_levels[state.repetition_levels.size()]);
+	if (state.repetition_levels.size() >= parent->repetition_levels.size()) {
+		return;
 	}
+	state.repetition_levels.insert(state.repetition_levels.end(),
+	                               parent->repetition_levels.begin() + state.repetition_levels.size(),
+	                               parent->repetition_levels.end());
 }
 
 void ColumnWriter::HandleDefineLevels(ColumnWriterState &state, ColumnWriterState *parent, const ValidityMask &validity,
@@ -200,36 +203,41 @@ void ColumnWriter::HandleDefineLevels(ColumnWriterState &state, ColumnWriterStat
 		while (state.definition_levels.size() < parent->definition_levels.size()) {
 			idx_t current_index = state.definition_levels.size();
 			if (parent->definition_levels[current_index] != PARQUET_DEFINE_VALID) {
+				//! Inherit nulls from parent
 				state.definition_levels.push_back(parent->definition_levels[current_index]);
 				state.parent_null_count++;
 			} else if (validity.RowIsValid(vector_index)) {
+				//! Produce a non-null define
 				state.definition_levels.push_back(define_value);
 			} else {
+				//! Produce a null define
 				if (!can_have_nulls) {
 					throw IOException("Parquet writer: map key column is not allowed to contain NULL values");
 				}
 				state.null_count++;
 				state.definition_levels.push_back(null_value);
 			}
+			D_ASSERT(parent->is_empty.empty() || current_index < parent->is_empty.size());
 			if (parent->is_empty.empty() || !parent->is_empty[current_index]) {
 				vector_index++;
 			}
 		}
+		return;
+	}
+
+	// no parent: set definition levels only from this validity mask
+	if (validity.AllValid()) {
+		state.definition_levels.insert(state.definition_levels.end(), count, define_value);
 	} else {
-		// no parent: set definition levels only from this validity mask
-		if (validity.AllValid()) {
-			state.definition_levels.insert(state.definition_levels.end(), count, define_value);
-		} else {
-			for (idx_t i = 0; i < count; i++) {
-				const auto is_null = !validity.RowIsValid(i);
-				state.definition_levels.emplace_back(is_null ? null_value : define_value);
-				state.null_count += is_null;
-			}
-		}
-		if (!can_have_nulls && state.null_count != 0) {
-			throw IOException("Parquet writer: map key column is not allowed to contain NULL values");
+		for (idx_t i = 0; i < count; i++) {
+			const auto is_null = !validity.RowIsValid(i);
+			state.definition_levels.emplace_back(is_null ? null_value : define_value);
+			state.null_count += is_null;
 		}
 	}
+	if (!can_have_nulls && state.null_count != 0) {
+		throw IOException("Parquet writer: map key column is not allowed to contain NULL values");
+	}
 }
 
 //===--------------------------------------------------------------------===//
@@ -368,6 +376,7 @@ ParquetColumnSchema ColumnWriter::FillParquetSchema(vector<duckdb_parquet::Schem
 		}
 		return map_column;
 	}
+
 	duckdb_parquet::SchemaElement schema_element;
 	schema_element.type = ParquetWriter::DuckDBTypeToParquetType(type);
 	schema_element.repetition_type = null_type;

src/duckdb/extension/parquet/include/column_writer.hpp

@@ -27,7 +27,7 @@ class ColumnWriterState {
 
 	unsafe_vector<uint16_t> definition_levels;
 	unsafe_vector<uint16_t> repetition_levels;
-	vector<bool> is_empty;
+	unsafe_vector<uint8_t> is_empty;
 	idx_t parent_null_count = 0;
 	idx_t null_count = 0;
 

src/duckdb/extension/parquet/include/writer/array_column_writer.hpp

@@ -25,6 +25,10 @@ class ArrayColumnWriter : public ListColumnWriter {
 	void Prepare(ColumnWriterState &state, ColumnWriterState *parent, Vector &vector, idx_t count,
 	             bool vector_can_span_multiple_pages) override;
 	void Write(ColumnWriterState &state, Vector &vector, idx_t count) override;
+
+protected:
+	void WriteArrayState(ListColumnWriterState &state, idx_t array_size, uint16_t first_repeat_level,
+	                     idx_t define_value, const bool is_empty = false);
 };
 
 } // namespace duckdb

src/duckdb/extension/parquet/writer/array_column_writer.cpp

@@ -9,6 +9,22 @@ void ArrayColumnWriter::Analyze(ColumnWriterState &state_p, ColumnWriterState *p
 	child_writer->Analyze(*state.child_state, &state_p, array_child, array_size * count);
 }
 
+void ArrayColumnWriter::WriteArrayState(ListColumnWriterState &state, idx_t array_size, uint16_t first_repeat_level,
+                                        idx_t define_value, const bool is_empty) {
+	state.definition_levels.push_back(define_value);
+	state.repetition_levels.push_back(first_repeat_level);
+	state.is_empty.push_back(is_empty);
+
+	if (is_empty) {
+		return;
+	}
+	for (idx_t k = 1; k < array_size; k++) {
+		state.repetition_levels.push_back(MaxRepeat() + 1);
+		state.definition_levels.push_back(define_value);
+		state.is_empty.push_back(false);
+	}
+}
+
 void ArrayColumnWriter::Prepare(ColumnWriterState &state_p, ColumnWriterState *parent, Vector &vector, idx_t count,
                                 bool vector_can_span_multiple_pages) {
 	auto &state = state_p.Cast<ListColumnWriterState>();
@@ -25,42 +41,20 @@ void ArrayColumnWriter::Prepare(ColumnWriterState &state_p, ColumnWriterState *p
 	for (idx_t i = start; i < vcount; i++) {
 		idx_t parent_index = state.parent_index + i;
 		if (parent && !parent->is_empty.empty() && parent->is_empty[parent_index]) {
-			state.definition_levels.push_back(parent->definition_levels[parent_index]);
-			state.repetition_levels.push_back(parent->repetition_levels[parent_index]);
-			state.is_empty.push_back(true);
+			WriteArrayState(state, array_size, parent->repetition_levels[parent_index],
+			                parent->definition_levels[parent_index], true);
 			continue;
 		}
 		auto first_repeat_level =
 		    parent && !parent->repetition_levels.empty() ? parent->repetition_levels[parent_index] : MaxRepeat();
 		if (parent && parent->definition_levels[parent_index] != PARQUET_DEFINE_VALID) {
-			state.definition_levels.push_back(parent->definition_levels[parent_index]);
-			state.repetition_levels.push_back(first_repeat_level);
-			state.is_empty.push_back(false);
-			for (idx_t k = 1; k < array_size; k++) {
-				state.repetition_levels.push_back(MaxRepeat() + 1);
-				state.definition_levels.push_back(parent->definition_levels[parent_index]);
-				state.is_empty.push_back(false);
-			}
+			WriteArrayState(state, array_size, first_repeat_level, parent->definition_levels[parent_index]);
 		} else if (validity.RowIsValid(vector_index)) {
 			// push the repetition levels
-			state.definition_levels.push_back(PARQUET_DEFINE_VALID);
-			state.is_empty.push_back(false);
-
-			state.repetition_levels.push_back(first_repeat_level);
-			for (idx_t k = 1; k < array_size; k++) {
-				state.repetition_levels.push_back(MaxRepeat() + 1);
-				state.definition_levels.push_back(PARQUET_DEFINE_VALID);
-				state.is_empty.push_back(false);
-			}
+			WriteArrayState(state, array_size, first_repeat_level, PARQUET_DEFINE_VALID);
 		} else {
-			state.definition_levels.push_back(MaxDefine() - 1);
-			state.repetition_levels.push_back(first_repeat_level);
-			state.is_empty.push_back(false);
-			for (idx_t k = 1; k < array_size; k++) {
-				state.repetition_levels.push_back(MaxRepeat() + 1);
-				state.definition_levels.push_back(MaxDefine() - 1);
-				state.is_empty.push_back(false);
-			}
+			//! Produce a null
+			WriteArrayState(state, array_size, first_repeat_level, MaxDefine() - 1);
 		}
 		vector_index++;
 	}

src/duckdb/extension/parquet/writer/struct_column_writer.cpp

@@ -62,8 +62,9 @@ void StructColumnWriter::Prepare(ColumnWriterState &state_p, ColumnWriterState *
 	auto &validity = FlatVector::Validity(vector);
 	if (parent) {
 		// propagate empty entries from the parent
-		while (state.is_empty.size() < parent->is_empty.size()) {
-			state.is_empty.push_back(parent->is_empty[state.is_empty.size()]);
+		if (state.is_empty.size() < parent->is_empty.size()) {
+			state.is_empty.insert(state.is_empty.end(), parent->is_empty.begin() + state.is_empty.size(),
+			                      parent->is_empty.end());
 		}
 	}
 	HandleRepeatLevels(state_p, parent, count, MaxRepeat());

src/duckdb/src/catalog/catalog.cpp

@@ -1135,6 +1135,9 @@ vector<reference<SchemaCatalogEntry>> Catalog::GetAllSchemas(ClientContext &cont
 	auto &db_manager = DatabaseManager::Get(context);
 	auto databases = db_manager.GetDatabases(context);
 	for (auto &database : databases) {
+		if (database->GetVisibility() == AttachVisibility::HIDDEN) {
+			continue;
+		}
 		auto &catalog = database->GetCatalog();
 		auto new_schemas = catalog.GetSchemas(context);
 		result.insert(result.end(), new_schemas.begin(), new_schemas.end());

src/duckdb/src/common/enum_util.cpp

@@ -60,6 +60,7 @@
 #include "duckdb/common/enums/set_scope.hpp"
 #include "duckdb/common/enums/set_type.hpp"
 #include "duckdb/common/enums/statement_type.hpp"
+#include "duckdb/common/enums/storage_block_prefetch.hpp"
 #include "duckdb/common/enums/stream_execution_result.hpp"
 #include "duckdb/common/enums/subquery_type.hpp"
 #include "duckdb/common/enums/tableref_type.hpp"
@@ -117,6 +118,7 @@
 #include "duckdb/function/table/arrow/enum/arrow_variable_size_type.hpp"
 #include "duckdb/function/table_function.hpp"
 #include "duckdb/function/window/window_merge_sort_tree.hpp"
+#include "duckdb/logging/log_storage.hpp"
 #include "duckdb/logging/logging.hpp"
 #include "duckdb/main/appender.hpp"
 #include "duckdb/main/capi/capi_internal.hpp"
@@ -2458,6 +2460,25 @@ LogMode EnumUtil::FromString<LogMode>(const char *value) {
 	return static_cast<LogMode>(StringUtil::StringToEnum(GetLogModeValues(), 3, "LogMode", value));
 }
 
+const StringUtil::EnumStringLiteral *GetLoggingTargetTableValues() {
+	static constexpr StringUtil::EnumStringLiteral values[] {
+		{ static_cast<uint32_t>(LoggingTargetTable::ALL_LOGS), "ALL_LOGS" },
+		{ static_cast<uint32_t>(LoggingTargetTable::LOG_ENTRIES), "LOG_ENTRIES" },
+		{ static_cast<uint32_t>(LoggingTargetTable::LOG_CONTEXTS), "LOG_CONTEXTS" }
+	};
+	return values;
+}
+
+template<>
+const char* EnumUtil::ToChars<LoggingTargetTable>(LoggingTargetTable value) {
+	return StringUtil::EnumToString(GetLoggingTargetTableValues(), 3, "LoggingTargetTable", static_cast<uint32_t>(value));
+}
+
+template<>
+LoggingTargetTable EnumUtil::FromString<LoggingTargetTable>(const char *value) {
+	return static_cast<LoggingTargetTable>(StringUtil::StringToEnum(GetLoggingTargetTableValues(), 3, "LoggingTargetTable", value));
+}
+
 const StringUtil::EnumStringLiteral *GetLogicalOperatorTypeValues() {
 	static constexpr StringUtil::EnumStringLiteral values[] {
 		{ static_cast<uint32_t>(LogicalOperatorType::LOGICAL_INVALID), "LOGICAL_INVALID" },
@@ -4235,6 +4256,26 @@ StatsInfo EnumUtil::FromString<StatsInfo>(const char *value) {
 	return static_cast<StatsInfo>(StringUtil::StringToEnum(GetStatsInfoValues(), 5, "StatsInfo", value));
 }
 
+const StringUtil::EnumStringLiteral *GetStorageBlockPrefetchValues() {
+	static constexpr StringUtil::EnumStringLiteral values[] {
+		{ static_cast<uint32_t>(StorageBlockPrefetch::REMOTE_ONLY), "REMOTE_ONLY" },
+		{ static_cast<uint32_t>(StorageBlockPrefetch::NEVER), "NEVER" },
+		{ static_cast<uint32_t>(StorageBlockPrefetch::ALWAYS_PREFETCH), "ALWAYS_PREFETCH" },
+		{ static_cast<uint32_t>(StorageBlockPrefetch::DEBUG_FORCE_ALWAYS), "DEBUG_FORCE_ALWAYS" }
+	};
+	return values;
+}
+
+template<>
+const char* EnumUtil::ToChars<StorageBlockPrefetch>(StorageBlockPrefetch value) {
+	return StringUtil::EnumToString(GetStorageBlockPrefetchValues(), 4, "StorageBlockPrefetch", static_cast<uint32_t>(value));
+}
+
+template<>
+StorageBlockPrefetch EnumUtil::FromString<StorageBlockPrefetch>(const char *value) {
+	return static_cast<StorageBlockPrefetch>(StringUtil::StringToEnum(GetStorageBlockPrefetchValues(), 4, "StorageBlockPrefetch", value));
+}
+
 const StringUtil::EnumStringLiteral *GetStrTimeSpecifierValues() {
 	static constexpr StringUtil::EnumStringLiteral values[] {
 		{ static_cast<uint32_t>(StrTimeSpecifier::ABBREVIATED_WEEKDAY_NAME), "ABBREVIATED_WEEKDAY_NAME" },