diff --git a/src/duckdb/src/catalog/catalog.cpp b/src/duckdb/src/catalog/catalog.cpp index 42353e3ce..60469e9cc 100644 --- a/src/duckdb/src/catalog/catalog.cpp +++ b/src/duckdb/src/catalog/catalog.cpp @@ -1235,4 +1235,9 @@ void Catalog::FinalizeLoad(optional_ptr context) { void Catalog::OnDetach(ClientContext &context) { } +bool Catalog::HasConflictingAttachOptions(const string &path, const AttachOptions &options) { + auto const db_type = options.db_type.empty() ? "duckdb" : options.db_type; + return GetDBPath() != path || GetCatalogType() != db_type; +} + } // namespace duckdb diff --git a/src/duckdb/src/common/vector_operations/vector_hash.cpp b/src/duckdb/src/common/vector_operations/vector_hash.cpp index 062ee1719..95551ecdb 100644 --- a/src/duckdb/src/common/vector_operations/vector_hash.cpp +++ b/src/duckdb/src/common/vector_operations/vector_hash.cpp @@ -483,8 +483,8 @@ void CombineHashTypeSwitch(Vector &hashes, Vector &input, const SelectionVector void VectorOperations::Hash(Vector &input, Vector &result, idx_t count) { if (input.GetVectorType() == VectorType::DICTIONARY_VECTOR && DictionaryVector::CanCacheHashes(input)) { - VectorOperations::Copy(DictionaryVector::GetCachedHashes(input), result, DictionaryVector::SelVector(input), - count, 0, 0); + Vector input_hashes(DictionaryVector::GetCachedHashes(input), DictionaryVector::SelVector(input), count); + TemplatedLoopHash(input_hashes, result, nullptr, count); } else { HashTypeSwitch(input, result, nullptr, count); } diff --git a/src/duckdb/src/execution/operator/schema/physical_attach.cpp b/src/duckdb/src/execution/operator/schema/physical_attach.cpp index d76cd1165..9ab71e379 100644 --- a/src/duckdb/src/execution/operator/schema/physical_attach.cpp +++ b/src/duckdb/src/execution/operator/schema/physical_attach.cpp @@ -51,10 +51,8 @@ SourceResultType PhysicalAttach::GetData(ExecutionContext &context, DataChunk &c existing_db->GetCatalog().SetDefaultTable(options.default_table.schema, options.default_table.name); } if (info->on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { - // same path, name and type, DB does not need replacing - auto const db_type = options.db_type.empty() ? "duckdb" : options.db_type; - if (existing_db->GetCatalog().GetDBPath() == path && - existing_db->GetCatalog().GetCatalogType() == db_type) { + // allow custom catalogs to override this behavior + if (!existing_db->GetCatalog().HasConflictingAttachOptions(path, options)) { return SourceResultType::FINISHED; } } else { diff --git a/src/duckdb/src/function/table/version/pragma_version.cpp b/src/duckdb/src/function/table/version/pragma_version.cpp index b7c1544a2..92a51f3db 100644 --- a/src/duckdb/src/function/table/version/pragma_version.cpp +++ b/src/duckdb/src/function/table/version/pragma_version.cpp @@ -1,5 +1,5 @@ #ifndef DUCKDB_PATCH_VERSION -#define DUCKDB_PATCH_VERSION "0-dev4076" +#define DUCKDB_PATCH_VERSION "0-dev4099" #endif #ifndef DUCKDB_MINOR_VERSION #define DUCKDB_MINOR_VERSION 4 @@ -8,10 +8,10 @@ #define DUCKDB_MAJOR_VERSION 1 #endif #ifndef DUCKDB_VERSION -#define DUCKDB_VERSION "v1.4.0-dev4076" +#define DUCKDB_VERSION "v1.4.0-dev4099" #endif #ifndef DUCKDB_SOURCE_ID -#define DUCKDB_SOURCE_ID "9cb1e3d3ca" +#define DUCKDB_SOURCE_ID "a00499ff6c" #endif #include "duckdb/function/table/system_functions.hpp" #include "duckdb/main/database.hpp" diff --git a/src/duckdb/src/include/duckdb.h b/src/duckdb/src/include/duckdb.h index 7196fc37e..a6cd9e265 100644 --- a/src/duckdb/src/include/duckdb.h +++ b/src/duckdb/src/include/duckdb.h @@ -35,11 +35,7 @@ #ifdef DUCKDB_STATIC_BUILD #define DUCKDB_EXTENSION_API #else -#ifdef DUCKDB_BUILD_LOADABLE_EXTENSION #define DUCKDB_EXTENSION_API __declspec(dllexport) -#else -#define DUCKDB_EXTENSION_API -#endif #endif #else #define DUCKDB_EXTENSION_API __attribute__((visibility("default"))) diff --git a/src/duckdb/src/include/duckdb/catalog/catalog.hpp b/src/duckdb/src/include/duckdb/catalog/catalog.hpp index 770bb3019..9817f6c03 100644 --- a/src/duckdb/src/include/duckdb/catalog/catalog.hpp +++ b/src/duckdb/src/include/duckdb/catalog/catalog.hpp @@ -26,6 +26,7 @@ #include namespace duckdb { +struct AttachOptions; struct CreateSchemaInfo; struct DropInfo; struct BoundCreateTableInfo; @@ -346,6 +347,9 @@ class Catalog { //! Returns the dependency manager of this catalog - if the catalog has anye virtual optional_ptr GetDependencyManager(); + //! Whether attaching a catalog with the given path and attach options would be considered a conflict + virtual bool HasConflictingAttachOptions(const string &path, const AttachOptions &options); + public: template static optional_ptr GetEntry(ClientContext &context, const string &catalog_name, const string &schema_name, diff --git a/src/duckdb/src/include/duckdb/common/winapi.hpp b/src/duckdb/src/include/duckdb/common/winapi.hpp index 625b60b89..67287565f 100644 --- a/src/duckdb/src/include/duckdb/common/winapi.hpp +++ b/src/duckdb/src/include/duckdb/common/winapi.hpp @@ -29,11 +29,7 @@ #ifdef DUCKDB_STATIC_BUILD #define DUCKDB_EXTENSION_API #else -#ifdef DUCKDB_BUILD_LOADABLE_EXTENSION #define DUCKDB_EXTENSION_API __declspec(dllexport) -#else -#define DUCKDB_EXTENSION_API -#endif #endif #else #define DUCKDB_EXTENSION_API __attribute__((visibility("default"))) diff --git a/src/duckdb/src/include/duckdb/main/extension_entries.hpp b/src/duckdb/src/include/duckdb/main/extension_entries.hpp index c710f5010..a32331c9b 100644 --- a/src/duckdb/src/include/duckdb/main/extension_entries.hpp +++ b/src/duckdb/src/include/duckdb/main/extension_entries.hpp @@ -558,6 +558,8 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = { {"st_area_spheroid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_asgeojson", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_ashexwkb", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, + {"st_asmvt", "spatial", CatalogType::AGGREGATE_FUNCTION_ENTRY}, + {"st_asmvtgeom", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_assvg", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_astext", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_aswkb", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, diff --git a/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp b/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp index 0860eb7c9..30dd141b3 100644 --- a/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp +++ b/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp @@ -30,7 +30,8 @@ class TableDataWriter { void WriteTableData(Serializer &metadata_serializer); virtual void WriteUnchangedTable(MetaBlockPointer pointer, idx_t total_rows) = 0; - virtual void FinalizeTable(const TableStatistics &global_stats, DataTableInfo *info, Serializer &serializer) = 0; + virtual void FinalizeTable(const TableStatistics &global_stats, DataTableInfo &info, RowGroupCollection &collection, + Serializer &serializer) = 0; virtual unique_ptr GetRowGroupWriter(RowGroup &row_group) = 0; virtual void AddRowGroup(RowGroupPointer &&row_group_pointer, unique_ptr writer); @@ -54,7 +55,8 @@ class SingleFileTableDataWriter : public TableDataWriter { public: void WriteUnchangedTable(MetaBlockPointer pointer, idx_t total_rows) override; - void FinalizeTable(const TableStatistics &global_stats, DataTableInfo *info, Serializer &serializer) override; + void FinalizeTable(const TableStatistics &global_stats, DataTableInfo &info, RowGroupCollection &collection, + Serializer &serializer) override; unique_ptr GetRowGroupWriter(RowGroup &row_group) override; CheckpointType GetCheckpointType() const override; MetadataManager &GetMetadataManager() override; diff --git a/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp b/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp index a3ff0cc58..6abfb5d5b 100644 --- a/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp +++ b/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp @@ -38,6 +38,9 @@ struct MetadataBlock { idx_t FreeBlocksToInteger(); void FreeBlocksFromInteger(idx_t blocks); + static vector BlocksFromInteger(idx_t free_list); + + string ToString() const; }; struct MetadataPointer { diff --git a/src/duckdb/src/include/duckdb/storage/table/in_memory_checkpoint.hpp b/src/duckdb/src/include/duckdb/storage/table/in_memory_checkpoint.hpp index 68896f664..e36e6169d 100644 --- a/src/duckdb/src/include/duckdb/storage/table/in_memory_checkpoint.hpp +++ b/src/duckdb/src/include/duckdb/storage/table/in_memory_checkpoint.hpp @@ -50,7 +50,8 @@ class InMemoryTableDataWriter : public TableDataWriter { public: void WriteUnchangedTable(MetaBlockPointer pointer, idx_t total_rows) override; - void FinalizeTable(const TableStatistics &global_stats, DataTableInfo *info, Serializer &serializer) override; + void FinalizeTable(const TableStatistics &global_stats, DataTableInfo &info, RowGroupCollection &collection, + Serializer &serializer) override; unique_ptr GetRowGroupWriter(RowGroup &row_group) override; CheckpointType GetCheckpointType() const override; MetadataManager &GetMetadataManager() override; diff --git a/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp b/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp index c943bca85..0a82bf0b4 100644 --- a/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +++ b/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp @@ -51,6 +51,7 @@ class RowGroupCollection { void Initialize(PersistentCollectionData &data); void Initialize(PersistentTableData &data); void InitializeEmpty(); + void FinalizeCheckpoint(MetaBlockPointer pointer); bool IsEmpty() const; diff --git a/src/duckdb/src/storage/checkpoint/table_data_writer.cpp b/src/duckdb/src/storage/checkpoint/table_data_writer.cpp index 0bde4acb3..342ea1ff5 100644 --- a/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +++ b/src/duckdb/src/storage/checkpoint/table_data_writer.cpp @@ -63,8 +63,8 @@ void SingleFileTableDataWriter::WriteUnchangedTable(MetaBlockPointer pointer, id existing_rows = total_rows; } -void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stats, DataTableInfo *info, - Serializer &serializer) { +void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stats, DataTableInfo &info, + RowGroupCollection &collection, Serializer &serializer) { MetaBlockPointer pointer; idx_t total_rows; if (!existing_pointer.IsValid()) { @@ -94,6 +94,7 @@ void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stat RowGroup::Serialize(row_group_pointer, row_group_serializer); row_group_serializer.End(); } + collection.FinalizeCheckpoint(pointer); } else { // we have existing metadata and the table is unchanged - write a pointer to the existing metadata pointer = existing_pointer; @@ -116,7 +117,7 @@ void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stat if (!v1_0_0_storage) { options.emplace("v1_0_0_storage", v1_0_0_storage); } - auto index_storage_infos = info->GetIndexes().SerializeToDisk(context, options); + auto index_storage_infos = info.GetIndexes().SerializeToDisk(context, options); #ifdef DUCKDB_BLOCK_VERIFICATION for (auto &entry : index_storage_infos) { diff --git a/src/duckdb/src/storage/data_table.cpp b/src/duckdb/src/storage/data_table.cpp index 983b35639..2b29b2820 100644 --- a/src/duckdb/src/storage/data_table.cpp +++ b/src/duckdb/src/storage/data_table.cpp @@ -1613,7 +1613,7 @@ void DataTable::Checkpoint(TableDataWriter &writer, Serializer &serializer) { // row-group pointers // table pointer // index data - writer.FinalizeTable(global_stats, info.get(), serializer); + writer.FinalizeTable(global_stats, *info, *row_groups, serializer); } void DataTable::CommitDropColumn(const idx_t column_index) { diff --git a/src/duckdb/src/storage/metadata/metadata_manager.cpp b/src/duckdb/src/storage/metadata/metadata_manager.cpp index d3efd4195..55b8790e4 100644 --- a/src/duckdb/src/storage/metadata/metadata_manager.cpp +++ b/src/duckdb/src/storage/metadata/metadata_manager.cpp @@ -31,6 +31,19 @@ MetadataBlock &MetadataBlock::operator=(MetadataBlock &&other) noexcept { return *this; } +string MetadataBlock::ToString() const { + string result; + for (idx_t i = 0; i < MetadataManager::METADATA_BLOCK_COUNT; i++) { + if (std::find(free_blocks.begin(), free_blocks.end(), i) != free_blocks.end()) { + if (!result.empty()) { + result += ", "; + } + result += to_string(i); + } + } + return "block_id: " + to_string(block_id) + " [" + result + "]"; +} + MetadataManager::MetadataManager(BlockManager &block_manager, BufferManager &buffer_manager) : block_manager(block_manager), buffer_manager(buffer_manager) { } @@ -294,18 +307,24 @@ idx_t MetadataBlock::FreeBlocksToInteger() { return result; } -void MetadataBlock::FreeBlocksFromInteger(idx_t free_list) { - free_blocks.clear(); - if (free_list == 0) { - return; - } +vector MetadataBlock::BlocksFromInteger(idx_t free_list) { + vector blocks; for (idx_t i = 64; i > 0; i--) { auto index = i - 1; idx_t mask = idx_t(1) << index; if (free_list & mask) { - free_blocks.push_back(UnsafeNumericCast(index)); + blocks.push_back(UnsafeNumericCast(index)); } } + return blocks; +} + +void MetadataBlock::FreeBlocksFromInteger(idx_t free_list) { + free_blocks.clear(); + if (free_list == 0) { + return; + } + free_blocks = BlocksFromInteger(free_list); } void MetadataManager::MarkBlocksAsModified() { @@ -339,6 +358,9 @@ void MetadataManager::MarkBlocksAsModified() { } void MetadataManager::ClearModifiedBlocks(const vector &pointers) { + if (pointers.empty()) { + return; + } for (auto &pointer : pointers) { auto block_id = pointer.GetBlockId(); auto block_index = pointer.GetBlockIndex(); diff --git a/src/duckdb/src/storage/table/in_memory_checkpoint.cpp b/src/duckdb/src/storage/table/in_memory_checkpoint.cpp index 911c86747..31a0170f3 100644 --- a/src/duckdb/src/storage/table/in_memory_checkpoint.cpp +++ b/src/duckdb/src/storage/table/in_memory_checkpoint.cpp @@ -89,8 +89,8 @@ InMemoryTableDataWriter::InMemoryTableDataWriter(InMemoryCheckpointer &checkpoin void InMemoryTableDataWriter::WriteUnchangedTable(MetaBlockPointer pointer, idx_t total_rows) { } -void InMemoryTableDataWriter::FinalizeTable(const TableStatistics &global_stats, DataTableInfo *info, - Serializer &serializer) { +void InMemoryTableDataWriter::FinalizeTable(const TableStatistics &global_stats, DataTableInfo &info, + RowGroupCollection &collection, Serializer &serializer) { // nop: no need to write anything } diff --git a/src/duckdb/src/storage/table/row_group.cpp b/src/duckdb/src/storage/table/row_group.cpp index c38d726e1..9fde2ed4c 100644 --- a/src/duckdb/src/storage/table/row_group.cpp +++ b/src/duckdb/src/storage/table/row_group.cpp @@ -1087,6 +1087,11 @@ RowGroupPointer RowGroup::Checkpoint(RowGroupWriteData write_data, RowGroupWrite // this metadata block is not stored - add it to the extra metadata blocks row_group_pointer.extra_metadata_blocks.push_back(column_pointer.block_pointer); } + // set up the pointers correctly within this row group for future operations + column_pointers = row_group_pointer.data_pointers; + has_metadata_blocks = true; + extra_metadata_blocks = row_group_pointer.extra_metadata_blocks; + if (metadata_manager) { row_group_pointer.deletes_pointers = CheckpointDeletes(*metadata_manager); } diff --git a/src/duckdb/src/storage/table/row_group_collection.cpp b/src/duckdb/src/storage/table/row_group_collection.cpp index 06ec400bd..4caada024 100644 --- a/src/duckdb/src/storage/table/row_group_collection.cpp +++ b/src/duckdb/src/storage/table/row_group_collection.cpp @@ -101,6 +101,10 @@ void RowGroupCollection::Initialize(PersistentTableData &data) { metadata_pointer = data.base_table_pointer; } +void RowGroupCollection::FinalizeCheckpoint(MetaBlockPointer pointer) { + metadata_pointer = pointer; +} + void RowGroupCollection::Initialize(PersistentCollectionData &data) { stats.InitializeEmpty(types); auto l = row_groups->Lock(); diff --git a/src/duckdb/third_party/httplib/httplib.hpp b/src/duckdb/third_party/httplib/httplib.hpp index 877849747..4aa0458dc 100644 --- a/src/duckdb/third_party/httplib/httplib.hpp +++ b/src/duckdb/third_party/httplib/httplib.hpp @@ -8555,7 +8555,11 @@ inline long SSLClient::get_openssl_verify_result() const { inline SSL_CTX *SSLClient::ssl_context() const { return ctx_; } inline bool SSLClient::create_and_connect_socket(Socket &socket, Error &error) { - return is_valid() && ClientImpl::create_and_connect_socket(socket, error); + if (!is_valid()) { + error = Error::SSLConnection; + return false; + } + return ClientImpl::create_and_connect_socket(socket, error); } // Assumes that socket_mutex_ is locked and that there are no requests in flight