Update vendored DuckDB sources to df0a3de744

src/duckdb/src/catalog/catalog.cpp

@@ -661,7 +661,7 @@ CatalogException Catalog::CreateMissingEntryException(CatalogEntryRetriever &ret
 		if (unseen_schemas.size() >= max_schema_count) {
 			break;
 		}
-		auto &catalog = database.get().GetCatalog();
+		auto &catalog = database->GetCatalog();
 		auto current_schemas = catalog.GetSchemas(context);
 		for (auto &current_schema : current_schemas) {
 			if (unseen_schemas.size() >= max_schema_count) {
@@ -1119,8 +1119,8 @@ vector<reference<SchemaCatalogEntry>> Catalog::GetAllSchemas(ClientContext &cont
 
 	auto &db_manager = DatabaseManager::Get(context);
 	auto databases = db_manager.GetDatabases(context);
-	for (auto database : databases) {
-		auto &catalog = database.get().GetCatalog();
+	for (auto &database : databases) {
+		auto &catalog = database->GetCatalog();
 		auto new_schemas = catalog.GetSchemas(context);
 		result.insert(result.end(), new_schemas.begin(), new_schemas.end());
 	}

src/duckdb/src/common/enum_util.cpp

@@ -4392,19 +4392,20 @@ const StringUtil::EnumStringLiteral *GetUndoFlagsValues() {
 		{ static_cast<uint32_t>(UndoFlags::INSERT_TUPLE), "INSERT_TUPLE" },
 		{ static_cast<uint32_t>(UndoFlags::DELETE_TUPLE), "DELETE_TUPLE" },
 		{ static_cast<uint32_t>(UndoFlags::UPDATE_TUPLE), "UPDATE_TUPLE" },
-		{ static_cast<uint32_t>(UndoFlags::SEQUENCE_VALUE), "SEQUENCE_VALUE" }
+		{ static_cast<uint32_t>(UndoFlags::SEQUENCE_VALUE), "SEQUENCE_VALUE" },
+		{ static_cast<uint32_t>(UndoFlags::ATTACHED_DATABASE), "ATTACHED_DATABASE" }
 	};
 	return values;
 }
 
 template<>
 const char* EnumUtil::ToChars<UndoFlags>(UndoFlags value) {
-	return StringUtil::EnumToString(GetUndoFlagsValues(), 6, "UndoFlags", static_cast<uint32_t>(value));
+	return StringUtil::EnumToString(GetUndoFlagsValues(), 7, "UndoFlags", static_cast<uint32_t>(value));
 }
 
 template<>
 UndoFlags EnumUtil::FromString<UndoFlags>(const char *value) {
-	return static_cast<UndoFlags>(StringUtil::StringToEnum(GetUndoFlagsValues(), 6, "UndoFlags", value));
+	return static_cast<UndoFlags>(StringUtil::StringToEnum(GetUndoFlagsValues(), 7, "UndoFlags", value));
 }
 
 const StringUtil::EnumStringLiteral *GetUnionInvalidReasonValues() {

src/duckdb/src/execution/operator/helper/physical_transaction.cpp

@@ -35,8 +35,8 @@ SourceResultType PhysicalTransaction::GetData(ExecutionContext &context, DataChu
 			if (config.options.immediate_transaction_mode) {
 				// if immediate transaction mode is enabled then start all transactions immediately
 				auto databases = DatabaseManager::Get(client).GetDatabases(client);
-				for (auto db : databases) {
-					context.client.transaction.ActiveTransaction().GetTransaction(db.get());
+				for (auto &db : databases) {
+					context.client.transaction.ActiveTransaction().GetTransaction(*db);
 				}
 			}
 		} else {

src/duckdb/src/execution/operator/schema/physical_attach.cpp

@@ -73,6 +73,8 @@ SourceResultType PhysicalAttach::GetData(ExecutionContext &context, DataChunk &c
 		attached_db->GetCatalog().SetDefaultTable(options.default_table.schema, options.default_table.name);
 	}
 	attached_db->FinalizeLoad(context.client);
+
+	db_manager.FinalizeAttach(context.client, *info, std::move(attached_db));
 	return SourceResultType::FINISHED;
 }
 

src/duckdb/src/function/table/system/duckdb_databases.cpp

@@ -8,7 +8,7 @@ struct DuckDBDatabasesData : public GlobalTableFunctionState {
 	DuckDBDatabasesData() : offset(0) {
 	}
 
-	vector<reference<AttachedDatabase>> entries;
+	vector<shared_ptr<AttachedDatabase>> entries;
 	idx_t offset;
 };
 
@@ -62,7 +62,7 @@ void DuckDBDatabasesFunction(ClientContext &context, TableFunctionInput &data_p,
 	while (data.offset < data.entries.size() && count < STANDARD_VECTOR_SIZE) {
 		auto &entry = data.entries[data.offset++];
 
-		auto &attached = entry.get().Cast<AttachedDatabase>();
+		auto &attached = *entry;
 		// return values:
 
 		idx_t col = 0;

src/duckdb/src/function/table/system/pragma_database_size.cpp

@@ -15,7 +15,7 @@ struct PragmaDatabaseSizeData : public GlobalTableFunctionState {
 	}
 
 	idx_t index;
-	vector<reference<AttachedDatabase>> databases;
+	vector<shared_ptr<AttachedDatabase>> databases;
 	Value memory_usage;
 	Value memory_limit;
 };
@@ -68,7 +68,7 @@ void PragmaDatabaseSizeFunction(ClientContext &context, TableFunctionInput &data
 	auto &data = data_p.global_state->Cast<PragmaDatabaseSizeData>();
 	idx_t row = 0;
 	for (; data.index < data.databases.size() && row < STANDARD_VECTOR_SIZE; data.index++) {
-		auto &db = data.databases[data.index].get();
+		auto &db = *data.databases[data.index];
 		if (db.IsSystem() || db.IsTemporary()) {
 			continue;
 		}

src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,5 +1,5 @@
 #ifndef DUCKDB_PATCH_VERSION
-#define DUCKDB_PATCH_VERSION "3-dev200"
+#define DUCKDB_PATCH_VERSION "3-dev227"
 #endif
 #ifndef DUCKDB_MINOR_VERSION
 #define DUCKDB_MINOR_VERSION 3
@@ -8,10 +8,10 @@
 #define DUCKDB_MAJOR_VERSION 1
 #endif
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "v1.3.3-dev200"
+#define DUCKDB_VERSION "v1.3.3-dev227"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "c5310ec83b"
+#define DUCKDB_SOURCE_ID "df0a3de744"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

src/duckdb/src/include/duckdb/common/enums/undo_flags.hpp

@@ -18,7 +18,8 @@ enum class UndoFlags : uint32_t { // far too big but aligned (TM)
 	INSERT_TUPLE = 2,
 	DELETE_TUPLE = 3,
 	UPDATE_TUPLE = 4,
-	SEQUENCE_VALUE = 5
+	SEQUENCE_VALUE = 5,
+	ATTACHED_DATABASE = 6
 };
 
 } // namespace duckdb

src/duckdb/src/include/duckdb/main/attached_database.hpp

@@ -32,6 +32,14 @@ enum class AttachedDatabaseType {
 	TEMP_DATABASE,
 };
 
+struct StoredDatabasePath {
+	StoredDatabasePath(DatabaseManager &manager, string path, const string &name);
+	~StoredDatabasePath();
+
+	DatabaseManager &manager;
+	string path;
+};
+
 //! AttachOptions holds information about a database we plan to attach. These options are generalized, i.e.,
 //! they have to apply to any database file type (duckdb, sqlite, etc.).
 struct AttachOptions {
@@ -51,7 +59,7 @@ struct AttachOptions {
 };
 
 //! The AttachedDatabase represents an attached database instance.
-class AttachedDatabase : public CatalogEntry {
+class AttachedDatabase : public CatalogEntry, public enable_shared_from_this<AttachedDatabase> {
 public:
 	//! Create the built-in system database (without storage).
 	explicit AttachedDatabase(DatabaseInstance &db, AttachedDatabaseType type = AttachedDatabaseType::SYSTEM_DATABASE);
@@ -94,8 +102,12 @@ class AttachedDatabase : public CatalogEntry {
 	static bool NameIsReserved(const string &name);
 	static string ExtractDatabaseName(const string &dbpath, FileSystem &fs);
 
+private:
+	void InsertDatabasePath(const string &path);
+
 private:
 	DatabaseInstance &db;
+	unique_ptr<StoredDatabasePath> stored_database_path;
 	unique_ptr<StorageManager> storage;
 	unique_ptr<Catalog> catalog;
 	unique_ptr<TransactionManager> transaction_manager;

src/duckdb/src/include/duckdb/main/database.hpp

@@ -75,7 +75,7 @@ class DatabaseInstance : public enable_shared_from_this<DatabaseInstance> {
 
 	DUCKDB_API SettingLookupResult TryGetCurrentSetting(const string &key, Value &result) const;
 
-	unique_ptr<AttachedDatabase> CreateAttachedDatabase(ClientContext &context, AttachInfo &info,
+	shared_ptr<AttachedDatabase> CreateAttachedDatabase(ClientContext &context, AttachInfo &info,
 	                                                    AttachOptions &options);
 
 	void AddExtensionInfo(const string &name, const ExtensionLoadedInfo &info);

src/duckdb/src/include/duckdb/main/database_manager.hpp

@@ -48,30 +48,37 @@ class DatabaseManager {
 	//! Get an attached database by its name
 	optional_ptr<AttachedDatabase> GetDatabase(ClientContext &context, const string &name);
 	//! Attach a new database
-	optional_ptr<AttachedDatabase> AttachDatabase(ClientContext &context, AttachInfo &info, AttachOptions &options);
+	shared_ptr<AttachedDatabase> AttachDatabase(ClientContext &context, AttachInfo &info, AttachOptions &options);
+
+	optional_ptr<AttachedDatabase> FinalizeAttach(ClientContext &context, AttachInfo &info,
+	                                              shared_ptr<AttachedDatabase> database);
 	//! Detach an existing database
 	void DetachDatabase(ClientContext &context, const string &name, OnEntryNotFound if_not_found);
+	//! Rollback the attach of a database
+	shared_ptr<AttachedDatabase> DetachInternal(const string &name);
 	//! Returns a reference to the system catalog
 	Catalog &GetSystemCatalog();
 
 	static const string &GetDefaultDatabase(ClientContext &context);
 	void SetDefaultDatabase(ClientContext &context, const string &new_value);
 
 	//! Inserts a path to name mapping to the database paths map
-	void InsertDatabasePath(ClientContext &context, const string &path, const string &name);
+	void InsertDatabasePath(const string &path, const string &name);
 	//! Erases a path from the database paths map
 	void EraseDatabasePath(const string &path);
+	//! Check if a path has a conflict
+	void CheckPathConflict(const string &path, const string &name);
 
 	//! Returns the database type. This might require checking the header of the file, in which case the file handle is
 	//! necessary. We can only grab the file handle, if it is not yet held, even for uncommitted changes. Thus, we have
 	//! to lock for this operation.
 	void GetDatabaseType(ClientContext &context, AttachInfo &info, const DBConfig &config, AttachOptions &options);
 	//! Scans the catalog set and adds each committed database entry, and each database entry of the current
 	//! transaction, to a vector holding AttachedDatabase references
-	vector<reference<AttachedDatabase>> GetDatabases(ClientContext &context,
-	                                                 const optional_idx max_db_count = optional_idx());
+	vector<shared_ptr<AttachedDatabase>> GetDatabases(ClientContext &context,
+	                                                  const optional_idx max_db_count = optional_idx());
 	//! Scans the catalog set and returns each committed database entry
-	vector<reference<AttachedDatabase>> GetDatabases();
+	vector<shared_ptr<AttachedDatabase>> GetDatabases();
 	//! Removes all databases from the catalog set. This is necessary for the database instance's destructor,
 	//! as the database manager has to be alive when destroying the catalog set objects.
 	void ResetDatabases(unique_ptr<TaskScheduler> &scheduler);
@@ -97,14 +104,13 @@ class DatabaseManager {
 	//! Gets a list of all attached database paths
 	vector<string> GetAttachedDatabasePaths();
 
-private:
-	void CheckPathConflict(ClientContext &context, const string &path);
-
 private:
 	//! The system database is a special database that holds system entries (e.g. functions)
-	unique_ptr<AttachedDatabase> system;
+	shared_ptr<AttachedDatabase> system;
+	//! Lock for databases
+	mutex databases_lock;
 	//! The set of attached databases
-	unique_ptr<CatalogSet> databases;
+	case_insensitive_map_t<shared_ptr<AttachedDatabase>> databases;
 	//! The next object id handed out by the NextOid method
 	atomic<idx_t> next_oid;
 	//! The current query number

src/duckdb/src/include/duckdb/transaction/duck_transaction.hpp

@@ -51,6 +51,7 @@ class DuckTransaction : public Transaction {
 	LocalStorage &GetLocalStorage();
 
 	void PushCatalogEntry(CatalogEntry &entry, data_ptr_t extra_data, idx_t extra_data_size);
+	void PushAttach(AttachedDatabase &db);
 
 	void SetReadWrite() override;
 

src/duckdb/src/include/duckdb/transaction/duck_transaction_manager.hpp

@@ -70,6 +70,7 @@ class DuckTransactionManager : public TransactionManager {
 
 	void PushCatalogEntry(Transaction &transaction_p, CatalogEntry &entry, data_ptr_t extra_data = nullptr,
 	                      idx_t extra_data_size = 0);
+	void PushAttach(Transaction &transaction_p, AttachedDatabase &db);
 
 protected:
 	struct CheckpointDecision {

src/duckdb/src/include/duckdb/transaction/meta_transaction.hpp

@@ -63,6 +63,7 @@ class MetaTransaction {
 	const vector<reference<AttachedDatabase>> &OpenedTransactions() const {
 		return all_transactions;
 	}
+	AttachedDatabase &UseDatabase(shared_ptr<AttachedDatabase> &database);
 
 private:
 	//! Lock to prevent all_transactions and transactions from getting out of sync
@@ -75,6 +76,8 @@ class MetaTransaction {
 	optional_ptr<AttachedDatabase> modified_database;
 	//! Whether or not the meta transaction is marked as read only
 	bool is_read_only;
+	//! The set of used / referenced databases
+	reference_map_t<AttachedDatabase, shared_ptr<AttachedDatabase>> referenced_databases;
 };
 
 } // namespace duckdb

src/duckdb/src/main/attached_database.cpp

@@ -13,6 +13,14 @@
 
 namespace duckdb {
 
+StoredDatabasePath::StoredDatabasePath(DatabaseManager &manager, string path_p, const string &name)
+    : manager(manager), path(std::move(path_p)) {
+	manager.InsertDatabasePath(path, name);
+}
+
+StoredDatabasePath::~StoredDatabasePath() {
+	manager.EraseDatabasePath(path);
+}
 //===--------------------------------------------------------------------===//
 // Attach Options
 //===--------------------------------------------------------------------===//
@@ -66,7 +74,6 @@ AttachOptions::AttachOptions(const unique_ptr<AttachInfo> &info, const AccessMod
 //===--------------------------------------------------------------------===//
 // Attached Database
 //===--------------------------------------------------------------------===//
-
 AttachedDatabase::AttachedDatabase(DatabaseInstance &db, AttachedDatabaseType type)
     : CatalogEntry(CatalogType::DATABASE_ENTRY,
                    type == AttachedDatabaseType::SYSTEM_DATABASE ? SYSTEM_CATALOG : TEMP_CATALOG, 0),
@@ -86,7 +93,6 @@ AttachedDatabase::AttachedDatabase(DatabaseInstance &db, AttachedDatabaseType ty
 AttachedDatabase::AttachedDatabase(DatabaseInstance &db, Catalog &catalog_p, string name_p, string file_path_p,
                                    AttachOptions &options)
     : CatalogEntry(CatalogType::DATABASE_ENTRY, catalog_p, std::move(name_p)), db(db), parent_catalog(&catalog_p) {
-
 	if (options.access_mode == AccessMode::READ_ONLY) {
 		type = AttachedDatabaseType::READ_ONLY_DATABASE;
 	} else {
@@ -107,8 +113,10 @@ AttachedDatabase::AttachedDatabase(DatabaseInstance &db, Catalog &catalog_p, str
 		}
 		throw BinderException("Unrecognized option for attach \"%s\"", entry.first);
 	}
+
 	// We create the storage after the catalog to guarantee we allow extensions to instantiate the DuckCatalog.
 	catalog = make_uniq<DuckCatalog>(*this);
+	InsertDatabasePath(file_path_p);
 	auto read_only = options.access_mode == AccessMode::READ_ONLY;
 	storage = make_uniq<SingleFileStorageManager>(*this, std::move(file_path_p), read_only);
 	transaction_manager = make_uniq<DuckTransactionManager>(*this);
@@ -131,6 +139,7 @@ AttachedDatabase::AttachedDatabase(DatabaseInstance &db, Catalog &catalog_p, Sto
 		throw InternalException("AttachedDatabase - attach function did not return a catalog");
 	}
 	if (catalog->IsDuckCatalog()) {
+		InsertDatabasePath(info.path);
 		// The attached database uses the DuckCatalog.
 		auto read_only = options.access_mode == AccessMode::READ_ONLY;
 		storage = make_uniq<SingleFileStorageManager>(*this, info.path, read_only);
@@ -159,6 +168,13 @@ bool AttachedDatabase::IsReadOnly() const {
 	return type == AttachedDatabaseType::READ_ONLY_DATABASE;
 }
 
+void AttachedDatabase::InsertDatabasePath(const string &path) {
+	if (path.empty() || path == IN_MEMORY_PATH) {
+		return;
+	}
+	stored_database_path = make_uniq<StoredDatabasePath>(db.GetDatabaseManager(), path, name);
+}
+
 bool AttachedDatabase::NameIsReserved(const string &name) {
 	return name == DEFAULT_SCHEMA || name == TEMP_CATALOG || name == SYSTEM_CATALOG;
 }
@@ -232,38 +248,31 @@ void AttachedDatabase::OnDetach(ClientContext &context) {
 }
 
 void AttachedDatabase::Close() {
-	D_ASSERT(catalog);
 	if (is_closed) {
 		return;
 	}
+	D_ASSERT(catalog);
 	is_closed = true;
 
-	if (!IsSystem() && !catalog->InMemory()) {
-		db.GetDatabaseManager().EraseDatabasePath(catalog->GetDBPath());
-	}
-
-	if (Exception::UncaughtException()) {
-		return;
-	}
-	if (!storage) {
-		return;
-	}
-
 	// shutting down: attempt to checkpoint the database
 	// but only if we are not cleaning up as part of an exception unwind
-	try {
-		if (!storage->InMemory()) {
+	if (!Exception::UncaughtException() && storage && !storage->InMemory()) {
+		try {
 			auto &config = DBConfig::GetConfig(db);
-			if (!config.options.checkpoint_on_shutdown) {
-				return;
+			if (config.options.checkpoint_on_shutdown) {
+				CheckpointOptions options;
+				options.wal_action = CheckpointWALAction::DELETE_WAL;
+				storage->CreateCheckpoint(nullptr, options);
 			}
-			CheckpointOptions options;
-			options.wal_action = CheckpointWALAction::DELETE_WAL;
-			storage->CreateCheckpoint(nullptr, options);
+		} catch (...) { // NOLINT
 		}
-	} catch (...) { // NOLINT
 	}
 
+	transaction_manager.reset();
+	catalog.reset();
+	storage.reset();
+	stored_database_path.reset();
+
 	if (Allocator::SupportsFlush()) {
 		Allocator::FlushAll();
 	}

src/duckdb/src/main/client_context.cpp

@@ -496,7 +496,8 @@ void ClientContext::CheckIfPreparedStatementIsExecutable(PreparedStatementData &
 		auto &modified_database = it.first;
 		auto entry = manager.GetDatabase(*this, modified_database);
 		if (!entry) {
-			throw InternalException("Database \"%s\" not found", modified_database);
+			// database has been detached
+			throw InvalidInputException("Database \"%s\" not found", modified_database);
 		}
 		if (entry->IsReadOnly()) {
 			throw InvalidInputException(StringUtil::Format(