From 8a41d7c53a1543d546733a4a67a4e22f75fec64e Mon Sep 17 00:00:00 2001 From: Kazuhiro Saito <32720872+ksaito7@users.noreply.github.com> Date: Tue, 19 Jun 2018 12:33:14 -0400 Subject: [PATCH 01/19] Replace tab with spaces (#1417) --- src/catalog/catalog.cpp | 26 ++++++++++----------- src/catalog/index_catalog.cpp | 12 +++++----- src/catalog/layout_catalog.cpp | 6 ++--- src/catalog/system_catalogs.cpp | 16 ++++++------- src/catalog/table_catalog.cpp | 14 +++++------ src/executor/drop_executor.cpp | 2 +- src/include/catalog/catalog_cache.h | 8 +++---- src/include/catalog/column_catalog.h | 4 ++-- src/include/catalog/index_catalog.h | 8 +++---- src/include/catalog/table_catalog.h | 2 +- src/include/storage/data_table.h | 2 +- src/include/storage/layout.h | 2 +- src/storage/layout.cpp | 12 +++++----- test/catalog/catalog_test.cpp | 22 ++++++++--------- test/codegen/table_scan_translator_test.cpp | 2 +- test/executor/drop_test.cpp | 6 ++--- test/sql/drop_sql_test.cpp | 4 ++-- 17 files changed, 74 insertions(+), 74 deletions(-) diff --git a/src/catalog/catalog.cpp b/src/catalog/catalog.cpp index 46f534dc84d..f2d30dd5ecf 100644 --- a/src/catalog/catalog.cpp +++ b/src/catalog/catalog.cpp @@ -369,13 +369,13 @@ ResultType Catalog::CreateTable(const std::string &database_name, // Update pg_table with table info pg_table->InsertTable(table_oid, table_name, schema_name, database_object->GetDatabaseOid(), - table->GetDefaultLayout()->GetOid(), pool_.get(), txn); + table->GetDefaultLayout()->GetOid(), pool_.get(), txn); oid_t column_id = 0; for (const auto &column : table->GetSchema()->GetColumns()) { pg_attribute->InsertColumn(table_oid, column.GetName(), column_id, column.GetOffset(), column.GetType(), column.GetLength(), column.IsInlined(), - column.GetConstraints(), pool_.get(), txn); + column.GetConstraints(), pool_.get(), txn); // Create index on unique single column if (column.IsUnique()) { @@ -393,11 +393,11 @@ ResultType Catalog::CreateTable(const std::string &database_name, // Create layout as default layout auto pg_layout = - catalog_map_[database_object->GetDatabaseOid()]->GetLayoutCatalog(); + catalog_map_[database_object->GetDatabaseOid()]->GetLayoutCatalog(); auto default_layout = table->GetDefaultLayout(); if (!pg_layout->InsertLayout(table_oid, default_layout, pool_.get(), txn)) throw CatalogException("Failed to create a new layout for table " - + table_name); + + table_name); return ResultType::SUCCESS; } @@ -594,10 +594,10 @@ std::shared_ptr Catalog::CreateLayout( // Add the layout the pg_layout table auto pg_layout = catalog_map_[database_oid]->GetLayoutCatalog(); if (pg_layout->GetLayoutWithOid(table_oid, new_layout->GetOid(), txn) - == nullptr && - !pg_layout->InsertLayout(table_oid, new_layout, pool_.get(), txn)) { - LOG_ERROR("Failed to create a new layout for table %u", table_oid); - return nullptr; + == nullptr && + !pg_layout->InsertLayout(table_oid, new_layout, pool_.get(), txn)) { + LOG_ERROR("Failed to create a new layout for table %u", table_oid); + return nullptr; } return new_layout; } @@ -615,7 +615,7 @@ std::shared_ptr Catalog::CreateDefaultLayout( // update table catalog catalog_map_[database_oid]->GetTableCatalog() - ->UpdateDefaultLayoutOid(new_layout->GetOid(), table_oid, txn); + ->UpdateDefaultLayoutOid(new_layout->GetOid(), table_oid, txn); } return new_layout; } @@ -853,16 +853,16 @@ ResultType Catalog::DropLayout(oid_t database_oid, oid_t table_oid, table->ResetDefaultLayout(); auto new_default_layout = table->GetDefaultLayout(); if (pg_layout->GetLayoutWithOid(table_oid, new_default_layout->GetOid(), - txn) == nullptr && - !pg_layout->InsertLayout(table_oid, new_default_layout, - pool_.get(), txn)) { + txn) == nullptr && + !pg_layout->InsertLayout(table_oid, new_default_layout, + pool_.get(), txn)) { LOG_DEBUG("Failed to create a new layout for table %d", table_oid); return ResultType::FAILURE; } // update table catalog catalog_map_[database_oid]->GetTableCatalog() - ->UpdateDefaultLayoutOid(new_default_layout->GetOid(), table_oid, txn); + ->UpdateDefaultLayoutOid(new_default_layout->GetOid(), table_oid, txn); } return ResultType::SUCCESS; diff --git a/src/catalog/index_catalog.cpp b/src/catalog/index_catalog.cpp index 5c66139e81f..066590ac4d1 100644 --- a/src/catalog/index_catalog.cpp +++ b/src/catalog/index_catalog.cpp @@ -174,11 +174,11 @@ bool IndexCatalog::DeleteIndex(oid_t database_oid, oid_t index_oid, values.push_back(type::ValueFactory::GetIntegerValue(index_oid).Copy()); auto index_object = txn->catalog_cache.GetCachedIndexObject(database_oid, - index_oid); + index_oid); if (index_object) { auto table_object = txn->catalog_cache.GetCachedTableObject(database_oid, - index_object->GetTableOid()); + index_object->GetTableOid()); table_object->EvictAllIndexObjects(); } @@ -192,7 +192,7 @@ std::shared_ptr IndexCatalog::GetIndexObject( } // try get from cache auto index_object = txn->catalog_cache.GetCachedIndexObject(database_oid, - index_oid); + index_oid); if (index_object) { return index_object; } @@ -227,15 +227,15 @@ std::shared_ptr IndexCatalog::GetIndexObject( } std::shared_ptr IndexCatalog::GetIndexObject( - const std::string &database_name, const std::string &index_name, - const std::string &schema_name, concurrency::TransactionContext *txn) { + const std::string &database_name, const std::string &index_name, + const std::string &schema_name, concurrency::TransactionContext *txn) { if (txn == nullptr) { throw CatalogException("Transaction is invalid!"); } // try get from cache auto index_object = txn->catalog_cache.GetCachedIndexObject(database_name, index_name, - schema_name); + schema_name); if (index_object) { return index_object; } diff --git a/src/catalog/layout_catalog.cpp b/src/catalog/layout_catalog.cpp index 3aa45ff566d..2189426fb0d 100644 --- a/src/catalog/layout_catalog.cpp +++ b/src/catalog/layout_catalog.cpp @@ -200,12 +200,12 @@ LayoutCatalog::GetLayouts(oid_t table_oid, column_map_type column_map; if (column_map_str.length() != 0) { - column_map = - storage::Layout::DeserializeColumnMap(num_columns, column_map_str); + column_map = + storage::Layout::DeserializeColumnMap(num_columns, column_map_str); } auto layout_object = std::make_shared(column_map, num_columns, - layout_oid); + layout_oid); table_object->InsertLayout(layout_object); } } diff --git a/src/catalog/system_catalogs.cpp b/src/catalog/system_catalogs.cpp index a2cbec24e60..611f6562c42 100644 --- a/src/catalog/system_catalogs.cpp +++ b/src/catalog/system_catalogs.cpp @@ -60,8 +60,8 @@ SystemCatalogs::SystemCatalogs(storage::Database *database, pg_attribute_->InsertColumn(shared_tables[i].second, column.GetName(), column_id, column.GetOffset(), column.GetType(), column.GetLength(), - column.IsInlined(), column.GetConstraints(), - pool, txn); + column.IsInlined(), column.GetConstraints(), + pool, txn); column_id++; } } @@ -110,12 +110,12 @@ void SystemCatalogs::Bootstrap(const std::string &database_name, // Reset oid of each catalog to avoid collisions between catalog // values added by system and users when checkpoint recovery. - pg_attribute_->UpdateOid(OID_FOR_USER_OFFSET); - pg_namespace_->UpdateOid(OID_FOR_USER_OFFSET); - pg_table_->UpdateOid(OID_FOR_USER_OFFSET); - pg_index_->UpdateOid(OID_FOR_USER_OFFSET); - pg_trigger_->UpdateOid(OID_FOR_USER_OFFSET); - // pg_proc->UpdateOid(OID_FOR_USER_OFFSET); + pg_attribute_->UpdateOid(OID_FOR_USER_OFFSET); + pg_namespace_->UpdateOid(OID_FOR_USER_OFFSET); + pg_table_->UpdateOid(OID_FOR_USER_OFFSET); + pg_index_->UpdateOid(OID_FOR_USER_OFFSET); + pg_trigger_->UpdateOid(OID_FOR_USER_OFFSET); + // pg_proc->UpdateOid(OID_FOR_USER_OFFSET); } } // namespace catalog diff --git a/src/catalog/table_catalog.cpp b/src/catalog/table_catalog.cpp index 8b8c85e4d76..478bb5f2d79 100644 --- a/src/catalog/table_catalog.cpp +++ b/src/catalog/table_catalog.cpp @@ -42,14 +42,14 @@ TableCatalogObject::TableCatalogObject(executor::LogicalTile *tile, version_id(tile->GetValue(tupleId, TableCatalog::ColumnId::VERSION_ID) .GetAs()), default_layout_oid(tile->GetValue(tupleId, - TableCatalog::ColumnId::DEFAULT_LAYOUT_OID).GetAs()), + TableCatalog::ColumnId::DEFAULT_LAYOUT_OID).GetAs()), index_objects(), index_names(), valid_index_objects(false), column_objects(), column_names(), valid_column_objects(false), - valid_layout_objects_(false), + valid_layout_objects_(false), txn(txn) {} /* @brief insert index catalog object into cache @@ -477,7 +477,7 @@ std::unique_ptr TableCatalog::InitializeSchema() { bool TableCatalog::InsertTable(oid_t table_oid, const std::string &table_name, const std::string &schema_name, oid_t database_oid, oid_t layout_oid, - type::AbstractPool *pool, + type::AbstractPool *pool, concurrency::TransactionContext *txn) { // Create the tuple first std::unique_ptr tuple( @@ -514,7 +514,7 @@ bool TableCatalog::DeleteTable(oid_t table_oid, // evict from cache auto table_object = txn->catalog_cache.GetCachedTableObject(database_oid, - table_oid); + table_oid); if (table_object) { auto database_object = DatabaseCatalog::GetInstance()->GetDatabaseObject(database_oid, txn); @@ -536,7 +536,7 @@ std::shared_ptr TableCatalog::GetTableObject( } // try get from cache auto table_object = txn->catalog_cache.GetCachedTableObject(database_oid, - table_oid); + table_oid); if (table_object) return table_object; // cache miss, get from pg_table @@ -677,7 +677,7 @@ bool TableCatalog::UpdateVersionId(oid_t update_val, oid_t table_oid, // get table object, then evict table object auto table_object = txn->catalog_cache.GetCachedTableObject(database_oid, - table_oid); + table_oid); if (table_object) { auto database_object = DatabaseCatalog::GetInstance()->GetDatabaseObject(database_oid, txn); @@ -708,7 +708,7 @@ bool TableCatalog::UpdateDefaultLayoutOid(oid_t update_val, oid_t table_oid, // get table object, then evict table object auto table_object = txn->catalog_cache.GetCachedTableObject(database_oid, - table_oid); + table_oid); if (table_object) { auto database_object = DatabaseCatalog::GetInstance()->GetDatabaseObject(database_oid, txn); diff --git a/src/executor/drop_executor.cpp b/src/executor/drop_executor.cpp index 77030acbe21..cf4073ce9d2 100644 --- a/src/executor/drop_executor.cpp +++ b/src/executor/drop_executor.cpp @@ -229,7 +229,7 @@ bool DropExecutor::DropIndex(const planner::DropPlan &node, ->GetSystemCatalogs(database_object->GetDatabaseOid()) ->GetIndexCatalog(); auto index_object = pg_index->GetIndexObject(database_object->GetDatabaseName(), - index_name, schema_name, txn); + index_name, schema_name, txn); if (index_object == nullptr) { throw CatalogException("Can't find index " + schema_name + "." + index_name + " to drop"); diff --git a/src/include/catalog/catalog_cache.h b/src/include/catalog/catalog_cache.h index 45fa1417b01..28ff9edacd0 100644 --- a/src/include/catalog/catalog_cache.h +++ b/src/include/catalog/catalog_cache.h @@ -50,12 +50,12 @@ class CatalogCache { const std::string &name); std::shared_ptr GetCachedTableObject(oid_t database_oid, - oid_t table_oid); + oid_t table_oid); std::shared_ptr GetCachedIndexObject(oid_t database_oid, - oid_t index_oid); + oid_t index_oid); std::shared_ptr GetCachedIndexObject( - const std::string &database_name, const std::string &index_name, - const std::string &schema_name); + const std::string &database_name, const std::string &index_name, + const std::string &schema_name); // database catalog cache interface bool InsertDatabaseObject( diff --git a/src/include/catalog/column_catalog.h b/src/include/catalog/column_catalog.h index 1cd9415b163..6a9173029fb 100644 --- a/src/include/catalog/column_catalog.h +++ b/src/include/catalog/column_catalog.h @@ -87,7 +87,7 @@ class ColumnCatalog : public AbstractCatalog { bool InsertColumn(oid_t table_oid, const std::string &column_name, oid_t column_id, oid_t column_offset, type::TypeId column_type, size_t column_length, - bool is_inlined, const std::vector &constraints, + bool is_inlined, const std::vector &constraints, type::AbstractPool *pool, concurrency::TransactionContext *txn); bool DeleteColumn(oid_t table_oid, const std::string &column_name, @@ -109,7 +109,7 @@ class ColumnCatalog : public AbstractCatalog { COLUMN_ID = 2, COLUMN_OFFSET = 3, COLUMN_TYPE = 4, - COLUMN_LENGTH = 5, + COLUMN_LENGTH = 5, IS_INLINED = 6, IS_PRIMARY = 7, IS_NOT_NULL = 8, diff --git a/src/include/catalog/index_catalog.h b/src/include/catalog/index_catalog.h index 466e0d6f94b..68e4d754f4c 100644 --- a/src/include/catalog/index_catalog.h +++ b/src/include/catalog/index_catalog.h @@ -90,17 +90,17 @@ class IndexCatalog : public AbstractCatalog { type::AbstractPool *pool, concurrency::TransactionContext *txn); bool DeleteIndex(oid_t database_oid, oid_t index_oid, - concurrency::TransactionContext *txn); + concurrency::TransactionContext *txn); /** Read Related API */ std::shared_ptr GetIndexObject( - const std::string &database_name, const std::string &index_name, - const std::string &schema_name, concurrency::TransactionContext *txn); + const std::string &database_name, const std::string &index_name, + const std::string &schema_name, concurrency::TransactionContext *txn); private: std::shared_ptr GetIndexObject( oid_t database_oid, oid_t index_oid, - concurrency::TransactionContext *txn); + concurrency::TransactionContext *txn); const std::unordered_map> GetIndexObjects(oid_t table_oid, concurrency::TransactionContext *txn); diff --git a/src/include/catalog/table_catalog.h b/src/include/catalog/table_catalog.h index e5571ebd62e..c4698f90a1c 100644 --- a/src/include/catalog/table_catalog.h +++ b/src/include/catalog/table_catalog.h @@ -194,7 +194,7 @@ class TableCatalog : public AbstractCatalog { SCHEMA_NAME = 2, DATABASE_OID = 3, VERSION_ID = 4, - DEFAULT_LAYOUT_OID = 5, + DEFAULT_LAYOUT_OID = 5, // Add new columns here in creation order }; std::vector all_column_ids = {0, 1, 2, 3, 4, 5}; diff --git a/src/include/storage/data_table.h b/src/include/storage/data_table.h index 01d14db6be1..1f0d4be29c4 100644 --- a/src/include/storage/data_table.h +++ b/src/include/storage/data_table.h @@ -267,7 +267,7 @@ class DataTable : public AbstractTable { } const std::shared_ptr GetDefaultLayout() const { - return default_layout_; + return default_layout_; } //===--------------------------------------------------------------------===// diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index 13a8a968248..8ba397f118c 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -60,7 +60,7 @@ class Layout : public Printable { * @param layout_oid Per-table unique OID. Generted by DataTable. */ Layout(const column_map_type &column_map, const oid_t num_columns, - const oid_t layout_oid); + const oid_t layout_oid); /** @brief Check whether this layout is a row store. */ bool IsRowStore() const { return (layout_type_ == LayoutType::ROW); } diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index 7ce3622eaab..3fd20c70cc7 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -26,14 +26,14 @@ namespace storage { // The default layout is always a ROW_STORE Layout::Layout(const oid_t num_columns, LayoutType layout_type) : num_columns_(num_columns), layout_type_(layout_type) { - // Assign the oid - if (layout_type == LayoutType::ROW) { + // Assign the oid + if (layout_type == LayoutType::ROW) { layout_oid_ = ROW_STORE_LAYOUT_OID; - } else if (layout_type == LayoutType::COLUMN) { + } else if (layout_type == LayoutType::COLUMN) { layout_oid_ = COLUMN_STORE_LAYOUT_OID; - } else { + } else { layout_oid_ = INVALID_OID; - } + } } // Constructor for the Layout class with column_map @@ -74,7 +74,7 @@ Layout::Layout(const column_map_type &column_map) // Constructor for Layout class with predefined layout_oid Layout::Layout(const column_map_type &column_map, const oid_t num_columns, - const oid_t layout_id) + const oid_t layout_id) : layout_oid_(layout_id), num_columns_(num_columns), column_layout_(column_map) { diff --git a/test/catalog/catalog_test.cpp b/test/catalog/catalog_test.cpp index ff1aa1f5ab6..d99ba49bf25 100644 --- a/test/catalog/catalog_test.cpp +++ b/test/catalog/catalog_test.cpp @@ -135,10 +135,10 @@ TEST_F(CatalogTests, TestingCatalogCache) { // check expected table object is acquired for (oid_t table_idx = 0; table_idx < user_database->GetTableCount(); table_idx++) { - auto table = user_database->GetTable(table_idx); - auto user_table_object = user_db_object->GetTableObject(table->GetOid()); - EXPECT_EQ(user_db_object->GetDatabaseOid(), - user_table_object->GetDatabaseOid()); + auto table = user_database->GetTable(table_idx); + auto user_table_object = user_db_object->GetTableObject(table->GetOid()); + EXPECT_EQ(user_db_object->GetDatabaseOid(), + user_table_object->GetDatabaseOid()); } txn_manager.CommitTransaction(txn); @@ -163,7 +163,7 @@ TEST_F(CatalogTests, TableObject) { EXPECT_EQ(0, column_objects[0]->GetColumnOffset()); EXPECT_EQ(type::TypeId::INTEGER, column_objects[0]->GetColumnType()); EXPECT_EQ(type::Type::GetTypeSize(type::TypeId::INTEGER), - column_objects[0]->GetColumnLength()); + column_objects[0]->GetColumnLength()); EXPECT_TRUE(column_objects[0]->IsInlined()); EXPECT_TRUE(column_objects[0]->IsPrimary()); EXPECT_FALSE(column_objects[0]->IsNotNull()); @@ -173,7 +173,7 @@ TEST_F(CatalogTests, TableObject) { EXPECT_EQ(1, column_objects[1]->GetColumnId()); EXPECT_EQ(4, column_objects[1]->GetColumnOffset()); EXPECT_EQ(type::TypeId::VARCHAR, column_objects[1]->GetColumnType()); - EXPECT_EQ(32, column_objects[1]->GetColumnLength()); + EXPECT_EQ(32, column_objects[1]->GetColumnLength()); EXPECT_TRUE(column_objects[1]->IsInlined()); EXPECT_FALSE(column_objects[1]->IsPrimary()); EXPECT_FALSE(column_objects[1]->IsNotNull()); @@ -397,7 +397,7 @@ TEST_F(CatalogTests, LayoutCatalogTest) { *(first_default_layout.get()), *(pg_layout->GetLayoutWithOid(table_oid, first_layout_oid, txn).get())); EXPECT_EQ(first_layout_oid, - catalog->GetTableObject(database_oid, table_oid, txn)->GetDefaultLayoutOid()); + catalog->GetTableObject(database_oid, table_oid, txn)->GetDefaultLayoutOid()); txn_manager.CommitTransaction(txn); // Change default layout. @@ -426,7 +426,7 @@ TEST_F(CatalogTests, LayoutCatalogTest) { *(default_layout.get()), *(pg_layout->GetLayoutWithOid(table_oid, default_layout_oid, txn).get())); EXPECT_EQ(default_layout_oid, - catalog->GetTableObject(database_oid, table_oid, txn)->GetDefaultLayoutOid()); + catalog->GetTableObject(database_oid, table_oid, txn)->GetDefaultLayoutOid()); txn_manager.CommitTransaction(txn); // Create additional layout. @@ -440,7 +440,7 @@ TEST_F(CatalogTests, LayoutCatalogTest) { auto other_layout = catalog->CreateLayout(database_oid, table_oid, non_default_map, txn); EXPECT_NE(nullptr, other_layout); - txn_manager.CommitTransaction(txn); + txn_manager.CommitTransaction(txn); // Check the created layout EXPECT_FALSE(other_layout->IsColumnStore()); @@ -457,7 +457,7 @@ TEST_F(CatalogTests, LayoutCatalogTest) { // Check that the default layout is still the same. EXPECT_NE(other_layout, table->GetDefaultLayout()); EXPECT_NE(other_layout_oid, - catalog->GetTableObject(database_oid, table_oid, txn)->GetDefaultLayoutOid()); + catalog->GetTableObject(database_oid, table_oid, txn)->GetDefaultLayoutOid()); txn_manager.CommitTransaction(txn); // Drop the default layout. @@ -478,7 +478,7 @@ TEST_F(CatalogTests, LayoutCatalogTest) { EXPECT_EQ(nullptr, pg_layout->GetLayoutWithOid(table_oid, default_layout_oid, txn)); EXPECT_EQ(ROW_STORE_LAYOUT_OID, - catalog->GetTableObject(database_oid, table_oid, txn)->GetDefaultLayoutOid()); + catalog->GetTableObject(database_oid, table_oid, txn)->GetDefaultLayoutOid()); // The additional layout must be present in pg_layout EXPECT_EQ( diff --git a/test/codegen/table_scan_translator_test.cpp b/test/codegen/table_scan_translator_test.cpp index 56027298749..f1080afda6d 100644 --- a/test/codegen/table_scan_translator_test.cpp +++ b/test/codegen/table_scan_translator_test.cpp @@ -716,7 +716,7 @@ TEST_F(TableScanTranslatorTest, MultiLayoutScan) { txn = txn_manager.BeginTransaction(); table->ResetDefaultLayout(LayoutType::COLUMN); catalog->GetSystemCatalogs(table->GetDatabaseOid())->GetTableCatalog() - ->UpdateDefaultLayoutOid(table->GetDefaultLayout()->GetOid(), + ->UpdateDefaultLayoutOid(table->GetDefaultLayout()->GetOid(), table->GetOid(), txn); txn_manager.CommitTransaction(txn); diff --git a/test/executor/drop_test.cpp b/test/executor/drop_test.cpp index 3573cd5210f..f7451d3dd8d 100644 --- a/test/executor/drop_test.cpp +++ b/test/executor/drop_test.cpp @@ -282,7 +282,7 @@ TEST_F(DropTests, DroppingIndexByName) { ->GetIndexCatalog(); auto index_object = pg_index->GetIndexObject(database_object->GetDatabaseName(), - index_name1, DEFAULT_SCHEMA_NAME, txn); + index_name1, DEFAULT_SCHEMA_NAME, txn); EXPECT_NE(nullptr, index_object); // Check the effect of drop // Most major check in this test case @@ -290,7 +290,7 @@ TEST_F(DropTests, DroppingIndexByName) { catalog->DropIndex(database_object->GetDatabaseOid(), index_object->GetIndexOid(), txn); EXPECT_EQ(pg_index->GetIndexObject(database_object->GetDatabaseName(), - index_name1, DEFAULT_SCHEMA_NAME, txn), + index_name1, DEFAULT_SCHEMA_NAME, txn), nullptr); txn_manager.CommitTransaction(txn); @@ -298,7 +298,7 @@ TEST_F(DropTests, DroppingIndexByName) { txn = txn_manager.BeginTransaction(); // Check the effect of drop index EXPECT_EQ(pg_index->GetIndexObject(database_object->GetDatabaseName(), - index_name1, DEFAULT_SCHEMA_NAME, txn), + index_name1, DEFAULT_SCHEMA_NAME, txn), nullptr); // Now dropping the table diff --git a/test/sql/drop_sql_test.cpp b/test/sql/drop_sql_test.cpp index d70fb978f4d..2e0dccc63f7 100644 --- a/test/sql/drop_sql_test.cpp +++ b/test/sql/drop_sql_test.cpp @@ -115,7 +115,7 @@ TEST_F(DropSQLTests, DropIndexTest) { txn = txn_manager.BeginTransaction(); try { index = pg_index->GetIndexObject(database_object->GetDatabaseName(), - "idx", DEFAULT_SCHEMA_NAME, txn); + "idx", DEFAULT_SCHEMA_NAME, txn); } catch (CatalogException &e) { index = nullptr; @@ -130,7 +130,7 @@ TEST_F(DropSQLTests, DropIndexTest) { // Check if index is not in catalog txn = txn_manager.BeginTransaction(); index = pg_index->GetIndexObject(database_object->GetDatabaseName(), - "idx", DEFAULT_SCHEMA_NAME, txn); + "idx", DEFAULT_SCHEMA_NAME, txn); EXPECT_EQ(index, nullptr); // Free the database just created From 671e4bd7e7b3c515cfe86344992da12e326194c1 Mon Sep 17 00:00:00 2001 From: tli2 Date: Tue, 19 Jun 2018 16:09:34 -0400 Subject: [PATCH 02/19] Network Layer Refactor (#1418) * Refactor I/O part out of connection handle and simplify state machine logic. * Run formatter on all code in network layer * Add in termination state --- src/include/common/notifiable_task.h | 30 +- .../network/connection_dispatcher_task.h | 2 +- src/include/network/connection_handle.h | 168 ++--- .../network/connection_handle_factory.h | 84 --- src/include/network/connection_handler_task.h | 2 +- src/include/network/marshal.h | 210 +++++- .../network/network_io_wrapper_factory.h | 66 ++ src/include/network/network_io_wrappers.h | 107 +++ src/include/network/network_state.h | 27 +- .../network/peloton_rpc_handler_task.h | 1 - src/include/network/peloton_server.h | 4 +- .../network/postgres_protocol_handler.h | 12 +- src/include/network/protocol_handler.h | 2 +- .../network/protocol_handler_factory.h | 4 +- src/network/README | 19 +- src/network/connection_dispatcher_task.cpp | 2 +- src/network/connection_handle.cpp | 699 ++++-------------- src/network/connection_handler_task.cpp | 21 +- src/network/marshal.cpp | 13 +- src/network/network_io_wrapper_factory.cpp | 78 ++ src/network/network_io_wrappers.cpp | 200 +++++ src/network/peloton_server.cpp | 28 +- src/network/postgres_protocol_handler.cpp | 110 ++- src/network/protocol_handler.cpp | 4 +- src/network/protocol_handler_factory.cpp | 6 +- test/network/exception_test.cpp | 14 +- test/network/prepare_stmt_test.cpp | 12 +- test/network/select_all_test.cpp | 9 +- test/network/simple_query_test.cpp | 10 +- test/network/ssl_test.cpp | 12 +- 30 files changed, 961 insertions(+), 995 deletions(-) delete mode 100644 src/include/network/connection_handle_factory.h create mode 100644 src/include/network/network_io_wrapper_factory.h create mode 100644 src/include/network/network_io_wrappers.h create mode 100644 src/network/network_io_wrapper_factory.cpp create mode 100644 src/network/network_io_wrappers.cpp diff --git a/src/include/common/notifiable_task.h b/src/include/common/notifiable_task.h index 8ea65efb26b..e1572ab63b9 100644 --- a/src/include/common/notifiable_task.h +++ b/src/include/common/notifiable_task.h @@ -62,7 +62,6 @@ class NotifiableTask { */ inline int Id() const { return task_id_; } - /** * @brief Register an event with the event base associated with this * notifiable task. @@ -140,22 +139,19 @@ class NotifiableTask { return RegisterEvent(-1, EV_PERSIST, callback, arg); } - // TODO(tianyu): The original network code seems to do this as an - // optimization. Specifically it avoids new memory allocation by reusing - // an existing event. I am leaving this out until we get numbers. - // void UpdateEvent(struct event *event, int fd, short flags, - // event_callback_fn callback, void *arg, - // const struct timeval *timeout = nullptr) { - // PELOTON_ASSERT(!(events_.find(event) == events_.end())); - // EventUtil::EventDel(event); - // EventUtil::EventAssign(event, base_, fd, flags, callback, arg); - // EventUtil::EventAdd(event, timeout); - // } - // - // void UpdateManualEvent(struct event *event, event_callback_fn callback, - // void *arg) { - // UpdateEvent(event, -1, EV_PERSIST, callback, arg); - // } + void UpdateEvent(struct event *event, int fd, short flags, + event_callback_fn callback, void *arg, + const struct timeval *timeout = nullptr) { + PELOTON_ASSERT(!(events_.find(event) == events_.end())); + EventUtil::EventDel(event); + EventUtil::EventAssign(event, base_, fd, flags, callback, arg); + EventUtil::EventAdd(event, timeout); + } + + void UpdateManualEvent(struct event *event, event_callback_fn callback, + void *arg) { + UpdateEvent(event, -1, EV_PERSIST, callback, arg); + } /** * @brief Unregister the event given. The event is no longer active and its diff --git a/src/include/network/connection_dispatcher_task.h b/src/include/network/connection_dispatcher_task.h index f7d2b62e6a3..0b97147622a 100644 --- a/src/include/network/connection_dispatcher_task.h +++ b/src/include/network/connection_dispatcher_task.h @@ -13,9 +13,9 @@ #pragma once #include "common/notifiable_task.h" -#include "network_state.h" #include "concurrency/epoch_manager_factory.h" #include "connection_handler_task.h" +#include "network_state.h" namespace peloton { namespace network { diff --git a/src/include/network/connection_handle.h b/src/include/network/connection_handle.h index c0ae311021c..84db833f102 100644 --- a/src/include/network/connection_handle.h +++ b/src/include/network/connection_handle.h @@ -32,6 +32,7 @@ #include "marshal.h" #include "network/connection_handler_task.h" +#include "network/network_io_wrappers.h" #include "network_state.h" #include "protocol_handler.h" @@ -41,56 +42,83 @@ namespace peloton { namespace network { -// TODO(tianyu) This class is not refactored in full as rewriting the logic is -// not cost-effective. However, readability -// improvement and other changes may become desirable in the future. Other than -// code clutter, responsibility assignment -// is not well thought-out in this class. Abstracting out some type of socket -// wrapper would be nice. /** * @brief A ConnectionHandle encapsulates all information about a client - * connection for its entire duration. - * One should not use the constructor to construct a new ConnectionHandle - * instance every time as it is expensive - * to allocate buffers. Instead, use the ConnectionHandleFactory. - * - * @see ConnectionHandleFactory + * connection for its entire duration. This includes a state machine and the + * necessary libevent infrastructure for a handler to work on this connection. */ class ConnectionHandle { public: /** - * Update the existing event to listen to the passed flags + * Constructs a new ConnectionHandle + * @param sock_fd Client's connection fd + * @param handler The handler responsible for this handle */ - void UpdateEventFlags(short flags); + ConnectionHandle(int sock_fd, ConnectionHandlerTask *handler); - WriteState WritePackets(); - - std::string WriteBufferToString(); + /** + * @brief Signal to libevent that this ConnectionHandle is ready to handle + * events + * + * This method needs to be called separately after initialization for the + * connection handle to do anything. The reason why this is not performed in + * the constructor is because it publishes pointers to this object. While the + * object should be fully initialized at that point, it's never a bad idea + * to be careful. + */ + inline void RegisterToReceiveEvents() { + workpool_event_ = conn_handler_->RegisterManualEvent( + METHOD_AS_CALLBACK(ConnectionHandle, HandleEvent), this); + + // TODO(Tianyi): should put the initialization else where.. check + // correctness first. + tcop_.SetTaskCallback( + [](void *arg) { + struct event *event = static_cast(arg); + event_active(event, EV_WRITE, 0); + }, + workpool_event_); + + network_event_ = conn_handler_->RegisterEvent( + io_wrapper_->GetSocketFd(), EV_READ | EV_PERSIST, + METHOD_AS_CALLBACK(ConnectionHandle, HandleEvent), this); + } + /** + * Handles a libevent event. This simply delegates the the state machine. + */ inline void HandleEvent(int, short) { state_machine_.Accept(Transition::WAKEUP, *this); } - // Exposed for testing - const std::unique_ptr &GetProtocolHandler() const { - return protocol_handler_; - } + /* State Machine Actions */ + // TODO(Tianyu): Write some documentation when feeling like it + inline Transition TryRead() { return io_wrapper_->FillReadBuffer(); } + Transition TryWrite(); + Transition Process(); + Transition GetResult(); + Transition TrySslHandshake(); + Transition TryCloseConnection(); - // State Machine actions /** - * refill_read_buffer - Used to repopulate read buffer with a fresh - * batch of data from the socket + * Updates the event flags of the network event. This configures how the + * handler reacts to client activity from this connection. + * @param flags new flags for the event handle. */ - Transition FillReadBuffer(); - Transition Wait(); - Transition Process(); - Transition ProcessWrite(); - Transition GetResult(); - Transition CloseSocket(); + inline void UpdateEventFlags(short flags) { + conn_handler_->UpdateEvent( + network_event_, io_wrapper_->GetSocketFd(), flags, + METHOD_AS_CALLBACK(ConnectionHandle, HandleEvent), this); + } + /** - * Flush out all the responses and do real SSL handshake + * Stops receiving network events from client connection. This is useful when + * we are waiting on peloton to return the result of a query and not handling + * client query. */ - Transition ProcessWrite_SSLHandshake(); + inline void StopReceivingNetworkEvent() { + EventUtil::EventDel(network_event_); + } private: /** @@ -145,55 +173,7 @@ class ConnectionHandle { }; friend class StateMachine; - friend class ConnectionHandleFactory; - - ConnectionHandle(int sock_fd, ConnectionHandlerTask *handler, - std::shared_ptr rbuf, std::shared_ptr wbuf); - - /** - * Writes a packet's header (type, size) into the write buffer - */ - WriteState BufferWriteBytesHeader(OutputPacket *pkt); - - /** - * Writes a packet's content into the write buffer - */ - WriteState BufferWriteBytesContent(OutputPacket *pkt); - - /** - * Used to invoke a write into the Socket, returns false if the socket is not - * ready for write - */ - WriteState FlushWriteBuffer(); - - /** - * @brief: process SSL handshake to generate valid SSL - * connection context for further communications - * @return FINISH when the SSL handshake failed - * PROCEED when the SSL handshake success - * NEED_DATA when the SSL handshake is partially done due to network - * latency - */ - Transition SSLHandshake(); - - /** - * Set the socket to non-blocking mode - */ - inline void SetNonBlocking(evutil_socket_t fd) { - auto flags = fcntl(fd, F_GETFL); - flags |= O_NONBLOCK; - if (fcntl(fd, F_SETFL, flags) < 0) { - LOG_ERROR("Failed to set non-blocking socket"); - } - } - - /** - * Set TCP No Delay for lower latency - */ - inline void SetTCPNoDelay(evutil_socket_t fd) { - int one = 1; - setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof one); - } + friend class NetworkIoWrapperFactory; /** * @brief: Determine if there is still responses in the buffer @@ -202,27 +182,17 @@ class ConnectionHandle { */ inline bool HasResponse() { return (protocol_handler_->responses_.size() != 0) || - (wbuf_->buf_size != 0); + (io_wrapper_->wbuf_->size_ != 0); } - int sock_fd_; // socket file descriptor - struct event *network_event = nullptr; // something to read from network - struct event *workpool_event = nullptr; // worker thread done the job - - SSL *conn_SSL_context = nullptr; // SSL context for the connection - - ConnectionHandlerTask *handler_; // reference to the network thread - std::unique_ptr - protocol_handler_; // Stores state for this socket - tcop::TrafficCop traffic_cop_; - - std::shared_ptr rbuf_; // Socket's read buffer - std::shared_ptr wbuf_; // Socket's write buffer - unsigned int next_response_ = 0; // The next response in the response buffer - + ConnectionHandlerTask *conn_handler_; + std::shared_ptr io_wrapper_; StateMachine state_machine_; - - short curr_event_flag_; // current libevent event flag + struct event *network_event_ = nullptr, *workpool_event_ = nullptr; + std::unique_ptr protocol_handler_ = nullptr; + tcop::TrafficCop tcop_; + // TODO(Tianyu): Put this into protocol handler in a later refactor + unsigned int next_response_ = 0; }; } // namespace network } // namespace peloton diff --git a/src/include/network/connection_handle_factory.h b/src/include/network/connection_handle_factory.h deleted file mode 100644 index 8f81d1e20dd..00000000000 --- a/src/include/network/connection_handle_factory.h +++ /dev/null @@ -1,84 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// connection_handle_factory.h -// -// Identification: src/include/network/connection_handle_factory.h -// -// Copyright (c) 2015-2018, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "network/connection_handle.h" -#include "peloton_server.h" - -namespace peloton { -namespace network { - -/** - * @brief Factory class for constructing ConnectionHandles - * The rationale behind using a factory is that buffers are expensive to - * allocate and allocating new - * ones every time is a bottleneck for throughput. - */ -class ConnectionHandleFactory { - public: - /** - * Creates or repurpose a ConnectionHandle to be run on the given handler, - * handling connection from conn_fd - * @param conn_fd Client connection fd. - * @param handler The handler this ConnectionHandle is assigned to - * @return - */ - std::shared_ptr GetConnectionHandle( - int conn_fd, ConnectionHandlerTask *handler) { - // TODO(tianyu): The use of a static variable here for testing purpose is - // VILE. Fix this in a later refactor - // (probably also to-do: beat up the person who wrote this) - PelotonServer::recent_connfd = conn_fd; - auto it = reusable_handles_.find(conn_fd); - if (it == reusable_handles_.end()) { - // We are not using std::make_shared here because we want to keep - // ConnectionHandle constructor - // private to avoid unintentional use. - auto handle = std::shared_ptr( - new ConnectionHandle(conn_fd, handler, std::make_shared(), - std::make_shared())); - reusable_handles_[conn_fd] = handle; - return handle; - } - - it->second->rbuf_->Reset(); - it->second->wbuf_->Reset(); - std::shared_ptr new_handle(new ConnectionHandle( - conn_fd, handler, it->second->rbuf_, it->second->wbuf_)); - reusable_handles_[conn_fd] = new_handle; - return new_handle; - } - - // TODO(tianyu) Again, this is VILE. Fix this in a later refactor. - /** - * Exposed for testing only. DO NOT USE ELSEWHERE IN CODE. - * @param conn_fd client socket fd - * @return ConnetionHandle object representing client connection at conn_fd - */ - std::shared_ptr ConnectionHandleAt(int conn_fd) { - return reusable_handles_[conn_fd]; - } - - // TODO(tianyu): This should removed with the rest of the singletons - // We are keeping this here as fixing singleton is not the focus of this - // refactor and fixing it would be pretty expensive. - static ConnectionHandleFactory &GetInstance() { - static ConnectionHandleFactory factory; - return factory; - } - - private: - std::unordered_map> reusable_handles_; -}; -} -} diff --git a/src/include/network/connection_handler_task.h b/src/include/network/connection_handler_task.h index c86ea1a24ce..44a34884e2e 100644 --- a/src/include/network/connection_handler_task.h +++ b/src/include/network/connection_handler_task.h @@ -19,9 +19,9 @@ #include +#include "common/container/lock_free_queue.h" #include "common/exception.h" #include "common/logger.h" -#include "common/container/lock_free_queue.h" #include "common/notifiable_task.h" namespace peloton { diff --git a/src/include/network/marshal.h b/src/include/network/marshal.h index 030231ec56d..56d29e57bbb 100644 --- a/src/include/network/marshal.h +++ b/src/include/network/marshal.h @@ -15,52 +15,206 @@ #include #include +#include +#include #include "common/internal_types.h" #include "common/logger.h" #include "common/macros.h" +#include "network/network_state.h" #define BUFFER_INIT_SIZE 100 namespace peloton { namespace network { -// Buffers used to batch messages at the socket +/** + * A plain old buffer with a movable cursor, the meaning of which is dependent + * on the use case. + * + * The buffer has a fix capacity and one can write a variable amount of + * meaningful bytes into it. We call this amount "size" of the buffer. + */ struct Buffer { - size_t buf_ptr; // buffer cursor - size_t buf_size; // buffer size - size_t buf_flush_ptr; // buffer cursor for write - ByteBuf buf; - - inline Buffer() : buf_ptr(0), buf_size(0), buf_flush_ptr(0) { - // capacity of the buffer - buf.reserve(SOCKET_BUFFER_SIZE); + public: + /** + * Instantiates a new buffer and reserve default many bytes. + */ + inline Buffer() { buf_.reserve(SOCKET_BUFFER_SIZE); } + + /** + * Reset the buffer pointer and clears content + */ + inline void Reset() { + size_ = 0; + offset_ = 0; } - inline void Reset() { - buf_ptr = 0; - buf_size = 0; - buf_flush_ptr = 0; + /** + * @param bytes The amount of bytes to check between the cursor and the end + * of the buffer (defaults to any) + * @return Whether there is any more bytes between the cursor and + * the end of the buffer + */ + inline bool HasMore(size_t bytes = 1) { return offset_ + bytes <= size_; } + + /** + * @return Whether the buffer is at capacity. (All usable space is filled + * with meaningful bytes) + */ + inline bool Full() { return size_ == Capacity(); } + + /** + * @return Iterator to the beginning of the buffer + */ + inline ByteBuf::const_iterator Begin() { return std::begin(buf_); } + + /** + * @return Capacity of the buffer (not actual size) + */ + inline size_t Capacity() const { return SOCKET_BUFFER_SIZE; } + + /** + * Shift contents to align the current cursor with start of the buffer, + * remove all bytes before the cursor. + */ + inline void MoveContentToHead() { + auto unprocessed_len = size_ - offset_; + std::memmove(&buf_[0], &buf_[offset_], unprocessed_len); + size_ = unprocessed_len; + offset_ = 0; } - // single buffer element accessor - inline uchar GetByte(size_t &index) { return buf[index]; } + // TODO(Tianyu): Make these protected once we refactor protocol handler + size_t size_ = 0, offset_ = 0; + ByteBuf buf_; +}; + +/** + * A buffer specialize for read + */ +class ReadBuffer : public Buffer { + public: + /** + * Read as many bytes as possible using SSL read + * @param context SSL context to read from + * @return the return value of ssl read + */ + inline int FillBufferFrom(SSL *context) { + ERR_clear_error(); + ssize_t bytes_read = SSL_read(context, &buf_[size_], Capacity() - size_); + int err = SSL_get_error(context, bytes_read); + if (err == SSL_ERROR_NONE) size_ += bytes_read; + return err; + }; + + /** + * Read as many bytes as possible using Posix from an fd + * @param fd the file descriptor to read from + * @return the return value of posix read + */ + inline int FillBufferFrom(int fd) { + ssize_t bytes_read = read(fd, &buf_[size_], Capacity() - size_); + if (bytes_read > 0) size_ += bytes_read; + return (int)bytes_read; + } - // Get pointer to index location - inline uchar *GetPtr(size_t index) { return &buf[index]; } + /** + * The number of bytes available to be consumed (i.e. meaningful bytes after + * current read cursor) + * @return The number of bytes available to be consumed + */ + inline size_t BytesAvailable() { return size_ - offset_; } + + /** + * Read the given number of bytes into destination, advancing cursor by that + * number + * @param bytes Number of bytes to read + * @param dest Desired memory location to read into + */ + inline void Read(size_t bytes, void *dest) { + std::copy(buf_.begin() + offset_, buf_.begin() + offset_ + bytes, + reinterpret_cast(dest)); + offset_ += bytes; + } - inline ByteBuf::const_iterator Begin() { return std::begin(buf); } + /** + * Read a value of type T off of the buffer, advancing cursor by appropriate + * amount. Does NOT convert from network bytes order. It is the caller's + * responsibility to do so. + * @tparam T type of value to read off. Preferably a primitive type + * @return the value of type T + */ + template + inline T ReadValue() { + T result; + Read(sizeof(result), &result); + return result; + } +}; - inline ByteBuf::const_iterator End() { return std::end(buf); } +/** + * A buffer specialized for write + */ +class WriteBuffer : public Buffer { + public: + /** + * Write as many bytes as possible using SSL write + * @param context SSL context to write out to + * @return return value of SSL write + */ + inline int WriteOutTo(SSL *context) { + ERR_clear_error(); + ssize_t bytes_written = SSL_write(context, &buf_[offset_], size_ - offset_); + int err = SSL_get_error(context, bytes_written); + if (err == SSL_ERROR_NONE) offset_ += bytes_written; + return err; + } - inline size_t GetMaxSize() { return SOCKET_BUFFER_SIZE; } + /** + * Write as many bytes as possible using Posix write to fd + * @param fd File descriptor to write out to + * @return return value of Posix write + */ + inline int WriteOutTo(int fd) { + ssize_t bytes_written = write(fd, &buf_[offset_], size_ - offset_); + if (bytes_written > 0) offset_ += bytes_written; + return (int)bytes_written; + } - // Get the 4 bytes Big endian uint32 and convert it to little endian - size_t GetUInt32BigEndian(); + /** + * The remaining capacity of this buffer. This value is equal to the + * maximum capacity minus the capacity already in use. + * @return Remaining capacity + */ + inline size_t RemainingCapacity() { return Capacity() - size_; } + + /** + * @param bytes Desired number of bytes to write + * @return Whether the buffer can accommodate the number of bytes given + */ + inline bool HasSpaceFor(size_t bytes) { return RemainingCapacity() >= bytes; } + + /** + * Append the desired range into current buffer. + * @tparam InputIt iterator type. + * @param first beginning of range + * @param len length of range + */ + template + inline void Append(InputIt first, size_t len) { + std::copy(first, first + len, std::begin(buf_) + size_); + size_ += len; + } - // Is the requested amount of data available from the current position in - // the reader buffer? - inline bool IsReadDataAvailable(size_t bytes) { - return ((buf_ptr - 1) + bytes < buf_size); + /** + * Append the given value into the current buffer. Does NOT convert to + * network byte order. It is up to the caller to do so. + * @tparam T input type + * @param val value to write into buffer + */ + template + inline void Append(T val) { + Append(reinterpret_cast(&val), sizeof(T)); } }; @@ -139,8 +293,8 @@ struct OutputPacket { NetworkMessageType msg_type; // header bool single_type_pkt; // there would be only a pkt type being written to the - // buffer when this flag is true - bool skip_header_write; // whether we should write header to socket wbuf + // buffer when this flag is true + bool skip_header_write; // whether we should write header to soc ket wbuf size_t write_ptr; // cursor used to write packet content to socket wbuf // TODO could packet be reused? diff --git a/src/include/network/network_io_wrapper_factory.h b/src/include/network/network_io_wrapper_factory.h new file mode 100644 index 00000000000..979e6a18afd --- /dev/null +++ b/src/include/network/network_io_wrapper_factory.h @@ -0,0 +1,66 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// network_io_wrapper_factory.h +// +// Identification: src/include/network/network_io_wrapper_factory.h +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "network/network_io_wrappers.h" +#include "network/peloton_server.h" + +namespace peloton { +namespace network { + +/** + * @brief Factory class for constructing NetworkIoWrapper objects + * Each NetworkIoWrapper is associated with read and write buffers that are + * expensive to reallocate on the fly. Thus, instead of destroying these wrapper + * objects when they are out of scope, we save them until we can transfer their + * buffers to other wrappers. + */ +// TODO(Tianyu): Make reuse more fine-grained and adjustable +// Currently there is no limit on the number of wrappers we save. This means +// that we never deallocated wrappers unless we shut down. Obviously this will +// be a memory overhead if we had a lot of connections at one point and dropped +// down after a while. Relying on OS fd values for reuse also can backfire. It +// shouldn't be hard to keep a pool of buffers with a size limit instead of a +// bunch of old wrapper objects. +class NetworkIoWrapperFactory { + public: + static inline NetworkIoWrapperFactory &GetInstance() { + static NetworkIoWrapperFactory factory; + return factory; + } + + /** + * @brief Creates or re-purpose a NetworkIoWrapper object for new use. + * The returned value always uses Posix I/O methods unles explicitly + * converted. + * @see NetworkIoWrapper for details + * @param conn_fd Client connection fd + * @return A new NetworkIoWrapper object + */ + std::shared_ptr NewNetworkIoWrapper(int conn_fd); + + /** + * @brief: process SSL handshake to generate valid SSL + * connection context for further communications + * @return FINISH when the SSL handshake failed + * PROCEED when the SSL handshake success + * NEED_DATA when the SSL handshake is partially done due to network + * latency + */ + Transition PerformSslHandshake(std::shared_ptr &io_wrapper); + + private: + std::unordered_map> reusable_wrappers_; +}; +} // namespace network +} // namespace peloton diff --git a/src/include/network/network_io_wrappers.h b/src/include/network/network_io_wrappers.h new file mode 100644 index 00000000000..1b100475ffd --- /dev/null +++ b/src/include/network/network_io_wrappers.h @@ -0,0 +1,107 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// network_io_wrappers.h +// +// Identification: src/include/network/network_io_wrappers.h +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include +#include +#include +#include "common/exception.h" +#include "common/utility.h" +#include "network/marshal.h" + +namespace peloton { +namespace network { + +/** + * A network io wrapper provides an interface for interacting with a client + * connection. + * + * Underneath the hood the wrapper buffers read and write, and can support posix + * and ssl reads and writes to the socket, depending on the concrete type at + * runtime. + * + * Because the buffers are large and expensive to allocate on fly, they are + * reused. Consequently, initialization of this class is handled by a factory + * class. @see NetworkIoWrapperFactory + */ +class NetworkIoWrapper { + friend class NetworkIoWrapperFactory; + + public: + virtual bool SslAble() const = 0; + // TODO(Tianyu): Change and document after we refactor protocol handler + virtual Transition FillReadBuffer() = 0; + virtual Transition FlushWriteBuffer() = 0; + virtual Transition Close() = 0; + + inline int GetSocketFd() { return sock_fd_; } + Transition WritePacket(OutputPacket *pkt); + // TODO(Tianyu): Make these protected when protocol handler refactor is + // complete + NetworkIoWrapper(int sock_fd, std::shared_ptr &rbuf, + std::shared_ptr &wbuf) + : sock_fd_(sock_fd), + rbuf_(std::move(rbuf)), + wbuf_(std::move(wbuf)) { + rbuf_->Reset(); + wbuf_->Reset(); + } + + DISALLOW_COPY(NetworkIoWrapper) + + NetworkIoWrapper(NetworkIoWrapper &&other) = default; + + int sock_fd_; + std::shared_ptr rbuf_; + std::shared_ptr wbuf_; +}; + +/** + * A Network IoWrapper specialized for dealing with posix sockets. + */ +class PosixSocketIoWrapper : public NetworkIoWrapper { + public: + PosixSocketIoWrapper(int sock_fd, std::shared_ptr rbuf, + std::shared_ptr wbuf); + + + inline bool SslAble() const override { return false; } + Transition FillReadBuffer() override; + Transition FlushWriteBuffer() override; + inline Transition Close() override { + peloton_close(sock_fd_); + return Transition::PROCEED; + } +}; + +/** + * NetworkIoWrapper specialized for dealing with ssl sockets. + */ +class SslSocketIoWrapper : public NetworkIoWrapper { + public: + // Realistically, an SslSocketIoWrapper is always derived from a + // PosixSocketIoWrapper, as the handshake process happens over posix sockets. + SslSocketIoWrapper(NetworkIoWrapper &&other, SSL *ssl) + : NetworkIoWrapper(std::move(other)), conn_ssl_context_(ssl) {} + + inline bool SslAble() const override { return true; } + Transition FillReadBuffer() override; + Transition FlushWriteBuffer() override; + Transition Close() override; + + private: + friend class NetworkIoWrapperFactory; + SSL *conn_ssl_context_; +}; +} // namespace network +} // namespace peloton diff --git a/src/include/network/network_state.h b/src/include/network/network_state.h index 9530f84bd7d..96373dbe919 100644 --- a/src/include/network/network_state.h +++ b/src/include/network/network_state.h @@ -19,19 +19,11 @@ namespace network { * @see ConnectionHandle::StateMachine */ enum class ConnState { - READ, // State that reads data from the network - WRITE, // State the writes data to the network - PROCESS, // State that runs the network protocol on received data - CLOSING, // State for closing the client connection - GET_RESULT, // State when triggered by worker thread that completes the task. - PROCESS_WRITE_SSL_HANDSHAKE, // State to flush out responses and doing (Real) - // SSL handshake -}; - -// TODO(tianyu): Convert use cases of this to just return Transition -enum class WriteState { - COMPLETE, // Write completed - NOT_READY, // Socket not ready to write + READ, // State that reads data from the network + WRITE, // State the writes data to the network + PROCESS, // State that runs the network protocol on received data + CLOSING, // State for closing the client connection + SSL_INIT, // State to flush out responses and doing (Real) SSL handshake }; /** @@ -43,12 +35,11 @@ enum class Transition { NONE, WAKEUP, PROCEED, - NEED_DATA, - // TODO(tianyu) generalize this symbol, this is currently only used in process - GET_RESULT, - FINISH, - RETRY, + NEED_READ, + NEED_RESULT, + TERMINATE, NEED_SSL_HANDSHAKE, + NEED_WRITE }; } // namespace network } // namespace peloton diff --git a/src/include/network/peloton_rpc_handler_task.h b/src/include/network/peloton_rpc_handler_task.h index 8abfa510af4..d32d236ffc3 100644 --- a/src/include/network/peloton_rpc_handler_task.h +++ b/src/include/network/peloton_rpc_handler_task.h @@ -29,7 +29,6 @@ class PelotonRpcServerImpl final : public PelotonService::Server { } }; - class PelotonRpcHandlerTask : public DedicatedThreadTask { public: explicit PelotonRpcHandlerTask(const char *address) : address_(address) {} diff --git a/src/include/network/peloton_server.h b/src/include/network/peloton_server.h index 6f592bd6166..e0baed54ef1 100644 --- a/src/include/network/peloton_server.h +++ b/src/include/network/peloton_server.h @@ -29,12 +29,12 @@ #include #include "common/container/lock_free_queue.h" +#include "common/dedicated_thread_owner.h" #include "common/exception.h" #include "common/logger.h" -#include "common/dedicated_thread_owner.h" +#include "common/notifiable_task.h" #include "connection_dispatcher_task.h" #include "network_state.h" -#include "common/notifiable_task.h" #include "protocol_handler.h" #include diff --git a/src/include/network/postgres_protocol_handler.h b/src/include/network/postgres_protocol_handler.h index ef75f0a4cb1..960e2fdfd46 100644 --- a/src/include/network/postgres_protocol_handler.h +++ b/src/include/network/postgres_protocol_handler.h @@ -27,7 +27,7 @@ #include "traffic_cop/traffic_cop.h" // Packet content macros -#define NULL_CONTENT_SIZE -1 +#define NULL_CONTENT_SIZE (-1) namespace peloton { @@ -51,7 +51,7 @@ class PostgresProtocolHandler : public ProtocolHandler { * to generate txn * @return @see ProcessResult */ - ProcessResult Process(Buffer &rbuf, size_t thread_id); + ProcessResult Process(ReadBuffer &rbuf, size_t thread_id); // Deserialize the parame types from packet static size_t ReadParamType(InputPacket *pkt, int num_params, @@ -85,7 +85,7 @@ class PostgresProtocolHandler : public ProtocolHandler { * (i.e. no type byte) * @return true if the parsing is complete */ - static bool ParseInputPacket(Buffer &rbuf, InputPacket &rpkt, + static bool ParseInputPacket(ReadBuffer &rbuf, InputPacket &rpkt, bool startup_format); /** @@ -95,14 +95,14 @@ class PostgresProtocolHandler : public ProtocolHandler { * @param rpkt the postgres rpkt we want to parse to * @return true if the parsing is complete */ - static bool ReadPacket(Buffer &rbuf, InputPacket &rpkt); + static bool ReadPacket(ReadBuffer &rbuf, InputPacket &rpkt); /** * @brief Helper function to extract the header of a Postgres packet from the * read buffer * @see ParseInputPacket from param and return value */ - static bool ReadPacketHeader(Buffer &rbuf, InputPacket &rpkt, + static bool ReadPacketHeader(ReadBuffer &rbuf, InputPacket &rpkt, bool startup_format); //===--------------------------------------------------------------------===// @@ -196,8 +196,6 @@ class PostgresProtocolHandler : public ProtocolHandler { NetworkProtocolType protocol_type_; - // Manage standalone queries - // The result-column format code std::vector result_format_; diff --git a/src/include/network/protocol_handler.h b/src/include/network/protocol_handler.h index 1f8b4d283fe..0a7ccef3898 100644 --- a/src/include/network/protocol_handler.h +++ b/src/include/network/protocol_handler.h @@ -35,7 +35,7 @@ class ProtocolHandler { * Main switch case wrapper to process every packet apart from the startup * packet. Avoid flushing the response for extended protocols. */ - virtual ProcessResult Process(Buffer &rbuf, const size_t thread_id); + virtual ProcessResult Process(ReadBuffer &rbuf, size_t thread_id); virtual void Reset(); diff --git a/src/include/network/protocol_handler_factory.h b/src/include/network/protocol_handler_factory.h index de52613c551..c13cca250b2 100644 --- a/src/include/network/protocol_handler_factory.h +++ b/src/include/network/protocol_handler_factory.h @@ -32,5 +32,5 @@ class ProtocolHandlerFactory { static std::unique_ptr CreateProtocolHandler( ProtocolHandlerType type, tcop::TrafficCop *trafficCop); }; -} -} +} // namespace network +} // namespace peloton diff --git a/src/network/README b/src/network/README index e2a8b3c6030..c3f8d1318b9 100644 --- a/src/network/README +++ b/src/network/README @@ -1,7 +1,19 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// README +// +// Identification: src/network/README +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + ##################################################### -# # -# PELOTON WIRE # -# # +# # +#PELOTON WIRE # +# # ##################################################### Description @@ -55,4 +67,3 @@ Packets supported * RowDescription (T) * DataRow (D) * CommandComplete (C) - diff --git a/src/network/connection_dispatcher_task.cpp b/src/network/connection_dispatcher_task.cpp index ce5ce18ffdd..4c800bf1440 100644 --- a/src/network/connection_dispatcher_task.cpp +++ b/src/network/connection_dispatcher_task.cpp @@ -6,7 +6,7 @@ // // Identification: src/network/connection_dispatcher_task.cpp // -// Copyright (c) 2015-2017, Carnegie Mellon University Database Group +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// diff --git a/src/network/connection_handle.cpp b/src/network/connection_handle.cpp index e79564b5c4d..e87eabd74c3 100644 --- a/src/network/connection_handle.cpp +++ b/src/network/connection_handle.cpp @@ -6,7 +6,7 @@ // // Identification: src/network/connection_handle.cpp // -// Copyright (c) 2015-2017, Carnegie Mellon University Database Group +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// @@ -15,12 +15,13 @@ #include "network/connection_dispatcher_task.h" #include "network/connection_handle.h" +#include "network/network_io_wrapper_factory.h" #include "network/peloton_server.h" #include "network/postgres_protocol_handler.h" #include "network/protocol_handler_factory.h" -#include "settings/settings_manager.h" #include "common/utility.h" +#include "settings/settings_manager.h" namespace peloton { namespace network { @@ -42,7 +43,7 @@ namespace network { * * state ::= DEFINE_STATE(ConnState) * transition list - * END_DEF + * END_STATE_DEF * * transition ::= * ON (Transition) SET_STATE_TO (ConnState) AND_INVOKE (ConnectionHandle @@ -50,6 +51,7 @@ namespace network { * * Note that all the symbols used must be defined in ConnState, Transition and * ClientSocketWrapper, respectively. + * */ namespace { // Underneath the hood these macro is defining the static method @@ -63,64 +65,89 @@ namespace { #define DEFINE_STATE(s) \ case ConnState::s: { \ switch (t) { -#define END_DEF \ - default: \ - throw std::runtime_error("undefined transition"); \ - } \ - } #define ON(t) \ case Transition::t: \ return #define SET_STATE_TO(s) \ { \ - ConnState::s, -#define AND_INVOKE(m) \ - ([](ConnectionHandle & w) { return w.m(); }) \ + ConnState::s, +#define AND_INVOKE(m) \ + ([](ConnectionHandle &w) { return w.m(); }) \ + } \ + ; +#define AND_WAIT_ON_READ \ + ([](ConnectionHandle &w) { \ + w.UpdateEventFlags(EV_READ | EV_PERSIST); \ + return Transition::NONE; \ + }) \ + } \ + ; +#define AND_WAIT_ON_WRITE \ + ([](ConnectionHandle &w) { \ + w.UpdateEventFlags(EV_WRITE | EV_PERSIST); \ + return Transition::NONE; \ + }) \ } \ ; -#define AND_WAIT \ - ([](ConnectionHandle &) { return Transition::NONE; }) \ - } \ +#define AND_WAIT_ON_PELOTON \ + ([](ConnectionHandle &w) { \ + w.StopReceivingNetworkEvent(); \ + return Transition::NONE; \ + }) \ + } \ ; -} +#define END_DEF \ + default: \ + throw std::runtime_error("undefined transition"); \ + } \ + } + +#define END_STATE_DEF \ + ON(TERMINATE) SET_STATE_TO(CLOSING) AND_INVOKE(TryCloseConnection) END_DEF +} // namespace // clang-format off DEF_TRANSITION_GRAPH - DEFINE_STATE(READ) - ON(WAKEUP) SET_STATE_TO(READ) AND_INVOKE(FillReadBuffer) - ON(PROCEED) SET_STATE_TO(PROCESS) AND_INVOKE(Process) - ON(NEED_DATA) SET_STATE_TO(READ) AND_WAIT - ON(FINISH) SET_STATE_TO(CLOSING) AND_INVOKE(CloseSocket) - END_DEF - - DEFINE_STATE(PROCESS_WRITE_SSL_HANDSHAKE) - ON(WAKEUP) SET_STATE_TO(PROCESS_WRITE_SSL_HANDSHAKE) - AND_INVOKE(ProcessWrite_SSLHandshake) - ON(NEED_DATA) SET_STATE_TO(PROCESS_WRITE_SSL_HANDSHAKE) AND_WAIT - ON(FINISH) SET_STATE_TO(CLOSING) AND_INVOKE(CloseSocket) - ON(PROCEED) SET_STATE_TO(PROCESS) AND_INVOKE(Process) - END_DEF - - DEFINE_STATE(PROCESS) - ON(PROCEED) SET_STATE_TO(WRITE) AND_INVOKE(ProcessWrite) - ON(NEED_DATA) SET_STATE_TO(READ) AND_INVOKE(FillReadBuffer) - ON(GET_RESULT) SET_STATE_TO(GET_RESULT) AND_WAIT - ON(FINISH) SET_STATE_TO(CLOSING) AND_INVOKE(CloseSocket) - ON(NEED_SSL_HANDSHAKE) SET_STATE_TO(PROCESS_WRITE_SSL_HANDSHAKE) - AND_INVOKE(ProcessWrite_SSLHandshake) - END_DEF - - DEFINE_STATE(WRITE) - ON(WAKEUP) SET_STATE_TO(WRITE) AND_INVOKE(ProcessWrite) - ON(NEED_DATA) SET_STATE_TO(PROCESS) AND_INVOKE(Process) - ON(PROCEED) SET_STATE_TO(PROCESS) AND_INVOKE(Process) - END_DEF - - DEFINE_STATE(GET_RESULT) - ON(WAKEUP) SET_STATE_TO(GET_RESULT) AND_INVOKE(GetResult) - ON(PROCEED) SET_STATE_TO(WRITE) AND_INVOKE(ProcessWrite) - END_DEF - + DEFINE_STATE(READ) + ON(WAKEUP) SET_STATE_TO(READ) AND_INVOKE(TryRead) + ON(PROCEED) SET_STATE_TO(PROCESS) AND_INVOKE(Process) + ON(NEED_READ) SET_STATE_TO(READ) AND_WAIT_ON_READ + // This case happens only when we use SSL and are blocked on a write + // during handshake. From peloton's perspective we are still waiting + // for reads. + ON(NEED_WRITE) SET_STATE_TO(READ) AND_WAIT_ON_WRITE + END_STATE_DEF + + DEFINE_STATE(SSL_INIT) + ON(WAKEUP) SET_STATE_TO(SSL_INIT) AND_INVOKE(TrySslHandshake) + ON(NEED_READ) SET_STATE_TO(SSL_INIT) AND_WAIT_ON_READ + ON(NEED_WRITE) SET_STATE_TO(SSL_INIT) AND_WAIT_ON_WRITE + ON(PROCEED) SET_STATE_TO(PROCESS) AND_INVOKE(Process) + END_STATE_DEF + + DEFINE_STATE(PROCESS) + ON(WAKEUP) SET_STATE_TO(PROCESS) AND_INVOKE(GetResult) + ON(PROCEED) SET_STATE_TO(WRITE) AND_INVOKE(TryWrite) + ON(NEED_READ) SET_STATE_TO(READ) AND_INVOKE(TryRead) + // Client connections are ignored while we wait on peloton + // to execute the query + ON(NEED_RESULT) SET_STATE_TO(PROCESS) AND_WAIT_ON_PELOTON + ON(NEED_SSL_HANDSHAKE) SET_STATE_TO(SSL_INIT) AND_INVOKE(TrySslHandshake) + END_STATE_DEF + + DEFINE_STATE(WRITE) + ON(WAKEUP) SET_STATE_TO(WRITE) AND_INVOKE(TryWrite) + // This happens when doing ssl-rehandshake with client + ON(NEED_READ) SET_STATE_TO(WRITE) AND_WAIT_ON_READ + ON(NEED_WRITE) SET_STATE_TO(WRITE) AND_WAIT_ON_WRITE + ON(PROCEED) SET_STATE_TO(PROCESS) AND_INVOKE(Process) + END_STATE_DEF + + DEFINE_STATE(CLOSING) + ON(WAKEUP) SET_STATE_TO(CLOSING) AND_INVOKE(TryCloseConnection) + ON(NEED_READ) SET_STATE_TO(WRITE) AND_WAIT_ON_READ + ON(NEED_WRITE) SET_STATE_TO(WRITE) AND_WAIT_ON_WRITE + END_STATE_DEF END_DEF // clang-format on @@ -134,537 +161,51 @@ void ConnectionHandle::StateMachine::Accept(Transition action, next = result.second(connection); } catch (NetworkProcessException &e) { LOG_ERROR("%s\n", e.what()); - connection.CloseSocket(); + connection.TryCloseConnection(); return; } } } -ConnectionHandle::ConnectionHandle(int sock_fd, ConnectionHandlerTask *handler, - std::shared_ptr rbuf, - std::shared_ptr wbuf) - : sock_fd_(sock_fd), - handler_(handler), - protocol_handler_(nullptr), - rbuf_(std::move(rbuf)), - wbuf_(std::move(wbuf)) { - SetNonBlocking(sock_fd_); - SetTCPNoDelay(sock_fd_); - - network_event = handler->RegisterEvent( - sock_fd_, EV_READ | EV_PERSIST, - METHOD_AS_CALLBACK(ConnectionHandle, HandleEvent), this); - workpool_event = handler->RegisterManualEvent( - METHOD_AS_CALLBACK(ConnectionHandle, HandleEvent), this); - - // TODO(Tianyu): should put the initialization else where.. check correctness - // first. - traffic_cop_.SetTaskCallback([](void *arg) { - struct event *event = static_cast(arg); - event_active(event, EV_WRITE, 0); - }, workpool_event); -} - -void ConnectionHandle::UpdateEventFlags(short flags) { - // TODO(tianyu): The original network code seems to do this as an - // optimization. I am leaving this out until we get numbers - // handler->UpdateEvent(network_event, sock_fd_, flags, - // METHOD_AS_CALLBACK(ConnectionHandle, HandleEvent), this); - - if (flags == curr_event_flag_) return; - - handler_->UnregisterEvent(network_event); - network_event = handler_->RegisterEvent( - sock_fd_, flags, METHOD_AS_CALLBACK(ConnectionHandle, HandleEvent), this); +ConnectionHandle::ConnectionHandle(int sock_fd, ConnectionHandlerTask *handler) + : conn_handler_(handler), + io_wrapper_(NetworkIoWrapperFactory::GetInstance().NewNetworkIoWrapper(sock_fd)) {} - curr_event_flag_ = flags; -} - -WriteState ConnectionHandle::WritePackets() { - // iterate through all the packets +Transition ConnectionHandle::TryWrite() { for (; next_response_ < protocol_handler_->responses_.size(); next_response_++) { - auto pkt = protocol_handler_->responses_[next_response_].get(); - LOG_TRACE("To send packet with type: %c, len %lu", - static_cast(pkt->msg_type), pkt->len); - // write is not ready during write. transit to WRITE - auto result = BufferWriteBytesHeader(pkt); - if (result == WriteState::NOT_READY) return result; - result = BufferWriteBytesContent(pkt); - if (result == WriteState::NOT_READY) return result; + auto result = io_wrapper_->WritePacket( + protocol_handler_->responses_[next_response_].get()); + if (result != Transition::PROCEED) return result; } - - // Done writing all packets. clear packets protocol_handler_->responses_.clear(); next_response_ = 0; - - if (protocol_handler_->GetFlushFlag()) { - return FlushWriteBuffer(); - } - - // we have flushed, disable force flush now + if (protocol_handler_->GetFlushFlag()) return io_wrapper_->FlushWriteBuffer(); protocol_handler_->SetFlushFlag(false); - - return WriteState::COMPLETE; -} - -Transition ConnectionHandle::FillReadBuffer() { - // This could be changed by SSL_ERROR_WANT_WRITE - // When we reenter, we need to recover. - UpdateEventFlags(EV_READ | EV_PERSIST); - - Transition result = Transition::NEED_DATA; - ssize_t bytes_read = 0; - bool done = false; - - // reset buffer if all the contents have been read - if (rbuf_->buf_ptr == rbuf_->buf_size) rbuf_->Reset(); - - // buf_ptr shouldn't overflow - PELOTON_ASSERT(rbuf_->buf_ptr <= rbuf_->buf_size); - - /* Do we have leftover data and are we at the end of the buffer? - * Move the data to the head of the buffer and clear out all the old data - * Note: The assumption here is that all the packets/headers till - * rbuf_.buf_ptr have been fully processed - */ - if (rbuf_->buf_ptr < rbuf_->buf_size && - rbuf_->buf_size == rbuf_->GetMaxSize()) { - auto unprocessed_len = rbuf_->buf_size - rbuf_->buf_ptr; - // Move this data to the head of rbuf_1 - std::memmove(rbuf_->GetPtr(0), rbuf_->GetPtr(rbuf_->buf_ptr), - unprocessed_len); - // update pointers - rbuf_->buf_ptr = 0; - rbuf_->buf_size = unprocessed_len; - } - - // return explicitly - while (!done) { - if (rbuf_->buf_size == rbuf_->GetMaxSize()) { - // we have filled the whole buffer, exit loop - done = true; - } else { - // try to fill the available space in the buffer - // if the connection is a SSL connection, we use SSL_read, otherwise - // we use general read function - if (conn_SSL_context != nullptr) { - ERR_clear_error(); - bytes_read = SSL_read(conn_SSL_context, rbuf_->GetPtr(rbuf_->buf_size), - rbuf_->GetMaxSize() - rbuf_->buf_size); - LOG_TRACE("SSL read successfully"); - int err = SSL_get_error(conn_SSL_context, bytes_read); - unsigned long ecode = - (err != SSL_ERROR_NONE || bytes_read < 0) ? ERR_get_error() : 0; - switch (err) { - case SSL_ERROR_NONE: { - // If successfully received, update buffer ptr and read status - // keep reading till no data is available or the buffer becomes full - rbuf_->buf_size += bytes_read; - result = Transition::PROCEED; - break; - } - - case SSL_ERROR_ZERO_RETURN: { - done = true; - result = Transition::FINISH; - break; - } - // The SSL packet is partially loaded to the SSL buffer only, - // More data is required in order to decode the whole packet. - case SSL_ERROR_WANT_READ: { - LOG_TRACE("SSL packet partially loaded to SSL buffer"); - done = true; - break; - } - // It happens when we're trying to rehandshake and we block on a write - // during the handshake. We need to wait on the socket to be writable - case SSL_ERROR_WANT_WRITE: { - LOG_TRACE("Rehandshake during write, block until writable"); - UpdateEventFlags(EV_WRITE | EV_PERSIST); - return Transition::NEED_DATA; - } - case SSL_ERROR_SYSCALL: { - // if interrupted, try again - if (errno == EINTR) { - LOG_INFO("Error SSL Reading: EINTR"); - break; - } - } - default: { - throw NetworkProcessException("SSL read error: %d, error code: " + - std::to_string(err) + " error code:" + - std::to_string(ecode)); - } - } - } else { - bytes_read = read(sock_fd_, rbuf_->GetPtr(rbuf_->buf_size), - rbuf_->GetMaxSize() - rbuf_->buf_size); - LOG_TRACE("When filling read buffer, read %ld bytes", bytes_read); - - if (bytes_read > 0) { - // read succeeded, update buffer size - rbuf_->buf_size += bytes_read; - result = Transition::PROCEED; - } else if (bytes_read == 0) { - return Transition::FINISH; - } else if (bytes_read < 0) { - // Nothing in the network pipe now - if (errno == EAGAIN || errno == EWOULDBLOCK) { - // return whatever results we have - done = true; - } else if (errno == EINTR) { - // interrupts are ok, try again - continue; - } else { - // some other error occured - LOG_ERROR("Error writing: %s", strerror(errno)); - throw NetworkProcessException("Error when filling read buffer " + - std::to_string(errno)); - } - } - } - } - } - return result; -} - -WriteState ConnectionHandle::FlushWriteBuffer() { - // This could be changed by unfinished write - // When we reenter, we need to recover it to read - UpdateEventFlags(EV_READ | EV_PERSIST); - - ssize_t written_bytes = 0; - // while we still have outstanding bytes to write - if (conn_SSL_context != nullptr) { - while (wbuf_->buf_size > 0) { - LOG_TRACE("SSL_write flush"); - ERR_clear_error(); - written_bytes = SSL_write( - conn_SSL_context, &wbuf_->buf[wbuf_->buf_flush_ptr], wbuf_->buf_size); - int err = SSL_get_error(conn_SSL_context, written_bytes); - unsigned long ecode = - (err != SSL_ERROR_NONE || written_bytes < 0) ? ERR_get_error() : 0; - switch (err) { - case SSL_ERROR_NONE: { - wbuf_->buf_flush_ptr += written_bytes; - wbuf_->buf_size -= written_bytes; - break; - } - case SSL_ERROR_WANT_WRITE: { - // The kernel will flush the network buffer automatically. What we - // need to do is to call SSL_write() again when the buffer becomes - // availble to write again(notified by Libevent). - UpdateEventFlags(EV_WRITE | EV_PERSIST); - LOG_TRACE("Flush write buffer, want write, not ready"); - return WriteState::NOT_READY; - } - case SSL_ERROR_WANT_READ: { - // It happens when doing rehandshake with client. - LOG_TRACE("Flush write buffer, want read, not ready"); - return WriteState::NOT_READY; - } - case SSL_ERROR_SYSCALL: { - // If interrupted, try again. - if (errno == EINTR) { - LOG_TRACE("Flush write buffer, eintr"); - break; - } - } - default: { - LOG_ERROR("SSL write error: %d, error code: %lu", err, ecode); - throw NetworkProcessException("SSL write error"); - } - } - } - } else { - while (wbuf_->buf_size > 0) { - written_bytes = 0; - while (written_bytes <= 0) { - LOG_TRACE("Normal write flush"); - written_bytes = - write(sock_fd_, &wbuf_->buf[wbuf_->buf_flush_ptr], wbuf_->buf_size); - // Write failed - if (written_bytes < 0) { - if (errno == EINTR) { - // interrupts are ok, try again - written_bytes = 0; - continue; - // Write would have blocked if the socket was - // in blocking mode. Wait till it's readable - } else if (errno == EAGAIN || errno == EWOULDBLOCK) { - // Listen for socket being enabled for write - UpdateEventFlags(EV_WRITE | EV_PERSIST); - // We should go to CONN_WRITE state - LOG_TRACE("WRITE NOT READY"); - return WriteState::NOT_READY; - } else { - // fatal errors - LOG_ERROR("Error writing: %s", strerror(errno)); - throw NetworkProcessException("Fatal error during write"); - } - } - - // weird edge case? - if (written_bytes == 0 && wbuf_->buf_size != 0) { - LOG_TRACE("Not all data is written"); - continue; - } - } - - // update book keeping - wbuf_->buf_flush_ptr += written_bytes; - wbuf_->buf_size -= written_bytes; - } - } - // buffer is empty - wbuf_->Reset(); - - // we are ok - return WriteState::COMPLETE; -} - -std::string ConnectionHandle::WriteBufferToString() { -#ifdef LOG_TRACE_ENABLED - LOG_TRACE("Write Buffer:"); - - for (size_t i = 0; i < wbuf_->buf_size; ++i) { - LOG_TRACE("%u", wbuf_->buf[i]); - } -#endif - - return std::string(wbuf_->buf.begin(), wbuf_->buf.end()); -} - -// TODO (Tianyi) Make this to be protocol specific -// Writes a packet's header (type, size) into the write buffer. -// Return false when the socket is not ready for write -WriteState ConnectionHandle::BufferWriteBytesHeader(OutputPacket *pkt) { - // If we should not write - if (pkt->skip_header_write) { - return WriteState::COMPLETE; - } - - size_t len = pkt->len; - unsigned char type = static_cast(pkt->msg_type); - int len_nb; // length in network byte order - - // check if we have enough space in the buffer - if (wbuf_->GetMaxSize() - wbuf_->buf_ptr < 1 + sizeof(int32_t)) { - // buffer needs to be flushed before adding header - auto result = FlushWriteBuffer(); - if (result == WriteState::NOT_READY) { - // Socket is not ready for write - return result; - } - } - - // assuming wbuf is now large enough to fit type and size fields in one go - if (type != 0) { - // type shouldn't be ignored - wbuf_->buf[wbuf_->buf_ptr++] = type; - } - - if (!pkt->single_type_pkt) { - // make len include its field size as well - len_nb = htonl(len + sizeof(int32_t)); - - // append the bytes of this integer in network-byte order - std::copy(reinterpret_cast(&len_nb), - reinterpret_cast(&len_nb) + 4, - std::begin(wbuf_->buf) + wbuf_->buf_ptr); - // move the write buffer pointer and update size of the socket buffer - wbuf_->buf_ptr += sizeof(int32_t); - } - - wbuf_->buf_size = wbuf_->buf_ptr; - - // Header is written to socket buf. No need to write it in the future - pkt->skip_header_write = true; - return WriteState::COMPLETE; -} - -// Writes a packet's content into the write buffer -// Return false when the socket is not ready for write -WriteState ConnectionHandle::BufferWriteBytesContent(OutputPacket *pkt) { - // the packet content to write - ByteBuf &pkt_buf = pkt->buf; - // the length of remaining content to write - size_t len = pkt->len; - // window is the size of remaining space in socket's wbuf - size_t window = 0; - - // fill the contents - while (len != 0) { - // calculate the remaining space in wbuf - window = wbuf_->GetMaxSize() - wbuf_->buf_ptr; - if (len <= window) { - // contents fit in the window, range copy "len" bytes - std::copy(std::begin(pkt_buf) + pkt->write_ptr, - std::begin(pkt_buf) + pkt->write_ptr + len, - std::begin(wbuf_->buf) + wbuf_->buf_ptr); - - // Move the cursor and update size of socket buffer - wbuf_->buf_ptr += len; - wbuf_->buf_size = wbuf_->buf_ptr; - LOG_TRACE("Content fit in window. Write content successful"); - return WriteState::COMPLETE; - } else { - // contents longer than socket buffer size, fill up the socket buffer - // with "window" bytes - - std::copy(std::begin(pkt_buf) + pkt->write_ptr, - std::begin(pkt_buf) + pkt->write_ptr + window, - std::begin(wbuf_->buf) + wbuf_->buf_ptr); - - // move the packet's cursor - pkt->write_ptr += window; - len -= window; - // Now the wbuf is full - wbuf_->buf_size = wbuf_->GetMaxSize(); - - LOG_TRACE("Content doesn't fit in window. Try flushing"); - auto result = FlushWriteBuffer(); - // flush before write the remaining content - if (result == WriteState::NOT_READY) { - // need to retry or close connection - return result; - } - } - } - return WriteState::COMPLETE; -} - -Transition ConnectionHandle::CloseSocket() { - LOG_DEBUG("Attempt to close the connection %d", sock_fd_); - // Remove listening event - handler_->UnregisterEvent(network_event); - handler_->UnregisterEvent(workpool_event); - - if (conn_SSL_context != nullptr) { - int shutdown_ret = 0; - ERR_clear_error(); - shutdown_ret = SSL_shutdown(conn_SSL_context); - if (shutdown_ret != 0) { - int err = SSL_get_error(conn_SSL_context, shutdown_ret); - if (err == SSL_ERROR_WANT_WRITE || err == SSL_ERROR_WANT_READ) { - LOG_TRACE("SSL shutdown is not finished yet"); - return Transition::NEED_DATA; - } else { - LOG_ERROR("Error shutting down ssl session, err: %d", err); - } - } - SSL_free(conn_SSL_context); - conn_SSL_context = nullptr; - } - - peloton_close(sock_fd_); - return Transition::NONE; - -} - -Transition ConnectionHandle::ProcessWrite_SSLHandshake() { - // Flush out all the response first - if (HasResponse()) { - auto write_ret = ProcessWrite(); - if (write_ret != Transition::PROCEED) { - return write_ret; - } - } - - return SSLHandshake(); -} - -Transition ConnectionHandle::SSLHandshake() { - if (conn_SSL_context == nullptr) { - conn_SSL_context = SSL_new(PelotonServer::ssl_context); - if (conn_SSL_context == nullptr) { - throw NetworkProcessException("ssl context for conn failed"); - } - SSL_set_session_id_context(conn_SSL_context, nullptr, 0); - if (SSL_set_fd(conn_SSL_context, sock_fd_) == 0) { - LOG_ERROR("Failed to set SSL fd"); - return Transition::FINISH; - } - } - - // TODO(Yuchen): post-connection verification? - // clear current thread's error queue before any OpenSSL call - ERR_clear_error(); - int ssl_accept_ret = SSL_accept(conn_SSL_context); - if (ssl_accept_ret > 0) return Transition::PROCEED; - - int err = SSL_get_error(conn_SSL_context, ssl_accept_ret); - int ecode = ERR_get_error(); - char error_string[120]; - ERR_error_string(ecode, error_string); - switch (err) { - case SSL_ERROR_SSL: { - if (ecode < 0) { - LOG_ERROR("Could not accept SSL connection"); - } else { - LOG_ERROR( - "Could not accept SSL connection: EOF detected, " - "ssl_error_ssl, %s", - error_string); - } - return Transition::FINISH; - } - case SSL_ERROR_ZERO_RETURN: { - LOG_ERROR( - "Could not accept SSL connection: EOF detected, " - "ssl_error_zero_return, %s", - error_string); - return Transition::FINISH; - } - case SSL_ERROR_SYSCALL: { - if (ecode < 0) { - LOG_ERROR("Could not accept SSL connection, %s", error_string); - } else { - LOG_ERROR( - "Could not accept SSL connection: EOF detected, " - "ssl_sys_call, %s", - error_string); - } - return Transition::FINISH; - } - case SSL_ERROR_WANT_READ: { - UpdateEventFlags(EV_READ | EV_PERSIST); - return Transition::NEED_DATA; - } - case SSL_ERROR_WANT_WRITE: { - UpdateEventFlags(EV_WRITE | EV_PERSIST); - return Transition::NEED_DATA; - } - default: { - LOG_ERROR("Unrecognized SSL error code: %d", err); - return Transition::FINISH; - } - } + return Transition::PROCEED; } Transition ConnectionHandle::Process() { - if (protocol_handler_ == nullptr) { + // TODO(Tianyu): Just use Transition instead of ProcessResult, this looks + // like a 1 - 1 mapping between the two types. + if (protocol_handler_ == nullptr) // TODO(Tianyi) Check the rbuf here before we create one if we have // another protocol handler protocol_handler_ = ProtocolHandlerFactory::CreateProtocolHandler( - ProtocolHandlerType::Postgres, &traffic_cop_); - } + ProtocolHandlerType::Postgres, &tcop_); - ProcessResult status = - protocol_handler_->Process(*rbuf_, (size_t)handler_->Id()); + ProcessResult status = protocol_handler_->Process( + *(io_wrapper_->rbuf_), (size_t)conn_handler_->Id()); switch (status) { case ProcessResult::MORE_DATA_REQUIRED: - return Transition::NEED_DATA; + return Transition::NEED_READ; case ProcessResult::COMPLETE: return Transition::PROCEED; case ProcessResult::PROCESSING: - EventUtil::EventDel(network_event); - LOG_TRACE("ProcessResult: queueing"); - return Transition::GET_RESULT; + return Transition::NEED_RESULT; case ProcessResult::TERMINATE: - return Transition::FINISH; + throw NetworkProcessException("Error when processing"); case ProcessResult::NEED_SSL_HANDSHAKE: return Transition::NEED_SSL_HANDSHAKE; default: @@ -673,27 +214,39 @@ Transition ConnectionHandle::Process() { } } -Transition ConnectionHandle::ProcessWrite() { - // TODO(tianyu): Should convert to use Transition in the top level method - switch (WritePackets()) { - case WriteState::COMPLETE: - UpdateEventFlags(EV_READ | EV_PERSIST); - return Transition::PROCEED; - case WriteState::NOT_READY: - return Transition::NONE; - } - throw NetworkProcessException("Unexpected write state"); -} - Transition ConnectionHandle::GetResult() { - // TODO(tianyu) We probably can collapse this state with some other state. - if (event_add(network_event, nullptr) < 0) { - LOG_ERROR("Failed to add event"); - PELOTON_ASSERT(false); - } + EventUtil::EventAdd(network_event_, nullptr); protocol_handler_->GetResult(); - traffic_cop_.SetQueuing(false); + tcop_.SetQueuing(false); return Transition::PROCEED; } + +Transition ConnectionHandle::TrySslHandshake() { + // Flush out all the response first + if (HasResponse()) { + auto write_ret = TryWrite(); + if (write_ret != Transition::PROCEED) return write_ret; + } + return NetworkIoWrapperFactory::GetInstance().PerformSslHandshake( + io_wrapper_); +} + +Transition ConnectionHandle::TryCloseConnection() { + LOG_DEBUG("Attempt to close the connection %d", io_wrapper_->GetSocketFd()); + // TODO(Tianyu): Handle close failure + Transition close = io_wrapper_->Close(); + if (close != Transition::PROCEED) return close; + // Remove listening event + // Only after the connection is closed is it safe to remove events, + // after this point no object in the system has reference to this + // connection handle and we will need to destruct and exit. + conn_handler_->UnregisterEvent(network_event_); + conn_handler_->UnregisterEvent(workpool_event_); + // This object is essentially managed by libevent (which unfortunately does + // not accept shared_ptrs.) and thus as we shut down we need to manually + // deallocate this object. + delete this; + return Transition::NONE; +} } // namespace network } // namespace peloton diff --git a/src/network/connection_handler_task.cpp b/src/network/connection_handler_task.cpp index 267c1e0ffd9..7d5a5114c78 100644 --- a/src/network/connection_handler_task.cpp +++ b/src/network/connection_handler_task.cpp @@ -6,13 +6,13 @@ // // Identification: src/network/connection_handler_task.cpp // -// Copyright (c) 2015-2017, Carnegie Mellon University Database Group +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// #include "network/connection_handler_task.h" #include "network/connection_handle.h" -#include "network/connection_handle_factory.h" +#include "network/network_io_wrapper_factory.h" namespace peloton { namespace network { @@ -41,22 +41,25 @@ void ConnectionHandlerTask::Notify(int conn_fd) { void ConnectionHandlerTask::HandleDispatch(int new_conn_recv_fd, short) { // buffer used to receive messages from the main thread char client_fd[sizeof(int)]; - std::shared_ptr conn; size_t bytes_read = 0; // read fully while (bytes_read < sizeof(int)) { - ssize_t result = read(new_conn_recv_fd, - client_fd + bytes_read, - sizeof(int) - bytes_read); + ssize_t result = read(new_conn_recv_fd, client_fd + bytes_read, + sizeof(int) - bytes_read); if (result < 0) { LOG_ERROR("Error when reading from dispatch"); } - bytes_read += (size_t) result; + bytes_read += (size_t)result; } - conn = ConnectionHandleFactory::GetInstance().GetConnectionHandle( - *((int *) client_fd), this); + // Smart pointers are not used here because libevent does not take smart + // pointers. During the life time of this object, the pointer to it will be + // maintained by libevent rather than by our own code. The object will have to + // be cleaned up by one of its methods (i.e. we call a method with "delete + // this" and have the object commit suicide from libevent. ) + (new ConnectionHandle(*reinterpret_cast(client_fd), this)) + ->RegisterToReceiveEvents(); } } // namespace network diff --git a/src/network/marshal.cpp b/src/network/marshal.cpp index 974d292330d..314dca1d5ea 100644 --- a/src/network/marshal.cpp +++ b/src/network/marshal.cpp @@ -6,14 +6,14 @@ // // Identification: src/network/marshal.cpp // -// Copyright (c) 2015-2017, Carnegie Mellon University Database Group +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// +#include "network/marshal.h" #include #include #include -#include "network/marshal.h" #include @@ -27,15 +27,6 @@ inline void CheckOverflow(UNUSED_ATTRIBUTE InputPacket *rpkt, PELOTON_ASSERT(rpkt->ptr + size - 1 < rpkt->len); } -size_t Buffer::GetUInt32BigEndian() { - size_t num = 0; - // directly converts from network byte order to little-endian - for (size_t i = buf_ptr; i < buf_ptr + sizeof(uint32_t); i++) { - num = (num << 8) | GetByte(i); - } - return num; -} - int PacketGetInt(InputPacket *rpkt, uchar base) { int value = 0; auto begin = rpkt->Begin() + rpkt->ptr; diff --git a/src/network/network_io_wrapper_factory.cpp b/src/network/network_io_wrapper_factory.cpp new file mode 100644 index 00000000000..2c675ea0d63 --- /dev/null +++ b/src/network/network_io_wrapper_factory.cpp @@ -0,0 +1,78 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// network_io_wrapper_factory.cpp +// +// Identification: src/network/network_io_wrapper_factory.cpp +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include +#include "network/network_io_wrapper_factory.h" + +namespace peloton { +namespace network { +std::shared_ptr NetworkIoWrapperFactory::NewNetworkIoWrapper( + int conn_fd) { + auto it = reusable_wrappers_.find(conn_fd); + if (it == reusable_wrappers_.end()) { + // No reusable wrappers + auto wrapper = std::make_shared( + conn_fd, std::make_shared(), + std::make_shared()); + reusable_wrappers_[conn_fd] = + std::static_pointer_cast( + wrapper); + return wrapper; + } + + // Construct new wrapper by reusing buffers from the old one. + // The old one will be deallocated as we replace the last reference to it + // in the reusable_wrappers_ map. We still need to explicitly call the + // constructor so the flags are set properly on the new file descriptor. + auto &reused_wrapper = it->second; + reused_wrapper = std::make_shared(conn_fd, + reused_wrapper->rbuf_, + reused_wrapper->wbuf_); + return reused_wrapper; +} + +Transition NetworkIoWrapperFactory::PerformSslHandshake( + std::shared_ptr &io_wrapper) { + SSL *context; + if (!io_wrapper->SslAble()) { + context = SSL_new(PelotonServer::ssl_context); + if (context == nullptr) + throw NetworkProcessException("ssl context for conn failed"); + SSL_set_session_id_context(context, nullptr, 0); + if (SSL_set_fd(context, io_wrapper->sock_fd_) == 0) + throw NetworkProcessException("Failed to set ssl fd"); + io_wrapper = + std::make_shared(std::move(*io_wrapper), context); + } else { + auto ptr = std::dynamic_pointer_cast( + io_wrapper); + context = ptr->conn_ssl_context_; + } + + // The wrapper already uses SSL methods. + // Yuchen: "Post-connection verification?" + ERR_clear_error(); + int ssl_accept_ret = SSL_accept(context); + if (ssl_accept_ret > 0) return Transition::PROCEED; + + int err = SSL_get_error(context, ssl_accept_ret); + switch (err) { + case SSL_ERROR_WANT_READ: + return Transition::NEED_READ; + case SSL_ERROR_WANT_WRITE: + return Transition::NEED_WRITE; + default: + throw NetworkProcessException("SSL Error, error code" + std::to_string(err)); + } +} +} // namespace network +} // namespace peloton \ No newline at end of file diff --git a/src/network/network_io_wrappers.cpp b/src/network/network_io_wrappers.cpp new file mode 100644 index 00000000000..80bad466c0c --- /dev/null +++ b/src/network/network_io_wrappers.cpp @@ -0,0 +1,200 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// network_io_wrappers.cpp +// +// Identification: src/network/network_io_wrappers.cpp +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "network/network_io_wrappers.h" +#include +#include +#include +#include +#include "network/peloton_server.h" + +namespace peloton { +namespace network { +Transition NetworkIoWrapper::WritePacket(OutputPacket *pkt) { + // Write Packet Header + if (!pkt->skip_header_write) { + if (!wbuf_->HasSpaceFor(1 + sizeof(int32_t))) { + auto result = FlushWriteBuffer(); + if (FlushWriteBuffer() != Transition::PROCEED) + // Unable to flush buffer, socket presumably not ready for write + return result; + } + + wbuf_->Append(static_cast(pkt->msg_type)); + if (!pkt->single_type_pkt) + // Need to convert bytes to network order + wbuf_->Append(htonl(pkt->len + sizeof(int32_t))); + pkt->skip_header_write = true; + } + + // Write Packet Content + for (size_t len = pkt->len; len != 0;) { + if (wbuf_->HasSpaceFor(len)) { + wbuf_->Append(std::begin(pkt->buf) + pkt->write_ptr, len); + break; + } else { + auto write_size = wbuf_->RemainingCapacity(); + wbuf_->Append(std::begin(pkt->buf) + pkt->write_ptr, write_size); + len -= write_size; + pkt->write_ptr += write_size; + auto result = FlushWriteBuffer(); + if (FlushWriteBuffer() != Transition::PROCEED) + // Unable to flush buffer, socket presumably not ready for write + return result; + } + } + return Transition::PROCEED; +} + +PosixSocketIoWrapper::PosixSocketIoWrapper(int sock_fd, + std::shared_ptr rbuf, + std::shared_ptr wbuf) + : NetworkIoWrapper(sock_fd, rbuf, wbuf) { + // Set Non Blocking + auto flags = fcntl(sock_fd_, F_GETFL); + flags |= O_NONBLOCK; + if (fcntl(sock_fd_, F_SETFL, flags) < 0) { + LOG_ERROR("Failed to set non-blocking socket"); + } + // Set TCP No Delay + int one = 1; + setsockopt(sock_fd_, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)); +} + +Transition PosixSocketIoWrapper::FillReadBuffer() { + if (!rbuf_->HasMore()) rbuf_->Reset(); + if (rbuf_->HasMore() && rbuf_->Full()) rbuf_->MoveContentToHead(); + Transition result = Transition::NEED_READ; + // Normal mode + while (!rbuf_->Full()) { + auto bytes_read = rbuf_->FillBufferFrom(sock_fd_); + if (bytes_read > 0) + result = Transition::PROCEED; + else if (bytes_read == 0) + return Transition::TERMINATE; + else + switch (errno) { + case EAGAIN: + // Equal to EWOULDBLOCK + return result; + case EINTR: + continue; + default: + LOG_ERROR("Error writing: %s", strerror(errno)); + throw NetworkProcessException("Error when filling read buffer " + + std::to_string(errno)); + } + } + return result; +} + +Transition PosixSocketIoWrapper::FlushWriteBuffer() { + while (wbuf_->HasMore()) { + auto bytes_written = wbuf_->WriteOutTo(sock_fd_); + if (bytes_written < 0) switch (errno) { + case EINTR: + continue; + case EAGAIN: + return Transition::NEED_WRITE; + default: + LOG_ERROR("Error writing: %s", strerror(errno)); + throw NetworkProcessException("Fatal error during write"); + } + } + wbuf_->Reset(); + return Transition::PROCEED; +} + +Transition SslSocketIoWrapper::FillReadBuffer() { + if (!rbuf_->HasMore()) rbuf_->Reset(); + if (rbuf_->HasMore() && rbuf_->Full()) rbuf_->MoveContentToHead(); + Transition result = Transition::NEED_READ; + while (!rbuf_->Full()) { + auto ret = rbuf_->FillBufferFrom(conn_ssl_context_); + switch (ret) { + case SSL_ERROR_NONE: + result = Transition::PROCEED; + break; + case SSL_ERROR_ZERO_RETURN: + return Transition::TERMINATE; + // The SSL packet is partially loaded to the SSL buffer only, + // More data is required in order to decode the wh`ole packet. + case SSL_ERROR_WANT_READ: + return result; + case SSL_ERROR_WANT_WRITE: + return Transition::NEED_WRITE; + case SSL_ERROR_SYSCALL: + if (errno == EINTR) { + LOG_INFO("Error SSL Reading: EINTR"); + break; + } + // Intentional fallthrough + default: + throw NetworkProcessException("SSL read error: " + std::to_string(ret)); + } + } + return result; +} + +Transition SslSocketIoWrapper::FlushWriteBuffer() { + while (wbuf_->HasMore()) { + auto ret = wbuf_->WriteOutTo(conn_ssl_context_); + switch (ret) { + case SSL_ERROR_NONE: + break; + case SSL_ERROR_WANT_WRITE: + return Transition::NEED_WRITE; + case SSL_ERROR_WANT_READ: + return Transition::NEED_READ; + case SSL_ERROR_SYSCALL: + // If interrupted, try again. + if (errno == EINTR) { + LOG_TRACE("Flush write buffer, eintr"); + break; + } + // Intentional Fallthrough + default: + LOG_ERROR("SSL write error: %d, error code: %lu", ret, ERR_get_error()); + throw NetworkProcessException("SSL write error"); + } + } + wbuf_->Reset(); + return Transition::PROCEED; +} + +Transition SslSocketIoWrapper::Close() { + ERR_clear_error(); + int ret = SSL_shutdown(conn_ssl_context_); + if (ret != 0) { + int err = SSL_get_error(conn_ssl_context_, ret); + switch (err) { + case SSL_ERROR_WANT_WRITE: + return Transition::NEED_WRITE; + case SSL_ERROR_WANT_READ: + // More work to do before shutdown + return Transition::NEED_READ; + default: + LOG_ERROR("Error shutting down ssl session, err: %d", err); + } + } + // SSL context is explicitly deallocated here because socket wrapper + // objects are saved reused for memory efficiency and the reuse might + // not happen immediately, and thus freeing it on reuse time can make this + // live on arbitrarily long. + SSL_free(conn_ssl_context_); + conn_ssl_context_ = nullptr; + peloton_close(sock_fd_); + return Transition::PROCEED; +} + +} // namespace network +} // namespace peloton diff --git a/src/network/peloton_server.cpp b/src/network/peloton_server.cpp index 6e72caefbbc..b667a42e932 100644 --- a/src/network/peloton_server.cpp +++ b/src/network/peloton_server.cpp @@ -10,7 +10,6 @@ // //===----------------------------------------------------------------------===// - #include #include #include "common/utility.h" @@ -80,16 +79,16 @@ void PelotonServer::SSLLockingFunction(int mode, int n, } unsigned long PelotonServer::SSLIdFunction(void) { - return ((unsigned long) THREAD_ID); + return ((unsigned long)THREAD_ID); } void PelotonServer::LoadSSLFileSettings() { private_key_file_ = DATA_DIR + settings::SettingsManager::GetString( - settings::SettingId::private_key_file); + settings::SettingId::private_key_file); certificate_file_ = DATA_DIR + settings::SettingsManager::GetString( - settings::SettingId::certificate_file); + settings::SettingId::certificate_file); root_cert_file_ = DATA_DIR + settings::SettingsManager::GetString( - settings::SettingId::root_cert_file); + settings::SettingId::root_cert_file); } void PelotonServer::SSLInit() { @@ -111,7 +110,8 @@ void PelotonServer::SSLInit() { // TODO(Yuchen): deal with returned error 0? SSLMutexSetup(); // set general-purpose version, actual protocol will be negotiated to the - // highest version mutually support between client and server during handshake + // highest version mutually support between client and server during + // handshake ssl_context = SSL_CTX_new(SSLv23_method()); if (ssl_context == nullptr) { SetSSLLevel(SSLLevel::SSL_DISABLE); @@ -162,10 +162,10 @@ void PelotonServer::SSLInit() { // automatically. set routine to filter the return status of the default // verification and returns new verification status. SSL_VERIFY_PEER: send // certificate request to client. Client may ignore the request. If the - // client sends back the certificate, it will be verified. Handshake will be - // terminated if the verification fails. SSL_VERIFY_FAIL_IF_NO_PEER_CERT: use - // with SSL_VERIFY_PEER, if client does not send back the certificate, - // terminate the handshake. + // client sends back the certificate, it will be verified. Handshake will + // be terminated if the verification fails. + // SSL_VERIFY_FAIL_IF_NO_PEER_CERT: use with SSL_VERIFY_PEER, if client does + // not send back the certificate, terminate the handshake. SSL_CTX_set_verify(ssl_context, SSL_VERIFY_PEER, VerifyCallback); SSL_CTX_set_verify_depth(ssl_context, 4); } else { @@ -223,7 +223,7 @@ int PelotonServer::VerifyCallback(int ok, X509_STORE_CTX *store) { return ok; } -template +template void PelotonServer::TrySslOperation(int (*func)(Ts...), Ts... arg) { if (func(arg...) < 0) { auto error_message = peloton_error_message(); @@ -238,7 +238,7 @@ PelotonServer &PelotonServer::SetupServer() { // This line is critical to performance for some reason evthread_use_pthreads(); if (settings::SettingsManager::GetString( - settings::SettingId::socket_family) != "AF_INET") + settings::SettingId::socket_family) != "AF_INET") throw ConnectionException("Unsupported socket family"); struct sockaddr_in sin; @@ -258,13 +258,13 @@ PelotonServer &PelotonServer::SetupServer() { setsockopt(listen_fd_, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); TrySslOperation( - bind, listen_fd_, (struct sockaddr *) &sin, sizeof(sin)); + bind, listen_fd_, (struct sockaddr *)&sin, sizeof(sin)); TrySslOperation(listen, listen_fd_, conn_backlog); dispatcher_task_ = std::make_shared( CONNECTION_THREAD_COUNT, listen_fd_); - LOG_INFO("Listening on port %llu", (unsigned long long) port_); + LOG_INFO("Listening on port %llu", (unsigned long long)port_); return *this; } diff --git a/src/network/postgres_protocol_handler.cpp b/src/network/postgres_protocol_handler.cpp index 644e8dfef16..6f03a617667 100644 --- a/src/network/postgres_protocol_handler.cpp +++ b/src/network/postgres_protocol_handler.cpp @@ -20,8 +20,8 @@ #include "common/portal.h" #include "expression/expression_util.h" #include "network/marshal.h" -#include "network/postgres_protocol_handler.h" #include "network/peloton_server.h" +#include "network/postgres_protocol_handler.h" #include "parser/postgresparser.h" #include "parser/statements.h" #include "planner/plan_util.h" @@ -32,7 +32,7 @@ #include "util/string_util.h" #define SSL_MESSAGE_VERNO 80877103 -#define PROTO_MAJOR_VERSION(x) (x >> 16) +#define PROTO_MAJOR_VERSION(x) ((x) >> 16) namespace peloton { namespace network { @@ -42,17 +42,17 @@ namespace network { const std::unordered_map // clang-format off PostgresProtocolHandler::parameter_status_map_ = - boost::assign::map_list_of("application_name", "psql") - ("client_encoding", "UTF8") - ("DateStyle", "ISO, MDY") - ("integer_datetimes", "on") - ("IntervalStyle", "postgres") - ("is_superuser", "on") - ("server_encoding", "UTF8") - ("server_version", "9.5devel") - ("session_authorization", "postgres") - ("standard_conforming_strings", "on") - ("TimeZone", "US/Eastern"); + boost::assign::map_list_of("application_name", "psql") + ("client_encoding", "UTF8") + ("DateStyle", "ISO, MDY") + ("integer_datetimes", "on") + ("IntervalStyle", "postgres") + ("is_superuser", "on") + ("server_encoding", "UTF8") + ("server_version", "9.5devel") + ("session_authorization", "postgres") + ("standard_conforming_strings", "on") + ("TimeZone", "US/Eastern"); // clang-format on PostgresProtocolHandler::PostgresProtocolHandler(tcop::TrafficCop *traffic_cop) @@ -89,13 +89,13 @@ bool PostgresProtocolHandler::HardcodedExecuteFilter(QueryType query_type) { case QueryType::QUERY_SET: case QueryType::QUERY_SHOW: return false; - // Skip duplicate BEGIN + // Skip duplicate BEGIN case QueryType::QUERY_BEGIN: if (txn_state_ == NetworkTransactionStateType::BLOCK) { return false; } break; - // Skip duplicate Commits and Rollbacks + // Skip duplicate Commits and Rollbacks case QueryType::QUERY_COMMIT: case QueryType::QUERY_ROLLBACK: if (txn_state_ == NetworkTransactionStateType::IDLE) { @@ -258,8 +258,8 @@ ResultType PostgresProtocolHandler::ExecQueryExplain( std::unique_ptr unnamed_sql_stmt_list( new parser::SQLStatementList()); unnamed_sql_stmt_list->PassInStatement(std::move(explain_stmt.real_sql_stmt)); - auto stmt = traffic_cop_->PrepareStatement( - "explain", query, std::move(unnamed_sql_stmt_list)); + auto stmt = traffic_cop_->PrepareStatement("explain", query, + std::move(unnamed_sql_stmt_list)); ResultType status = ResultType::UNKNOWN; if (stmt != nullptr) { traffic_cop_->SetStatement(stmt); @@ -631,7 +631,8 @@ size_t PostgresProtocolHandler::ReadParamValue( .CastAs(PostgresValueTypeToPelotonValueType( (PostgresValueType)param_types[param_idx])); } - PELOTON_ASSERT(param_values[param_idx].GetTypeId() != type::TypeId::INVALID); + PELOTON_ASSERT(param_values[param_idx].GetTypeId() != + type::TypeId::INVALID); } else { // BINARY mode PostgresValueType pg_value_type = @@ -711,7 +712,8 @@ size_t PostgresProtocolHandler::ReadParamValue( break; } } - PELOTON_ASSERT(param_values[param_idx].GetTypeId() != type::TypeId::INVALID); + PELOTON_ASSERT(param_values[param_idx].GetTypeId() != + type::TypeId::INVALID); } } } @@ -901,17 +903,11 @@ void PostgresProtocolHandler::ExecCloseMessage(InputPacket *pkt) { responses_.push_back(std::move(response)); } -bool PostgresProtocolHandler::ParseInputPacket(Buffer &rbuf, InputPacket &rpkt, +bool PostgresProtocolHandler::ParseInputPacket(ReadBuffer &rbuf, + InputPacket &rpkt, bool startup_format) { - if (rpkt.header_parsed == false) { - // parse out the header first - if (ReadPacketHeader(rbuf, rpkt, startup_format) == false) { - // need more data - return false; - } - } - - PELOTON_ASSERT(rpkt.header_parsed == true); + if (!rpkt.header_parsed && !ReadPacketHeader(rbuf, rpkt, startup_format)) + return false; if (rpkt.is_initialized == false) { // packet needs to be initialized with rest of the contents @@ -926,57 +922,46 @@ bool PostgresProtocolHandler::ParseInputPacket(Buffer &rbuf, InputPacket &rpkt, // The function tries to do a preliminary read to fetch the size value and // then reads the rest of the packet. // Assume: Packet length field is always 32-bit int -bool PostgresProtocolHandler::ReadPacketHeader(Buffer &rbuf, InputPacket &rpkt, +bool PostgresProtocolHandler::ReadPacketHeader(ReadBuffer &rbuf, + InputPacket &rpkt, bool startup) { // All packets other than the startup packet have a 5 bytes header - size_t initial_read_size = startup ? sizeof(int32_t) : sizeof(int32_t) + 1; + size_t header_size = startup ? sizeof(int32_t) : sizeof(int32_t) + 1; // check if header bytes are available - if (!rbuf.IsReadDataAvailable(initial_read_size)) { - // nothing more to read - return false; - } - - if (!startup) { - // Header also contains msg type - rpkt.msg_type = static_cast(rbuf.GetByte(rbuf.buf_ptr)); - // Skip the message type byte - rbuf.buf_ptr++; - } + if (!rbuf.HasMore(header_size)) return false; + if (!startup) rpkt.msg_type = rbuf.ReadValue(); // get packet size from the header // extract packet contents size // content lengths should exclude the length bytes - rpkt.len = rbuf.GetUInt32BigEndian() - sizeof(uint32_t); + rpkt.len = ntohl(rbuf.ReadValue()) - sizeof(uint32_t); // do we need to use the extended buffer for this packet? - rpkt.is_extended = (rpkt.len > rbuf.GetMaxSize()); + rpkt.is_extended = (rpkt.len > rbuf.Capacity()); if (rpkt.is_extended) { LOG_TRACE("Using extended buffer for pkt size:%ld", rpkt.len); // reserve space for the extended buffer rpkt.ReserveExtendedBuffer(); } - // we have processed the data, move buffer pointer - rbuf.buf_ptr += sizeof(int32_t); rpkt.header_parsed = true; - return true; } // Tries to read the contents of a single packet, returns true on success, false // on failure. -bool PostgresProtocolHandler::ReadPacket(Buffer &rbuf, InputPacket &rpkt) { +bool PostgresProtocolHandler::ReadPacket(ReadBuffer &rbuf, InputPacket &rpkt) { if (rpkt.is_extended) { // extended packet mode - auto bytes_available = rbuf.buf_size - rbuf.buf_ptr; + auto bytes_available = rbuf.BytesAvailable(); auto bytes_required = rpkt.ExtendedBytesRequired(); // read minimum of the two ranges auto read_size = std::min(bytes_available, bytes_required); - rpkt.AppendToExtendedBuffer(rbuf.Begin() + rbuf.buf_ptr, - rbuf.Begin() + rbuf.buf_ptr + read_size); + rpkt.AppendToExtendedBuffer(rbuf.Begin() + rbuf.offset_, + rbuf.Begin() + rbuf.offset_ + read_size); // data has been copied, move ptr - rbuf.buf_ptr += read_size; + rbuf.offset_ += read_size; if (bytes_required > bytes_available) { // more data needs to be read return false; @@ -985,14 +970,14 @@ bool PostgresProtocolHandler::ReadPacket(Buffer &rbuf, InputPacket &rpkt) { rpkt.InitializePacket(); return true; } else { - if (rbuf.IsReadDataAvailable(rpkt.len) == false) { + if (rbuf.HasMore(rpkt.len) == false) { // data not available yet, return return false; } // Initialize the packet's "contents" - rpkt.InitializePacket(rbuf.buf_ptr, rbuf.Begin()); + rpkt.InitializePacket(rbuf.offset_, rbuf.Begin()); // We have processed the data, move buffer pointer - rbuf.buf_ptr += rpkt.len; + rbuf.offset_ += rpkt.len; } return true; @@ -1059,17 +1044,14 @@ ProcessResult PostgresProtocolHandler::ProcessStartupPacket( return ProcessResult::COMPLETE; } -ProcessResult PostgresProtocolHandler::Process(Buffer &rbuf, +ProcessResult PostgresProtocolHandler::Process(ReadBuffer &rbuf, const size_t thread_id) { if (!ParseInputPacket(rbuf, request_, init_stage_)) return ProcessResult::MORE_DATA_REQUIRED; - ProcessResult process_status; - if (init_stage_) { - process_status = ProcessInitialPacket(&request_); - } else { - process_status = ProcessNormalPacket(&request_, thread_id); - } + ProcessResult process_status = + init_stage_ ? ProcessInitialPacket(&request_) + : ProcessNormalPacket(&request_, thread_id); request_.Reset(); @@ -1205,9 +1187,9 @@ void PostgresProtocolHandler::CompleteCommand(const QueryType &query_type, case QueryType::QUERY_BEGIN: txn_state_ = NetworkTransactionStateType::BLOCK; break; - /* After commit, we end the txn block */ + /* After commit, we end the txn block */ case QueryType::QUERY_COMMIT: - /* After rollback, the txn block is ended */ + /* After rollback, the txn block is ended */ case QueryType::QUERY_ROLLBACK: txn_state_ = NetworkTransactionStateType::IDLE; break; diff --git a/src/network/protocol_handler.cpp b/src/network/protocol_handler.cpp index a6bbb110c22..20a56351f85 100644 --- a/src/network/protocol_handler.cpp +++ b/src/network/protocol_handler.cpp @@ -23,9 +23,7 @@ ProtocolHandler::ProtocolHandler(tcop::TrafficCop *traffic_cop) { ProtocolHandler::~ProtocolHandler() {} -ProcessResult ProtocolHandler::Process(UNUSED_ATTRIBUTE Buffer &rbuf, - UNUSED_ATTRIBUTE const size_t - thread_id) { +ProcessResult ProtocolHandler::Process(ReadBuffer &, const size_t) { return ProcessResult::TERMINATE; } diff --git a/src/network/protocol_handler_factory.cpp b/src/network/protocol_handler_factory.cpp index 9e05939e8b5..9df0d5fad86 100644 --- a/src/network/protocol_handler_factory.cpp +++ b/src/network/protocol_handler_factory.cpp @@ -6,7 +6,7 @@ // // Identification: src/network/protocol_handler_factory.cpp // -// Copyright (c) 2015-2017, Carnegie Mellon University Database Group +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// @@ -26,5 +26,5 @@ std::unique_ptr ProtocolHandlerFactory::CreateProtocolHandler( return nullptr; } } -} -} +} // namespace network +} // namespace peloton diff --git a/test/network/exception_test.cpp b/test/network/exception_test.cpp index 73118f0478e..08ecc98c9a5 100644 --- a/test/network/exception_test.cpp +++ b/test/network/exception_test.cpp @@ -16,14 +16,12 @@ #include "common/harness.h" #include "common/logger.h" #include "gtest/gtest.h" -#include "network/connection_handle_factory.h" +#include "network/network_io_wrapper_factory.h" #include "network/peloton_server.h" #include "network/postgres_protocol_handler.h" #include "network/protocol_handler_factory.h" #include "util/string_util.h" -#define NUM_THREADS 1 - namespace peloton { namespace test { @@ -74,16 +72,6 @@ void *ParserExceptionTest(int port) { "sslmode=disable application_name=psql", port)); - peloton::network::ConnectionHandle *conn = - peloton::network::ConnectionHandleFactory::GetInstance() - .ConnectionHandleAt(peloton::network::PelotonServer::recent_connfd) - .get(); - - network::PostgresProtocolHandler *handler = - dynamic_cast( - conn->GetProtocolHandler().get()); - EXPECT_NE(handler, nullptr); - // If an exception occurs on one transaction, we can not use this // transaction anymore int exception_count = 0, total = 6; diff --git a/test/network/prepare_stmt_test.cpp b/test/network/prepare_stmt_test.cpp index 3e11472dd54..4c76a37ecbb 100644 --- a/test/network/prepare_stmt_test.cpp +++ b/test/network/prepare_stmt_test.cpp @@ -17,7 +17,7 @@ #include "network/peloton_server.h" #include "network/postgres_protocol_handler.h" #include "util/string_util.h" -#include "network/connection_handle_factory.h" +#include "network/network_io_wrapper_factory.h" namespace peloton { namespace test { @@ -41,16 +41,6 @@ void *PrepareStatementTest(int port) { LOG_INFO("[PrepareStatementTest] Connected to %s", C.dbname()); pqxx::work txn1(C); - peloton::network::ConnectionHandle *conn = - peloton::network::ConnectionHandleFactory::GetInstance().ConnectionHandleAt( - peloton::network::PelotonServer::recent_connfd).get(); - - //Check type of protocol handler - network::PostgresProtocolHandler* handler = - dynamic_cast(conn->GetProtocolHandler().get()); - - EXPECT_NE(handler, nullptr); - // create table and insert some data txn1.exec("DROP TABLE IF EXISTS employee;"); txn1.exec("CREATE TABLE employee(id INT, name VARCHAR(100));"); diff --git a/test/network/select_all_test.cpp b/test/network/select_all_test.cpp index d601537c585..1f5552b7aa9 100644 --- a/test/network/select_all_test.cpp +++ b/test/network/select_all_test.cpp @@ -15,7 +15,7 @@ #include "common/logger.h" #include "network/peloton_server.h" #include "network/protocol_handler_factory.h" -#include "network/connection_handle_factory.h" +#include "network/network_io_wrapper_factory.h" #include "util/string_util.h" #include /* libpqxx is used to instantiate C++ client */ #include "network/postgres_protocol_handler.h" @@ -40,13 +40,6 @@ void *SelectAllTest(int port) { pqxx::connection C(StringUtil::Format( "host=127.0.0.1 port=%d user=default_database sslmode=disable application_name=psql", port)); pqxx::work txn1(C); - peloton::network::ConnectionHandle *conn = - peloton::network::ConnectionHandleFactory::GetInstance().ConnectionHandleAt( - peloton::network::PelotonServer::recent_connfd).get(); - - network::PostgresProtocolHandler *handler = - dynamic_cast(conn->GetProtocolHandler().get()); - EXPECT_NE(handler, nullptr); // create table and insert some data txn1.exec("DROP TABLE IF EXISTS template;"); diff --git a/test/network/simple_query_test.cpp b/test/network/simple_query_test.cpp index eb728e3fd6e..8e2409f2621 100644 --- a/test/network/simple_query_test.cpp +++ b/test/network/simple_query_test.cpp @@ -18,7 +18,7 @@ #include "util/string_util.h" #include /* libpqxx is used to instantiate C++ client */ #include "network/postgres_protocol_handler.h" -#include "network/connection_handle_factory.h" +#include "network/network_io_wrapper_factory.h" #define NUM_THREADS 1 @@ -41,14 +41,6 @@ void *SimpleQueryTest(int port) { "host=127.0.0.1 port=%d user=default_database sslmode=disable application_name=psql", port)); pqxx::work txn1(C); - peloton::network::ConnectionHandle *conn = - peloton::network::ConnectionHandleFactory::GetInstance().ConnectionHandleAt( - peloton::network::PelotonServer::recent_connfd).get(); - - network::PostgresProtocolHandler *handler = - dynamic_cast(conn->GetProtocolHandler().get()); - EXPECT_NE(handler, nullptr); - // EXPECT_EQ(conn->state, peloton::network::READ); // create table and insert some data txn1.exec("DROP TABLE IF EXISTS employee;"); diff --git a/test/network/ssl_test.cpp b/test/network/ssl_test.cpp index 555f069afbd..b9399ce7757 100644 --- a/test/network/ssl_test.cpp +++ b/test/network/ssl_test.cpp @@ -14,7 +14,7 @@ #include "common/harness.h" #include "common/logger.h" #include "gtest/gtest.h" -#include "network/connection_handle_factory.h" +#include "network/network_io_wrapper_factory.h" #include "network/peloton_server.h" #include "network/postgres_protocol_handler.h" #include "network/protocol_handler_factory.h" @@ -58,16 +58,6 @@ void *TestRoutine(int port) { pqxx::work txn1(C); - peloton::network::ConnectionHandle *conn = - peloton::network::ConnectionHandleFactory::GetInstance() - .ConnectionHandleAt(peloton::network::PelotonServer::recent_connfd) - .get(); - - network::PostgresProtocolHandler *handler = - dynamic_cast( - conn->GetProtocolHandler().get()); - EXPECT_NE(handler, nullptr); - // basic test // create table and insert some data txn1.exec("DROP TABLE IF EXISTS employee;"); From bf7ff625e86f2917bc939f85c41e810eec3588ec Mon Sep 17 00:00:00 2001 From: Marcel Kost Date: Tue, 19 Jun 2018 18:24:04 -0400 Subject: [PATCH 03/19] Remove wrong check in QueryCompiler::IsSupported (#1378) * Remove wrong check in QueryCompiler::IsSupported * Replace psxx extended procotol call with SQL for prepared statement to avoid type inference, which is not implemented yet in Peloton. --- src/codegen/query_compiler.cpp | 1 - test/network/prepare_stmt_test.cpp | 14 +++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/codegen/query_compiler.cpp b/src/codegen/query_compiler.cpp index d6aa9912d51..bc7c87dd7e6 100644 --- a/src/codegen/query_compiler.cpp +++ b/src/codegen/query_compiler.cpp @@ -125,7 +125,6 @@ bool QueryCompiler::IsExpressionSupported( const expression::AbstractExpression &expr) { switch (expr.GetExpressionType()) { case ExpressionType::STAR: - case ExpressionType::VALUE_PARAMETER: return false; default: break; diff --git a/test/network/prepare_stmt_test.cpp b/test/network/prepare_stmt_test.cpp index 4c76a37ecbb..07e7ebb76c8 100644 --- a/test/network/prepare_stmt_test.cpp +++ b/test/network/prepare_stmt_test.cpp @@ -52,15 +52,23 @@ void *PrepareStatementTest(int port) { txn2.exec("INSERT INTO employee VALUES (3, 'Yilei CHU');"); // test prepare statement - C.prepare("searchstmt", "SELECT name FROM employee WHERE id=$1;"); + + // REPLACED by SQL to avoid type inference bug + //C.prepare("searchstmt", "SELECT name FROM employee WHERE id=$1;"); + txn2.exec("PREPARE searchstmt (INT) as SELECT name FROM employee WHERE id=$1"); + // invocation as in variable binding - pqxx::result R = txn2.prepared("searchstmt")(1).exec(); + + // REPLACED by SQL to avoid type inference bug + //pqxx::result R = txn2.prepared("searchstmt")(1).exec(); + auto result = txn2.exec("EXECUTE searchstmt(1)"); + txn2.commit(); // test prepared statement already in statement cache // LOG_INFO("[Prepare statement cache] // %d",conn->protocol_handler_.ExistCachedStatement("searchstmt")); - EXPECT_EQ(R.size(), 1); + EXPECT_EQ(result.size(), 1); } catch (const std::exception &e) { LOG_INFO("[PrepareStatementTest] Exception occurred: %s", e.what()); From 55abc0cb7af252a12f89d90e41546ed09b578bbc Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Sun, 24 Jun 2018 14:40:51 -0400 Subject: [PATCH 04/19] TileGroupHeader refactor (#1423) * TileGroupHeader refactor to get rid of hardcoded pointer arithmetic and casts. * Don't repeatedly look up the SpinLatch field. * clang-format. * Don't replace the SpinLatch on resetting of tuple. * Fix in TileGroupHeader assignment operator. * Update Field Descriptions based on refactor. --- ...timestamp_ordering_transaction_manager.cpp | 70 +++----- src/gc/transaction_level_gc_manager.cpp | 7 +- .../timestamp_ordering_transaction_manager.h | 50 +----- src/include/storage/tile_group_header.h | 168 +++++++++--------- src/storage/tile_group_header.cpp | 31 +--- 5 files changed, 112 insertions(+), 214 deletions(-) diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index 3de2620a275..26f2eb5f331 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -27,30 +27,15 @@ namespace peloton { namespace concurrency { -common::synchronization::SpinLatch * -TimestampOrderingTransactionManager::GetSpinLatchField( - const storage::TileGroupHeader *const tile_group_header, - const oid_t &tuple_id) { - return ( - common::synchronization::SpinLatch - *)(tile_group_header->GetReservedFieldRef(tuple_id) + LOCK_OFFSET); -} - -cid_t TimestampOrderingTransactionManager::GetLastReaderCommitId( - const storage::TileGroupHeader *const tile_group_header, - const oid_t &tuple_id) { - return *(cid_t *)(tile_group_header->GetReservedFieldRef(tuple_id) + - LAST_READER_OFFSET); -} - bool TimestampOrderingTransactionManager::SetLastReaderCommitId( const storage::TileGroupHeader *const tile_group_header, const oid_t &tuple_id, const cid_t ¤t_cid, const bool is_owner) { // get the pointer to the last_reader_cid field. - cid_t *ts_ptr = (cid_t *)(tile_group_header->GetReservedFieldRef(tuple_id) + - LAST_READER_OFFSET); + cid_t read_ts = tile_group_header->GetLastReaderCommitId(tuple_id); + + auto latch = tile_group_header->GetSpinLatch(tuple_id); - GetSpinLatchField(tile_group_header, tuple_id)->Lock(); + latch->Lock(); txn_id_t tuple_txn_id = tile_group_header->GetTransactionId(tuple_id); @@ -58,29 +43,20 @@ bool TimestampOrderingTransactionManager::SetLastReaderCommitId( // if the write lock has already been acquired by some concurrent // transactions, // then return without setting the last_reader_cid. - GetSpinLatchField(tile_group_header, tuple_id)->Unlock(); + latch->Unlock(); return false; } else { // if current_cid is larger than the current value of last_reader_cid field, // then set last_reader_cid to current_cid. - if (*ts_ptr < current_cid) { - *ts_ptr = current_cid; + if (read_ts < current_cid) { + tile_group_header->SetLastReaderCommitId(tuple_id, current_cid); } - GetSpinLatchField(tile_group_header, tuple_id)->Unlock(); + latch->Unlock(); return true; } } -void TimestampOrderingTransactionManager::InitTupleReserved( - const storage::TileGroupHeader *const tile_group_header, - const oid_t tuple_id) { - auto reserved_area = tile_group_header->GetReservedFieldRef(tuple_id); - - new ((reserved_area + LOCK_OFFSET)) common::synchronization::SpinLatch(); - *(cid_t *)(reserved_area + LAST_READER_OFFSET) = 0; -} - TimestampOrderingTransactionManager & TimestampOrderingTransactionManager::GetInstance( const ProtocolType protocol, const IsolationLevelType isolation, @@ -138,25 +114,26 @@ bool TimestampOrderingTransactionManager::AcquireOwnership( // to acquire the ownership, // we must guarantee that no transaction that has read // the tuple has a larger timestamp than the current transaction. - GetSpinLatchField(tile_group_header, tuple_id)->Lock(); + auto latch = tile_group_header->GetSpinLatch(tuple_id); + latch->Lock(); // change timestamp - cid_t last_reader_cid = GetLastReaderCommitId(tile_group_header, tuple_id); + cid_t last_reader_cid = tile_group_header->GetLastReaderCommitId(tuple_id); // must compare last_reader_cid with a transaction's commit_id // (rather than read_id). // consider a transaction that is executed under snapshot isolation. // in this case, commit_id is not equal to read_id. if (last_reader_cid > current_txn->GetCommitId()) { - GetSpinLatchField(tile_group_header, tuple_id)->Unlock(); + tile_group_header->GetSpinLatch(tuple_id)->Unlock(); return false; } else { if (tile_group_header->SetAtomicTransactionId(tuple_id, txn_id) == false) { - GetSpinLatchField(tile_group_header, tuple_id)->Unlock(); + latch->Unlock(); return false; } else { - GetSpinLatchField(tile_group_header, tuple_id)->Unlock(); + latch->Unlock(); return true; } @@ -328,9 +305,9 @@ bool TimestampOrderingTransactionManager::PerformRead(TransactionContext *const // if we have already owned the version. PELOTON_ASSERT(IsOwner(current_txn, tile_group_header, tuple_id) == true); - PELOTON_ASSERT(GetLastReaderCommitId(tile_group_header, tuple_id) == + PELOTON_ASSERT(tile_group_header->GetLastReaderCommitId(tuple_id) == current_txn->GetCommitId() || - GetLastReaderCommitId(tile_group_header, tuple_id) == 0); + tile_group_header->GetLastReaderCommitId(tuple_id) == 0); return true; } else { @@ -352,9 +329,9 @@ bool TimestampOrderingTransactionManager::PerformRead(TransactionContext *const } else { // if the current transaction has already owned this tuple, // then perform read directly. - PELOTON_ASSERT(GetLastReaderCommitId(tile_group_header, tuple_id) == + PELOTON_ASSERT(tile_group_header->GetLastReaderCommitId(tuple_id) == current_txn->GetCommitId() || - GetLastReaderCommitId(tile_group_header, tuple_id) == 0); + tile_group_header->GetLastReaderCommitId(tuple_id) == 0); // this version must already be in the read/write set. // so no need to update read set. @@ -391,8 +368,6 @@ void TimestampOrderingTransactionManager::PerformInsert( // Add the new tuple into the insert set current_txn->RecordInsert(location); - InitTupleReserved(tile_group_header, tuple_id); - // Write down the head pointer's address in tile group header tile_group_header->SetIndirection(tuple_id, index_entry_ptr); } @@ -446,8 +421,6 @@ void TimestampOrderingTransactionManager::PerformUpdate( // newer version to older version. COMPILER_MEMORY_FENCE; - InitTupleReserved(new_tile_group_header, new_location.offset); - // we must be updating the latest version. // Set the header information for the new version ItemPointer *index_entry_ptr = @@ -520,9 +493,8 @@ void TimestampOrderingTransactionManager::PerformDelete( auto transaction_id = current_txn->GetTransactionId(); - PELOTON_ASSERT( - GetLastReaderCommitId(tile_group_header, old_location.offset) == - current_txn->GetCommitId()); + PELOTON_ASSERT(tile_group_header->GetLastReaderCommitId( + old_location.offset) == current_txn->GetCommitId()); // if we can perform delete, then we must have already locked the older // version. @@ -554,8 +526,6 @@ void TimestampOrderingTransactionManager::PerformDelete( // newer version to older version. COMPILER_MEMORY_FENCE; - InitTupleReserved(new_tile_group_header, new_location.offset); - // we must be deleting the latest version. // Set the header information for the new version ItemPointer *index_entry_ptr = diff --git a/src/gc/transaction_level_gc_manager.cpp b/src/gc/transaction_level_gc_manager.cpp index 804ae21f05b..e6630aa6cf6 100644 --- a/src/gc/transaction_level_gc_manager.cpp +++ b/src/gc/transaction_level_gc_manager.cpp @@ -36,13 +36,12 @@ bool TransactionLevelGCManager::ResetTuple(const ItemPointer &location) { // Reset the header tile_group_header->SetTransactionId(location.offset, INVALID_TXN_ID); + tile_group_header->SetLastReaderCommitId(location.offset, INVALID_CID); tile_group_header->SetBeginCommitId(location.offset, MAX_CID); tile_group_header->SetEndCommitId(location.offset, MAX_CID); - tile_group_header->SetPrevItemPointer(location.offset, INVALID_ITEMPOINTER); tile_group_header->SetNextItemPointer(location.offset, INVALID_ITEMPOINTER); - - PELOTON_MEMSET(tile_group_header->GetReservedFieldRef(location.offset), 0, - storage::TileGroupHeader::GetReservedSize()); + tile_group_header->SetPrevItemPointer(location.offset, INVALID_ITEMPOINTER); + tile_group_header->SetIndirection(location.offset, nullptr); // Reclaim the varlen pool CheckAndReclaimVarlenColumns(tile_group, location.offset); diff --git a/src/include/concurrency/timestamp_ordering_transaction_manager.h b/src/include/concurrency/timestamp_ordering_transaction_manager.h index ec3a5af5861..64fe0f34885 100644 --- a/src/include/concurrency/timestamp_ordering_transaction_manager.h +++ b/src/include/concurrency/timestamp_ordering_transaction_manager.h @@ -238,41 +238,7 @@ class TimestampOrderingTransactionManager : public TransactionManager { */ virtual ResultType AbortTransaction(TransactionContext *const current_txn); - -private: - static const int LOCK_OFFSET = 0; - static const int LAST_READER_OFFSET = (LOCK_OFFSET + 8); - - /** - * @brief Gets the spin latch field. - * - * Timestamp ordering requires a spinlock field for protecting the atomic access - * to txn_id field and last_reader_cid field. - * - * @param[in] tile_group_header The tile group header - * @param[in] tuple_id The tuple identifier - * - * @return The spin latch field. - */ - common::synchronization::SpinLatch *GetSpinLatchField( - const storage::TileGroupHeader *const tile_group_header, - const oid_t &tuple_id); - - /** - * @brief Gets the last reader commit identifier. - * - * In timestamp ordering, the last_reader_cid records the timestamp of the last - * transaction that reads the tuple. - * - * @param[in] tile_group_header The tile group header - * @param[in] tuple_id The tuple identifier - * - * @return The last reader commit identifier. - */ - cid_t GetLastReaderCommitId( - const storage::TileGroupHeader *const tile_group_header, - const oid_t &tuple_id); - + private: /** * @brief Sets the last reader commit identifier. * @@ -285,19 +251,7 @@ class TimestampOrderingTransactionManager : public TransactionManager { */ bool SetLastReaderCommitId( const storage::TileGroupHeader *const tile_group_header, - const oid_t &tuple_id, - const cid_t ¤t_cid, - const bool is_owner); - - /** - * Initialize reserved area of a tuple. - * - * @param[in] tile_group_header The tile group header - * @param[in] tuple_id The tuple identifier - */ - void InitTupleReserved( - const storage::TileGroupHeader *const tile_group_header, - const oid_t tuple_id); + const oid_t &tuple_id, const cid_t ¤t_cid, const bool is_owner); }; } } diff --git a/src/include/storage/tile_group_header.h b/src/include/storage/tile_group_header.h index c7e8c010530..0c851ee1414 100644 --- a/src/include/storage/tile_group_header.h +++ b/src/include/storage/tile_group_header.h @@ -28,6 +28,34 @@ namespace storage { class TileGroup; +//===--------------------------------------------------------------------===// +// Tuple Header +//===--------------------------------------------------------------------===// + +struct TupleHeader { + std::unique_ptr latch; + std::atomic txn_id; + cid_t read_ts; + cid_t begin_ts; + cid_t end_ts; + ItemPointer next; + ItemPointer prev; + ItemPointer *indirection; +} __attribute__((aligned(64))); + +/** + * FIELD DESCRIPTIONS: + * =================== + * latch: Tuple header latch used to acquire ownership or update read_ts + * txn_id: serve as a write lock on the tuple version + * read_ts: the last txn to read this tuple + * begin_ts: the lower bound of the version visibility range. + * end_ts: the upper bound of the version visibility range. + * next: the pointer pointing to the next (older) version in the version chain. + * prev: the pointer pointing to the prev (newer) version in the version chain. + * indirection: the pointer pointing to the index entry that holds the address of the version chain header. +*/ + //===--------------------------------------------------------------------===// // Tile Group Header //===--------------------------------------------------------------------===// @@ -37,24 +65,6 @@ class TileGroup; * This contains information related to MVCC. * It is shared by all tiles in a tile group. * - * Layout : - * - * ----------------------------------------------------------------------------- - * | TxnID (8 bytes) | BeginTimeStamp (8 bytes) | EndTimeStamp (8 bytes) | - * | NextItemPointer (8 bytes) | PrevItemPointer (8 bytes) | - * | Indirection (8 bytes) | ReservedField (16 bytes) - * ----------------------------------------------------------------------------- - * - * FIELD DESCRIPTIONS: - * =================== - * TxnID: serve as a write lock on the tuple version. - * BeginTimeStamp: the lower bound of the version visibility range. - * EndTimeStamp: the upper bound of the version visibility range. - * NextItemPointer: the pointer pointing to the next (older) version in the version chain. - * PrevItemPointer: the pointer pointing to the prev (newer) version in the version chain. - * Indirection: the pointer pointing to the index entry that holds the address of the version chain header. - * ReservedField: unused space for future usage. - * * STATUS: * =================== * TxnID == INITIAL_TXN_ID, BeginTS == MAX_CID, EndTS == MAX_CID --> empty version @@ -63,8 +73,6 @@ class TileGroup; * TxnID != INITIAL_TXN_ID, BeginTS == MAX_CID, EndTS == INVALID_CID --> to-be-installed deleted version */ -#define TUPLE_HEADER_LOCATION data + (tuple_slot_id * header_entry_size) - class TileGroupHeader : public Printable { TileGroupHeader() = delete; @@ -75,21 +83,34 @@ class TileGroupHeader : public Printable { // check for self-assignment if (&other == this) return *this; - header_size = other.header_size; - - // copy over all the data - PELOTON_MEMCPY(data, other.data, header_size); - + backend_type = other.backend_type; + tile_group = other.tile_group; num_tuple_slots = other.num_tuple_slots; - oid_t val = other.next_tuple_slot; - next_tuple_slot = val; + next_tuple_slot.store(other.next_tuple_slot); + immutable = other.immutable; + + // copy tuple header values + for (oid_t tuple_slot_id = START_OID; tuple_slot_id < num_tuple_slots; + tuple_slot_id++) { + tuple_headers_[tuple_slot_id].latch.reset( + new common::synchronization::SpinLatch); + SetTransactionId(tuple_slot_id, other.GetTransactionId(tuple_slot_id)); + SetLastReaderCommitId(tuple_slot_id, + other.GetLastReaderCommitId(tuple_slot_id)); + SetBeginCommitId(tuple_slot_id, other.GetBeginCommitId(tuple_slot_id)); + SetEndCommitId(tuple_slot_id, other.GetEndCommitId(tuple_slot_id)); + SetNextItemPointer(tuple_slot_id, + other.GetNextItemPointer(tuple_slot_id)); + SetPrevItemPointer(tuple_slot_id, + other.GetPrevItemPointer(tuple_slot_id)); + SetIndirection(tuple_slot_id, other.GetIndirection(tuple_slot_id)); + } return *this; } - ~TileGroupHeader(); + ~TileGroupHeader() = default; - // this function is only called by DataTable::GetEmptyTupleSlot(). oid_t GetNextEmptyTupleSlot() { if (next_tuple_slot >= num_tuple_slots) { return INVALID_OID; @@ -145,36 +166,37 @@ class TileGroupHeader : public Printable { return tile_group; } - // it is possible that some other transactions are modifying the txn_id, - // but the current transaction reads the txn_id. - // the returned value seems to be uncertain. + inline common::synchronization::SpinLatch *GetSpinLatch( + const oid_t &tuple_slot_id) const { + return tuple_headers_[tuple_slot_id].latch.get(); + } + inline txn_id_t GetTransactionId(const oid_t &tuple_slot_id) const { - return *((txn_id_t *)(TUPLE_HEADER_LOCATION)); + return tuple_headers_[tuple_slot_id].txn_id; + } + + inline cid_t GetLastReaderCommitId(const oid_t &tuple_slot_id) const { + return tuple_headers_[tuple_slot_id].read_ts; } inline cid_t GetBeginCommitId(const oid_t &tuple_slot_id) const { - return *((cid_t *)(TUPLE_HEADER_LOCATION + begin_cid_offset)); + return tuple_headers_[tuple_slot_id].begin_ts; } inline cid_t GetEndCommitId(const oid_t &tuple_slot_id) const { - return *((cid_t *)(TUPLE_HEADER_LOCATION + end_cid_offset)); + return tuple_headers_[tuple_slot_id].end_ts; } inline ItemPointer GetNextItemPointer(const oid_t &tuple_slot_id) const { - return *((ItemPointer *)(TUPLE_HEADER_LOCATION + next_pointer_offset)); + return tuple_headers_[tuple_slot_id].next; } inline ItemPointer GetPrevItemPointer(const oid_t &tuple_slot_id) const { - return *((ItemPointer *)(TUPLE_HEADER_LOCATION + prev_pointer_offset)); + return tuple_headers_[tuple_slot_id].prev; } inline ItemPointer *GetIndirection(const oid_t &tuple_slot_id) const { - return *(ItemPointer **)(TUPLE_HEADER_LOCATION + indirection_offset); - } - - // constraint: at most 16 bytes. - inline char *GetReservedFieldRef(const oid_t &tuple_slot_id) const { - return (char *)(TUPLE_HEADER_LOCATION + reserved_field_offset); + return tuple_headers_[tuple_slot_id].indirection; } // Setters @@ -182,49 +204,47 @@ class TileGroupHeader : public Printable { inline void SetTileGroup(TileGroup *tile_group) { this->tile_group = tile_group; } + inline void SetTransactionId(const oid_t &tuple_slot_id, const txn_id_t &transaction_id) const { - *((txn_id_t *)(TUPLE_HEADER_LOCATION)) = transaction_id; + tuple_headers_[tuple_slot_id].txn_id = transaction_id; + } + + inline void SetLastReaderCommitId(const oid_t &tuple_slot_id, + const cid_t &read_cid) const { + tuple_headers_[tuple_slot_id].read_ts = read_cid; } inline void SetBeginCommitId(const oid_t &tuple_slot_id, const cid_t &begin_cid) { - *((cid_t *)(TUPLE_HEADER_LOCATION + begin_cid_offset)) = begin_cid; + tuple_headers_[tuple_slot_id].begin_ts = begin_cid; } inline void SetEndCommitId(const oid_t &tuple_slot_id, const cid_t &end_cid) const { - *((cid_t *)(TUPLE_HEADER_LOCATION + end_cid_offset)) = end_cid; + tuple_headers_[tuple_slot_id].end_ts = end_cid; } inline void SetNextItemPointer(const oid_t &tuple_slot_id, const ItemPointer &item) const { - *((ItemPointer *)(TUPLE_HEADER_LOCATION + next_pointer_offset)) = item; + tuple_headers_[tuple_slot_id].next = item; } inline void SetPrevItemPointer(const oid_t &tuple_slot_id, const ItemPointer &item) const { - *((ItemPointer *)(TUPLE_HEADER_LOCATION + prev_pointer_offset)) = item; + tuple_headers_[tuple_slot_id].prev = item; } inline void SetIndirection(const oid_t &tuple_slot_id, - const ItemPointer *indirection) const { - *((const ItemPointer **)(TUPLE_HEADER_LOCATION + indirection_offset)) = - indirection; - } - - inline txn_id_t SetAtomicTransactionId(const oid_t &tuple_slot_id, - const txn_id_t &old_txn_id, - const txn_id_t &new_txn_id) const { - txn_id_t *txn_id_ptr = (txn_id_t *)(TUPLE_HEADER_LOCATION); - return __sync_val_compare_and_swap(txn_id_ptr, old_txn_id, new_txn_id); + ItemPointer *indirection) const { + tuple_headers_[tuple_slot_id].indirection = indirection; } inline bool SetAtomicTransactionId(const oid_t &tuple_slot_id, const txn_id_t &transaction_id) const { - txn_id_t *txn_id_ptr = (txn_id_t *)(TUPLE_HEADER_LOCATION); - return __sync_bool_compare_and_swap(txn_id_ptr, INITIAL_TXN_ID, - transaction_id); + auto old_val = INITIAL_TXN_ID; + return tuple_headers_[tuple_slot_id].txn_id.compare_exchange_strong( + old_val, transaction_id); } /* @@ -250,9 +270,6 @@ class TileGroupHeader : public Printable { // Getter for spin lock common::synchronization::SpinLatch &GetHeaderLock() { return tile_header_lock; } - // Sync the contents - void Sync(); - //===--------------------------------------------------------------------===// // Utilities //===--------------------------------------------------------------------===// @@ -260,24 +277,6 @@ class TileGroupHeader : public Printable { // Get a string representation for debugging const std::string GetInfo() const; - static inline size_t GetReservedSize() { return reserved_size; } - - // header entry size is the size of the layout described above - static const size_t reserved_size = 16; - static const size_t header_entry_size = sizeof(txn_id_t) + 2 * sizeof(cid_t) + - 2 * sizeof(ItemPointer) + - sizeof(ItemPointer *) + reserved_size; - static const size_t txn_id_offset = 0; - static const size_t begin_cid_offset = txn_id_offset + sizeof(txn_id_t); - static const size_t end_cid_offset = begin_cid_offset + sizeof(cid_t); - static const size_t next_pointer_offset = end_cid_offset + sizeof(cid_t); - static const size_t prev_pointer_offset = - next_pointer_offset + sizeof(ItemPointer); - static const size_t indirection_offset = - prev_pointer_offset + sizeof(ItemPointer); - static const size_t reserved_field_offset = - indirection_offset + sizeof(ItemPointer); - private: //===--------------------------------------------------------------------===// // Data members @@ -289,10 +288,7 @@ class TileGroupHeader : public Printable { // Associated tile_group TileGroup *tile_group; - size_t header_size; - - // set of fixed-length tuple slots - char *data; + std::unique_ptr tuple_headers_; // number of tuple slots allocated oid_t num_tuple_slots; diff --git a/src/storage/tile_group_header.cpp b/src/storage/tile_group_header.cpp index 1e0b450144e..ee30fe82dff 100644 --- a/src/storage/tile_group_header.cpp +++ b/src/storage/tile_group_header.cpp @@ -34,44 +34,29 @@ TileGroupHeader::TileGroupHeader(const BackendType &backend_type, const int &tuple_count) : backend_type(backend_type), tile_group(nullptr), - data(nullptr), num_tuple_slots(tuple_count), next_tuple_slot(0), tile_header_lock() { - header_size = num_tuple_slots * header_entry_size; - - // allocate storage space for header - // auto &storage_manager = storage::StorageManager::GetInstance(); - // data = reinterpret_cast( - // storage_manager.Allocate(backend_type, header_size)); - data = new char[header_size]; - PELOTON_ASSERT(data != nullptr); - - // zero out the data - PELOTON_MEMSET(data, 0, header_size); + tuple_headers_.reset(new TupleHeader[tuple_count]); // Set MVCC Initial Value for (oid_t tuple_slot_id = START_OID; tuple_slot_id < num_tuple_slots; tuple_slot_id++) { + tuple_headers_[tuple_slot_id].latch.reset( + new common::synchronization::SpinLatch); SetTransactionId(tuple_slot_id, INVALID_TXN_ID); + SetLastReaderCommitId(tuple_slot_id, INVALID_CID); SetBeginCommitId(tuple_slot_id, MAX_CID); SetEndCommitId(tuple_slot_id, MAX_CID); SetNextItemPointer(tuple_slot_id, INVALID_ITEMPOINTER); SetPrevItemPointer(tuple_slot_id, INVALID_ITEMPOINTER); + SetIndirection(tuple_slot_id, nullptr); } // Initially immutabile flag to false initially. immutable = false; } -TileGroupHeader::~TileGroupHeader() { - // reclaim the space - // auto &storage_manager = storage::StorageManager::GetInstance(); - // storage_manager.Release(backend_type, data); - delete[] data; - data = nullptr; -} - //===--------------------------------------------------------------------===// // Tile Group Header //===--------------------------------------------------------------------===// @@ -166,12 +151,6 @@ const std::string TileGroupHeader::GetInfo() const { return os.str(); } -void TileGroupHeader::Sync() { - // Sync the tile group data - // auto &storage_manager = storage::StorageManager::GetInstance(); - // storage_manager.Sync(backend_type, data, header_size); -} - void TileGroupHeader::PrintVisibility(txn_id_t txn_id, cid_t at_cid) { oid_t active_tuple_slots = GetCurrentNextTupleSlot(); std::stringstream os; From 29f92eb0f38e8a1cd4baf060d8ebc705541f89e4 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Sun, 24 Jun 2018 16:47:10 -0400 Subject: [PATCH 05/19] Skip recording READs in the RWSet. (#1425) --- ...timestamp_ordering_transaction_manager.cpp | 6 +----- src/concurrency/transaction_context.cpp | 21 +++---------------- src/include/concurrency/transaction_context.h | 2 -- 3 files changed, 4 insertions(+), 25 deletions(-) diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index 26f2eb5f331..3adf5dc69b4 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -201,8 +201,7 @@ bool TimestampOrderingTransactionManager::PerformRead(TransactionContext *const return true; } else { - // if it's not select for update, then update read set and return true. - current_txn->RecordRead(location); + // if it's not select for update, then return true. return true; } @@ -243,7 +242,6 @@ bool TimestampOrderingTransactionManager::PerformRead(TransactionContext *const // a transaction can never read an uncommitted version. if (IsOwner(current_txn, tile_group_header, tuple_id) == false) { if (IsOwned(current_txn, tile_group_header, tuple_id) == false) { - current_txn->RecordRead(location); return true; } else { @@ -316,8 +314,6 @@ bool TimestampOrderingTransactionManager::PerformRead(TransactionContext *const // then attempt to set last reader cid. if (SetLastReaderCommitId(tile_group_header, tuple_id, current_txn->GetCommitId(), false) == true) { - // update read set. - current_txn->RecordRead(location); return true; } else { // if the tuple has been owned by some concurrent transactions, diff --git a/src/concurrency/transaction_context.cpp b/src/concurrency/transaction_context.cpp index 498cc927e60..535ba703137 100644 --- a/src/concurrency/transaction_context.cpp +++ b/src/concurrency/transaction_context.cpp @@ -94,35 +94,20 @@ RWType TransactionContext::GetRWType(const ItemPointer &location) { return RWType::INVALID; } -void TransactionContext::RecordRead(const ItemPointer &location) { - PELOTON_ASSERT(rw_set_.find(location) == rw_set_.end() || - (rw_set_[location] != RWType::DELETE && - rw_set_[location] != RWType::INS_DEL)); - auto rw_set_it = rw_set_.find(location); - if (rw_set_it != rw_set_.end()) { - return; - } - rw_set_[location] = RWType::READ; -} - void TransactionContext::RecordReadOwn(const ItemPointer &location) { PELOTON_ASSERT(rw_set_.find(location) == rw_set_.end() || (rw_set_[location] != RWType::DELETE && rw_set_[location] != RWType::INS_DEL)); rw_set_[location] = RWType::READ_OWN; + is_written_ = true; } void TransactionContext::RecordUpdate(const ItemPointer &location) { PELOTON_ASSERT(rw_set_.find(location) == rw_set_.end() || (rw_set_[location] != RWType::DELETE && rw_set_[location] != RWType::INS_DEL)); - auto rw_set_it = rw_set_.find(location); - if (rw_set_it != rw_set_.end() && (rw_set_it->second == RWType::READ || - rw_set_it->second == RWType::READ_OWN)) { - rw_set_it->second = RWType::UPDATE; - is_written_ = true; - } - PELOTON_ASSERT(is_written_); + rw_set_[location] = RWType::UPDATE; + is_written_ = true; } void TransactionContext::RecordInsert(const ItemPointer &location) { diff --git a/src/include/concurrency/transaction_context.h b/src/include/concurrency/transaction_context.h index 04419082825..511e0bd38f7 100644 --- a/src/include/concurrency/transaction_context.h +++ b/src/include/concurrency/transaction_context.h @@ -159,8 +159,6 @@ class TransactionContext : public Printable { index_oid, DDLType::DROP)); } - void RecordRead(const ItemPointer &); - void RecordReadOwn(const ItemPointer &); void RecordUpdate(const ItemPointer &); From 9cca1b7f1291c4f655dd21909a8f5a22bac384bc Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Mon, 25 Jun 2018 13:08:58 -0400 Subject: [PATCH 06/19] Inline tuple SpinLatch after TileGroupHeader refactor (#1423) moved it out. (#1426) --- .../timestamp_ordering_transaction_manager.cpp | 18 +++++++++--------- src/include/storage/tile_group_header.h | 8 +++----- src/storage/tile_group_header.cpp | 2 -- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index 3adf5dc69b4..3a39ccc8877 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -33,9 +33,9 @@ bool TimestampOrderingTransactionManager::SetLastReaderCommitId( // get the pointer to the last_reader_cid field. cid_t read_ts = tile_group_header->GetLastReaderCommitId(tuple_id); - auto latch = tile_group_header->GetSpinLatch(tuple_id); + auto &latch = tile_group_header->GetSpinLatch(tuple_id); - latch->Lock(); + latch.Lock(); txn_id_t tuple_txn_id = tile_group_header->GetTransactionId(tuple_id); @@ -43,7 +43,7 @@ bool TimestampOrderingTransactionManager::SetLastReaderCommitId( // if the write lock has already been acquired by some concurrent // transactions, // then return without setting the last_reader_cid. - latch->Unlock(); + latch.Unlock(); return false; } else { // if current_cid is larger than the current value of last_reader_cid field, @@ -52,7 +52,7 @@ bool TimestampOrderingTransactionManager::SetLastReaderCommitId( tile_group_header->SetLastReaderCommitId(tuple_id, current_cid); } - latch->Unlock(); + latch.Unlock(); return true; } } @@ -114,8 +114,8 @@ bool TimestampOrderingTransactionManager::AcquireOwnership( // to acquire the ownership, // we must guarantee that no transaction that has read // the tuple has a larger timestamp than the current transaction. - auto latch = tile_group_header->GetSpinLatch(tuple_id); - latch->Lock(); + auto &latch = tile_group_header->GetSpinLatch(tuple_id); + latch.Lock(); // change timestamp cid_t last_reader_cid = tile_group_header->GetLastReaderCommitId(tuple_id); @@ -124,16 +124,16 @@ bool TimestampOrderingTransactionManager::AcquireOwnership( // consider a transaction that is executed under snapshot isolation. // in this case, commit_id is not equal to read_id. if (last_reader_cid > current_txn->GetCommitId()) { - tile_group_header->GetSpinLatch(tuple_id)->Unlock(); + latch.Unlock(); return false; } else { if (tile_group_header->SetAtomicTransactionId(tuple_id, txn_id) == false) { - latch->Unlock(); + latch.Unlock(); return false; } else { - latch->Unlock(); + latch.Unlock(); return true; } diff --git a/src/include/storage/tile_group_header.h b/src/include/storage/tile_group_header.h index 0c851ee1414..12ede331232 100644 --- a/src/include/storage/tile_group_header.h +++ b/src/include/storage/tile_group_header.h @@ -33,7 +33,7 @@ class TileGroup; //===--------------------------------------------------------------------===// struct TupleHeader { - std::unique_ptr latch; + common::synchronization::SpinLatch latch; std::atomic txn_id; cid_t read_ts; cid_t begin_ts; @@ -92,8 +92,6 @@ class TileGroupHeader : public Printable { // copy tuple header values for (oid_t tuple_slot_id = START_OID; tuple_slot_id < num_tuple_slots; tuple_slot_id++) { - tuple_headers_[tuple_slot_id].latch.reset( - new common::synchronization::SpinLatch); SetTransactionId(tuple_slot_id, other.GetTransactionId(tuple_slot_id)); SetLastReaderCommitId(tuple_slot_id, other.GetLastReaderCommitId(tuple_slot_id)); @@ -166,9 +164,9 @@ class TileGroupHeader : public Printable { return tile_group; } - inline common::synchronization::SpinLatch *GetSpinLatch( + inline common::synchronization::SpinLatch &GetSpinLatch( const oid_t &tuple_slot_id) const { - return tuple_headers_[tuple_slot_id].latch.get(); + return tuple_headers_[tuple_slot_id].latch; } inline txn_id_t GetTransactionId(const oid_t &tuple_slot_id) const { diff --git a/src/storage/tile_group_header.cpp b/src/storage/tile_group_header.cpp index ee30fe82dff..4ff67e670f9 100644 --- a/src/storage/tile_group_header.cpp +++ b/src/storage/tile_group_header.cpp @@ -42,8 +42,6 @@ TileGroupHeader::TileGroupHeader(const BackendType &backend_type, // Set MVCC Initial Value for (oid_t tuple_slot_id = START_OID; tuple_slot_id < num_tuple_slots; tuple_slot_id++) { - tuple_headers_[tuple_slot_id].latch.reset( - new common::synchronization::SpinLatch); SetTransactionId(tuple_slot_id, INVALID_TXN_ID); SetLastReaderCommitId(tuple_slot_id, INVALID_CID); SetBeginCommitId(tuple_slot_id, MAX_CID); From a045cfc95bf349742a8101aee65e22efd9ec8096 Mon Sep 17 00:00:00 2001 From: Marcel Kost Date: Tue, 26 Jun 2018 10:11:55 -0400 Subject: [PATCH 07/19] Bytecode Interpreter (#1244) * Introducing Bytecode Interpreter * interpreter deactivated if not requested by command line * no supprt for OrderBy yet * Change Compile/Verify functions to void * throw an Exception in case of errors * Small changes from review on #1244 * Add explicit function calls, small changes * add template system to generate handler for explicit function calls * refactor InsertBytecodeFunction * add missingFP truct/extend instructions --- cmake/Dependencies.cmake | 5 + cmake/Modules/FindLibffi.cmake | 39 + script/installation/packages.sh | 3 + src/codegen/code_context.cpp | 99 +- src/codegen/codegen.cpp | 24 +- src/codegen/compilation_context.cpp | 10 +- src/codegen/interpreter/bytecode_builder.cpp | 1885 +++++++++++++++++ src/codegen/interpreter/bytecode_function.cpp | 302 +++ .../interpreter/bytecode_interpreter.cpp | 190 ++ src/codegen/query.cpp | 188 +- src/codegen/updateable_storage.cpp | 3 +- src/executor/plan_executor.cpp | 2 +- src/include/codegen/code_context.h | 40 +- src/include/codegen/codegen.h | 11 +- .../codegen/interpreter/bytecode_builder.h | 470 ++++ .../codegen/interpreter/bytecode_function.h | 359 ++++ .../interpreter/bytecode_instructions.def | 412 ++++ .../interpreter/bytecode_interpreter.h | 1260 +++++++++++ src/include/codegen/proxy/proxy.h | 2 +- src/include/codegen/query.h | 51 +- src/include/codegen/query_compiler.h | 4 +- src/include/common/macros.h | 25 + src/include/common/overflow_builtins.h | 78 + src/include/settings/settings.h | 4 + src/include/util/math_util.h | 35 + test/codegen/bloom_filter_test.cpp | 3 +- test/codegen/bytecode_interpreter_test.cpp | 242 +++ test/codegen/function_builder_test.cpp | 4 +- test/codegen/if_test.cpp | 8 +- test/codegen/testing_codegen_util.cpp | 20 +- test/codegen/value_integrity_test.cpp | 4 +- test/common/overflow_builtins_test.cpp | 212 ++ test/include/codegen/testing_codegen_util.h | 11 +- 33 files changed, 5889 insertions(+), 116 deletions(-) create mode 100644 cmake/Modules/FindLibffi.cmake create mode 100644 src/codegen/interpreter/bytecode_builder.cpp create mode 100644 src/codegen/interpreter/bytecode_function.cpp create mode 100644 src/codegen/interpreter/bytecode_interpreter.cpp create mode 100644 src/include/codegen/interpreter/bytecode_builder.h create mode 100644 src/include/codegen/interpreter/bytecode_function.h create mode 100644 src/include/codegen/interpreter/bytecode_instructions.def create mode 100644 src/include/codegen/interpreter/bytecode_interpreter.h create mode 100644 src/include/common/overflow_builtins.h create mode 100644 src/include/util/math_util.h create mode 100644 test/codegen/bytecode_interpreter_test.cpp create mode 100644 test/common/overflow_builtins_test.cpp diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index e9acc86bae6..e86fae410bd 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -86,6 +86,11 @@ llvm_map_components_to_libnames(LLVM_LIBRARIES core mcjit nativecodegen native) include_directories(SYSTEM ${LLVM_INCLUDE_DIRS}) list(APPEND Peloton_LINKER_LIBS ${LLVM_LIBRARIES}) +# --[ FFI +find_package(Libffi) +include_directories(SYSTEM ${LIBFFI_INCLUDE_DIRS}) +list(APPEND Peloton_LINKER_LIBS ${LIBFFI_LIBRARIES}) + # --[ IWYU # Generate clang compilation database diff --git a/cmake/Modules/FindLibffi.cmake b/cmake/Modules/FindLibffi.cmake new file mode 100644 index 00000000000..8e9883967bc --- /dev/null +++ b/cmake/Modules/FindLibffi.cmake @@ -0,0 +1,39 @@ +# - Try to find Libffi +# +# A Portable Foreign Function Interface Library (https://sourceware.org/libffi) +# +# Usage: +# LIBFFI_INCLUDE_DIRS, location of header files +# LIBFFI_LIBRARIES, location of library +# LIBFFI_FOUND, indicates if libffi was found + +# Look for the header file. +execute_process(COMMAND brew --prefix libffi OUTPUT_VARIABLE LIBFFI_BREW_PREFIX) + +find_library(LIBFFI_LIBRARY NAMES ffi libffi + PATHS /usr /usr/local /opt/local + PATH_SUFFIXES lib lib64 x86_64-linux-gnu lib/x86_64-linux-gnu + ) + +find_path(LIBFFI_INCLUDE_DIR ffi.h + PATHS /usr /usr/local /opt/local /usr/include/ffi + PATH_SUFFIXES include include/ffi include/x86_64-linux-gnu x86_64-linux-gnu + HINT LIBFFI_BREW_PREFIX + ) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(LIBFFI DEFAULT_MSG LIBFFI_LIBRARY LIBFFI_INCLUDE_DIR) + + +# Copy the results to the output variables. +IF(LIBFFI_FOUND) + SET(LIBFFI_LIBRARIES ${LIBFFI_LIBRARY}) + SET(LIBFFI_INCLUDE_DIRS ${LIBFFI_INCLUDE_DIR}) +ELSE(LIBFFI_FOUND) + SET(LIBFFI_LIBRARIES) + SET(LIBFFI_INCLUDE_DIRS) +ENDIF(LIBFFI_FOUND) + +MARK_AS_ADVANCED(LIBFFI_INCLUDE_DIRS LIBFFI_LIBRARIES) + +message(STATUS "Found Libffi (include: ${LIBFFI_INCLUDE_DIRS}, library: ${LIBFFI_LIBRARIES})") \ No newline at end of file diff --git a/script/installation/packages.sh b/script/installation/packages.sh index dec631fab98..1dff718b7d6 100755 --- a/script/installation/packages.sh +++ b/script/installation/packages.sh @@ -176,6 +176,8 @@ if [ "$DISTRO" = "UBUNTU" ]; then libedit-dev \ libssl-dev \ postgresql-client \ + libffi6 \ + libffi-dev \ libtbb-dev \ python3-pip \ curl \ @@ -219,6 +221,7 @@ elif [ "$DISTRO" = "DARWIN" ]; then brew install libedit brew install llvm@3.7 brew install postgresql + brew install libffi brew install tbb brew install curl brew install wget diff --git a/src/codegen/code_context.cpp b/src/codegen/code_context.cpp index cffdd6e22f2..14731e4fdc1 100644 --- a/src/codegen/code_context.cpp +++ b/src/codegen/code_context.cpp @@ -44,8 +44,9 @@ namespace { class PelotonMemoryManager : public llvm::SectionMemoryManager { public: explicit PelotonMemoryManager( - const std::unordered_map &symbols) - : symbols_(symbols) {} + const std::unordered_map> &builtins) + : builtins_(builtins) {} #if LLVM_VERSION_GE(4, 0) #define RET_TYPE llvm::JITSymbol @@ -56,8 +57,6 @@ class PelotonMemoryManager : public llvm::SectionMemoryManager { #define BUILD_RET_TYPE(addr) \ (RET_TYPE{(uint64_t)addr, llvm::JITSymbolFlags::Exported}) #endif - - /// Find the address of the function with the given name RET_TYPE findSymbol(const std::string &name) override { LOG_TRACE("Looking up symbol '%s' ...", name.c_str()); if (auto *builtin = LookupSymbol(name)) { @@ -68,23 +67,22 @@ class PelotonMemoryManager : public llvm::SectionMemoryManager { LOG_TRACE("--> Not builtin, use fallback resolution ..."); return llvm::SectionMemoryManager::findSymbol(name); } - #undef RET_TYPE #undef BUILD_RET_TYPE private: void *LookupSymbol(const std::string &name) const { // Check for a builtin with the exact name - auto symbol_iter = symbols_.find(name); - if (symbol_iter != symbols_.end()) { - return symbol_iter->second; + auto symbol_iter = builtins_.find(name); + if (symbol_iter != builtins_.end()) { + return symbol_iter->second.second; } // Check for a builtin with the leading '_' removed if (!name.empty() && name[0] == '_') { - symbol_iter = symbols_.find(name.substr(1)); - if (symbol_iter != symbols_.end()) { - return symbol_iter->second; + symbol_iter = builtins_.find(name.substr(1)); + if (symbol_iter != builtins_.end()) { + return symbol_iter->second.second; } } @@ -94,7 +92,9 @@ class PelotonMemoryManager : public llvm::SectionMemoryManager { private: // The code context - const std::unordered_map &symbols_; + const std::unordered_map> + &builtins_; }; //////////////////////////////////////////////////////////////////////////////// @@ -177,7 +177,8 @@ CodeContext::CodeContext() func_(nullptr), udf_func_ptr_(nullptr), pass_manager_(nullptr), - engine_(nullptr) { + engine_(nullptr), + is_verified_(false) { // Initialize JIT stuff llvm::InitializeNativeTarget(); llvm::InitializeNativeTargetAsmPrinter(); @@ -200,8 +201,7 @@ CodeContext::CodeContext() engine_.reset( llvm::EngineBuilder(std::move(m)) .setEngineKind(llvm::EngineKind::JIT) - .setMCJITMemoryManager( - llvm::make_unique(function_symbols_)) + .setMCJITMemoryManager(llvm::make_unique(builtins_)) .setMCPU(llvm::sys::getHostCPUName()) .setErrorStr(&err_str_) .create()); @@ -223,6 +223,7 @@ CodeContext::CodeContext() int32_type_ = llvm::Type::getInt32Ty(*context_); int64_type_ = llvm::Type::getInt64Ty(*context_); double_type_ = llvm::Type::getDoubleTy(*context_); + float_type_ = llvm::Type::getFloatTy(*context_); void_type_ = llvm::Type::getVoidTy(*context_); void_ptr_type_ = llvm::Type::getInt8PtrTy(*context_); char_ptr_type_ = llvm::Type::getInt8PtrTy(*context_); @@ -251,14 +252,13 @@ void CodeContext::RegisterExternalFunction(llvm::Function *func_decl, PELOTON_ASSERT(func_impl != nullptr && "The function pointer cannot be NULL"); functions_.emplace_back(func_decl, func_impl); - // Register the builtin symbol by name - function_symbols_[func_decl->getName()] = func_impl; + builtins_[func_decl->getName()] = std::make_pair(func_decl, func_impl); } void CodeContext::RegisterBuiltin(llvm::Function *func_decl, CodeContext::FuncPtr func_impl) { const auto name = func_decl->getName(); - if (LookupBuiltin(name) != nullptr) { + if (LookupBuiltin(name).first != nullptr) { LOG_DEBUG("Builtin '%s' already registered, skipping ...", name.data()); return; } @@ -268,36 +268,50 @@ void CodeContext::RegisterBuiltin(llvm::Function *func_decl, func_decl->isDeclaration() && "You cannot provide a function definition for a builtin function"); - // Register the builtin function - builtins_[name] = func_decl; - - // Register the builtin symbol by name - function_symbols_[name] = func_impl; + // Register the builtin function with type and implementation + builtins_[name] = std::make_pair(func_decl, func_impl); } -llvm::Function *CodeContext::LookupBuiltin(const std::string &name) const { +std::pair CodeContext::LookupBuiltin(const std::string &name) const { auto iter = builtins_.find(name); - return (iter == builtins_.end() ? nullptr : iter->second); + return (iter == builtins_.end() ? std::make_pair(nullptr, nullptr) : iter->second); } -/// Optimize and JIT compile all the functions that were created in this context -bool CodeContext::Compile() { +/// Verify all the functions that were created in this context +void CodeContext::Verify() { // Verify the module is okay llvm::raw_ostream &errors = llvm::errs(); if (llvm::verifyModule(*module_, &errors)) { - // There is an error in the module that failed compilation. + // There is an error in the module. // Dump the crappy IR to the log ... LOG_ERROR("ERROR IN MODULE:\n%s\n", GetIR().c_str()); - return false; + + throw Exception("The generated LLVM code contains errors. "); } + // All is well + is_verified_ = true; +} + +/// Optimize all the functions that were created in this context +void CodeContext::Optimize() { + // make sure the code is verified + if (!is_verified_) Verify(); + // Run the optimization passes over each function in this module pass_manager_->doInitialization(); for (auto &func_iter : functions_) { pass_manager_->run(*func_iter.first); } pass_manager_->doFinalization(); +} + +/// JIT compile all the functions that were created in this context +void CodeContext::Compile() { + // make sure the code is verified + if (!is_verified_) Verify(); + // Print some IR stats if (settings::SettingsManager::GetBool(settings::SettingId::print_ir_stats)) { char name[] = "inst count"; InstructionCounts inst_count(*name); @@ -305,7 +319,7 @@ bool CodeContext::Compile() { inst_count.DumpStats(); } - // Functions and module have been optimized, now JIT compile the module + // JIT compile the module engine_->finalizeObject(); // Pull out the compiled function implementations @@ -314,14 +328,34 @@ bool CodeContext::Compile() { } // Log the module + LOG_TRACE("%s\n", GetIR().c_str()); if (settings::SettingsManager::GetBool(settings::SettingId::dump_ir)) { LOG_DEBUG("%s\n", GetIR().c_str()); } +} - // All is well - return true; +size_t CodeContext::GetTypeSize(llvm::Type *type) const { + auto size = GetDataLayout().getTypeSizeInBits(type) / 8; + return size != 0 ? size : 1; +} + +size_t CodeContext::GetTypeSizeInBits(llvm::Type *type) const { + return GetDataLayout().getTypeSizeInBits(type); +} + +size_t CodeContext::GetTypeAllocSize(llvm::Type *type) const { + return GetDataLayout().getTypeAllocSize(type); +} + +size_t CodeContext::GetTypeAllocSizeInBits(llvm::Type *type) const { + return GetDataLayout().getTypeAllocSizeInBits(type); +} + +size_t CodeContext::GetStructElementOffset(llvm::StructType *type, size_t index) const { + return GetDataLayout().getStructLayout(type)->getElementOffset(index); } +// TODO(marcel) same as LookupBuiltin? CodeContext::FuncPtr CodeContext::GetRawFunctionPointer( llvm::Function *fn) const { for (const auto &iter : functions_) { @@ -334,6 +368,7 @@ CodeContext::FuncPtr CodeContext::GetRawFunctionPointer( return nullptr; } +/// Get the module's layout const llvm::DataLayout &CodeContext::GetDataLayout() const { return module_->getDataLayout(); } diff --git a/src/codegen/codegen.cpp b/src/codegen/codegen.cpp index b810fd4c092..9b93049cfe1 100644 --- a/src/codegen/codegen.cpp +++ b/src/codegen/codegen.cpp @@ -150,7 +150,7 @@ llvm::Value *CodeGen::CallFunc(llvm::Value *fn, llvm::Value *CodeGen::Printf(const std::string &format, const std::vector &args) { - auto *printf_fn = LookupBuiltin("printf"); + auto *printf_fn = LookupBuiltin("printf").first; if (printf_fn == nullptr) { #if GCC_AT_LEAST_6 // In newer GCC versions (i.e., GCC 6+), function attributes are part of the @@ -183,7 +183,7 @@ llvm::Value *CodeGen::Printf(const std::string &format, llvm::Value *CodeGen::Memcmp(llvm::Value *ptr1, llvm::Value *ptr2, llvm::Value *len) { static constexpr char kMemcmpFnName[] = "memcmp"; - auto *memcmp_fn = LookupBuiltin(kMemcmpFnName); + auto *memcmp_fn = LookupBuiltin(kMemcmpFnName).first; if (memcmp_fn == nullptr) { #if GCC_AT_LEAST_6 // In newer GCC versions (i.e., GCC 6+), function attributes are part of the @@ -311,7 +311,7 @@ llvm::Function *CodeGen::RegisterBuiltin(const std::string &fn_name, llvm::FunctionType *fn_type, void *func_impl) { // Check if this is already registered as a built in, quit if to - auto *builtin = LookupBuiltin(fn_name); + auto *builtin = LookupBuiltin(fn_name).first; if (builtin != nullptr) { return builtin; } @@ -332,6 +332,10 @@ llvm::Type *CodeGen::LookupType(const std::string &name) const { return GetModule().getTypeByName(name); } +std::pair CodeGen::LookupBuiltin(const std::string &name) const { + return code_context_.LookupBuiltin(name); +}; + llvm::Value *CodeGen::GetState() const { auto *func_builder = code_context_.GetCurrentFunction(); PELOTON_ASSERT(func_builder != nullptr); @@ -346,6 +350,20 @@ uint64_t CodeGen::SizeOf(llvm::Type *type) const { return size != 0 ? size : 1; } +std::string CodeGen::Dump(const llvm::Value *value) { + std::string string; + llvm::raw_string_ostream llvm_stream(string); + llvm_stream << *value; + return llvm_stream.str(); +} + +std::string CodeGen::Dump(llvm::Type *type) { + std::string string; + llvm::raw_string_ostream llvm_stream(string); + llvm_stream << *type; + return llvm_stream.str(); +} + uint64_t CodeGen::ElementOffset(llvm::Type *type, uint32_t element_idx) const { PELOTON_ASSERT(llvm::isa(type)); auto &data_layout = code_context_.GetDataLayout(); diff --git a/src/codegen/compilation_context.cpp b/src/codegen/compilation_context.cpp index d7f65dafcda..82cd32b38d9 100644 --- a/src/codegen/compilation_context.cpp +++ b/src/codegen/compilation_context.cpp @@ -97,17 +97,13 @@ void CompilationContext::GeneratePlan(Query &query, } // Next, we prepare the query statement with the functions we've generated - Query::QueryFunctions funcs = { - .init_func = init, .plan_func = plan, .tear_down_func = tear_down}; - bool prepared = query.Prepare(funcs); - if (!prepared) { - throw Exception{"There was an error preparing the compiled query"}; - } + Query::LLVMFunctions funcs = {init, plan, tear_down}; + query.Prepare(funcs); // We're done if (stats != nullptr) { timer.Stop(); - stats->jit_ms = timer.GetDuration(); + stats->optimize_ms = timer.GetDuration(); } } diff --git a/src/codegen/interpreter/bytecode_builder.cpp b/src/codegen/interpreter/bytecode_builder.cpp new file mode 100644 index 00000000000..57295da2567 --- /dev/null +++ b/src/codegen/interpreter/bytecode_builder.cpp @@ -0,0 +1,1885 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// bytecode_builder.cpp +// +// Identification: src/codegen/interpreter/bytecode_builder.cpp +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "codegen/interpreter/bytecode_builder.h" + +#include +#include + +#include "codegen/codegen.h" +#include "common/exception.h" +#include "util/math_util.h" + +namespace peloton { +namespace codegen { +namespace interpreter { + +BytecodeBuilder::BytecodeBuilder(const CodeContext &code_context, + const llvm::Function *function) + : bytecode_function_(function->getName().str()), + number_value_slots_(0), + number_temporary_value_slots_(0), + rpo_traversal_(function), + code_context_(code_context), + llvm_function_(function) {} + +BytecodeFunction BytecodeBuilder::CreateBytecodeFunction( + const CodeContext &code_context, const llvm::Function *function, + bool use_naive_register_allocator) { + BytecodeBuilder builder(code_context, function); + builder.AnalyseFunction(); + + if (use_naive_register_allocator) { + builder.PerformNaiveRegisterAllocation(); + } else { + builder.PerformGreedyRegisterAllocation(); + } + + builder.TranslateFunction(); + builder.Finalize(); + + return std::move(builder.bytecode_function_); +} + +Opcode BytecodeBuilder::GetOpcodeForTypeAllTypes(Opcode untyped_op, + llvm::Type *type) const { + index_t id = BytecodeFunction::GetOpcodeId(untyped_op); + + // This function highly depends on the macros in bytecode_instructions.def! + + if (type == code_context_.bool_type_ || type == code_context_.int8_type_) { + return BytecodeFunction::GetOpcodeFromId(id + 0); + } else if (type == code_context_.int16_type_) { + return BytecodeFunction::GetOpcodeFromId(id + 1); + } else if (type == code_context_.int32_type_) { + return BytecodeFunction::GetOpcodeFromId(id + 2); + } else if (type == code_context_.int64_type_ || + type == code_context_.char_ptr_type_ || type->isPointerTy()) { + return BytecodeFunction::GetOpcodeFromId(id + 3); + } else if (type == code_context_.float_type_) { + return BytecodeFunction::GetOpcodeFromId(id + 4); + } else if (type == code_context_.double_type_) { + return BytecodeFunction::GetOpcodeFromId(id + 5); + } else { + throw NotSupportedException("llvm type not supported: " + + CodeGen::Dump(type)); + } +} + +Opcode BytecodeBuilder::GetOpcodeForTypeIntTypes(Opcode untyped_op, + llvm::Type *type) const { + index_t id = BytecodeFunction::GetOpcodeId(untyped_op); + + // This function highly depends on the macros in bytecode_instructions.def! + + if (type == code_context_.bool_type_ || type == code_context_.int8_type_) { + return BytecodeFunction::GetOpcodeFromId(id + 0); + } else if (type == code_context_.int16_type_) { + return BytecodeFunction::GetOpcodeFromId(id + 1); + } else if (type == code_context_.int32_type_) { + return BytecodeFunction::GetOpcodeFromId(id + 2); + } else if (type == code_context_.int64_type_ || + type == code_context_.char_ptr_type_ || type->isPointerTy()) { + return BytecodeFunction::GetOpcodeFromId(id + 3); + } else { + throw NotSupportedException("llvm type not supported: " + + CodeGen::Dump(type)); + } +} + +Opcode BytecodeBuilder::GetOpcodeForTypeFloatTypes(Opcode untyped_op, + llvm::Type *type) const { + index_t id = BytecodeFunction::GetOpcodeId(untyped_op); + + // This function highly depends on the macros in bytecode_instructions.def! + + // float is missing! + if (type == code_context_.float_type_) { + return BytecodeFunction::GetOpcodeFromId(id + 0); + } else if (type == code_context_.double_type_) { + return BytecodeFunction::GetOpcodeFromId(id + 1); + } else { + throw NotSupportedException("llvm type not supported: " + + CodeGen::Dump(type)); + } +} + +Opcode BytecodeBuilder::GetOpcodeForTypeSizeIntTypes(Opcode untyped_op, + llvm::Type *type) const { + index_t id = BytecodeFunction::GetOpcodeId(untyped_op); + + // This function highly depends on the macros in bytecode_instructions.def! + + switch (code_context_.GetTypeSize(type)) { + case 1: + return BytecodeFunction::GetOpcodeFromId(id + 0); + + case 2: + return BytecodeFunction::GetOpcodeFromId(id + 1); + + case 4: + return BytecodeFunction::GetOpcodeFromId(id + 2); + + case 8: + return BytecodeFunction::GetOpcodeFromId(id + 3); + + default: + throw NotSupportedException("llvm type size not supported: " + + CodeGen::Dump(type)); + } +} + +Instruction &BytecodeBuilder::InsertBytecodeInstruction( + const llvm::Instruction *llvm_instruction, Opcode opcode, + const std::vector &args) { + PELOTON_ASSERT(opcode != Opcode::undefined); + + // calculate number of required instruction slots + // args.size() + 1 because of the Opcode + const size_t number_instruction_slots = MathUtil::DivRoundUp( + sizeof(uint16_t) * (1 + args.size()), sizeof(instr_slot_t)); + + bytecode_function_.bytecode_.insert(bytecode_function_.bytecode_.end(), + number_instruction_slots, 0); + Instruction &instruction = *reinterpret_cast( + &*(bytecode_function_.bytecode_.end() - number_instruction_slots)); + instruction.op = opcode; + for (size_t i = 0; i < args.size(); i++) instruction.args[i] = args[i]; + + AddInstructionToTrace(llvm_instruction, number_instruction_slots); + + return instruction; +} + +Instruction &BytecodeBuilder::InsertBytecodeInstruction( + const llvm::Instruction *llvm_instruction, Opcode opcode, + const std::vector &args) { + PELOTON_ASSERT(opcode != Opcode::undefined); + + std::vector args_transformed(args.size()); + std::transform( + args.begin(), args.end(), args_transformed.begin(), + [this](const llvm::Value *value) { return GetValueSlot(value); }); + + return InsertBytecodeInstruction(llvm_instruction, opcode, args_transformed); +} + +ExternalCallInstruction &BytecodeBuilder::InsertBytecodeExternalCallInstruction( + const llvm::Instruction *llvm_instruction, index_t call_context, + void *function) { + // calculate number of required instructionsslots and assert it is 2 + // (this way we recognise if any unintended size changes) + const size_t number_instruction_slots = MathUtil::DivRoundUp( + sizeof(ExternalCallInstruction), sizeof(instr_slot_t)); + PELOTON_ASSERT(number_instruction_slots == 2); + + bytecode_function_.bytecode_.insert(bytecode_function_.bytecode_.end(), + number_instruction_slots, 0); + + ExternalCallInstruction instruction = { + Opcode::call_external, call_context, + reinterpret_cast(function)}; + + instr_slot_t *instruction_slot = + &*(bytecode_function_.bytecode_.end() - number_instruction_slots); + ExternalCallInstruction *call_instruction_slot = + reinterpret_cast(instruction_slot); + *call_instruction_slot = instruction; + + AddInstructionToTrace(llvm_instruction, number_instruction_slots); + + return reinterpret_cast( + bytecode_function_.bytecode_[bytecode_function_.bytecode_.size() - + number_instruction_slots]); +} + +InternalCallInstruction &BytecodeBuilder::InsertBytecodeInternalCallInstruction( + const llvm::Instruction *llvm_instruction, index_t sub_function, + index_t dest_slot, size_t number_arguments) { + // calculate number of required instruction slots + // number_arguments + 4 because of the number of fixed arguments + // (see structure of InternalCallInstruction) + const size_t number_instruction_slots = MathUtil::DivRoundUp( + sizeof(uint16_t) * (4 + number_arguments), sizeof(instr_slot_t)); + + bytecode_function_.bytecode_.insert(bytecode_function_.bytecode_.end(), + number_instruction_slots, 0); + InternalCallInstruction &instruction = + *reinterpret_cast( + &*(bytecode_function_.bytecode_.end() - number_instruction_slots)); + instruction.op = Opcode::call_internal; + instruction.sub_function = sub_function; + instruction.dest_slot = dest_slot; + instruction.number_args = static_cast(number_arguments); + + PELOTON_ASSERT( + &instruction.args[number_arguments - 1] < + reinterpret_cast(&bytecode_function_.bytecode_.back() + 1)); + + AddInstructionToTrace(llvm_instruction, number_instruction_slots); + + return reinterpret_cast( + *(bytecode_function_.bytecode_.end() - number_instruction_slots)); +} + +#ifndef NDEBUG +void BytecodeBuilder::AddInstructionToTrace( + const llvm::Instruction *llvm_instruction, + size_t number_instruction_slots) { + bytecode_function_.instruction_trace_.insert( + bytecode_function_.instruction_trace_.end(), number_instruction_slots, + llvm_instruction); +} +#endif + +BytecodeBuilder::value_index_t BytecodeBuilder::GetValueIndex( + const llvm::Value *value) { + auto result = value_mapping_.find(value); + + // If the index already exists, just return it + if (result != value_mapping_.end()) { + return result->second; + } + + // Otherwise create a new index + + // Special case for constants + if (auto *llvm_constant = llvm::dyn_cast(value)) { + return GetConstantIndex(llvm_constant); + } + + value_index_t value_index = value_liveness_.size(); + value_mapping_[value] = value_index; + value_liveness_.emplace_back(std::numeric_limits::max(), + std::numeric_limits::max()); + return value_index; +} + +BytecodeBuilder::value_index_t BytecodeBuilder::CreateValueAlias( + const llvm::Value *alias, value_index_t value_index) { + PELOTON_ASSERT(value_mapping_.find(alias) == value_mapping_.end()); + value_mapping_[alias] = value_index; + + return value_index; +} + +value_t BytecodeBuilder::GetConstantValue( + const llvm::Constant *constant) const { + llvm::Type *type = constant->getType(); + + if (constant->isNullValue() || constant->isZeroValue() || llvm::isa(constant)) { + return 0; + } else { + switch (type->getTypeID()) { + case llvm::Type::IntegerTyID: { + int64_t value_signed = + llvm::cast(constant)->getSExtValue(); + return *reinterpret_cast(&value_signed); + } + + case llvm::Type::FloatTyID: { + float value_float = llvm::cast(constant) + ->getValueAPF() + .convertToFloat(); + return *reinterpret_cast(&value_float); + } + + case llvm::Type::DoubleTyID: { + double value_double = llvm::cast(constant) + ->getValueAPF() + .convertToDouble(); + + return *reinterpret_cast(&value_double); + } + + case llvm::Type::PointerTyID: { + if (constant->getNumOperands() > 0) { + if (auto *constant_int = + llvm::dyn_cast(constant->getOperand(0))) { + return reinterpret_cast(constant_int->getZExtValue()); + } + } + + PELOTON_FALLTHROUGH; + } + + default: + throw NotSupportedException("unsupported constant type: " + + CodeGen::Dump(constant->getType())); + } + } +} + +BytecodeBuilder::value_index_t BytecodeBuilder::GetConstantIndex( + const llvm::Constant *constant) { + auto value_mapping_result = value_mapping_.find(constant); + if (value_mapping_result != value_mapping_.end()) { + return value_mapping_result->second; + } + + value_t value = GetConstantValue(constant); + value_index_t value_index; + + // We merge all constants that share the same value (not the type!) + + // Check if entry with this value already exists + auto constant_result = std::find(bytecode_function_.constants_.begin(), + bytecode_function_.constants_.end(), value); + + if (constant_result == bytecode_function_.constants_.end()) { + // create new constant with that value + value_index = value_liveness_.size(); + value_mapping_[constant] = value_index; + value_liveness_.emplace_back(0, 0); // constant liveness starts at 0 + + bytecode_function_.constants_.push_back(value); + constant_value_indexes_.push_back(value_index); + + // constants liveness starts at program start + value_liveness_[value_index].first = 0; + } else { + // value already exists, create alias + auto constant_index = + constant_result - bytecode_function_.constants_.begin(); + value_index = constant_value_indexes_[constant_index]; + CreateValueAlias(constant, value_index); + } + + return value_index; +}; + +index_t BytecodeBuilder::GetValueSlot(const llvm::Value *value) const { + auto result = value_mapping_.find(value); + PELOTON_ASSERT(result != value_mapping_.end()); + + return value_slots_[result->second]; +} + +void BytecodeBuilder::ExtendValueLiveness( + const llvm::Value *llvm_value, instruction_index_t instruction_index) { + value_index_t value_index = GetValueIndex(llvm_value); + + // Special case if no liveness information is available yet + if (value_liveness_[value_index].first == + std::numeric_limits::max()) { + value_liveness_[value_index].first = instruction_index; + value_liveness_[value_index].second = instruction_index; + return; + } + + if (instruction_index < value_liveness_[value_index].first) { + value_liveness_[value_index].first = instruction_index; + } else if (instruction_index > value_liveness_[value_index].second) { + value_liveness_[value_index].second = instruction_index; + } +} + +index_t BytecodeBuilder::GetTemporaryValueSlot(const llvm::BasicBlock *bb) { + // we basically count the number of additional value slots that are + // requested per basic block + + // new entry in map is created automatically if necessary + number_temporary_values_[bb]++; + + number_temporary_value_slots_ = + std::max(number_temporary_value_slots_, + static_cast(number_temporary_values_[bb])); + return number_value_slots_ + number_temporary_values_[bb] - 1; +} + +ffi_type *BytecodeBuilder::GetFFIType(llvm::Type *type) const { + if (type->isVoidTy()) { + return &ffi_type_void; + } else if (type->isPointerTy()) { + return &ffi_type_pointer; + } else if (type == code_context_.double_type_) { + return &ffi_type_double; + } + + // exact type not necessary, only size is important + switch (code_context_.GetTypeSize(type)) { + case 1: + return &ffi_type_uint8; + + case 2: + return &ffi_type_uint16; + + case 4: + return &ffi_type_uint32; + + case 8: + return &ffi_type_uint64; + + default: + throw NotSupportedException( + std::string("can't find a ffi_type for type: ") + + CodeGen::Dump(type)); + } +} + +bool BytecodeBuilder::IsConstantValue(const llvm::Value *value) const { + auto *constant = llvm::dyn_cast(value); + return (constant != nullptr); +} + +int64_t BytecodeBuilder::GetConstantIntegerValueSigned( + llvm::Value *constant) const { + return llvm::cast(constant)->getSExtValue(); +} + +uint64_t BytecodeBuilder::GetConstantIntegerValueUnsigned( + llvm::Value *constant) const { + return llvm::cast(constant)->getZExtValue(); +} + +bool BytecodeBuilder::BasicBlockIsRPOSucc(const llvm::BasicBlock *bb, + const llvm::BasicBlock *succ) const { + // walk the vector where we saved the basic block pointers in R + // reverse post order (RPO) + for (size_t i = 0; i < bb_reverse_post_order_.size() - 1; i++) { + if (bb_reverse_post_order_[i] == bb && + bb_reverse_post_order_[i + 1] == succ) { + return true; + } + } + + return false; +} + +void BytecodeBuilder::AnalyseFunction() { + std::unordered_map> + bb_instruction_index_range; + + /* The analyse pass does: + * - determine the liveness of all values + * - merge values of instructions that translate to nop + * - merge constants and create list of constants + * - extract some additional information, e.g. for overflow aware operations + */ + + // Process function arguments + for (auto &argument : llvm_function_->args()) { + // DEF: function arguments are already defined at function start + ExtendValueLiveness(&argument, 0); + } + + instruction_index_t instruction_index = 0; + for (llvm::ReversePostOrderTraversal::rpo_iterator + traversal_iterator = rpo_traversal_.begin(); + traversal_iterator != rpo_traversal_.end(); ++traversal_iterator) { + const llvm::BasicBlock *bb = *traversal_iterator; + + // Add this basic block to the rpo vector for pred/succ lookups + bb_reverse_post_order_.push_back(bb); + + bb_instruction_index_range[bb].first = instruction_index; + + // Iterate all instructions to collect the liveness information + // There are exceptions for several instructions, + // which are labeled and explained below. + for (llvm::BasicBlock::const_iterator instr_iterator = bb->begin(); + instr_iterator != bb->end(); ++instr_iterator, ++instruction_index) { + const llvm::Instruction *instruction = instr_iterator; + + bool is_non_zero_gep = false; + if (instruction->getOpcode() == llvm::Instruction::GetElementPtr && + !llvm::cast(instruction) + ->hasAllZeroIndices()) { + is_non_zero_gep = true; + } + + // PHI-Handling: + // We do not process the PHI instructions directly, but at the end of a + // basic block, we process all PHI instructions of the successor blocks, + // that refer to the currect basic block. This is the position where we + // will insert the mov instructions when we resolve the PHIs later. + + // Skip PHI instructions + if (instruction->getOpcode() == llvm::Instruction::PHI) { + continue; + } + + // If next instruction is a terminator instruction, process + // PHIs of succeeding basic blocks first + if (llvm::isa(instruction)) { + bool found_back_edge = false; + + // For all successor basic blocks + for (auto succ_iterator = llvm::succ_begin(bb); + succ_iterator != llvm::succ_end(bb); ++succ_iterator) { + // Iterate phi instructions + for (llvm::BasicBlock::const_iterator instr_iterator = + succ_iterator->begin(); + auto *phi_instruction = + llvm::dyn_cast(&*instr_iterator); + ++instr_iterator) { + // extend lifetime of phi value itself + ExtendValueLiveness(phi_instruction, instruction_index); + + // extend lifetime of its operand + llvm::Value *phi_operand = + phi_instruction->getIncomingValueForBlock(bb); + // Similar to Exception 3, we extend the lifetime by one, to ensure + // the other phi operations do not overwrite the operand + ExtendValueLiveness(phi_operand, instruction_index + 1); + } // end iterate phi instructions + + // We also use iterating the basic block successors to find + // back edges. If we have seen a successor basic block before, it + // must be a back edge. + if (!found_back_edge) { + auto instruction_index_range = + bb_instruction_index_range.find(*succ_iterator); + if (instruction_index_range != bb_instruction_index_range.end()) { + index_t back_edge_instruction_index = + instruction_index_range->second.first; + + // For all values that are live at that time... + for (auto &liveness : value_liveness_) { + if (liveness.first < back_edge_instruction_index && + liveness.second >= back_edge_instruction_index) { + // ...extend lifetime of this value to survive back edge + // instruction_index + 1 is the index of the last + // instruction in this basic block + liveness.second = instruction_index + 1; + } + } + + found_back_edge = true; + } + } + } // end iterate successor basic blocks + + instruction_index++; + + // fall through (continue with terminator instruction) + } + + // Exception 1: Skip the ExtractValue instructions we already + // processed in Exception 6 + if (instruction->getOpcode() == llvm::Instruction::ExtractValue) { + auto *extractvalue_instruction = + llvm::cast(instruction); + + // Check if this extract refers to a overflow call instruction + auto result = overflow_results_mapping_.find( + llvm::cast(instruction->getOperand(0))); + if (result != overflow_results_mapping_.end() && + (result->second.first == extractvalue_instruction || + result->second.second == extractvalue_instruction)) { + continue; + } + + // fall through + } + + // USE: Iterate operands of instruction and extend their liveness + for (llvm::Instruction::const_op_iterator op_iterator = + instruction->op_begin(); + op_iterator != instruction->op_end(); ++op_iterator) { + llvm::Value *operand = op_iterator->get(); + + // constant operands + if (IsConstantValue(operand)) { + // Exception 2: the called function in a CallInst is also a constant + // but we want to skip this one + auto *call_instruction = llvm::dyn_cast(instruction); + if (call_instruction != nullptr && + call_instruction->getCalledFunction() == &*operand) { + continue; + } + + // Exception 3: constant operands from GEP and extractvalue are not + // needed, as they get encoded in the instruction itself + if (instruction->getOpcode() == llvm::Instruction::GetElementPtr || + instruction->getOpcode() == llvm::Instruction::ExtractValue) { + continue; + } + + // USE: extend liveness of constant value + ExtendValueLiveness(operand, instruction_index); + + // Exception 4: We extend the lifetime of GEP operands of GEPs + // that don't translate to nop, by one, to make sure that the operands + // don't get overridden when we split the GEP into several + // instructions. + } else if (is_non_zero_gep) { + ExtendValueLiveness(operand, instruction_index + 1); // extended! + + // A BasicBlock may be a label operand, but we don't need to track + // them + } else if (!llvm::isa(operand)) { + ExtendValueLiveness(operand, instruction_index); + } + } + + // Exception 5: For some instructions we know in advance that they will + // produce a nop, so we merge their value and their operand here + if (instruction->getOpcode() == llvm::Instruction::BitCast || + instruction->getOpcode() == llvm::Instruction::Trunc || + instruction->getOpcode() == llvm::Instruction::PtrToInt || + (instruction->getOpcode() == llvm::Instruction::GetElementPtr && + llvm::cast(instruction) + ->hasAllZeroIndices())) { + // merge operand resulting value + CreateValueAlias(instruction, + GetValueIndex(instruction->getOperand(0))); + continue; + } + + // Exception 6: Call instructions to any overflow aware operation + // have to be tracked, because we save their results directly in + // the destination slots of the ExtractValue instructions referring + // to them. + if (instruction->getOpcode() == llvm::Instruction::Call) { + // Check if the call instruction calls a overflow aware operation + // (unfortunately there is no better way to check this) + auto *call_instruction = llvm::cast(instruction); + llvm::Function *function = call_instruction->getCalledFunction(); + if (function->isDeclaration()) { + std::string function_name = function->getName().str(); + + if (function_name.size() >= 13 && + function_name.substr(10, 13) == "with.overflow") { + // create entry for this call + overflow_results_mapping_[call_instruction] = + std::make_pair(nullptr, nullptr); + + // Find the first ExtractValue instruction referring to this call + // instruction for result and overflow each and put it in the + // value_liveness vector here. The liveness of those + // instructions has to be extended to the definition of the call + // instruction, and this way we ensure that the vector is sorted + // by lifetime start index and we avoid sorting it later. + for (auto *user : call_instruction->users()) { + auto *extract_instruction = + llvm::cast(user); + size_t extract_index = *extract_instruction->idx_begin(); + + if (extract_index == 0) { + PELOTON_ASSERT( + overflow_results_mapping_[call_instruction].first == + nullptr); + overflow_results_mapping_[call_instruction].first = + extract_instruction; + + } else if (extract_index == 1) { + PELOTON_ASSERT( + overflow_results_mapping_[call_instruction].second == + nullptr); + overflow_results_mapping_[call_instruction].second = + extract_instruction; + } + + ExtendValueLiveness(extract_instruction, instruction_index); + } + + // Do not process the result of this instruction, + // as this value (the overflow result struct) doesn't exist + // later in the bytecode. + + continue; + } + } + } + + // DEF: save the instruction index as the liveness starting point + if (!instruction->getType()->isVoidTy()) { + ExtendValueLiveness(instruction, instruction_index); + } + } + + bb_instruction_index_range[bb].second = instruction_index - 1; + } +} + +void BytecodeBuilder::PerformNaiveRegisterAllocation() { + // assign a value slot to every liveness range in value_liveness_ + value_slots_.resize(value_liveness_.size(), 0); + index_t reg = 0; + + // process constants + for (auto &constant_value_index : constant_value_indexes_) { + value_slots_[constant_value_index] = reg++ + 1; + } + + // process function arguments + for (auto &argument : llvm_function_->args()) { + value_index_t argument_value_index = GetValueIndex(&argument); + value_slots_[argument_value_index] = reg++ + 1; + } + + // iterate over other entries, which are already sorted + for (value_index_t i = 0; i < value_liveness_.size(); ++i) { + // skip values that are never used (get assigned to dummy slot) + if (value_liveness_[i].first == value_liveness_[i].second) { + continue; + } + + // some values (constants, function arguments) are processed already + if (value_slots_[i] == 0) { + value_slots_[i] = reg++ + 1; // + 1 because 0 is dummy slot + } + } + + number_value_slots_ = reg + 1; +} + +void BytecodeBuilder::PerformGreedyRegisterAllocation() { + // assign a value slot to every liveness range in value_liveness_ + + value_slots_.resize(value_liveness_.size(), 0); + std::vector registers(constant_value_indexes_.size() + + llvm_function_->arg_size()); + index_t reg = 0; + + auto findEmptyRegister = [®isters](ValueLiveness liveness) { + for (index_t i = 0; i < registers.size(); ++i) { + if (registers[i].second <= liveness.first) { + registers[i] = liveness; + return i; + } + } + + // no empty register found, create new one + registers.push_back(liveness); + return static_cast(registers.size() - 1); + }; + + // process constants + for (auto &constant_value_index : constant_value_indexes_) { + registers[reg] = value_liveness_[constant_value_index]; + value_slots_[constant_value_index] = + reg++ + 1; // + 1 because 0 is dummy slot + } + + // process function arguments + for (auto &argument : llvm_function_->args()) { + value_index_t argument_value_index = GetValueIndex(&argument); + registers[reg] = value_liveness_[argument_value_index]; + value_slots_[argument_value_index] = + reg++ + 1; // + 1 because 0 is dummy slot + } + + PELOTON_ASSERT(registers.size() == reg); + +// The vector value_liveness_ is already sorted by lifetime start index +// except for the constant values, which are already processed + +#ifndef NDEBUG + // additional check in debug mode, to ensure that our assertion that the + // vector is already sorted by lifetime start index (except zero) is correct + instruction_index_t instruction_index = 1; + + for (value_index_t i = 0; i < value_liveness_.size(); ++i) { + if (value_liveness_[i].first != 0) { + PELOTON_ASSERT(value_liveness_[i].first >= instruction_index); + instruction_index = value_liveness_[i].first; + } + } +#endif + + // iterate over other entries, which are already sorted + for (value_index_t i = 0; i < value_liveness_.size(); ++i) { + // skip values that are never used + if (value_liveness_[i].first == value_liveness_[i].second) { + continue; + } + + if (value_slots_[i] == 0) { + value_slots_[i] = findEmptyRegister(value_liveness_[i]) + + 1; // + 1 because 0 is dummy slot + } + } + + number_value_slots_ = registers.size() + 1; // + 1 because 0 is dummy slot +} + +void BytecodeBuilder::TranslateFunction() { + // Map every basic block an index in the resulting bytecode stream. This + // is needed to perform the relocations in the branch instructions. + std::unordered_map bb_mapping; + + // Collect all bytecode relocations that have to be performed after + // translation, when the mapping information in bb_mapping is complete. + std::vector bytecode_relocations; + + // Iterate the basic blocks in reverse post order (RPO) + // Linear scan register allocation requires RPO traversal + // Initializing the RPO traversal is expensice, so we initialize it once + // for the BytecodeBuilder object and reuse it. + for (llvm::ReversePostOrderTraversal::rpo_iterator + traversal_iterator = rpo_traversal_.begin(); + traversal_iterator != rpo_traversal_.end(); ++traversal_iterator) { + const llvm::BasicBlock *bb = *traversal_iterator; + + // add basic block mapping + bb_mapping[bb] = bytecode_function_.bytecode_.size(); + + // Interate all instruction in the basic block + for (llvm::BasicBlock::const_iterator instr_iterator = bb->begin(); + instr_iterator != bb->end(); ++instr_iterator) { + const llvm::Instruction *instruction = instr_iterator; + + // Dispatch to the respective translator function + switch (instruction->getOpcode()) { + // Terminators + case llvm::Instruction::Br: + ProcessPHIsForBasicBlock(bb); + TranslateBranch(instruction, bytecode_relocations); + break; + + case llvm::Instruction::Ret: + ProcessPHIsForBasicBlock(bb); + TranslateReturn(instruction); + break; + + // Standard binary operators + // Logical operators + case llvm::Instruction::Add: + case llvm::Instruction::Sub: + case llvm::Instruction::Mul: + case llvm::Instruction::UDiv: + case llvm::Instruction::SDiv: + case llvm::Instruction::URem: + case llvm::Instruction::SRem: + case llvm::Instruction::Shl: + case llvm::Instruction::LShr: + case llvm::Instruction::And: + case llvm::Instruction::Or: + case llvm::Instruction::Xor: + case llvm::Instruction::AShr: + case llvm::Instruction::FAdd: + case llvm::Instruction::FSub: + case llvm::Instruction::FMul: + case llvm::Instruction::FDiv: + case llvm::Instruction::FRem: + TranslateBinaryOperator(instruction); + break; + + // Memory instructions + case llvm::Instruction::Load: + TranslateLoad(instruction); + break; + + case llvm::Instruction::Store: + TranslateStore(instruction); + break; + + case llvm::Instruction::Alloca: + TranslateAlloca(instruction); + break; + + case llvm::Instruction::GetElementPtr: + TranslateGetElementPtr(instruction); + break; + + // Cast instructions + case llvm::Instruction::BitCast: + // bit casts translate to nop + // values got already merged in analysis pass + break; + + case llvm::Instruction::SExt: + case llvm::Instruction::ZExt: + case llvm::Instruction::IntToPtr: + TranslateIntExt(instruction); + break; + + case llvm::Instruction::Trunc: + case llvm::Instruction::PtrToInt: + // trunc translates to nop + // values got already merged in analysis pass + break; + case llvm::Instruction::FPExt: + case llvm::Instruction::FPTrunc: + TranslateFloatTruncExt(instruction); + + case llvm::Instruction::UIToFP: + case llvm::Instruction::SIToFP: + case llvm::Instruction::FPToUI: + case llvm::Instruction::FPToSI: + TranslateFloatIntCast(instruction); + break; + + // Other instructions + case llvm::Instruction::ICmp: + case llvm::Instruction::FCmp: + TranslateCmp(instruction); + break; + + case llvm::Instruction::PHI: + // PHIs are handled before every terminating instruction + break; + + case llvm::Instruction::Call: + TranslateCall(instruction); + break; + + case llvm::Instruction::Select: + TranslateSelect(instruction); + break; + + case llvm::Instruction::ExtractValue: + TranslateExtractValue(instruction); + break; + + case llvm::Instruction::Unreachable: + // nop + break; + + // Instruction is not supported + default: { throw NotSupportedException("instruction not supported"); } + } + } + } + + // apply the relocations required by the placed branch instructions + for (auto &relocation : bytecode_relocations) { + reinterpret_cast( + &bytecode_function_.bytecode_[relocation.instruction_slot]) + ->args[relocation.argument] = bb_mapping[relocation.bb]; + } +} + +void BytecodeBuilder::Finalize() { + // calculate final number of value slots during runtime + bytecode_function_.number_values_ = + number_value_slots_ + number_temporary_value_slots_; + + // check if number values exceeds bit range (unrealistic) + if (bytecode_function_.number_values_ >= + std::numeric_limits::max()) { + throw NotSupportedException("number of values exceeds max number of bits"); + } + + // prepare arguments + bytecode_function_.number_function_arguments_ = llvm_function_->arg_size(); +} + +void BytecodeBuilder::ProcessPHIsForBasicBlock(const llvm::BasicBlock *bb) { + struct AdditionalMove { + const llvm::Instruction *instruction; + index_t dest; + index_t src; + }; + + // Takes track of additional moves (du to PHI swap problem) that have to be + // applied after all PHI nodes have been processed. + std::vector additional_moves; + + for (auto succ_iterator = llvm::succ_begin(bb); + succ_iterator != llvm::succ_end(bb); ++succ_iterator) { + // If the basic block is its own successor, we take risk to run into the PHI + // swap problem (lost copy problem). To avoid this, we move the values in + // temporary registers and move them to their destination after processing + // all other PHI nodes. + if (*succ_iterator == bb) { + for (auto instruction_iterator = succ_iterator->begin(); + auto *phi_node = + llvm::dyn_cast(&*instruction_iterator); + ++instruction_iterator) { + index_t temp_slot = GetTemporaryValueSlot(bb); + + InsertBytecodeInstruction( + phi_node, Opcode::phi_mov, + {temp_slot, GetValueSlot(phi_node->getIncomingValueForBlock(bb))}); + additional_moves.push_back( + {phi_node, GetValueSlot(phi_node), temp_slot}); + } + + // Common case: create mov instruction to destination slot + } else { + for (auto instruction_iterator = succ_iterator->begin(); + auto *phi_node = + llvm::dyn_cast(&*instruction_iterator); + ++instruction_iterator) { + if (GetValueSlot(phi_node) == + GetValueSlot(phi_node->getIncomingValueForBlock(bb))) { + continue; + } + + InsertBytecodeInstruction( + phi_node, Opcode::phi_mov, + {phi_node, phi_node->getIncomingValueForBlock(bb)}); + } + } + } + + // Place additional moves if needed + for (auto &entry : additional_moves) { + InsertBytecodeInstruction(entry.instruction, Opcode::phi_mov, + {entry.dest, entry.src}); + } +} + +void BytecodeBuilder::TranslateBranch( + const llvm::Instruction *instruction, + std::vector &bytecode_relocations) { + auto *branch_instruction = llvm::cast(&*instruction); + + // conditional branch + if (branch_instruction->isConditional()) { + // The first operand in the IR is the false branch, while the second one + // is the true one (printed llvm assembly is the other way round). + // To be consistent, we use the order of the memory representation + // in our bytecode. + + // If false branch is next basic block, we can use a fall through branch + if (BasicBlockIsRPOSucc( + branch_instruction->getParent(), + llvm::cast(branch_instruction->getOperand(1)))) { + InsertBytecodeInstruction( + instruction, Opcode::branch_cond_ft, + std::vector{GetValueSlot(branch_instruction->getOperand(0)), + 0}); + + BytecodeRelocation relocation_false{ + static_cast(bytecode_function_.bytecode_.size() - 1), 1, + llvm::cast(branch_instruction->getOperand(2))}; + + // add relocation entry, to insert missing information of destination + // later + bytecode_relocations.push_back(relocation_false); + + // no fall through + } else { + InsertBytecodeInstruction( + instruction, Opcode::branch_cond, + {GetValueSlot(branch_instruction->getOperand(0)), 0, 0}); + + BytecodeRelocation relocation_false{ + static_cast(bytecode_function_.bytecode_.size() - 1), 1, + llvm::cast(branch_instruction->getOperand(1))}; + + // add relocation entry, to insert missing information of destination + // later + bytecode_relocations.push_back(relocation_false); + + BytecodeRelocation relocation_true{ + static_cast(bytecode_function_.bytecode_.size() - 1), 2, + llvm::cast(branch_instruction->getOperand(2))}; + + // add relocation entry, to insert missing information of destination + // later + bytecode_relocations.push_back(relocation_true); + } + + // unconditional branch + } else { + // If the unconditional branch points to the next basic block, + // we can omit the branch instruction + if (!BasicBlockIsRPOSucc( + branch_instruction->getParent(), + llvm::cast(branch_instruction->getOperand(0)))) { + InsertBytecodeInstruction(instruction, Opcode::branch_uncond, + std::vector{0}); + + BytecodeRelocation relocation{ + static_cast(bytecode_function_.bytecode_.size() - 1), 0, + llvm::cast(branch_instruction->getOperand(0))}; + + // add relocation entry, to insert missing information of destination + // later + bytecode_relocations.push_back(relocation); + } + } +} + +void BytecodeBuilder::TranslateReturn(const llvm::Instruction *instruction) { + auto *return_instruction = llvm::cast(&*instruction); + + // We only have one ret bytecode instruction. If the function returns void, + // the instruction will return the value of the dummy value slot zero, + // but no one will every pick up that value. + + index_t return_slot = 0; + if (return_instruction->getNumOperands() > 0) { + return_slot = GetValueSlot(return_instruction->getOperand(0)); + } + + InsertBytecodeInstruction(instruction, Opcode::ret, + std::vector{return_slot}); +} + +void BytecodeBuilder::TranslateBinaryOperator( + const llvm::Instruction *instruction) { + auto *binary_operator = llvm::cast(&*instruction); + auto *type = binary_operator->getType(); + Opcode opcode; + + switch (binary_operator->getOpcode()) { + case llvm::Instruction::Add: + case llvm::Instruction::FAdd: + opcode = GetOpcodeForTypeAllTypes(GET_FIRST_ALL_TYPES(Opcode::add), type); + break; + + case llvm::Instruction::Sub: + case llvm::Instruction::FSub: + opcode = GetOpcodeForTypeAllTypes(GET_FIRST_ALL_TYPES(Opcode::sub), type); + break; + + case llvm::Instruction::Mul: + case llvm::Instruction::FMul: + opcode = GetOpcodeForTypeAllTypes(GET_FIRST_ALL_TYPES(Opcode::mul), type); + break; + + case llvm::Instruction::UDiv: + case llvm::Instruction::FDiv: + opcode = GetOpcodeForTypeAllTypes(GET_FIRST_ALL_TYPES(Opcode::div), type); + break; + + case llvm::Instruction::SDiv: + opcode = + GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode::sdiv), type); + break; + + case llvm::Instruction::URem: + opcode = + GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode::urem), type); + break; + + case llvm::Instruction::FRem: + opcode = + GetOpcodeForTypeFloatTypes(GET_FIRST_FLOAT_TYPES(Opcode::frem), type); + break; + + case llvm::Instruction::SRem: + opcode = + GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode::srem), type); + break; + + case llvm::Instruction::Shl: + opcode = GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode::shl), type); + break; + + case llvm::Instruction::LShr: + opcode = + GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode::lshr), type); + break; + + case llvm::Instruction::AShr: + opcode = + GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode::ashr), type); + break; + + case llvm::Instruction::And: + opcode = GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode::and), type); + break; + + case llvm::Instruction::Or: + opcode = GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode:: or), type); + break; + + case llvm::Instruction::Xor: + opcode = + GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode:: xor), type); + break; + + default: + throw NotSupportedException("binary operation not supported"); + } + + InsertBytecodeInstruction(instruction, opcode, + {binary_operator, binary_operator->getOperand(0), + binary_operator->getOperand(1)}); +} + +void BytecodeBuilder::TranslateAlloca(const llvm::Instruction *instruction) { + auto *alloca_instruction = llvm::cast(&*instruction); + Opcode opcode; + + // get type to allocate + llvm::Type *type = alloca_instruction->getAllocatedType(); + + // get type size in bytes + size_t type_size = code_context_.GetTypeSize(type); + + if (alloca_instruction->isArrayAllocation()) { + index_t array_size = GetValueSlot(alloca_instruction->getArraySize()); + opcode = + GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode::alloca_array), + alloca_instruction->getArraySize()->getType()); + + // type size is immediate value! + InsertBytecodeInstruction(instruction, opcode, + {GetValueSlot(alloca_instruction), + static_cast(type_size), array_size}); + } else { + opcode = Opcode::alloca; + // type size is immediate value! + InsertBytecodeInstruction( + instruction, opcode, + {GetValueSlot(alloca_instruction), static_cast(type_size)}); + } +} + +void BytecodeBuilder::TranslateLoad(const llvm::Instruction *instruction) { + auto *load_instruction = llvm::cast(&*instruction); + + Opcode opcode = GetOpcodeForTypeSizeIntTypes( + GET_FIRST_INT_TYPES(Opcode::load), load_instruction->getType()); + InsertBytecodeInstruction( + instruction, opcode, + {load_instruction, load_instruction->getPointerOperand()}); +} + +void BytecodeBuilder::TranslateStore(const llvm::Instruction *instruction) { + auto *store_instruction = llvm::cast(&*instruction); + + Opcode opcode = + GetOpcodeForTypeSizeIntTypes(GET_FIRST_INT_TYPES(Opcode::store), + store_instruction->getOperand(0)->getType()); + InsertBytecodeInstruction( + instruction, opcode, + std::vector{store_instruction->getPointerOperand(), + store_instruction->getValueOperand()}); +} + +void BytecodeBuilder::TranslateGetElementPtr( + const llvm::Instruction *instruction) { + auto *gep_instruction = llvm::cast(&*instruction); + int64_t overall_offset = 0; + + // If the GEP translates to a nop, the values have been already merged + // during the analysis pass + if (gep_instruction->hasAllZeroIndices()) { + return; + } + + // The offset is an immediate constant, not a slot index + // instruction is created here, but offset will be filled in later, + // because we may merge it with constant array accesses + auto &gep_offset_bytecode_instruction_ref = InsertBytecodeInstruction( + gep_instruction, Opcode::gep_offset, + {GetValueSlot(gep_instruction), + GetValueSlot(gep_instruction->getPointerOperand()), 0}); + size_t gep_offset_bytecode_instruction_index = + bytecode_function_.GetIndexFromIP(&gep_offset_bytecode_instruction_ref); + + // First index operand of the instruction is the array index for the + // source type + + // Get type of struct/array which will be processed + llvm::Type *type = gep_instruction->getSourceElementType(); + + if (IsConstantValue(gep_instruction->getOperand(1))) { + overall_offset += + code_context_.GetTypeSize(type) * + GetConstantIntegerValueSigned(gep_instruction->getOperand(1)); + } else { + index_t index = GetValueSlot(instruction->getOperand(1)); + Opcode opcode = + GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode::gep_array), + instruction->getOperand(1)->getType()); + + // size of array element is an immediate constant, not a slot index! + InsertBytecodeInstruction( + gep_instruction, opcode, + {GetValueSlot(gep_instruction), index, + static_cast(code_context_.GetTypeSize(type))}); + } + + // Iterate remaining Indexes + for (unsigned int operand_index = 2; + operand_index < instruction->getNumOperands(); ++operand_index) { + auto *operand = instruction->getOperand(operand_index); + + if (auto *array_type = llvm::dyn_cast(type)) { + if (IsConstantValue(operand)) { + overall_offset += + code_context_.GetTypeSize(array_type->getElementType()) * + GetConstantIntegerValueSigned(operand); + } else { + index_t index = GetValueSlot(operand); + Opcode opcode = GetOpcodeForTypeIntTypes( + GET_FIRST_INT_TYPES(Opcode::gep_array), operand->getType()); + + // size of array element is an immediate constant, not a slot index! + InsertBytecodeInstruction( + gep_instruction, opcode, + {GetValueSlot(gep_instruction), index, + static_cast( + code_context_.GetTypeSize(array_type->getElementType()))}); + } + + // get inner type for next iteration + type = array_type->getElementType(); + + } else if (auto *struct_type = llvm::dyn_cast(type)) { + uint64_t index = GetConstantIntegerValueUnsigned(operand); + PELOTON_ASSERT(index < struct_type->getNumElements()); + + // get element offset + overall_offset += code_context_.GetStructElementOffset(struct_type, index); + + // get inner type for next iteration + type = struct_type->getElementType(index); + + } else { + throw NotSupportedException( + "unexpected type in getelementptr instruction"); + } + } + + // make sure that resulting type is correct + PELOTON_ASSERT(type == gep_instruction->getResultElementType()); + + // fill in calculated overall offset in previously placed gep_offset + // bytecode instruction + // (use index instead of reference, as vector may has been relocated!) + reinterpret_cast( + &bytecode_function_.bytecode_[gep_offset_bytecode_instruction_index]) + ->args[2] = static_cast(overall_offset); +} + +void BytecodeBuilder::TranslateFloatIntCast( + const llvm::Instruction *instruction) { + auto *cast_instruction = llvm::dyn_cast(&*instruction); + + // These instruction basically exist from every integer type to every + // floating point type and the other way round. + // We can only expand instructions in one dimension, so we expand the + // integer dimension and create the floating point instances manually + // (float and double) + + Opcode opcode = Opcode::undefined; + + if (instruction->getOpcode() == llvm::Instruction::FPToSI) { + if (cast_instruction->getOperand(0)->getType() == + code_context_.float_type_) { + opcode = GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode::floattosi), + cast_instruction->getType()); + } else if (cast_instruction->getOperand(0)->getType() == + code_context_.double_type_) { + opcode = GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode::doubletosi), + cast_instruction->getType()); + } else { + throw NotSupportedException("unsupported cast instruction"); + } + + } else if (instruction->getOpcode() == llvm::Instruction::FPToUI) { + if (cast_instruction->getOperand(0)->getType() == + code_context_.float_type_) { + opcode = GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode::floattoui), + cast_instruction->getType()); + } else if (cast_instruction->getOperand(0)->getType() == + code_context_.double_type_) { + opcode = GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode::doubletoui), + cast_instruction->getType()); + } else { + throw NotSupportedException("unsupported cast instruction"); + } + + } else if (instruction->getOpcode() == llvm::Instruction::SIToFP) { + if (cast_instruction->getType() == code_context_.float_type_) { + opcode = + GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode::sitofloat), + cast_instruction->getOperand(0)->getType()); + } else if (cast_instruction->getType() == code_context_.double_type_) { + opcode = + GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode::sitodouble), + cast_instruction->getOperand(0)->getType()); + } else { + throw NotSupportedException("unsupported cast instruction"); + } + + } else if (instruction->getOpcode() == llvm::Instruction::UIToFP) { + if (cast_instruction->getType() == code_context_.float_type_) { + opcode = + GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode::uitofloat), + cast_instruction->getOperand(0)->getType()); + } else if (cast_instruction->getType() == code_context_.double_type_) { + opcode = + GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode::uitodouble), + cast_instruction->getOperand(0)->getType()); + } else { + throw NotSupportedException("unsupported cast instruction"); + } + + } else { + throw NotSupportedException("unsupported cast instruction"); + } + + InsertBytecodeInstruction( + cast_instruction, opcode, + {cast_instruction, cast_instruction->getOperand(0)}); +} + +void BytecodeBuilder::TranslateIntExt(const llvm::Instruction *instruction) { + auto *cast_instruction = llvm::dyn_cast(&*instruction); + + size_t src_type_size = + code_context_.GetTypeSize(cast_instruction->getSrcTy()); + size_t dest_type_size = + code_context_.GetTypeSize(cast_instruction->getDestTy()); + + if (src_type_size == dest_type_size) { + if (GetValueSlot(instruction) != GetValueSlot(instruction->getOperand(0))) + InsertBytecodeInstruction(instruction, Opcode::nop_mov, + {instruction, instruction->getOperand(0)}); + return; + } + + Opcode opcode = Opcode::undefined; + + if (instruction->getOpcode() == llvm::Instruction::SExt) { + if (src_type_size == 1 && dest_type_size == 2) { + opcode = Opcode::sext_i8_i16; + + } else if (src_type_size == 1 && dest_type_size == 4) { + opcode = Opcode::sext_i8_i32; + + } else if (src_type_size == 1 && dest_type_size == 8) { + opcode = Opcode::sext_i8_i64; + + } else if (src_type_size == 2 && dest_type_size == 4) { + opcode = Opcode::sext_i16_i32; + + } else if (src_type_size == 2 && dest_type_size == 8) { + opcode = Opcode::sext_i16_i64; + + } else if (src_type_size == 4 && dest_type_size == 8) { + opcode = Opcode::sext_i32_i64; + + } else { + throw NotSupportedException("unsupported sext instruction"); + } + + } else if (instruction->getOpcode() == llvm::Instruction::ZExt || + instruction->getOpcode() == llvm::Instruction::IntToPtr) { + if (src_type_size == 1 && dest_type_size == 2) { + opcode = Opcode::zext_i8_i16; + + } else if (src_type_size == 1 && dest_type_size == 4) { + opcode = Opcode::zext_i8_i32; + + } else if (src_type_size == 1 && dest_type_size == 8) { + opcode = Opcode::zext_i8_i64; + + } else if (src_type_size == 2 && dest_type_size == 4) { + opcode = Opcode::zext_i16_i32; + + } else if (src_type_size == 2 && dest_type_size == 8) { + opcode = Opcode::zext_i16_i64; + + } else if (src_type_size == 4 && dest_type_size == 8) { + opcode = Opcode::zext_i32_i64; + + } else { + throw NotSupportedException("unsupported zext instruction"); + } + + } else { + throw NotSupportedException("unexpected ext instruction"); + } + + InsertBytecodeInstruction( + cast_instruction, opcode, + {cast_instruction, cast_instruction->getOperand(0)}); +} + +void BytecodeBuilder::TranslateFloatTruncExt( + const llvm::Instruction *instruction) { + auto *cast_instruction = llvm::dyn_cast(&*instruction); + + auto src_type = cast_instruction->getSrcTy(); + auto dest_type = cast_instruction->getDestTy(); + + if (src_type == dest_type) { + if (GetValueSlot(instruction) != GetValueSlot(instruction->getOperand(0))) { + InsertBytecodeInstruction(instruction, Opcode::nop_mov, + {instruction, instruction->getOperand(0)}); + } + return; + } + + if (src_type == code_context_.double_type_ && + dest_type == code_context_.float_type_) { + InsertBytecodeInstruction( + cast_instruction, Opcode::doubletofloat, + {cast_instruction, cast_instruction->getOperand(0)}); + } else if (src_type == code_context_.float_type_ && + dest_type == code_context_.double_type_) { + InsertBytecodeInstruction( + cast_instruction, Opcode::floattodouble, + {cast_instruction, cast_instruction->getOperand(0)}); + } else { + throw NotSupportedException("unsupported FPTrunc/PFExt instruction"); + } +} + +void BytecodeBuilder::TranslateCmp(const llvm::Instruction *instruction) { + auto *cmp_instruction = llvm::cast(&*instruction); + auto *type = cmp_instruction->getOperand(0)->getType(); + Opcode opcode = Opcode::undefined; + + switch (cmp_instruction->getPredicate()) { + case llvm::CmpInst::Predicate::ICMP_EQ: + case llvm::CmpInst::Predicate::FCMP_OEQ: + case llvm::CmpInst::Predicate::FCMP_UEQ: + opcode = + GetOpcodeForTypeAllTypes(GET_FIRST_ALL_TYPES(Opcode::cmp_eq), type); + break; + + case llvm::CmpInst::Predicate::ICMP_NE: + case llvm::CmpInst::Predicate::FCMP_ONE: + case llvm::CmpInst::Predicate::FCMP_UNE: + opcode = + GetOpcodeForTypeAllTypes(GET_FIRST_ALL_TYPES(Opcode::cmp_ne), type); + break; + + case llvm::CmpInst::Predicate::ICMP_UGT: + case llvm::CmpInst::Predicate::FCMP_OGT: + case llvm::CmpInst::Predicate::FCMP_UGT: + opcode = + GetOpcodeForTypeAllTypes(GET_FIRST_ALL_TYPES(Opcode::cmp_gt), type); + break; + + case llvm::CmpInst::Predicate::ICMP_UGE: + case llvm::CmpInst::Predicate::FCMP_OGE: + case llvm::CmpInst::Predicate::FCMP_UGE: + opcode = + GetOpcodeForTypeAllTypes(GET_FIRST_ALL_TYPES(Opcode::cmp_ge), type); + break; + + case llvm::CmpInst::Predicate::ICMP_ULT: + case llvm::CmpInst::Predicate::FCMP_OLT: + case llvm::CmpInst::Predicate::FCMP_ULT: + opcode = + GetOpcodeForTypeAllTypes(GET_FIRST_ALL_TYPES(Opcode::cmp_lt), type); + break; + + case llvm::CmpInst::Predicate::ICMP_ULE: + case llvm::CmpInst::Predicate::FCMP_OLE: + case llvm::CmpInst::Predicate::FCMP_ULE: + opcode = + GetOpcodeForTypeAllTypes(GET_FIRST_ALL_TYPES(Opcode::cmp_le), type); + break; + + case llvm::CmpInst::Predicate::ICMP_SGT: + opcode = + GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode::cmp_sgt), type); + break; + + case llvm::CmpInst::Predicate::ICMP_SGE: + opcode = + GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode::cmp_sge), type); + break; + + case llvm::CmpInst::Predicate::ICMP_SLT: + opcode = + GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode::cmp_slt), type); + break; + + case llvm::CmpInst::Predicate::ICMP_SLE: + opcode = + GetOpcodeForTypeIntTypes(GET_FIRST_INT_TYPES(Opcode::cmp_sle), type); + break; + + default: + throw NotSupportedException("compare operand not supported"); + } + + InsertBytecodeInstruction(cmp_instruction, opcode, + {cmp_instruction, cmp_instruction->getOperand(0), + cmp_instruction->getOperand(1)}); +} + +void BytecodeBuilder::TranslateCall(const llvm::Instruction *instruction) { + auto *call_instruction = llvm::cast(&*instruction); + + llvm::Function *function = call_instruction->getCalledFunction(); + + if (function->isDeclaration()) { + // The only way to find out about the called function (even if its an + // intrinsic) is to check the function name string + std::string function_name = function->getName().str(); + + if (function_name.find("llvm.memcpy") == 0) { + if (call_instruction->getOperand(2)->getType() != + code_context_.int64_type_) { + throw NotSupportedException( + "memcpy with different size type than i64 not supported"); + } + + InsertBytecodeInstruction( + call_instruction, Opcode::llvm_memcpy, + {call_instruction->getOperand(0), call_instruction->getOperand(1), + call_instruction->getOperand(2)}); + + } else if (function_name.find("llvm.memmove") == 0) { + if (call_instruction->getOperand(2)->getType() != + code_context_.int64_type_) + throw NotSupportedException( + "memmove with different size type than i64 not supported"); + + InsertBytecodeInstruction( + call_instruction, Opcode::llvm_memmove, + {call_instruction->getOperand(0), call_instruction->getOperand(1), + call_instruction->getOperand(2)}); + + } else if (function_name.find("llvm.memset") == 0) { + if (call_instruction->getOperand(2)->getType() != + code_context_.int64_type_) + throw NotSupportedException( + "memset with different size type than i64 not supported"); + + InsertBytecodeInstruction( + call_instruction, Opcode::llvm_memset, + {call_instruction->getOperand(0), call_instruction->getOperand(1), + call_instruction->getOperand(2)}); + + } else if (function_name.find("with.overflow") == 10) { + index_t result = 0; + index_t overflow = 0; + auto *type = call_instruction->getOperand(0)->getType(); + Opcode opcode = Opcode::undefined; + + // The destination slots have been already prepared from the analysis pass + PELOTON_ASSERT(overflow_results_mapping_.find(call_instruction) != + overflow_results_mapping_.end()); + + if (overflow_results_mapping_[call_instruction].first != nullptr) { + result = + GetValueSlot(overflow_results_mapping_[call_instruction].first); + } + + if (overflow_results_mapping_[call_instruction].second != nullptr) { + overflow = + GetValueSlot(overflow_results_mapping_[call_instruction].second); + } + + if (function_name.substr(5, 4) == "uadd") { + opcode = GetOpcodeForTypeIntTypes( + GET_FIRST_INT_TYPES(Opcode::llvm_uadd_overflow), type); + } else if (function_name.substr(5, 4) == "sadd") { + opcode = GetOpcodeForTypeIntTypes( + GET_FIRST_INT_TYPES(Opcode::llvm_sadd_overflow), type); + } else if (function_name.substr(5, 4) == "usub") { + opcode = GetOpcodeForTypeIntTypes( + GET_FIRST_INT_TYPES(Opcode::llvm_usub_overflow), type); + } else if (function_name.substr(5, 4) == "ssub") { + opcode = GetOpcodeForTypeIntTypes( + GET_FIRST_INT_TYPES(Opcode::llvm_ssub_overflow), type); + } else if (function_name.substr(5, 4) == "umul") { + opcode = GetOpcodeForTypeIntTypes( + GET_FIRST_INT_TYPES(Opcode::llvm_umul_overflow), type); + } else if (function_name.substr(5, 4) == "smul") { + opcode = GetOpcodeForTypeIntTypes( + GET_FIRST_INT_TYPES(Opcode::llvm_smul_overflow), type); + } else { + throw NotSupportedException( + "the requested operation with overflow is not supported"); + } + + InsertBytecodeInstruction( + call_instruction, opcode, + {result, overflow, GetValueSlot(call_instruction->getOperand(0)), + GetValueSlot(call_instruction->getOperand(1))}); + + } else if (function_name.find("llvm.x86.sse42.crc32") == 0) { + if (call_instruction->getType() != code_context_.int64_type_) { + throw NotSupportedException( + "sse42.crc32 with different size type than i64 not supported"); + } + + InsertBytecodeInstruction( + call_instruction, Opcode::llvm_sse42_crc32, + {call_instruction, call_instruction->getOperand(0), + call_instruction->getOperand(1)}); + + } else { + Opcode opcode = + BytecodeFunction::GetExplicitCallOpcodeByString(function_name); + + // call explicit instantiation of this function if available + if (opcode != Opcode::undefined) { + std::vector args; + args.reserve(call_instruction->getNumArgOperands()); + + if (!instruction->getType()->isVoidTy()) { + args.push_back(call_instruction); + } + + for (unsigned int i = 0; i < call_instruction->getNumArgOperands(); + i++) { + args.push_back(call_instruction->getArgOperand(i)); + } + + InsertBytecodeInstruction(call_instruction, opcode, args); + + } else { + // Function is not available in IR context, so we have to make an + // external function call + + // lookup function pointer in code context + void *raw_pointer = code_context_.LookupBuiltin(function_name).second; + + if (raw_pointer == nullptr) { + throw NotSupportedException("could not find external function: " + + function_name); + } + + // libffi is used for external function calls + // Here we collect all the information that will be needed at runtime + // (function activation time) to create the libffi call interface. + + // Show a hint, that an explicit wrapper could be created for this + // function + LOG_DEBUG("The interpreter will call the C++ function '%s' per libffi. " + "Consider adding an explicit wrapper for this function in " + "bytecode_instructions.def\n", function_name.c_str()); + + index_t dest_slot = 0; + if (!instruction->getType()->isVoidTy()) { + dest_slot = GetValueSlot(call_instruction); + } + + size_t arguments_num = call_instruction->getNumArgOperands(); + ExternalCallContext call_context{ + dest_slot, GetFFIType(instruction->getType()), + std::vector(arguments_num), + std::vector(arguments_num)}; + + for (unsigned int i = 0; i < call_instruction->getNumArgOperands(); + i++) { + call_context.args[i] = + GetValueSlot(call_instruction->getArgOperand(i)); + call_context.arg_types[i] = + GetFFIType(call_instruction->getArgOperand(i)->getType()); + } + + // add call context to bytecode function + bytecode_function_.external_call_contexts_.push_back(call_context); + + // insert bytecode instruction referring to this call context + InsertBytecodeExternalCallInstruction( + call_instruction, + static_cast( + bytecode_function_.external_call_contexts_.size() - 1), + raw_pointer); + } + } + } else { + // Internal function call to another IR function in this code context + + index_t dest_slot = 0; + if (!instruction->getType()->isVoidTy()) { + dest_slot = GetValueSlot(call_instruction); + } + + // Translate the bytecode function we want to call + index_t sub_function_index; + const auto result = sub_function_mapping_.find(function); + if (result != sub_function_mapping_.end()) { + sub_function_index = result->second; + } else { + auto sub_function = + BytecodeBuilder::CreateBytecodeFunction(code_context_, function); + + bytecode_function_.sub_functions_.push_back(std::move(sub_function)); + sub_function_index = bytecode_function_.sub_functions_.size() - 1; + sub_function_mapping_[function] = sub_function_index; + } + + InternalCallInstruction &bytecode_instruction = + InsertBytecodeInternalCallInstruction( + call_instruction, sub_function_index, dest_slot, + call_instruction->getNumArgOperands()); + + for (unsigned int i = 0; i < call_instruction->getNumArgOperands(); i++) { + bytecode_instruction.args[i] = + GetValueSlot(call_instruction->getArgOperand(i)); + + // just to make sure, we check that no function argument is bigger + // than 8 Bytes + if (code_context_.GetTypeSize( + call_instruction->getArgOperand(i)->getType()) > 8) { + throw NotSupportedException("argument for internal call too big"); + } + } + } +} + +void BytecodeBuilder::TranslateSelect(const llvm::Instruction *instruction) { + auto *select_instruction = llvm::cast(&*instruction); + + InsertBytecodeInstruction( + select_instruction, Opcode::select, + {select_instruction, select_instruction->getCondition(), + select_instruction->getTrueValue(), + select_instruction->getFalseValue()}); +} + +void BytecodeBuilder::TranslateExtractValue( + const llvm::Instruction *instruction) { + auto *extract_instruction = llvm::cast(&*instruction); + + // Skip, if this ExtractValue instruction belongs to an overflow operation + auto call_result = overflow_results_mapping_.find( + llvm::cast(instruction->getOperand(0))); + if (call_result != overflow_results_mapping_.end()) { + return; + } + + // Get value type + llvm::Type *type = extract_instruction->getAggregateOperand()->getType(); + size_t offset_bits = 0; + + // make sure the result type fits in a value_t + if (code_context_.GetTypeSize(instruction->getType()) <= sizeof(value_t)) { + throw NotSupportedException("extracted value too big for register size"); + } + + // Iterate indexes + for (auto index_it = extract_instruction->idx_begin(), + index_end = extract_instruction->idx_end(); + index_it != index_end; index_it++) { + uint32_t index = *index_it; + + if (auto *array_type = llvm::dyn_cast(type)) { + // Advance offset + offset_bits += + code_context_.GetTypeAllocSizeInBits(array_type->getElementType()) * + index; + + // get inner type for next iteration + type = array_type->getElementType(); + } else if (auto *struct_type = llvm::dyn_cast(type)) { + PELOTON_ASSERT(index < struct_type->getNumElements()); + + // get element offset + offset_bits += code_context_.GetStructElementOffset(struct_type, index) * 8; + + // get inner type for next iteration + type = struct_type->getElementType(index); + } else { + throw NotSupportedException( + "unexpected type in extractvalue instruction"); + } + } + + // assure that resulting type is correct + PELOTON_ASSERT(type == extract_instruction->getType()); + + // number if bits to shift is an immediate value! + InsertBytecodeInstruction( + extract_instruction, Opcode::extractvalue, + {GetValueSlot(extract_instruction), + GetValueSlot(extract_instruction->getAggregateOperand()), + static_cast(offset_bits)}); +} + +} // namespace interpreter +} // namespace codegen +} // namespace peloton diff --git a/src/codegen/interpreter/bytecode_function.cpp b/src/codegen/interpreter/bytecode_function.cpp new file mode 100644 index 00000000000..2e29fe47e55 --- /dev/null +++ b/src/codegen/interpreter/bytecode_function.cpp @@ -0,0 +1,302 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// bytecode_function.cpp +// +// Identification: src/codegen/interpreter/bytecode_function.cpp +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "codegen/interpreter/bytecode_function.h" + +#include +#include +#include +#include + +#include "codegen/codegen.h" + +// Includes for explicit function calls +#include "codegen/bloom_filter_accessor.h" +#include "codegen/util/bloom_filter.h" +#include "codegen/buffering_consumer.h" +#include "codegen/deleter.h" +#include "codegen/inserter.h" +#include "codegen/query_parameters.h" +#include "codegen/runtime_functions.h" +#include "codegen/transaction_runtime.h" +#include "codegen/updater.h" +#include "codegen/util/oa_hash_table.h" +#include "codegen/util/hash_table.h" +#include "codegen/util/sorter.h" +#include "codegen/values_runtime.h" +#include "executor/executor_context.h" +#include "function/date_functions.h" +#include "function/numeric_functions.h" +#include "function/string_functions.h" +#include "function/timestamp_functions.h" +#include "planner/project_info.h" +#include "storage/data_table.h" +#include "storage/storage_manager.h" +#include "storage/tile_group.h" +#include "storage/zone_map_manager.h" +#include "codegen/util/buffer.h" + +namespace peloton { +namespace codegen { +namespace interpreter { + +/** + * This lambda function serves as an init function to fill the const mapping + * of function names to opcodes. + */ +const std::unordered_map + BytecodeFunction::explicit_call_opcode_mapping_ = []() { + std::unordered_map mapping; + +#define HANDLE_INST(op) +#define HANDLE_EXPLICIT_CALL_INST(op, func) mapping[#func] = Opcode::op; + +#include "codegen/interpreter/bytecode_instructions.def" + + return mapping; + }(); + +const char *BytecodeFunction::GetOpcodeString(Opcode opcode) { + switch (opcode) { +#define HANDLE_INST(opcode) \ + case Opcode::opcode: \ + return #opcode; + +#include "codegen/interpreter/bytecode_instructions.def" + + default: + return "(invalid)"; + } +} + +#ifndef NDEBUG +const llvm::Instruction *BytecodeFunction::GetIRInstructionFromIP( + index_t instr_slot) const { + return instruction_trace_.at(instr_slot); +} +#endif + +size_t BytecodeFunction::GetInstructionSlotSize( + const Instruction *instruction) { + switch (instruction->op) { +#define HANDLE_INST(op) \ + case Opcode::op: \ + return 1; +#define HANDLE_EXTERNAL_CALL_INST(op) \ + case Opcode::op: \ + return 2; +#define HANDLE_INTERNAL_CALL_INST(op) \ + case Opcode::op: \ + return GetInteralCallInstructionSlotSize( \ + reinterpret_cast(instruction)); +#define HANDLE_SELECT_INST(op) \ + case Opcode::op: \ + return 2; +#define HANDLE_OVERFLOW_TYPED_INST(op, type) \ + case Opcode::op##_##type: \ + return 2; +#define HANDLE_EXPLICIT_CALL_INST(op, func) \ + case Opcode::op: \ + return GetExplicitCallInstructionSlotSize( \ + GetFunctionRequiredArgSlotsNum(&func)); + +#include "codegen/interpreter/bytecode_instructions.def" + + default: + PELOTON_ASSERT(false); + return 0; + } +} + +Opcode BytecodeFunction::GetExplicitCallOpcodeByString( + std::string function_name) { + auto result = explicit_call_opcode_mapping_.find(function_name); + + if (result != explicit_call_opcode_mapping_.end()) + return result->second; + else + return Opcode::undefined; +} + +void BytecodeFunction::DumpContents() const { + std::ofstream output; + output.open(function_name_ + ".bf"); + +#ifndef NDEBUG + const llvm::BasicBlock *bb; +#endif + + // Print Bytecode + output << "Bytecode:" << std::endl; + for (index_t i = 0; i < bytecode_.size();) { + auto *instruction = GetIPFromIndex(i); + +#ifndef NDEBUG + const llvm::Instruction *llvm_instruction = GetIRInstructionFromIP(i); + if (llvm_instruction->getOpcode() != llvm::Instruction::PHI) { + if (i > 0 && bb != llvm_instruction->getParent()) { + output << llvm_instruction->getParent()->getName().str() << ":" + << std::endl; + } + bb = llvm_instruction->getParent(); + } +#endif + + output << Dump(instruction) << std::endl; + i += GetInstructionSlotSize(instruction); + } + + // Print Constants + if (constants_.size() > 0) output << "Constants:" << std::endl; + for (size_t i = 0; i < constants_.size(); i++) { + output << "[" << std::setw(3) << std::dec << (i + 1) + << "] = " << *reinterpret_cast(&constants_[i]) + << " 0x" << std::hex << constants_[i] << std::endl; + } + + output << std::endl; + + output.close(); +} + +std::string BytecodeFunction::Dump(const Instruction *instruction) const { + std::ostringstream output; + output << "[" << std::setw(3) << GetIndexFromIP(instruction) << "] "; + output << std::setw(18) << GetOpcodeString(instruction->op) << " "; + + switch (instruction->op) { +#define HANDLE_INST(opcode) \ + case Opcode::opcode: \ + output << "[" << std::setw(3) << instruction->args[0] << "] "; \ + output << "[" << std::setw(3) << instruction->args[1] << "] "; \ + output << "[" << std::setw(3) << instruction->args[2] << "] "; \ + break; + +#ifndef NDEBUG +#define HANDLE_EXTERNAL_CALL_INST(opcode) \ + case Opcode::opcode: \ + output \ + << "[" << std::setw(3) \ + << external_call_contexts_ \ + [reinterpret_cast(instruction) \ + ->external_call_context] \ + .dest_slot \ + << "] "; \ + for (auto arg : external_call_contexts_[instruction->args[0]].args) { \ + output << "[" << std::setw(3) << arg << "] "; \ + } \ + output << "(" \ + << static_cast( \ + instruction_trace_[GetIndexFromIP(instruction)]) \ + ->getCalledFunction() \ + ->getName() \ + .str() \ + << ") "; \ + break; +#else +#define HANDLE_CALL_INST(opcode) \ + case Opcode::opcode: \ + output << "[" << std::setw(3) \ + << call_contexts_[reinterpret_cast( \ + instruction) \ + ->call_context] \ + .dest_slot \ + << "] "; \ + for (auto arg : call_contexts_[instruction->args[0]].args) { \ + output << "[" << std::setw(3) << arg << "] "; \ + } \ + break; +#endif + +#ifndef NDEBUG +#define HANDLE_INTERNAL_CALL_INST(opcode) \ + case Opcode::opcode: \ + output << "[" << std::setw(3) \ + << reinterpret_cast(instruction) \ + ->dest_slot \ + << "] "; \ + for (size_t i = 0; \ + i < reinterpret_cast(instruction) \ + ->number_args; \ + i++) { \ + output << "[" << std::setw(3) \ + << reinterpret_cast(instruction) \ + ->args[i] \ + << "] "; \ + } \ + output << "(" \ + << static_cast( \ + instruction_trace_[GetIndexFromIP(instruction)]) \ + ->getCalledFunction() \ + ->getName() \ + .str() \ + << ") "; \ + break; +#else +#define HANDLE_INTERNAL_CALL_INST(opcode) \ + case Opcode::opcode: \ + output << "[" << std::setw(3) \ + << reinterpret_cast(instruction) \ + ->dest_slot \ + << "] "; \ + for (size_t i = 0; \ + i < reinterpret_cast(instruction) \ + ->number_args; \ + i++) { \ + output << "[" << std::setw(3) \ + << reinterpret_cast(instruction) \ + ->args[i] \ + << "] "; \ + } \ + break; +#endif + +#define HANDLE_SELECT_INST(opcode) \ + case Opcode::opcode: \ + output << "[" << std::setw(3) << instruction->args[0] << "] "; \ + output << "[" << std::setw(3) << instruction->args[1] << "] "; \ + output << "[" << std::setw(3) << instruction->args[2] << "] "; \ + output << "[" << std::setw(3) << instruction->args[3] << "] "; \ + break; + +#define HANDLE_OVERFLOW_TYPED_INST(op, type) \ + case Opcode::op##_##type: \ + output << "[" << std::setw(3) << instruction->args[0] << "] "; \ + output << "[" << std::setw(3) << instruction->args[1] << "] "; \ + output << "[" << std::setw(3) << instruction->args[2] << "] "; \ + output << "[" << std::setw(3) << instruction->args[3] << "] "; \ + break; + +#define HANDLE_EXPLICIT_CALL_INST(opcode, func) \ + case Opcode::opcode: \ + for (size_t i = 0; i < GetFunctionRequiredArgSlotsNum(&func); i++) \ + output << "[" << std::setw(3) << instruction->args[i] << "] "; \ + break; + +#include "codegen/interpreter/bytecode_instructions.def" + + default: + break; + } + +#ifndef NDEBUG + output << "(" + << CodeGen::Dump(GetIRInstructionFromIP(GetIndexFromIP(instruction))) + << ")"; +#endif + + return output.str(); +} + +} // namespace interpreter +} // namespace codegen +} // namespace peloton diff --git a/src/codegen/interpreter/bytecode_interpreter.cpp b/src/codegen/interpreter/bytecode_interpreter.cpp new file mode 100644 index 00000000000..ccd51f495d8 --- /dev/null +++ b/src/codegen/interpreter/bytecode_interpreter.cpp @@ -0,0 +1,190 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// bytecode_interpreter.cpp +// +// Identification: src/codegen/interpreter/bytecode_interpreter.cpp +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "codegen/interpreter/bytecode_interpreter.h" +#include "codegen/interpreter/bytecode_function.h" + +namespace peloton { +namespace codegen { +namespace interpreter { + +/** This is the actual dispatch code: It lookups the destination handler address + * in the label_pointers_ array and performs a direct jump there. + */ +#define INTERPRETER_DISPATCH_GOTO(ip) \ + goto *(label_pointers_[BytecodeFunction::GetOpcodeId( \ + reinterpret_cast(ip)->op)]) + +/** + * The array with the label pointers has to be zero initialized to make sure, + * that we fill it with the actual values on the first execution. + */ +void * + BytecodeInterpreter::label_pointers_[BytecodeFunction::GetNumberOpcodes()] = + {nullptr}; + +BytecodeInterpreter::BytecodeInterpreter( + const BytecodeFunction &bytecode_function) + : bytecode_function_(bytecode_function) {} + +value_t BytecodeInterpreter::ExecuteFunction( + const BytecodeFunction &bytecode_function, + const std::vector &arguments) { + BytecodeInterpreter interpreter(bytecode_function); + interpreter.ExecuteFunction(arguments); + + return interpreter.GetReturnValue(); +} + +void BytecodeInterpreter::ExecuteFunction( + const BytecodeFunction &bytecode_function, char *param) { + BytecodeInterpreter interpreter(bytecode_function); + interpreter.ExecuteFunction({reinterpret_cast(param)}); +} + +NEVER_INLINE NO_CLONE void BytecodeInterpreter::ExecuteFunction( + const std::vector &arguments) { + // Fill the label_pointers_ array with the handler addresses at first + // startup. (This can't be done outside of this function, as the labels are + // not visible there. + if (label_pointers_[0] == nullptr) { +#define HANDLE_INST(op) \ + label_pointers_[BytecodeFunction::GetOpcodeId(Opcode::op)] = &&_##op; + +#include "codegen/interpreter/bytecode_instructions.def" + } + + InitializeActivationRecord(arguments); + + // Get initial instruction pointer + const Instruction *bytecode = + reinterpret_cast(&bytecode_function_.bytecode_[0]); + const Instruction *ip = bytecode; + + // Start execution with first instruction + INTERPRETER_DISPATCH_GOTO(ip); + +//--------------------------------------------------------------------------// +// Dispatch area +// +// This is the actual dispatch area of the interpreter. Because we use +// threaded interpretation, this is not a dispatch loop, but a long list of +// labels, and the control flow jumps from one handler to the next with +// goto's -> INTERPRETER_DISPATCH_GOTO(ip) +// +// The whole dispatch area gets generated using the bytecode_instructions.def +// file. All instruction handlers from query_interpreter.h will get inlined +// here for all their types. Even though the function looks small here, +// it will be over 13kB in the resulting binary! +//--------------------------------------------------------------------------// + +#ifdef LOG_TRACE_ENABLED +#define TRACE_CODE_PRE LOG_TRACE("%s", bytecode_function_.Dump(ip).c_str()) +#else +#define TRACE_CODE_PRE +#endif + +#define HANDLE_RET_INST(op) \ + _ret: \ + TRACE_CODE_PRE; \ + GetValueReference(0) = GetValue(ip->args[0]); \ + return; + +#define HANDLE_TYPED_INST(op, type) \ + _##op##_##type : TRACE_CODE_PRE; \ + ip = op##Handler(ip); \ + INTERPRETER_DISPATCH_GOTO(ip); + +#define HANDLE_INST(op) \ + _##op : TRACE_CODE_PRE; \ + ip = op##Handler(ip); \ + INTERPRETER_DISPATCH_GOTO(ip); + +#define HANDLE_EXPLICIT_CALL_INST(op, func) \ + _##op : TRACE_CODE_PRE; \ + ip = explicit_callHandler(ip, &func); \ + INTERPRETER_DISPATCH_GOTO(ip); + +#include "codegen/interpreter/bytecode_instructions.def" + + //--------------------------------------------------------------------------// +} + +template +type_t BytecodeInterpreter::GetReturnValue() { + // the ret instruction saves the return value in value slot 0 by definition + return GetValue(0); +} + +void BytecodeInterpreter::InitializeActivationRecord( + const std::vector &arguments) { + // resize vector to required number of value slots + values_.resize(bytecode_function_.number_values_); + + index_t value_slot = 1; + + // fill in constants + for (auto &constant : bytecode_function_.constants_) { + SetValue(value_slot++, constant); + } + + // check if provided number of arguments matches the number required by + // the function + if (bytecode_function_.number_function_arguments_ != arguments.size()) { + throw Exception( + "llvm function called through interpreter with wrong number of " + "arguments"); + } + + // fill in function arguments + for (auto &argument : arguments) { + SetValue(value_slot++, argument); + } + + // prepare call activations + call_activations_.resize(bytecode_function_.external_call_contexts_.size()); + for (size_t i = 0; i < bytecode_function_.external_call_contexts_.size(); + i++) { + auto &call_context = bytecode_function_.external_call_contexts_[i]; + auto &call_activation = call_activations_[i]; + + // initialize libffi call interface + if (ffi_prep_cif(&call_activation.call_interface, FFI_DEFAULT_ABI, + call_context.args.size(), call_context.dest_type, + const_cast(call_context.arg_types.data())) != + FFI_OK) { + throw Exception("initializing ffi call interface failed "); + } + + // save the pointers to the value slots in the continuous arrays + for (const auto &arg : call_context.args) { + call_activation.value_pointers.push_back(&values_[arg]); + } + call_activation.return_pointer = &values_[call_context.dest_slot]; + } +} + +uintptr_t BytecodeInterpreter::AllocateMemory(size_t number_bytes) { + // allocate memory + std::unique_ptr pointer = + std::unique_ptr(new char[number_bytes]); + + // get raw pointer before moving pointer object! + auto raw_pointer = reinterpret_cast(pointer.get()); + + allocations_.emplace_back(std::move(pointer)); + return raw_pointer; +} + +} // namespace interpreter +} // namespace codegen +} // namespace peloton \ No newline at end of file diff --git a/src/codegen/query.cpp b/src/codegen/query.cpp index 87ed5ab572d..c69601b3b7f 100644 --- a/src/codegen/query.cpp +++ b/src/codegen/query.cpp @@ -11,11 +11,15 @@ //===----------------------------------------------------------------------===// #include "codegen/query.h" -#include "codegen/execution_consumer.h" +#include "codegen/interpreter/bytecode_builder.h" +#include "codegen/interpreter/bytecode_interpreter.h" +#include "codegen/query_compiler.h" #include "common/timer.h" #include "executor/plan_executor.h" +#include "codegen/execution_consumer.h" #include "executor/executor_context.h" #include "storage/storage_manager.h" +#include "settings/settings_manager.h" namespace peloton { namespace codegen { @@ -31,36 +35,103 @@ void Query::Execute(executor::ExecutorContext &executor_context, llvm::Type *query_state_type = query_state_.GetType(); size_t parameter_size = codegen.SizeOf(query_state_type); PELOTON_ASSERT((parameter_size % 8 == 0) && - "parameter size not multiple of 8"); + "parameter size not multiple of 8"); // Allocate some space for the function arguments std::unique_ptr param_data{new char[parameter_size]}; char *param = param_data.get(); PELOTON_MEMSET(param, 0, parameter_size); - // We use this handy class to avoid complex casting and pointer manipulation - struct FunctionArguments { - executor::ExecutorContext *executor_context; - char *consumer_arg; - char rest[0]; - } PACKED; - // Set up the function arguments auto *func_args = reinterpret_cast(param_data.get()); func_args->executor_context = &executor_context; func_args->consumer_arg = consumer.GetConsumerState(); + bool force_interpreter = settings::SettingsManager::GetBool( + settings::SettingId::codegen_interpreter); + + if (is_compiled_ && !force_interpreter) { + ExecuteNative(func_args, stats); + } else { + try { + ExecuteInterpreter(func_args, stats); + } catch (interpreter::NotSupportedException e) { + LOG_ERROR("query not supported by interpreter: %s", e.what()); + } + } +} + +void Query::Prepare(const LLVMFunctions &query_funcs) { + llvm_functions_ = query_funcs; + + // verify the functions + // will also be done by Optimize() or Compile() if not done before, + // but we do not want to mix up the timings, so do it here + code_context_.Verify(); + + // optimize the functions + // TODO(marcel): add switch to enable/disable optimization + // TODO(marcel): add timer to measure time used for optimization (see + // RuntimeStats) + code_context_.Optimize(); + + is_compiled_ = false; +} + +void Query::Compile(CompileStats *stats) { // Timer Timer timer; - timer.Start(); + if (stats != nullptr) { + timer.Start(); + } + + // Compile all functions in context + LOG_TRACE("Starting Query compilation ..."); + code_context_.Compile(); + + // Get pointers to the JITed functions + compiled_functions_.init_func = + (compiled_function_t)code_context_.GetRawFunctionPointer( + llvm_functions_.init_func); + PELOTON_ASSERT(compiled_functions_.init_func != nullptr); + + compiled_functions_.plan_func = + (compiled_function_t)code_context_.GetRawFunctionPointer( + llvm_functions_.plan_func); + PELOTON_ASSERT(compiled_functions_.plan_func != nullptr); + + compiled_functions_.tear_down_func = + (compiled_function_t)code_context_.GetRawFunctionPointer( + llvm_functions_.tear_down_func); + PELOTON_ASSERT(compiled_functions_.tear_down_func != nullptr); + + is_compiled_ = true; + + LOG_TRACE("Compilation finished."); + + // Timer for JIT compilation + if (stats != nullptr) { + timer.Stop(); + stats->compile_ms = timer.GetDuration(); + timer.Reset(); + } +} + +void Query::ExecuteNative(FunctionArguments *function_arguments, + RuntimeStats *stats) { + // Start timer + Timer timer; + if (stats != nullptr) { + timer.Start(); + } // Call init LOG_TRACE("Calling query's init() ..."); try { - init_func_(param); + compiled_functions_.init_func(function_arguments); } catch (...) { // Cleanup if an exception is encountered - tear_down_func_(param); + compiled_functions_.tear_down_func(function_arguments); throw; } @@ -75,10 +146,10 @@ void Query::Execute(executor::ExecutorContext &executor_context, // Execute the query! LOG_TRACE("Calling query's plan() ..."); try { - plan_func_(param); + compiled_functions_.plan_func(function_arguments); } catch (...) { // Cleanup if an exception is encountered - tear_down_func_(param); + compiled_functions_.tear_down_func(function_arguments); throw; } @@ -92,7 +163,7 @@ void Query::Execute(executor::ExecutorContext &executor_context, // Clean up LOG_TRACE("Calling query's tearDown() ..."); - tear_down_func_(param); + compiled_functions_.tear_down_func(function_arguments); // No need to cleanup if we get an exception while cleaning up... if (stats != nullptr) { @@ -101,33 +172,82 @@ void Query::Execute(executor::ExecutorContext &executor_context, } } -bool Query::Prepare(const QueryFunctions &query_funcs) { - LOG_TRACE("Going to JIT the query ..."); +void Query::ExecuteInterpreter(FunctionArguments *function_arguments, + RuntimeStats *stats) { + LOG_INFO("Using codegen interpreter to execute plan"); - // Compile the code - if (!code_context_.Compile()) { - return false; + // Timer + Timer timer; + if (stats != nullptr) { + timer.Start(); } - LOG_TRACE("Setting up Query ..."); + // Create Bytecode + interpreter::BytecodeFunction init_bytecode = + interpreter::BytecodeBuilder::CreateBytecodeFunction( + code_context_, llvm_functions_.init_func); + interpreter::BytecodeFunction plan_bytecode = + interpreter::BytecodeBuilder::CreateBytecodeFunction( + code_context_, llvm_functions_.plan_func); + interpreter::BytecodeFunction tear_down_bytecode = + interpreter::BytecodeBuilder::CreateBytecodeFunction( + code_context_, llvm_functions_.tear_down_func); - // Get pointers to the JITed functions - init_func_ = (compiled_function_t)code_context_.GetRawFunctionPointer( - query_funcs.init_func); - PELOTON_ASSERT(init_func_ != nullptr); + // Time initialization + if (stats != nullptr) { + timer.Stop(); + stats->interpreter_prepare_ms = timer.GetDuration(); + timer.Reset(); + timer.Start(); + } - plan_func_ = (compiled_function_t)code_context_.GetRawFunctionPointer( - query_funcs.plan_func); - PELOTON_ASSERT(plan_func_ != nullptr); + // Call init + LOG_TRACE("Calling query's init() ..."); + try { + interpreter::BytecodeInterpreter::ExecuteFunction( + init_bytecode, reinterpret_cast(function_arguments)); + } catch (...) { + interpreter::BytecodeInterpreter::ExecuteFunction( + tear_down_bytecode, reinterpret_cast(function_arguments)); + throw; + } + + if (stats != nullptr) { + timer.Stop(); + stats->init_ms = timer.GetDuration(); + timer.Reset(); + timer.Start(); + } + + // Execute the query! + LOG_TRACE("Calling query's plan() ..."); + try { + interpreter::BytecodeInterpreter::ExecuteFunction( + plan_bytecode, reinterpret_cast(function_arguments)); + } catch (...) { + interpreter::BytecodeInterpreter::ExecuteFunction( + tear_down_bytecode, reinterpret_cast(function_arguments)); + throw; + } - tear_down_func_ = (compiled_function_t)code_context_.GetRawFunctionPointer( - query_funcs.tear_down_func); - PELOTON_ASSERT(tear_down_func_ != nullptr); + // Timer plan execution + if (stats != nullptr) { + timer.Stop(); + stats->plan_ms = timer.GetDuration(); + timer.Reset(); + timer.Start(); + } - LOG_TRACE("Query has been setup ..."); + // Clean up + LOG_TRACE("Calling query's tearDown() ..."); + interpreter::BytecodeInterpreter::ExecuteFunction( + tear_down_bytecode, reinterpret_cast(function_arguments)); - // All is well - return true; + // No need to cleanup if we get an exception while cleaning up... + if (stats != nullptr) { + timer.Stop(); + stats->tear_down_ms = timer.GetDuration(); + } } } // namespace codegen diff --git a/src/codegen/updateable_storage.cpp b/src/codegen/updateable_storage.cpp index f911f13a8a1..ba699aeea25 100644 --- a/src/codegen/updateable_storage.cpp +++ b/src/codegen/updateable_storage.cpp @@ -16,6 +16,7 @@ #include "codegen/lang/if.h" #include "codegen/type/sql_type.h" +#include "util/math_util.h" namespace peloton { namespace codegen { @@ -248,7 +249,7 @@ UpdateableStorage::NullBitmap::NullBitmap(CodeGen &codegen, bitmap_ptr_ = codegen->CreateConstInBoundsGEP2_32( storage.GetNullBitmapType(), bitmap_arr, 0, 0); } - uint32_t num_bytes = (storage_.GetNumElements() + 7) >> 3; + uint32_t num_bytes = MathUtil::DivRoundUp(storage_.GetNumElements(), 8); bytes_.resize(num_bytes, nullptr); dirty_.resize(num_bytes, false); } diff --git a/src/executor/plan_executor.cpp b/src/executor/plan_executor.cpp index 6226e3a26cf..a945c46bc59 100644 --- a/src/executor/plan_executor.cpp +++ b/src/executor/plan_executor.cpp @@ -56,10 +56,10 @@ static void CompileAndExecutePlan( // Check if we have a cached compiled plan already codegen::Query *query = codegen::QueryCache::Instance().Find(plan); if (query == nullptr) { - // Cached plan doesn't exist, let's compile the query codegen::QueryCompiler compiler; auto compiled_query = compiler.Compile( *plan, executor_context.GetParams().GetQueryParametersMap(), consumer); + compiled_query->Compile(); // Grab an instance to the plan query = compiled_query.get(); diff --git a/src/include/codegen/code_context.h b/src/include/codegen/code_context.h index be41f2f536d..2c6c1b97a5d 100644 --- a/src/include/codegen/code_context.h +++ b/src/include/codegen/code_context.h @@ -34,6 +34,10 @@ namespace codegen { class FunctionBuilder; +namespace interpreter { +class BytecodeBuilder; +} // namespace interpreter + //===----------------------------------------------------------------------===// // The context where all generated LLVM query code resides. We create a context // instance for every query we see. We keep instances of these around in the @@ -43,6 +47,7 @@ class FunctionBuilder; class CodeContext { friend class CodeGen; friend class FunctionBuilder; + friend class interpreter::BytecodeBuilder; public: using FuncPtr = void *; @@ -63,7 +68,7 @@ class CodeContext { void RegisterBuiltin(llvm::Function *func_decl, FuncPtr func_impl); /// Lookup a builtin function that has been registered in this context - llvm::Function *LookupBuiltin(const std::string &name) const; + std::pair LookupBuiltin(const std::string &name) const; /// Return the LLVM function for UDF that has been registered in this context llvm::Function *GetUDF() const { return udf_func_ptr_; } @@ -71,13 +76,37 @@ class CodeContext { /// Sets UDF function ptr void SetUDF(llvm::Function *func_ptr) { udf_func_ptr_ = func_ptr; } + /// Verify all the code contained in this context + void Verify(); + + /// Optimize all the code contained in this context + void Optimize(); + /// Compile all the code contained in this context - bool Compile(); + void Compile(); /// Retrieve the raw function pointer to the provided compiled LLVM function FuncPtr GetRawFunctionPointer(llvm::Function *fn) const; + /// Get the number of bytes that are needed to store this type + size_t GetTypeSize(llvm::Type *type) const; + + /// Get the number of bits that are needed to store this type + size_t GetTypeSizeInBits(llvm::Type *type) const; + + /// Get the number of bytes between two elements of this type + /// This also includes the padding + size_t GetTypeAllocSize(llvm::Type *type) const; + + /// Get the number of bits between two elements of this type + /// This also includes the padding + size_t GetTypeAllocSizeInBits(llvm::Type *type) const; + + /// Get the offset of element inside a struct in byte + size_t GetStructElementOffset(llvm::StructType *type, size_t index) const; + /// Dump the contents of all the code in this context + /// Attention: this function may change the IR! void DumpContents() const; ////////////////////////////////////////////////////////////////////////////// @@ -143,20 +172,23 @@ class CodeContext { llvm::Type *int16_type_; llvm::Type *int32_type_; llvm::Type *int64_type_; + llvm::Type *float_type_; llvm::Type *double_type_; llvm::Type *void_type_; llvm::Type *void_ptr_type_; llvm::PointerType *char_ptr_type_; // All C/C++ builtin functions and their implementations - std::unordered_map builtins_; + std::unordered_map> + builtins_; // The functions needed in this module, and their implementations. If the // function has not been compiled yet, the function pointer will be NULL. The // function pointers are populated in Compile() std::vector> functions_; - std::unordered_map function_symbols_; + // Shows if the Verify() has been run + bool is_verified_; }; } // namespace codegen diff --git a/src/include/codegen/codegen.h b/src/include/codegen/codegen.h index 9a56edf5dfd..07952d2c4a6 100644 --- a/src/include/codegen/codegen.h +++ b/src/include/codegen/codegen.h @@ -157,9 +157,7 @@ class CodeGen { // Function lookup and registration //===--------------------------------------------------------------------===// llvm::Type *LookupType(const std::string &name) const; - llvm::Function *LookupBuiltin(const std::string &fn_name) const { - return code_context_.LookupBuiltin(fn_name); - } + std::pair LookupBuiltin(const std::string &name) const; llvm::Function *RegisterBuiltin(const std::string &fn_name, llvm::FunctionType *fn_type, void *func_impl); @@ -182,6 +180,13 @@ class CodeGen { return code_context_.GetCurrentFunction(); } + //===--------------------------------------------------------------------===// + // DEBUG OUTPUT + //===--------------------------------------------------------------------===// + + static std::string Dump(const llvm::Value *value); + static std::string Dump(llvm::Type *type); + private: friend class Hash; friend class Value; diff --git a/src/include/codegen/interpreter/bytecode_builder.h b/src/include/codegen/interpreter/bytecode_builder.h new file mode 100644 index 00000000000..20571a0a248 --- /dev/null +++ b/src/include/codegen/interpreter/bytecode_builder.h @@ -0,0 +1,470 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// bytecode_builder.h +// +// Identification: src/include/codegen/interpreter/bytecode_builder.h +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "codegen/interpreter/bytecode_function.h" + +namespace llvm { +class Instruction; +class Function; +class Value; +class BasicBlock; +class Type; +class Constant; +class CallInst; +class ExtractValueInst; +} // namespace llvm + +namespace peloton { +namespace codegen { + +class CodeContext; + +namespace interpreter { + +class BytecodeBuilder { + public: + /** + * Static method to create a bytecode function from a code context. + * @param code_context CodeContext containing the LLVM function + * @param function LLVM function that shall be interpreted later + * @return A BytecodeFunction object that can be passed to the + * BytecodeInterpreter (several times). + */ + static BytecodeFunction CreateBytecodeFunction( + const CodeContext &code_context, const llvm::Function *function, + bool use_naive_register_allocator = false); + + private: + // These types definitions have the purpose to make the code better + // understandable. The bytecode builder creates indexes to identify the + // LLVM types, which usually are only accessed by raw pointers. + // Those types shall indicate which index is meant by a function. + // None of these indexes end up in the bytecode function! + using value_index_t = index_t; + using instruction_index_t = index_t; + + /** + * Describes a bytecode relocation that has to be applied to add the + * destination of a branch instruction once its value is available. + * It gets created by TranslateBranch() and is processed after + * TranslateFunction() processed all instructions. + */ + struct BytecodeRelocation { + index_t instruction_slot; + index_t argument; + const llvm::BasicBlock *bb; + }; + + /** + * Describes the liveness of a value by start and end instruction index. + */ + using ValueLiveness = std::pair; + + private: + BytecodeBuilder(const CodeContext &code_context, + const llvm::Function *function); + + /** + * Analyses the function to collect values and constants and gets + * value liveness information + */ + void AnalyseFunction(); + + /** + * Naive register allocation that just assings a unique value slot to + * every value + */ + void PerformNaiveRegisterAllocation(); + + /** + * Greedy register allocation, that for each value tries to find the next free + * value slot, that is not occupied anymore. + */ + void PerformGreedyRegisterAllocation(); + + /** + * Translates all instructions into bytecode. + */ + void TranslateFunction(); + + /** + * Do some final conversations to make the created BytecodeFunction usable. + */ + void Finalize(); + + private: + //===--------------------------------------------------------------------===// + // Methods for Value Handling + //===--------------------------------------------------------------------===// + + /** + * Gets the value index for a given LLVM value. If no value index exists + * for this LLVM value, a new one is created. + * @param value LLVM Value + * @return the value index that is mapped to this LLVM value + */ + value_index_t GetValueIndex(const llvm::Value *value); + + /** + * Maps a given LLVM value to the same value index as another LLVM Value. + * @param alias LLVM value + * @param value_index the value index to map the LLVM value to. The + * value index must already exist. + * @return the value index that was given as parameter + */ + value_index_t CreateValueAlias(const llvm::Value *alias, + value_index_t value_index); + + /** + * Returns the value_index for a LLVM constant. + * In LLVM several Constant Objects with the same value can exist. This + * function tries to find an existing constant with the same value or creates + * a new one if necessary. + * @param constant LLVM constant + * @return a value index that refers to a constant with the same value. If + * no internal constant with this value exists before, a new value index + * is created. + */ + value_index_t GetConstantIndex(const llvm::Constant *constant); + + /** + * Returns the value slot (register) for a given LLVM value + * @param value LLVM value + * @return the value slot (register) assigned by the register allocation + * This function must not be called before the Analysis pass and the + * Register Allocation! + */ + index_t GetValueSlot(const llvm::Value *value) const; + + /** + * Extends the liveness range of a value to cover the given instruction index. + * The will be extended to the "left" or the "right" if necessary, or not at + * all, if it already covers this index. + * This function calls GetValueIndex, which may create a new value index. + * @param llvm_value LLVM value for which the liveness should be extended + * @param instruction_index position in the + */ + void ExtendValueLiveness(const llvm::Value *llvm_value, + instruction_index_t instruction_index); + + /** + * Returns the index for a additional temporary value slot in that + * basic block. Due to the phi swap problem (lost copy) it can happen, + * that during translation additional value slots are needed that have not + * been mapped by the register allocation. The number of additional temporary + * value slots is tracked and added to the overall number of value + * slots during finalization. + * @param bb basic block the temporary value slot shall be created in + * @return a temporary value slot index, that can be used only in + * this basic block + */ + index_t GetTemporaryValueSlot(const llvm::BasicBlock *bb); + + //===--------------------------------------------------------------------===// + // Helper Functions (const) + //===--------------------------------------------------------------------===// + + /** + * Returns the matching FFI type for a given LLVM type + * @param type LLVM type + * @return FFI type + */ + ffi_type *GetFFIType(llvm::Type *type) const; + + /** + * Checks if a LLVM Value is a constant + * @param value LLVM Value + * @return true, if the given LLVM value is a constant + */ + bool IsConstantValue(const llvm::Value *value) const; + + /** + * Extracts the actual constant value of a LLVM constant + * @param constant LLVM constant + * @return the actual value of the constant, sign or zero extended to + * the size of value_t + */ + value_t GetConstantValue(const llvm::Constant *constant) const; + + /** + * Directly extracts the signed integer value of a integer constant + * @param constant LLVM Constant that is a instance of llvm::ConstantInt + * @return signed integer value of the LLVM constant + */ + int64_t GetConstantIntegerValueSigned(llvm::Value *constant) const; + + /** + * Directly extracts the unsigned integer value of a integer constant + * @param constant LLVM Constant that is a instance of llvm::ConstantInt + * @return unsigned integer value of the LLVM constant + */ + uint64_t GetConstantIntegerValueUnsigned(llvm::Value *constant) const; + + /** + * Checks if one basic block is the successor of another basic block + * when walking all basic blocks in reverse post order. + * (Because ->nextNode doesn't work then) + * @param bb current LLVM basic block + * @param succ LLVM basic block that shall be checked to be the successor + * @return true, if succ is the successor of bb + */ + bool BasicBlockIsRPOSucc(const llvm::BasicBlock *bb, + const llvm::BasicBlock *succ) const; + + /** + * Creates the typed opcode for a bytecode instruction that is defined for + * _all_ types + * @param untyped_op untyped opcode for a byte instruction, retrieved using + * GET_FIRST_ALL_TYPES(op), where op must be defined for all types. + * @param type LLVM type to take the type information from + * @return typed opcode _ + */ + Opcode GetOpcodeForTypeAllTypes(Opcode untyped_op, llvm::Type *type) const; + + /** + * Creates the typed opcode for a bytecode instruction that is defined only + * for _integer_ types + * @param untyped_op untyped opcode for a byte instruction, retrieved using + * GET_FIRST_INT_TYPES(op), where op must be defined only for integer types. + * @param type LLVM type to take the type information from + * @return typed opcode _ + */ + Opcode GetOpcodeForTypeIntTypes(Opcode untyped_op, llvm::Type *type) const; + + /** + * Creates the typed opcode for a bytecode instruction that is defined only + * for _floating point_ types + * @param untyped_op untyped opcode for a byte instruction, retrieved using + * GET_FIRST_FLOAT_TYPES(op), where op must be defined only for float types. + * @param type LLVM type to take the type information from + * @return typed opcode _ + */ + Opcode GetOpcodeForTypeFloatTypes(Opcode untyped_op, llvm::Type *type) const; + + /** + * Creates the typed opcode for a bytecode instruction that is defined only + * for _integer_ types. In difference to the other function, this one only + * considers the type size to determine the opcode type. + * @param untyped_op untyped opcode for a byte instruction, retrieved using + * GET_FIRST_INT_TYPES(op), where op must be defined only for integer types. + * @param type LLVM type to take the size information from + * @return typed opcode _ + */ + Opcode GetOpcodeForTypeSizeIntTypes(Opcode untyped_op, + llvm::Type *type) const; + + //===--------------------------------------------------------------------===// + // Methods for creating Bytecode Instructions + //===--------------------------------------------------------------------===// + + /** + * Insert a bytecode instruction into the bytecode stream. + */ + Instruction &InsertBytecodeInstruction( + const llvm::Instruction *llvm_instruction, Opcode opcode, + const std::vector &args); + + /** + * Insert a bytecode instruction into the bytecode stream. + * Wrapper that automatically gets the value slots for the LLVM values + * provided. + */ + Instruction &InsertBytecodeInstruction( + const llvm::Instruction *llvm_instruction, Opcode opcode, + const std::vector &args); + + /** + * Insert a external call bytecode instruction into the bytecode stream. + * @param llvm_instruction LLVM function this instruction is created from. + * (Only needed for tracing information, not used in Release mode!) + * @param call_context index of the call context created for this external + * call instruction + * @param function function pointer to the external function + * @return Reference to the created instruction in the bytecode stream. + */ + ExternalCallInstruction &InsertBytecodeExternalCallInstruction( + const llvm::Instruction *llvm_instruction, index_t call_context, + void *function); + + /** + * Insert a internal call bytecode instruction into the bytecode stream. + * @param llvm_instruction LLVM function this instruction is created from. + * (Only needed for tracing information, not used in Release mode!) + * @param sub_function index to the sub function (bytecode function) for + * this LLVM function + * @param dest_slot Destination slot for the return value. Set zero if + * internal function returns void. + * @param number_arguments number of arguments provided in this function call. + * The internal call instruction has variadic size, depending on the number + * of arguments! + * @return Reference to the created instruction in the bytecode stream. + */ + InternalCallInstruction &InsertBytecodeInternalCallInstruction( + const llvm::Instruction *llvm_instruction, index_t sub_function, + index_t dest_slot, size_t number_arguments); + +/** + * Helper function, that adds the given instruction to the instruction trace. + * (Should only be called from InsertBytecode instructions) + * In Release mode this function compiles to a stub. + * @param llvm_instruction LLVM instruction the just created bytecode + * instruction originates from + * @param number_instruction_slots size of the bytecode instruction + */ +#ifndef NDEBUG + void AddInstructionToTrace(const llvm::Instruction *llvm_instruction, + size_t number_instruction_slots = 1); +#else + void AddInstructionToTrace( + UNUSED_ATTRIBUTE const llvm::Instruction *llvm_instruction, + UNUSED_ATTRIBUTE size_t number_instruction_slots = 1) {} +#endif + + //===--------------------------------------------------------------------===// + // Methods for Translating LLVM Instructions (called by TranslateFunction()) + //===--------------------------------------------------------------------===// + + /** + * Resolves the PHI nodes referring to this basic block, by placing mov + * instructions. Must be called just before the terminating LLVM instruction + * in a basic block. If the PHI swap / lost copy problem can occur, the + * function creates additional mov instructions and value slots. + * @param bb current basic block + */ + void ProcessPHIsForBasicBlock(const llvm::BasicBlock *bb); + + void TranslateBranch(const llvm::Instruction *instruction, + std::vector &bytecode_relocations); + void TranslateReturn(const llvm::Instruction *instruction); + void TranslateBinaryOperator(const llvm::Instruction *instruction); + void TranslateAlloca(const llvm::Instruction *instruction); + void TranslateLoad(const llvm::Instruction *instruction); + void TranslateStore(const llvm::Instruction *instruction); + void TranslateGetElementPtr(const llvm::Instruction *instruction); + void TranslateIntExt(const llvm::Instruction *instruction); + void TranslateFloatTruncExt(const llvm::Instruction *instruction); + void TranslateFloatIntCast(const llvm::Instruction *instruction); + void TranslateCmp(const llvm::Instruction *instruction); + void TranslateCall(const llvm::Instruction *instruction); + void TranslateSelect(const llvm::Instruction *instruction); + void TranslateExtractValue(const llvm::Instruction *instruction); + + private: + /** + * The bytecode function that is created (and then moved). All other + * members are helping data structures that don't end up in the resulting + * bytecode function + */ + BytecodeFunction bytecode_function_; + + /** + * Mapping from Value* to internal value index (includes merged + * values/constants). The value index is used to access the vectors below. + */ + std::unordered_map value_mapping_; + + /** + * Holds the value liveness per value (after analysis) + */ + std::vector value_liveness_; + + /** + * Holds the assigned value slot per value (after register allocation) + */ + std::vector value_slots_; + + /** + * Overall number of value slots needed (from register allocation) + * without temporary value slots (added during translation) + */ + size_t number_value_slots_; + + /** + * Holds the value_index of the constants in bytecode_function_.constants_, + * accessed with the same index. + */ + std::vector constant_value_indexes_; + + /** + * Additional temporary value slots (created due to phi swap problem). + * Mapping from instruction index to number of temporary slots needed + * at that time (specified by instruction index). + */ + std::unordered_map + number_temporary_values_; + + /** + * Maximum number of temporary value slots needed at all time points. + */ + size_t number_temporary_value_slots_; + + /** + * Keep track of all Call instructions that refer to a overflow aware + * operation, as their results get directly saved in the destination slots + * of the ExtractValue instructions refering to them. + */ + std::unordered_map< + const llvm::CallInst *, + std::pair> + overflow_results_mapping_; + + /** + * Mapping of LLVM functions to bytecode functions to avoid duplicated + * functions in case a internal function is called several times + */ + std::unordered_map sub_function_mapping_; + + /** + * ReversePostOrderTraversal, which is used for all BB traversals + * Initialization is very expensive, so we reuse it + * cannot be const, because the class doesn't provide const iterators + */ + llvm::ReversePostOrderTraversal rpo_traversal_; + + /** + * A vector holding the the basic block pointers in reverse post order. + * This vector is retrieved from the RPO traversal and necessary + * to make pred/pred lookups. + */ + std::vector bb_reverse_post_order_; + + /** + * Original code context the bytecode function is build from + */ + const CodeContext &code_context_; + + /** + * LLVM function that shall be translated + */ + const llvm::Function *llvm_function_; +}; + +class NotSupportedException : public std::runtime_error { + public: + NotSupportedException(std::string message) : std::runtime_error(message) {} +}; + +} // namespace interpreter +} // namespace codegen +} // namespace peloton diff --git a/src/include/codegen/interpreter/bytecode_function.h b/src/include/codegen/interpreter/bytecode_function.h new file mode 100644 index 00000000000..3bbb077e4f0 --- /dev/null +++ b/src/include/codegen/interpreter/bytecode_function.h @@ -0,0 +1,359 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// bytecode_function.h +// +// Identification: src/include/codegen/interpreter/bytecode_function.h +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/macros.h" +#include "util/math_util.h" + +namespace llvm { +class Instruction; +} // namespace llvm + +namespace peloton { +namespace codegen { + +class CodeContext; + +namespace interpreter { + +class BytecodeInterpreter; +class BytecodeBuilder; + +// Type definitions to match the LLVM terminology +using i8 = uint8_t; +using i16 = uint16_t; +using i32 = uint32_t; +using i64 = uint64_t; +using value_t = uint64_t; +using index_t = uint16_t; +using instr_slot_t = uint64_t; + +// Type template that converts any type into the matching interpreter type +template +using bytecode_type = typename std::conditional< + sizeof(type) == 1, i8, + typename std::conditional< + sizeof(type) == 2, i16, + typename std::conditional::type>::type>::type; + +/** + * Enum holding all Opcodes for all instructions. + */ +enum class Opcode : index_t { + undefined, + +#define HANDLE_INST(opcode) opcode, +#include "codegen/interpreter/bytecode_instructions.def" +#undef HANDLE_INST + + NUMBER_OPCODES +}; + +/** + * Struct to access a generic bytecode instruction. + * Every bytecode instruction starts with a 2 byte Opcode, followed by a + * variable number of 2 byte arguments. (Exception: ExternalCallInstruction) + * + * This struct is only for accessing Instructions, not for saving them! + * (sizeof returns a wrong value) All bytecode instructions are saved in + * one or more 8 byte instructions slots (instr_slot_t) in the bytecode stream. + */ +struct Instruction { + Opcode op; + index_t args[]; +}; + +/** + * Specialized struct for accessing a InternalCallInstruction. The number of + * arguments in .args[] is variable and must match the value .number_args . + * GetInteralCallInstructionSlotSize uses this information to calculate the + * number of occupied instruction slots. + */ +struct InternalCallInstruction { + Opcode op; + index_t sub_function; + index_t dest_slot; + index_t number_args; + index_t args[]; +}; + +/** + * Specialized struct for accessing a ExternalCallInstruction. It is the only + * instruction, that contains a field that is greater than 2 byte. + * Because libffi requires pointers to value slots of the current + * activation record, the instruction itself only contains an index for + * accessing the proper call context. During interpretation a call activation + * is created for every call context, holding the actual runtime pointers, + * which can be accessed with the same index. + */ +struct ExternalCallInstruction { + Opcode op; + index_t external_call_context; + void (*function)(void); +}; + +/** + * Call context holding information needed to create a runtime call activation + * for a ExternalCallInstruction in the bytecode stream. + */ +struct ExternalCallContext { + index_t dest_slot; + ffi_type *dest_type; + std::vector args; + std::vector arg_types; +}; + +/** + * A BytecodeFunction contains all information necessary to run a LLVM + * function in the interpreter and is completely independent from the + * CodeContext it was created from (except for the tracing information in debug + * mode). It can be moved and copied. + */ +class BytecodeFunction { + public: + /** + * Returns the Opcode enum for a given Opcode Id (to avoid plain casting) + * @param id Opcode Id + * @return Opcode enum + */ + ALWAYS_INLINE inline static constexpr Opcode GetOpcodeFromId(index_t id) { + return static_cast(id); + } + + /** + * Returns the Opcode Id to a given Opcode enum (to avoid plain casting) + * @param opcode Opcode enum + * @return Opcode Id + */ + ALWAYS_INLINE inline static constexpr index_t GetOpcodeId(Opcode opcode) { + return static_cast(opcode); + } + + /** + * Returns a numan readable string to a given Opcode + * @param opcode Opcode enum + * @return String representation if the Opcode + */ + static const char *GetOpcodeString(Opcode opcode); + + /** + * Returns the overall number of existing Opcodes (not trivial, as the Opcodes + * are created with expanding macros) + * @return overall number of existing Opcodes + */ + inline static constexpr size_t GetNumberOpcodes() { + return static_cast(Opcode::NUMBER_OPCODES); + } + + /** + * Return the instruction pointer to a given instruction index (from this + * bytecode function) + * @param index instruction index + * @return pointer to the instruction at that index inside the bytecode + */ + ALWAYS_INLINE inline const Instruction *GetIPFromIndex(index_t index) const { + return reinterpret_cast( + const_cast(bytecode_.data()) + index); + } + + /** + * Returns the instruction index for a given instruction pointer (from this + * bytecode function) + * @param instruction pointer to a given instruction inside the bytecode + * @return index to the instruction the pointer is pointing to + */ + ALWAYS_INLINE inline index_t GetIndexFromIP( + const Instruction *instruction) const { + index_t index = + reinterpret_cast(instruction) - bytecode_.data(); + return index; + } + +#ifndef NDEBUG + const llvm::Instruction *GetIRInstructionFromIP(index_t instr_slot) const; +#endif + + /** + * Returns the number of slots a given instruction occupies in the bytecode + * stream. + * @param instruction pointer to the instruction inside the bytecode + * @return number of slots (each 8 Byte) that are used by this instruction + */ + static size_t GetInstructionSlotSize(const Instruction *instruction); + + /** + * Returns the number of slots a given internal call instruction occupies in + * the bytecode stream. Internal instructions have a variable length, so the + * size has to be calculated. + * @param instruction pointer to instruction of type internal call + * @return number of slots (each 8 Byte) that are used by this instruction + */ + static ALWAYS_INLINE inline size_t GetInteralCallInstructionSlotSize( + const InternalCallInstruction *instruction) { + const size_t number_slots = + MathUtil::DivRoundUp(sizeof(uint16_t) * (4 + instruction->number_args), + sizeof(instr_slot_t)); + PELOTON_ASSERT(number_slots > 0); + return number_slots; + } + + /** + * Returns the number of slots an explicit call instruction occupies, + * given the number of argument slots. (return value and/or object pointer + * also need a slot!) + * @param number_args number of needed argument slots + * @return number of slots (each 8 Byte) that are used by this instruction + */ + static constexpr ALWAYS_INLINE inline size_t + GetExplicitCallInstructionSlotSize(size_t number_args) { + return MathUtil::DivRoundUp(sizeof(uint16_t) * (1 + number_args), + sizeof(instr_slot_t)); + } + + /** + * Returns the number of required argument slots that are needed in an + * explicit call bytecode instruction for this function. + * @param func pointer/reference to the function (declaration must be visible) + * @return number of required argument slots + * = arguments + return value + object pointer + */ + template + static constexpr ALWAYS_INLINE inline size_t GetFunctionRequiredArgSlotsNum( + UNUSED_ATTRIBUTE return_type (*func)(arg_types...)) { + return (std::is_void::value) ? sizeof...(arg_types) + : sizeof...(arg_types) + 1; + } + + template + static constexpr ALWAYS_INLINE inline size_t GetFunctionRequiredArgSlotsNum( + UNUSED_ATTRIBUTE return_type (class_type::*func)(arg_types...)) { + return (std::is_void::value) ? sizeof...(arg_types) + 1 + : sizeof...(arg_types) + 2; + } + + template + static constexpr ALWAYS_INLINE inline size_t GetFunctionRequiredArgSlotsNum( + UNUSED_ATTRIBUTE return_type (class_type::*func)(arg_types...) const) { + return (std::is_void::value) ? sizeof...(arg_types) + 1 + : sizeof...(arg_types) + 2; + } + + /** + * Returns the opcode for a fiven function name string (lookup in hash map). + * @param function_name string of function name with namespace + * @return the matching opcode or Opcode::undefined + */ + static Opcode GetExplicitCallOpcodeByString(std::string function_name); + + /*** + * Dumps the bytecode and the constants of this bytecode function to a + * file, identified by function name. + */ + void DumpContents() const; + + /** + * Gives a textual representation of the given instruction. (and the + * LLVM instruction it originates from, if Debug mode is enabled) + * @param instruction instruction from this bytecode function + * @return string containing a textual representatino of the instruction + */ + std::string Dump(const Instruction *instruction) const; + + private: + /** + * Creates a new empty BytecodeFunction object. + * @param id identifier for this bytecode function, usually inherited + * from code context. + */ + BytecodeFunction(std::string function_name) : function_name_(function_name) {} + + private: + /** + * Function name of the original function (used only for output). + */ + std::string function_name_; + + /** + * Number of needed value slots at runtime. + */ + size_t number_values_; + + /** + * Number of function arguments (to check correct number given to interpreter) + */ + size_t number_function_arguments_; + + /** + * Constants needed during runtime. + */ + std::vector constants_; + + /** + * This array of instruction slots holds the actual bytecode that is + * interpreted. Usually one instruction occupies one slot, but some + * instruction require several slots. Except for InternalCallInstruction, + * all instruction have a static size. The number of occipied instruction + * slots for an instruction can be obtained by GetInstructionSlotSize() + * + * The "Instruction" struct can be used to access every instruction in a + * generic way. + * + * It can be accessed by index (instruction index) oder a direct pointer + * to a instruction slot (IP). + */ + std::vector bytecode_; + + /** + * Call contexts that belong to ExternalCallInstructions in the bytecode, + * accessed by index. + */ + std::vector external_call_contexts_; + + /** + * Hierarchical array of further bytecode functions belonging to + * InteralFunctionCalls, accessed by index. + */ + std::vector sub_functions_; + + /** + * Constant map created at system startup, that maps the function name string + * of explicit defined function to their opcode. This way the function name + * lookup is implicitly made with a hash table. + */ + static const std::unordered_map + explicit_call_opcode_mapping_; + +#ifndef NDEBUG + /** + * In Debug mode: Maps every bytecode instruction slot to the + * LLVM instruction it was created from. + */ + std::vector instruction_trace_; +#endif + + private: + friend BytecodeInterpreter; + friend BytecodeBuilder; +}; + +} // namespace interpreter +} // namespace codegen +} // namespace peloton diff --git a/src/include/codegen/interpreter/bytecode_instructions.def b/src/include/codegen/interpreter/bytecode_instructions.def new file mode 100644 index 00000000000..83ec605ee0b --- /dev/null +++ b/src/include/codegen/interpreter/bytecode_instructions.def @@ -0,0 +1,412 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// bytecode_instructions.def +// +// Identification: src/include/codegen/interpreter/bytecode_instructions.def +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +//----------------------------------------------------------------------------// +// Instruction Definitions +// +// This file contains the definitions for all bytecode instructions. +// +// The definitions can be used by defining one of the HANDLE functions below +// before including this definition file (see X-Macros). This way the +// definitions can be used to generate the Opcode enum, the dispatch area, etc. +// +// Most instructions are automatically expanded to all their supported types. +// +// When adding a bytecode instruction here, the instruction at least needs a +// Translate-function in the BytecodeBuilder and a Handler-function in the +// BytecodeInterpreter. +//----------------------------------------------------------------------------// + +#ifndef HANDLE_INST +#define HANDLE_INST(op) +#endif + +#ifndef HANDLE_TYPED_INST +#define HANDLE_TYPED_INST(op, type) HANDLE_INST(op##_##type) +#endif + +#ifndef HANDLE_OVERFLOW_TYPED_INST +#define HANDLE_OVERFLOW_TYPED_INST(op, type) HANDLE_TYPED_INST(op, type) +#endif + +#ifndef HANDLE_SELECT_INST +#define HANDLE_SELECT_INST(op) HANDLE_INST(op) +#endif + +#ifndef HANDLE_RET_INST +#define HANDLE_RET_INST(op) HANDLE_INST(op) +#endif + +#ifndef HANDLE_EXTERNAL_CALL_INST +#define HANDLE_EXTERNAL_CALL_INST(op) HANDLE_INST(op) +#endif + +#ifndef HANDLE_INTERNAL_CALL_INST +#define HANDLE_INTERNAL_CALL_INST(op) HANDLE_INST(op) +#endif + +#ifndef HANDLE_EXPLICIT_CALL_INST +#define HANDLE_EXPLICIT_CALL_INST(op, func) HANDLE_INST(op) +#endif + +// Takes a function and a opcode and calls the function for all type instances +// of that opcode +#define CREATE_FOR_ALL_TYPES(func, op) \ + func(op, i8) func(op, i16) func(op, i32) func(op, i64) func(op, float) \ + func(op, double) + +// Returns the first type used when expanding to all types +// (needed for use of GetOpcodeForTypeAllTypes) +#define GET_FIRST_ALL_TYPES(op) (op##_i8) + +// Takes a function and a opcode and calls the function for all integer +// instances of that opcode +#define CREATE_FOR_INT_TYPES(func, op) \ + func(op, i8) func(op, i16) func(op, i32) func(op, i64) + +// Returns the first type used when expanding to integer types +// (needed for use of GetOpcodeForTypeIntTypes) +#define GET_FIRST_INT_TYPES(op) (op##_i8) + +// Takes a function and a opcode and calls the function for all floating point +// instances of that opcode +#define CREATE_FOR_FLOAT_TYPES(func, op) func(op, float) func(op, double) + +// Returns the first type used when expanding to floating point types +// (needed for use of GetOpcodeForTypeFloatTypes) +#define GET_FIRST_FLOAT_TYPES(op) (op##_float) + +//------ Bytecode Instruction Definitions ------// + +CREATE_FOR_ALL_TYPES(HANDLE_TYPED_INST, add) +CREATE_FOR_ALL_TYPES(HANDLE_TYPED_INST, sub) +CREATE_FOR_ALL_TYPES(HANDLE_TYPED_INST, mul) +CREATE_FOR_ALL_TYPES(HANDLE_TYPED_INST, + div) // division for unsigned integer and floating point +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, sdiv) // division for signed integer +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, urem) // remainder for unsigned integer +CREATE_FOR_FLOAT_TYPES(HANDLE_TYPED_INST, frem) // remainder for floating point +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, srem) // remainder for signed integer +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, shl) +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, lshr) +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, ashr) +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, and) +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, or) +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, xor) + +HANDLE_INST(extractvalue) +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, load) +CREATE_FOR_ALL_TYPES(HANDLE_TYPED_INST, store) +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, alloca_array) +HANDLE_INST(alloca) + +CREATE_FOR_ALL_TYPES(HANDLE_TYPED_INST, + cmp_eq) // compare for unsigned integer and floating point +CREATE_FOR_ALL_TYPES(HANDLE_TYPED_INST, + cmp_ne) // compare for unsigned integer and floating point +CREATE_FOR_ALL_TYPES(HANDLE_TYPED_INST, + cmp_gt) // compare for unsigned integer and floating point +CREATE_FOR_ALL_TYPES(HANDLE_TYPED_INST, + cmp_lt) // compare for unsigned integer and floating point +CREATE_FOR_ALL_TYPES(HANDLE_TYPED_INST, + cmp_ge) // compare for unsigned integer and floating point +CREATE_FOR_ALL_TYPES(HANDLE_TYPED_INST, + cmp_le) // compare for unsigned integer and floating point +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, cmp_sgt) // compare for signed integer +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, cmp_slt) // compare for signed integer +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, cmp_sge) // compare for signed integer +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, cmp_sle) // compare for signed integer + +HANDLE_INST(sext_i8_i16) // there is no handy way to expand this relationship +HANDLE_INST(sext_i8_i32) +HANDLE_INST(sext_i8_i64) +HANDLE_INST(sext_i16_i32) +HANDLE_INST(sext_i16_i64) +HANDLE_INST(sext_i32_i64) +HANDLE_INST(zext_i8_i16) +HANDLE_INST(zext_i8_i32) +HANDLE_INST(zext_i8_i64) +HANDLE_INST(zext_i16_i32) +HANDLE_INST(zext_i16_i64) +HANDLE_INST(zext_i32_i64) + +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, + doubletosi) // we can only expand in one dimension, so we + // expand the integer dimension and write down + // all floating point instances manually +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, doubletoui) +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, sitodouble) +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, uitodouble) +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, floattosi) +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, floattoui) +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, sitofloat) +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, uitofloat) +HANDLE_INST(doubletofloat) +HANDLE_INST(floattodouble) + +HANDLE_INST(gep_offset) // struct access of GEP instruction (accumulated) +CREATE_FOR_INT_TYPES(HANDLE_TYPED_INST, + gep_array) // array access of GEP instruction (inplace) +HANDLE_INST(phi_mov) +HANDLE_INST(nop_mov) +HANDLE_SELECT_INST(select) +HANDLE_EXTERNAL_CALL_INST(call_external) // external function call +HANDLE_INTERNAL_CALL_INST(call_internal) // internal function call + +HANDLE_RET_INST(ret) +HANDLE_INST(branch_uncond) +HANDLE_INST(branch_cond) +HANDLE_INST(branch_cond_ft) // conditional branch with fall through + +HANDLE_INST(llvm_memcpy) +HANDLE_INST(llvm_memmove) +HANDLE_INST(llvm_memset) + +CREATE_FOR_INT_TYPES(HANDLE_OVERFLOW_TYPED_INST, llvm_uadd_overflow) +CREATE_FOR_INT_TYPES(HANDLE_OVERFLOW_TYPED_INST, llvm_sadd_overflow) +CREATE_FOR_INT_TYPES(HANDLE_OVERFLOW_TYPED_INST, llvm_usub_overflow) +CREATE_FOR_INT_TYPES(HANDLE_OVERFLOW_TYPED_INST, llvm_ssub_overflow) +CREATE_FOR_INT_TYPES(HANDLE_OVERFLOW_TYPED_INST, llvm_umul_overflow) +CREATE_FOR_INT_TYPES(HANDLE_OVERFLOW_TYPED_INST, llvm_smul_overflow) + +HANDLE_INST(llvm_sse42_crc32) + +//------ Explicit Call Instructions ------// +// +// Usually external functions are called using libffi. +// However, for often used functions, an explicit bytecode instruction for that +// specific function call can be created. For every function listed below, +// a bytecode instruction _and_ a matching handler function are generated +// automatically using heavy template-macro-magic. +// +// Further functions can easily be added here. The regarding headers have to be +// added in these files manually: bytecode_interpreter.h, bytecode_function.cpp + +HANDLE_EXPLICIT_CALL_INST( + peloton_transactionruntime_performvectorizedread, + peloton::codegen::TransactionRuntime::PerformVectorizedRead) + +HANDLE_EXPLICIT_CALL_INST(peloton_oahashtable_init, + peloton::codegen::util::OAHashTable::Init) +HANDLE_EXPLICIT_CALL_INST(peloton_oahashtable_storetuple, + peloton::codegen::util::OAHashTable::StoreTuple) +HANDLE_EXPLICIT_CALL_INST(peloton_oahashtable_destroy, + peloton::codegen::util::OAHashTable::Destroy) + +HANDLE_EXPLICIT_CALL_INST(peloton_deleter_init, peloton::codegen::Deleter::Init) +HANDLE_EXPLICIT_CALL_INST(peloton_deleter_delete, + peloton::codegen::Deleter::Delete) + +HANDLE_EXPLICIT_CALL_INST(peloton_updater_init, peloton::codegen::Updater::Init) +HANDLE_EXPLICIT_CALL_INST(peloton_updater_prepare, + peloton::codegen::Updater::Prepare) +HANDLE_EXPLICIT_CALL_INST(peloton_updater_preparepk, + peloton::codegen::Updater::PreparePK) +HANDLE_EXPLICIT_CALL_INST(peloton_updater_getpool, + peloton::codegen::Updater::GetPool) +HANDLE_EXPLICIT_CALL_INST(peloton_updater_update, + peloton::codegen::Updater::Update) +HANDLE_EXPLICIT_CALL_INST(peloton_updater_updatepk, + peloton::codegen::Updater::UpdatePK) +HANDLE_EXPLICIT_CALL_INST(peloton_updater_teardown, + peloton::codegen::Updater::TearDown) + +HANDLE_EXPLICIT_CALL_INST(peloton_inserter_init, + peloton::codegen::Inserter::Init) +HANDLE_EXPLICIT_CALL_INST(peloton_inserter_allocatetuplestorage, + peloton::codegen::Inserter::AllocateTupleStorage) +HANDLE_EXPLICIT_CALL_INST(peloton_inserter_getpool, + peloton::codegen::Inserter::GetPool) +HANDLE_EXPLICIT_CALL_INST(peloton_inserter_insert, + peloton::codegen::Inserter::Insert) +HANDLE_EXPLICIT_CALL_INST(peloton_inserter_teardown, + peloton::codegen::Inserter::TearDown) + +HANDLE_EXPLICIT_CALL_INST(peloton_sorter_init, + peloton::codegen::util::Sorter::Init) +HANDLE_EXPLICIT_CALL_INST(peloton_sorter_storeinputtuple, + peloton::codegen::util::Sorter::StoreInputTuple) +HANDLE_EXPLICIT_CALL_INST(peloton_sorter_sort, + peloton::codegen::util::Sorter::Sort) +HANDLE_EXPLICIT_CALL_INST(peloton_sorter_sortparallel, + peloton::codegen::util::Sorter::SortParallel) +HANDLE_EXPLICIT_CALL_INST(peloton_sorter_destroy, + peloton::codegen::util::Sorter::Destroy) + +HANDLE_EXPLICIT_CALL_INST(peloton_zonemap_shouldscantilegroup, + peloton::storage::ZoneMapManager::ShouldScanTileGroup) +HANDLE_EXPLICIT_CALL_INST(peloton_zonemap_getinstance, + peloton::storage::ZoneMapManager::GetInstance) + +HANDLE_EXPLICIT_CALL_INST(peloton_valuesruntime_outputboolean, + peloton::codegen::ValuesRuntime::OutputBoolean) +HANDLE_EXPLICIT_CALL_INST(peloton_valuesruntime_outputtinyint, + peloton::codegen::ValuesRuntime::OutputTinyInt) +HANDLE_EXPLICIT_CALL_INST(peloton_valuesruntime_outputsmallint, + peloton::codegen::ValuesRuntime::OutputSmallInt) +HANDLE_EXPLICIT_CALL_INST(peloton_valuesruntime_outputinteger, + peloton::codegen::ValuesRuntime::OutputInteger) +HANDLE_EXPLICIT_CALL_INST(peloton_valuesruntime_outputbigint, + peloton::codegen::ValuesRuntime::OutputBigInt) +HANDLE_EXPLICIT_CALL_INST(peloton_valuesruntime_outputdate, + peloton::codegen::ValuesRuntime::OutputDate) +HANDLE_EXPLICIT_CALL_INST(peloton_valuesruntime_outputtimestamp, + peloton::codegen::ValuesRuntime::OutputTimestamp) +HANDLE_EXPLICIT_CALL_INST(peloton_valuesruntime_outputdecimal, + peloton::codegen::ValuesRuntime::OutputDecimal) +HANDLE_EXPLICIT_CALL_INST(peloton_valuesruntime_outputvarchar, + peloton::codegen::ValuesRuntime::OutputVarchar) +HANDLE_EXPLICIT_CALL_INST(peloton_valuesruntime_outputvarbinary, + peloton::codegen::ValuesRuntime::OutputVarbinary) + +HANDLE_EXPLICIT_CALL_INST(peloton_executorcontext_gettransaction, + peloton::executor::ExecutorContext::GetTransaction) + +HANDLE_EXPLICIT_CALL_INST(peloton_stringfunctions_ascii, + peloton::function::StringFunctions::Ascii) +HANDLE_EXPLICIT_CALL_INST(peloton_stringfunctions_like, + peloton::function::StringFunctions::Like) +HANDLE_EXPLICIT_CALL_INST(peloton_stringfunctions_length, + peloton::function::StringFunctions::Length) +HANDLE_EXPLICIT_CALL_INST(peloton_stringfunctions_btrim, + peloton::function::StringFunctions::BTrim) +HANDLE_EXPLICIT_CALL_INST(peloton_stringfunctions_trim, + peloton::function::StringFunctions::Trim) +HANDLE_EXPLICIT_CALL_INST(peloton_stringfunctions_ltrim, + peloton::function::StringFunctions::LTrim) +HANDLE_EXPLICIT_CALL_INST(peloton_stringfunctions_rtrim, + peloton::function::StringFunctions::RTrim) +HANDLE_EXPLICIT_CALL_INST(peloton_stringfunctions_substr, + peloton::function::StringFunctions::Substr) +HANDLE_EXPLICIT_CALL_INST(peloton_stringfunctions_repeat, + peloton::function::StringFunctions::Repeat) +HANDLE_EXPLICIT_CALL_INST(peloton_stringfunctions_comparestrings, + peloton::function::StringFunctions::CompareStrings) + +HANDLE_EXPLICIT_CALL_INST(peloton_buffer_init, + peloton::codegen::util::Buffer::Init) +HANDLE_EXPLICIT_CALL_INST(peloton_buffer_append, + peloton::codegen::util::Buffer::Append) +HANDLE_EXPLICIT_CALL_INST(peloton_buffer_reset, + peloton::codegen::util::Buffer::Reset) +HANDLE_EXPLICIT_CALL_INST(peloton_buffer_destroy, + peloton::codegen::util::Buffer::Destroy) + +HANDLE_EXPLICIT_CALL_INST(peloton_numericfunctions_abs, + peloton::function::NumericFunctions::Abs) +HANDLE_EXPLICIT_CALL_INST(peloton_numericfunctions_floor, + peloton::function::NumericFunctions::Floor) +HANDLE_EXPLICIT_CALL_INST(peloton_numericfunctions_round, + peloton::function::NumericFunctions::Round) +HANDLE_EXPLICIT_CALL_INST(peloton_numericfunctions_ceil, + peloton::function::NumericFunctions::Ceil) + +HANDLE_EXPLICIT_CALL_INST(peloton_bloomfilteraccessor_init, + peloton::codegen::BloomFilterAccessor::Init) +HANDLE_EXPLICIT_CALL_INST(peloton_bloomfilteraccessor_destroy, + peloton::codegen::BloomFilterAccessor::Destroy) + +HANDLE_EXPLICIT_CALL_INST(peloton_bloomfilter_init, + peloton::codegen::util::BloomFilter::Init) +HANDLE_EXPLICIT_CALL_INST(peloton_bloomfilter_destroy, + peloton::codegen::util::BloomFilter::Destroy) + +HANDLE_EXPLICIT_CALL_INST(peloton_datatable_gettilegroupcount, + peloton::storage::DataTable::GetTileGroupCount) + +HANDLE_EXPLICIT_CALL_INST(peloton_datefunctions_now, + peloton::function::DateFunctions::Now) + +HANDLE_EXPLICIT_CALL_INST(peloton_hashtable_init, + peloton::codegen::util::HashTable::Init) +HANDLE_EXPLICIT_CALL_INST(peloton_hashtable_insert, + peloton::codegen::util::HashTable::Insert) +HANDLE_EXPLICIT_CALL_INST(peloton_hashtable_insertlazy, + peloton::codegen::util::HashTable::InsertLazy) +HANDLE_EXPLICIT_CALL_INST(peloton_hashtable_buildlazy, + peloton::codegen::util::HashTable::BuildLazy) +HANDLE_EXPLICIT_CALL_INST(peloton_hashtable_reservelazy, + peloton::codegen::util::HashTable::ReserveLazy) +HANDLE_EXPLICIT_CALL_INST(peloton_hashtable_mergelazyunfinished, + peloton::codegen::util::HashTable::MergeLazyUnfinished) +HANDLE_EXPLICIT_CALL_INST(peloton_hashtable_destroy, + peloton::codegen::util::HashTable::Destroy) + +HANDLE_EXPLICIT_CALL_INST(peloton_storagemanager_gettablewithoid, + peloton::storage::StorageManager::GetTableWithOid) + +HANDLE_EXPLICIT_CALL_INST(peloton_tilegroup_getnexttupleslot, + peloton::storage::TileGroup::GetNextTupleSlot) +HANDLE_EXPLICIT_CALL_INST(peloton_tilegroup_gettilegroupid, + peloton::storage::TileGroup::GetTileGroupId) + +HANDLE_EXPLICIT_CALL_INST(peloton_timestampfunctions_datetrunc, + peloton::function::TimestampFunctions::DateTrunc) +HANDLE_EXPLICIT_CALL_INST(peloton_timestampfunctions_datepart, + peloton::function::TimestampFunctions::DatePart) + +HANDLE_EXPLICIT_CALL_INST(peloton_queryparameters_getboolean, + peloton::codegen::QueryParameters::GetBoolean) +HANDLE_EXPLICIT_CALL_INST(peloton_queryparameters_gettinyint, + peloton::codegen::QueryParameters::GetTinyInt) +HANDLE_EXPLICIT_CALL_INST(peloton_queryparameters_getsmallint, + peloton::codegen::QueryParameters::GetSmallInt) +HANDLE_EXPLICIT_CALL_INST(peloton_queryparameters_getinteger, + peloton::codegen::QueryParameters::GetInteger) +HANDLE_EXPLICIT_CALL_INST(peloton_queryparameters_getbigint, + peloton::codegen::QueryParameters::GetBigInt) +HANDLE_EXPLICIT_CALL_INST(peloton_queryparameters_getdouble, + peloton::codegen::QueryParameters::GetDouble) +HANDLE_EXPLICIT_CALL_INST(peloton_queryparameters_getdate, + peloton::codegen::QueryParameters::GetDate) +HANDLE_EXPLICIT_CALL_INST(peloton_queryparameters_gettimestamp, + peloton::codegen::QueryParameters::GetTimestamp) +HANDLE_EXPLICIT_CALL_INST(peloton_queryparameters_getvarcharval, + peloton::codegen::QueryParameters::GetVarcharVal) +HANDLE_EXPLICIT_CALL_INST(peloton_queryparameters_getvarcharlen, + peloton::codegen::QueryParameters::GetVarcharLen) +HANDLE_EXPLICIT_CALL_INST(peloton_queryparameters_getvarbinaryval, + peloton::codegen::QueryParameters::GetVarbinaryVal) +HANDLE_EXPLICIT_CALL_INST(peloton_queryparameters_getvarbinarylen, + peloton::codegen::QueryParameters::GetVarbinaryLen) +HANDLE_EXPLICIT_CALL_INST(peloton_queryparameters_isnull, + peloton::codegen::QueryParameters::IsNull) + +HANDLE_EXPLICIT_CALL_INST(peloton_runtimefunctions_hashcrc64, + peloton::codegen::RuntimeFunctions::HashCrc64) +HANDLE_EXPLICIT_CALL_INST(peloton_runtimefunctions_gettilegroup, + peloton::codegen::RuntimeFunctions::GetTileGroup) +HANDLE_EXPLICIT_CALL_INST( + peloton_runtimefunctions_gettilegrouplayout, + peloton::codegen::RuntimeFunctions::GetTileGroupLayout) +HANDLE_EXPLICIT_CALL_INST( + peloton_runtimefunctions_fillpredicatearray, + peloton::codegen::RuntimeFunctions::FillPredicateArray) +HANDLE_EXPLICIT_CALL_INST( + peloton_runtimefunctions_throwdividebyzeroexception, + peloton::codegen::RuntimeFunctions::ThrowDivideByZeroException) +HANDLE_EXPLICIT_CALL_INST( + peloton_runtimefunctions_throwoverflowexception, + peloton::codegen::RuntimeFunctions::ThrowOverflowException) + +HANDLE_EXPLICIT_CALL_INST(peloton_bufferingconsumer_buffertuple, + peloton::codegen::BufferingConsumer::BufferTuple) + +// undefine all handlers +#undef HANDLE_INST +#undef HANDLE_TYPED_INST +#undef HANDLE_OVERFLOW_TYPED_INST +#undef HANDLE_SELECT_INST +#undef HANDLE_RET_INST +#undef HANDLE_EXTERNAL_CALL_INST +#undef HANDLE_INTERNAL_CALL_INST +#undef HANDLE_EXPLICIT_CALL_INST \ No newline at end of file diff --git a/src/include/codegen/interpreter/bytecode_interpreter.h b/src/include/codegen/interpreter/bytecode_interpreter.h new file mode 100644 index 00000000000..16193efbc5c --- /dev/null +++ b/src/include/codegen/interpreter/bytecode_interpreter.h @@ -0,0 +1,1260 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// bytecode_interpreter.h +// +// Identification: src/include/codegen/interpreter/bytecode_interpreter.h +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "codegen/interpreter/bytecode_function.h" + +#include + +#include "codegen/query.h" +#include "common/exception.h" +#include "common/overflow_builtins.h" + +// Includes for explicit function calls +#include "codegen/bloom_filter_accessor.h" +#include "codegen/util/bloom_filter.h" +#include "codegen/buffering_consumer.h" +#include "codegen/deleter.h" +#include "codegen/inserter.h" +#include "codegen/query_parameters.h" +#include "codegen/runtime_functions.h" +#include "codegen/transaction_runtime.h" +#include "codegen/updater.h" +#include "codegen/util/oa_hash_table.h" +#include "codegen/util/hash_table.h" +#include "codegen/util/sorter.h" +#include "codegen/values_runtime.h" +#include "executor/executor_context.h" +#include "function/date_functions.h" +#include "function/numeric_functions.h" +#include "function/string_functions.h" +#include "function/timestamp_functions.h" +#include "planner/project_info.h" +#include "storage/data_table.h" +#include "storage/storage_manager.h" +#include "storage/tile_group.h" +#include "storage/zone_map_manager.h" +#include "codegen/util/buffer.h" + +namespace peloton { +namespace codegen { +namespace interpreter { + +/** + * Holds the runtime information for a external funtion call. Because libffi + * requires pointers to the actual value slots, this information is different + * for every function activation, and can not be stored in the bytecode + * function. + */ +struct CallActivation { + ffi_cif call_interface; + std::vector value_pointers; + value_t *return_pointer; +}; + +//----------------------------------------------------------------------------// +// Template Helper Functions // +//----------------------------------------------------------------------------// + +/** + * The seq-types allow to create a template sequence of integers, e.g. for + * indexed access. (std::integer_sequence is only available in C++14) + */ +template +struct seq { + using type = seq; +}; +template +struct concat; +template +struct concat, seq> : seq {}; + +template +struct gen_seq; +template +struct gen_seq : concat::type, + typename gen_seq::type>::type {}; +template <> +struct gen_seq<0> : seq<> {}; +template <> +struct gen_seq<1> : seq<0> {}; + +/** + * This function converts references to pointers to make value handling + * possible. The function is tagged with the a bool type that indicates + * whether the type is a reference. + * Non-reference types are returned without changes. + */ +template +static ALWAYS_INLINE inline constexpr + typename std::remove_pointer::type & + ConvertPointerToReference(type_t source, + UNUSED_ATTRIBUTE std::true_type is_reference) { + return *source; +}; + +template +static ALWAYS_INLINE inline constexpr type_t ConvertPointerToReference( + type_t source, UNUSED_ATTRIBUTE std::false_type not_a_reference) { + return source; +}; + +class BytecodeInterpreter { + public: + /** + * Executes a translated function with the interpreter + * @param bytecode_function bytecode function that shall be executed + * @param arguments vector of function arguments (stored as value_t). The + * number of arguments must match the number expected by the executed + * function. + * @return return Value of the LLVM function or undefined if void. + */ + static value_t ExecuteFunction(const BytecodeFunction &bytecode_function, + const std::vector &arguments); + /** + * Executes a translated function with the interpreter + * (Wrapper for usage with a single char* argument) + * @param bytecode_function bytecode function that shall be executed, must + * expect one argument. + * @param arguments Char pointer argument of the function. + */ + static void ExecuteFunction(const BytecodeFunction &bytecode_function, + char *param); + + private: + explicit BytecodeInterpreter(const BytecodeFunction &bytecode_function); + + /** + * Executes a function with the given arguments. The return value can + * afterwards retrieved with GetReturnValue(). This function is also called + * for internal function calls during execution. + * @param arguments Vector of function arguments (stored as value_t). The + * number of arguments must match the number expected by the executed + * function. + */ + void ExecuteFunction(const std::vector &arguments); + + /** + * Initializes the activation record by allocating the value slots, placing + * function arguments and constants and preparing call contexts. + * @param arguments Vector of function arguments (stored as value_t). The + * number of arguments must match the number expected by the executed + * function. + */ + void InitializeActivationRecord(const std::vector &arguments); + + /** + * Returns the function return value _after_ execution. + * @tparam type_t Expected return type. + * @return Return value of executed function or undefined if void. + */ + template + type_t GetReturnValue(); + + /** + * Get the current value of a value slot. + * @tparam type_t requested type + * @param index value slot index + * @return value as requested type + */ + template + ALWAYS_INLINE inline type_t GetValue(const index_t index) { + using type_noref_t = typename std::conditional< + std::is_reference::value, + typename std::remove_reference::type *, type_t>::type; + static_assert(sizeof(type_noref_t) <= sizeof(value_t), + "The interpreter can only handle values that fit in 8 bytes"); + + PELOTON_ASSERT(index >= 0 && index < bytecode_function_.number_values_); + return ConvertPointerToReference( + *reinterpret_cast(&values_[index]), + std::is_reference()); + } + + /** + * Get the reference to a value slot. Usually SetValue() should be used + * to set the values, but some use cases require pointers/references to the + * slots. + * @tparam type_t requested type + * @param index value slot index + * @return typed reference to the requested slot + */ + template + ALWAYS_INLINE inline type_t &GetValueReference(const index_t index) { + PELOTON_ASSERT(index >= 0 && index < bytecode_function_.number_values_); + return reinterpret_cast(values_[index]); + } + + /** + * Set the current value of a slot + * @tparam type_t requested type + * @param index value slot index + * @param value value of type type_t, that shall be set + */ + template + ALWAYS_INLINE inline void SetValue(const index_t index, const type_t value) { + using type_noref_t = typename std::conditional< + std::is_reference::value, + typename std::remove_reference::type *, type_t>::type; + + PELOTON_ASSERT(index >= 0 && index < bytecode_function_.number_values_); + *reinterpret_cast(&values_[index]) = value; + + DumpValue(index); + } + + /** + * Advance the instruction pointer by a compile-time value. + * @tparam number_instruction_slots size of current instruction + * @param instruction current instruction pointer + * @return new instruction pointer + */ + template + ALWAYS_INLINE inline const Instruction *AdvanceIP( + const Instruction *instruction) { + auto next = reinterpret_cast( + const_cast( + reinterpret_cast(instruction)) + + number_instruction_slots); + return next; + } + + /** + * Advance the instruction pointer by a run-time value. + * @tparam number_instruction_slots size of current instruction + * @param instruction current instruction pointer + * @return new instruction pointer + */ + ALWAYS_INLINE inline const Instruction *AdvanceIP( + const Instruction *instruction, size_t number_instruction_slots) { + auto next = reinterpret_cast( + const_cast( + reinterpret_cast(instruction)) + + number_instruction_slots); + return next; + } + + /** + * Allocate memory and return a pointer to it. (Memory is managed and gets + * freed after the interpreter exits) + * @param number_bytes number of bytes to allocate + * @return pointer to the allocated memory + */ + uintptr_t AllocateMemory(size_t number_bytes); + +/** + * Dump the value of the given as value slot for debug purposes. + * If LOG_TRACE is not enabled, this function compiles to a stub. + * @param index value index of value slot to dump + */ +#ifdef LOG_TRACE_ENABLED + template + void DumpValue(const index_t index) { + std::ostringstream output; + output << " [" << std::dec << std::setw(3) << index + << "] <= " << GetValue>(index) << "/0x" + << std::hex << GetValue>(index); + LOG_TRACE("%s", output.str().c_str()); + } +#else + template + void DumpValue(UNUSED_ATTRIBUTE const index_t index) {} +#endif + + //--------------------------------------------------------------------------// + // Instruction Handlers + // + // - The following functions are the instruction handlers for the bytecode + // instructions defined in bytecode_instructions.def . + // - The signatures of those functions are not code style conform, as they are + // generated from the opcode mnemonic + // - If the instruction is marked as a typed instruction in the .def file, + // it has a templated handler. Some handlers only support floating point or + // integer types, some both. Static asserts ensure this. + // - Because all the handlers will get inlined in the dispatch area, their + // definition must be in this header file. + //--------------------------------------------------------------------------// + + template + ALWAYS_INLINE inline const Instruction *addHandler( + const Instruction *instruction) { + SetValue(instruction->args[0], + (GetValue(instruction->args[1]) + + GetValue(instruction->args[2]))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *subHandler( + const Instruction *instruction) { + SetValue(instruction->args[0], + (GetValue(instruction->args[1]) - + GetValue(instruction->args[2]))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *mulHandler( + const Instruction *instruction) { + SetValue(instruction->args[0], + (GetValue(instruction->args[1]) * + GetValue(instruction->args[2]))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *divHandler( + const Instruction *instruction) { + SetValue(instruction->args[0], + (GetValue(instruction->args[1]) / + GetValue(instruction->args[2]))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *sdivHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ must only be used with integer types"); + using type_signed_t = typename std::make_signed::type; + SetValue(instruction->args[0], + (GetValue(instruction->args[1]) / + GetValue(instruction->args[2]))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *uremHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ must only be used with integer types"); + SetValue(instruction->args[0], + (GetValue(instruction->args[1]) % + GetValue(instruction->args[2]))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *fremHandler( + const Instruction *instruction) { + static_assert(std::is_floating_point::value, + "__func__ must only be used with floating point types"); + SetValue(instruction->args[0], + (std::fmod(GetValue(instruction->args[1]), + GetValue(instruction->args[2])))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *sremHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ must only be used with integer types"); + using type_signed_t = typename std::make_signed::type; + SetValue(instruction->args[0], + (GetValue(instruction->args[1]) % + GetValue(instruction->args[2]))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *shlHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ must only be used with integer types"); + SetValue(instruction->args[0], + (GetValue(instruction->args[1]) + << GetValue(instruction->args[2]))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *lshrHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ must only be used with integer types"); + SetValue(instruction->args[0], + (GetValue(instruction->args[1]) >> + GetValue(instruction->args[2]))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *ashrHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ must only be used with integer types"); + using type_signed_t = typename std::make_signed::type; + SetValue(instruction->args[0], + (GetValue(instruction->args[1]) >> + GetValue(instruction->args[2]))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *andHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ must only be used with integer types"); + SetValue(instruction->args[0], + (GetValue(instruction->args[1]) & + GetValue(instruction->args[2]))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *orHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ must only be used with integer types"); + SetValue(instruction->args[0], + (GetValue(instruction->args[1]) | + GetValue(instruction->args[2]))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *xorHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ must only be used with integer types"); + SetValue(instruction->args[0], + (GetValue(instruction->args[1]) ^ + GetValue(instruction->args[2]))); + return AdvanceIP<1>(instruction); + } + + ALWAYS_INLINE inline const Instruction *extractvalueHandler( + const Instruction *instruction) { + SetValue( + instruction->args[0], + (GetValue(instruction->args[1]) >> instruction->args[2])); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *loadHandler( + const Instruction *instruction) { + SetValue(instruction->args[0], + (*GetValue(instruction->args[1]))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *storeHandler( + const Instruction *instruction) { + *GetValue(instruction->args[0]) = + GetValue(instruction->args[1]); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *alloca_arrayHandler( + const Instruction *instruction) { + size_t number_bytes = + instruction->args[1] * GetValue(instruction->args[2]); + SetValue(instruction->args[0], (AllocateMemory(number_bytes))); + return AdvanceIP<1>(instruction); + } + + ALWAYS_INLINE inline const Instruction *allocaHandler( + const Instruction *instruction) { + size_t number_bytes = instruction->args[1]; + SetValue(instruction->args[0], (AllocateMemory(number_bytes))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *cmp_eqHandler( + const Instruction *instruction) { + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1]) == + GetValue(instruction->args[2])))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *cmp_neHandler( + const Instruction *instruction) { + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1]) != + GetValue(instruction->args[2])))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *cmp_gtHandler( + const Instruction *instruction) { + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1]) > + GetValue(instruction->args[2])))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *cmp_ltHandler( + const Instruction *instruction) { + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1]) < + GetValue(instruction->args[2])))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *cmp_geHandler( + const Instruction *instruction) { + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1]) >= + GetValue(instruction->args[2])))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *cmp_leHandler( + const Instruction *instruction) { + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1]) <= + GetValue(instruction->args[2])))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *cmp_sgtHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ must only be used with integer types"); + using type_signed_t = typename std::make_signed::type; + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1]) > + GetValue(instruction->args[2])))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *cmp_sltHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ must only be used with integer types"); + using type_signed_t = typename std::make_signed::type; + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1]) < + GetValue(instruction->args[2])))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *cmp_sgeHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ must only be used with integer types"); + using type_signed_t = typename std::make_signed::type; + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1]) >= + GetValue(instruction->args[2])))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *cmp_sleHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ must only be used with integer types"); + using type_signed_t = typename std::make_signed::type; + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1]) <= + GetValue(instruction->args[2])))); + return AdvanceIP<1>(instruction); + } + + ALWAYS_INLINE inline const Instruction *sext_i8_i16Handler( + const Instruction *instruction) { + using src_t = typename std::make_signed::type; + using dest_t = typename std::make_signed::type; + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1])))); + return AdvanceIP<1>(instruction); + } + + ALWAYS_INLINE inline const Instruction *sext_i8_i32Handler( + const Instruction *instruction) { + using src_t = typename std::make_signed::type; + using dest_t = typename std::make_signed::type; + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1])))); + return AdvanceIP<1>(instruction); + } + + ALWAYS_INLINE inline const Instruction *sext_i8_i64Handler( + const Instruction *instruction) { + using src_t = typename std::make_signed::type; + using dest_t = typename std::make_signed::type; + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1])))); + return AdvanceIP<1>(instruction); + } + + ALWAYS_INLINE inline const Instruction *sext_i16_i32Handler( + const Instruction *instruction) { + using src_t = typename std::make_signed::type; + using dest_t = typename std::make_signed::type; + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1])))); + return AdvanceIP<1>(instruction); + } + + ALWAYS_INLINE inline const Instruction *sext_i16_i64Handler( + const Instruction *instruction) { + using src_t = typename std::make_signed::type; + using dest_t = typename std::make_signed::type; + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1])))); + return AdvanceIP<1>(instruction); + } + + ALWAYS_INLINE inline const Instruction *sext_i32_i64Handler( + const Instruction *instruction) { + using src_t = typename std::make_signed::type; + using dest_t = typename std::make_signed::type; + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1])))); + return AdvanceIP<1>(instruction); + } + + ALWAYS_INLINE inline const Instruction *zext_i8_i16Handler( + const Instruction *instruction) { + using src_t = i8; + using dest_t = i16; + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1])))); + return AdvanceIP<1>(instruction); + } + + ALWAYS_INLINE inline const Instruction *zext_i8_i32Handler( + const Instruction *instruction) { + using src_t = i8; + using dest_t = i32; + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1])))); + return AdvanceIP<1>(instruction); + } + + ALWAYS_INLINE inline const Instruction *zext_i8_i64Handler( + const Instruction *instruction) { + using src_t = i8; + using dest_t = i64; + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1])))); + return AdvanceIP<1>(instruction); + } + + ALWAYS_INLINE inline const Instruction *zext_i16_i32Handler( + const Instruction *instruction) { + using src_t = i16; + using dest_t = i32; + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1])))); + return AdvanceIP<1>(instruction); + } + + ALWAYS_INLINE inline const Instruction *zext_i16_i64Handler( + const Instruction *instruction) { + using src_t = i16; + using dest_t = i64; + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1])))); + return AdvanceIP<1>(instruction); + } + + ALWAYS_INLINE inline const Instruction *zext_i32_i64Handler( + const Instruction *instruction) { + using src_t = i32; + using dest_t = i64; + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1])))); + return AdvanceIP<1>(instruction); + } + + // The FP<>Int casts are created in a two-level hierarchy + // eg. the generated call to floattosiHandler is redirected to + // tosiHandler + + template + ALWAYS_INLINE inline const Instruction *tosiHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ dest_type must be an integer type"); + static_assert(std::is_floating_point::value, + "__func__ src_type must be a floating point type"); + using dest_type_signed_t = typename std::make_signed::type; + + SetValue( + instruction->args[0], (static_cast( + GetValue(instruction->args[1])))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *touiHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ dest_type must be an integer type"); + static_assert(std::is_floating_point::value, + "__func__ src_type must be a floating point type"); + + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1])))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *sitoHandler( + const Instruction *instruction) { + static_assert(std::is_floating_point::value, + "__func__ dest_type must be a floating point type"); + static_assert(std::is_integral::value, + "__func__ src_type must be an integer type"); + using src_type_signed_t = typename std::make_signed::type; + + SetValue(instruction->args[0], + (static_cast(GetValue( + instruction->args[1])))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *uitoHandler( + const Instruction *instruction) { + static_assert(std::is_floating_point::value, + "__func__ dest_type must be a floating point type"); + static_assert(std::is_integral::value, + "__func__ src_type must be an integer type"); + + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1])))); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *floattosiHandler( + const Instruction *instruction) { + return tosiHandler(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *floattouiHandler( + const Instruction *instruction) { + return touiHandler(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *sitofloatHandler( + const Instruction *instruction) { + return sitoHandler(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *uitofloatHandler( + const Instruction *instruction) { + return uitoHandler(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *doubletosiHandler( + const Instruction *instruction) { + return tosiHandler(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *doubletouiHandler( + const Instruction *instruction) { + return touiHandler(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *sitodoubleHandler( + const Instruction *instruction) { + return sitoHandler(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *uitodoubleHandler( + const Instruction *instruction) { + return uitoHandler(instruction); + } + + ALWAYS_INLINE inline const Instruction *doubletofloatHandler( + const Instruction *instruction) { + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1])))); + return AdvanceIP<1>(instruction); + } + + ALWAYS_INLINE inline const Instruction *floattodoubleHandler( + const Instruction *instruction) { + SetValue( + instruction->args[0], + (static_cast(GetValue(instruction->args[1])))); + return AdvanceIP<1>(instruction); + } + + ALWAYS_INLINE inline const Instruction *gep_offsetHandler( + const Instruction *instruction) { + uintptr_t sum = GetValue(instruction->args[1]) + + static_cast(instruction->args[2]); + SetValue(instruction->args[0], (sum)); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *gep_arrayHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ must only be used with integer types"); + uintptr_t product = + GetValue(instruction->args[1]) * instruction->args[2]; + SetValue(instruction->args[0], + (GetValue(instruction->args[0]) + product)); + return AdvanceIP<1>(instruction); + } + + ALWAYS_INLINE inline const Instruction *phi_movHandler( + const Instruction *instruction) { + SetValue(instruction->args[0], + (GetValue(instruction->args[1]))); + return AdvanceIP<1>(instruction); + } + + ALWAYS_INLINE inline const Instruction *selectHandler( + const Instruction *instruction) { + value_t result; + if (GetValue(instruction->args[1]) > 0) + result = GetValue(instruction->args[2]); + else + result = GetValue(instruction->args[3]); + + SetValue(instruction->args[0], (result)); + return AdvanceIP<2>(instruction); // bigger slot size! + } + + ALWAYS_INLINE inline const Instruction *call_externalHandler( + const Instruction *instruction) { + const ExternalCallInstruction *call_instruction = + reinterpret_cast(instruction); + CallActivation &call_activation = + call_activations_[call_instruction->external_call_context]; + + // call external function + ffi_call(&call_activation.call_interface, call_instruction->function, + call_activation.return_pointer, + reinterpret_cast(call_activation.value_pointers.data())); + + if (bytecode_function_ + .external_call_contexts_[call_instruction->external_call_context] + .dest_type != &ffi_type_void) { + DumpValue( + bytecode_function_ + .external_call_contexts_[call_instruction->external_call_context] + .dest_slot); + } + + return AdvanceIP<2>(instruction); // bigger slot size! + } + + ALWAYS_INLINE inline const Instruction *call_internalHandler( + const Instruction *instruction) { + const InternalCallInstruction *call_instruction = + reinterpret_cast(instruction); + + std::vector arguments(call_instruction->number_args); + for (size_t i = 0; i < call_instruction->number_args; i++) { + arguments[i] = GetValue(call_instruction->args[i]); + } + + value_t result = ExecuteFunction( + bytecode_function_.sub_functions_[call_instruction->sub_function], + arguments); + SetValue(call_instruction->dest_slot, result); + + return AdvanceIP( + instruction, + bytecode_function_.GetInteralCallInstructionSlotSize(call_instruction)); + } + + ALWAYS_INLINE inline const Instruction *nop_movHandler( + const Instruction *instruction) { + SetValue(instruction->args[0], + (GetValue(instruction->args[1]))); + return AdvanceIP<1>(instruction); + } + + ALWAYS_INLINE inline const Instruction *branch_uncondHandler( + const Instruction *instruction) { + return bytecode_function_.GetIPFromIndex(instruction->args[0]); + } + + ALWAYS_INLINE inline const Instruction *branch_condHandler( + const Instruction *instruction) { + index_t next_bb; + if (GetValue(instruction->args[0]) > 0) + next_bb = instruction->args[2]; + else + next_bb = instruction->args[1]; + + return bytecode_function_.GetIPFromIndex(next_bb); + } + + ALWAYS_INLINE inline const Instruction *branch_cond_ftHandler( + const Instruction *instruction) { + const Instruction *ip; + if ((GetValue(instruction->args[0]) & 0x1) > 0) + ip = bytecode_function_.GetIPFromIndex(instruction->args[1]); + else + ip = AdvanceIP<1>(instruction); + + return ip; + } + + ALWAYS_INLINE inline const Instruction *llvm_memcpyHandler( + const Instruction *instruction) { + PELOTON_MEMCPY(GetValue(instruction->args[0]), + GetValue(instruction->args[1]), + GetValue(instruction->args[2])); + return AdvanceIP<1>(instruction); + } + + ALWAYS_INLINE inline const Instruction *llvm_memmoveHandler( + const Instruction *instruction) { + std::memmove(GetValue(instruction->args[0]), + GetValue(instruction->args[1]), + GetValue(instruction->args[2])); + return AdvanceIP<1>(instruction); + } + + ALWAYS_INLINE inline const Instruction *llvm_memsetHandler( + const Instruction *instruction) { + PELOTON_MEMSET(GetValue(instruction->args[0]), + GetValue(instruction->args[1]), + GetValue(instruction->args[2])); + return AdvanceIP<1>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *llvm_uadd_overflowHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ must only be used with integer types"); + bool overflow = __builtin_add_overflow( + GetValue(instruction->args[2]), + GetValue(instruction->args[3]), + &GetValueReference(instruction->args[0])); + + DumpValue(instruction->args[0]); + + SetValue(instruction->args[1], (static_cast(overflow))); + return AdvanceIP<2>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *llvm_sadd_overflowHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ must only be used with integer types"); + using type_signed_t = typename std::make_signed::type; + bool overflow = __builtin_add_overflow( + GetValue(instruction->args[2]), + GetValue(instruction->args[3]), + &GetValueReference(instruction->args[0])); + + DumpValue(instruction->args[0]); + + SetValue(instruction->args[1], (static_cast(overflow))); + return AdvanceIP<2>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *llvm_usub_overflowHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ must only be used with integer types"); + bool overflow = __builtin_sub_overflow( + GetValue(instruction->args[2]), + GetValue(instruction->args[3]), + &GetValueReference(instruction->args[0])); + + DumpValue(instruction->args[0]); + + SetValue(instruction->args[1], (static_cast(overflow))); + return AdvanceIP<2>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *llvm_ssub_overflowHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ must only be used with integer types"); + using type_signed_t = typename std::make_signed::type; + bool overflow = __builtin_sub_overflow( + GetValue(instruction->args[2]), + GetValue(instruction->args[3]), + &GetValueReference(instruction->args[0])); + + DumpValue(instruction->args[0]); + + SetValue(instruction->args[1], (static_cast(overflow))); + return AdvanceIP<2>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *llvm_umul_overflowHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ must only be used with integer types"); + bool overflow = __builtin_mul_overflow( + GetValue(instruction->args[2]), + GetValue(instruction->args[3]), + &GetValueReference(instruction->args[0])); + + DumpValue(instruction->args[0]); + + SetValue(instruction->args[1], (static_cast(overflow))); + return AdvanceIP<2>(instruction); + } + + template + ALWAYS_INLINE inline const Instruction *llvm_smul_overflowHandler( + const Instruction *instruction) { + static_assert(std::is_integral::value, + "__func__ must only be used with integer types"); + using type_signed_t = typename std::make_signed::type; + bool overflow = __builtin_mul_overflow( + GetValue(instruction->args[2]), + GetValue(instruction->args[3]), + &GetValueReference(instruction->args[0])); + + DumpValue(instruction->args[0]); + + SetValue(instruction->args[1], (static_cast(overflow))); + return AdvanceIP<2>(instruction); + } + + ALWAYS_INLINE inline const Instruction *llvm_sse42_crc32Handler( + const Instruction *instruction) { + SetValue( + instruction->args[0], + (__builtin_ia32_crc32di(GetValue(instruction->args[1]), + GetValue(instruction->args[2])))); + return AdvanceIP<1>(instruction); + } + + // The handlers for explicit calls are generated using templates. + // + // The call arrives in explicit_callHandler(...), which is overloaded for + // 1. static functions + // 2. class methods and + // 3. const class methods + // and is then forwarded to explicit_call_wrapperHandler(...) which is tagged + // by a bool type, whether the called function returns void or not, which + // makes it 6 instances of that function. + + // 1. static function + template + ALWAYS_INLINE inline const Instruction *explicit_callHandler( + const Instruction *instruction, return_type (*func)(arg_types...)) { + // forward call depending on whether func returns void or not + return explicit_call_wrapperHandler(instruction, func, + gen_seq(), + std::is_void()); + } + + // 2. class method + template + ALWAYS_INLINE inline const Instruction *explicit_callHandler( + const Instruction *instruction, + return_type (class_type::*func)(arg_types...)) { + // forward call depending on whether func returns void or not + return explicit_call_wrapperHandler(instruction, func, + gen_seq(), + std::is_void()); + } + + // 3. const class method + template + ALWAYS_INLINE inline const Instruction *explicit_callHandler( + const Instruction *instruction, + return_type (class_type::*func)(arg_types...) const) { + // forward call depending on whether func returns void or not + return explicit_call_wrapperHandler(instruction, func, + gen_seq(), + std::is_void()); + } + + // 1. static function a) returns void + template + ALWAYS_INLINE inline const Instruction *explicit_call_wrapperHandler( + const Instruction *instruction, + UNUSED_ATTRIBUTE return_type (*func)(arg_types...), + const seq &, UNUSED_ATTRIBUTE std::false_type returns_void) { + // call the actual function + auto ret = func(GetValue(instruction->args[indexes + 1])...); + SetValue(instruction->args[0], ret); + + return AdvanceIP(instruction); + } + + // 1. static function b) returns non-void + template + ALWAYS_INLINE inline const Instruction *explicit_call_wrapperHandler( + const Instruction *instruction, + UNUSED_ATTRIBUTE return_type (*func)(arg_types...), + const seq &, + UNUSED_ATTRIBUTE std::true_type returns_not_void) { + // call the actual function + func(GetValue(instruction->args[indexes])...); + + return AdvanceIP(instruction); + } + + // 2. class method a) returns void + template + ALWAYS_INLINE inline const Instruction *explicit_call_wrapperHandler( + const Instruction *instruction, + UNUSED_ATTRIBUTE return_type (class_type::*func)(arg_types...), + const seq &, UNUSED_ATTRIBUTE std::false_type returns_void) { + // call the actual function + auto *obj = GetValue(instruction->args[1]); + return_type ret = + (obj->*func)(GetValue(instruction->args[indexes + 2])...); + SetValue(instruction->args[0], ret); + + return AdvanceIP(instruction); + } + + // 2. class method b) returns non-void + template + ALWAYS_INLINE inline const Instruction *explicit_call_wrapperHandler( + const Instruction *instruction, + UNUSED_ATTRIBUTE return_type (class_type::*func)(arg_types...), + const seq &, + UNUSED_ATTRIBUTE std::true_type returns_not_void) { + // call the actual function + auto *obj = GetValue(instruction->args[0]); + (obj->*func)(GetValue(instruction->args[indexes + 1])...); + + return AdvanceIP(instruction); + } + + // 3. const class method a) returns void + template + ALWAYS_INLINE inline const Instruction *explicit_call_wrapperHandler( + const Instruction *instruction, + UNUSED_ATTRIBUTE return_type (class_type::*func)(arg_types...) const, + const seq &, UNUSED_ATTRIBUTE std::false_type returns_void) { + // call the actual function + auto *obj = GetValue(instruction->args[1]); + return_type ret = + (obj->*func)(GetValue(instruction->args[indexes + 2])...); + SetValue(instruction->args[0], ret); + + return AdvanceIP(instruction); + } + + // 3. const class method b) returns non-void + template + ALWAYS_INLINE inline const Instruction *explicit_call_wrapperHandler( + const Instruction *instruction, + UNUSED_ATTRIBUTE return_type (class_type::*func)(arg_types...) const, + const seq &, + UNUSED_ATTRIBUTE std::true_type returns_not_void) { + // call the actual function + auto *obj = GetValue(instruction->args[0]); + (obj->*func)(GetValue(instruction->args[indexes + 1])...); + + return AdvanceIP(instruction); + } + + //--------------------------------------------------------------------------// + + private: + /** + * This static array holds the goto-pointer for the dispatch area for each + * Opcode. It will be filled once, when the interpreter is called the + * first time. + */ + static void *label_pointers_[BytecodeFunction::GetNumberOpcodes()]; + + /** + * Value slots (register) for the current function activation. + * (Aligned by something that is most likely the cache line size) + */ + alignas(64) std::vector values_; + + /** + * Holds all allocations made with alloca. We do not need to access them, + * but the unique pointer ensures they will be released at the end. + */ + std::vector> allocations_; + + /** + * Holds the call activation records for all external call instructions. + * (Created during initialization) + */ + std::vector call_activations_; + + /** + * Bytecode function used for execution. + */ + const BytecodeFunction &bytecode_function_; + + private: + // This class cannot be copy or move-constructed + DISALLOW_COPY_AND_MOVE(BytecodeInterpreter); +}; + +} // namespace interpreter +} // namespace codegen +} // namespace peloton \ No newline at end of file diff --git a/src/include/codegen/proxy/proxy.h b/src/include/codegen/proxy/proxy.h index 8e647e9e2ac..ab518bd59ae 100644 --- a/src/include/codegen/proxy/proxy.h +++ b/src/include/codegen/proxy/proxy.h @@ -237,7 +237,7 @@ struct MemFn { ::peloton::codegen::CodeGen &codegen) { \ static constexpr const char *kFnName = STR(NS::C::F); \ /* If the function has already been defined, return it. */ \ - if (::llvm::Function *func = codegen.LookupBuiltin(kFnName)) { \ + if (::llvm::Function *func = codegen.LookupBuiltin(kFnName).first) { \ return func; \ } \ \ diff --git a/src/include/codegen/query.h b/src/include/codegen/query.h index eecb5a175a5..141b5303b7d 100644 --- a/src/include/codegen/query.h +++ b/src/include/codegen/query.h @@ -13,9 +13,9 @@ #pragma once #include "codegen/code_context.h" +#include "codegen/parameter_cache.h" #include "codegen/query_parameters.h" #include "codegen/query_state.h" -#include "codegen/parameter_cache.h" namespace peloton { @@ -41,18 +41,39 @@ class ExecutionConsumer; //===----------------------------------------------------------------------===// class Query { public: + struct CompileStats { + double compile_ms = 0.0; + }; + struct RuntimeStats { + double interpreter_prepare_ms = 0.0; double init_ms = 0.0; double plan_ms = 0.0; double tear_down_ms = 0.0; }; - struct QueryFunctions { + // We use this handy class for the parameters to the llvm functions + // to avoid complex casting and pointer manipulation + struct FunctionArguments { + executor::ExecutorContext *executor_context; + char *consumer_arg; + char rest[0]; + } PACKED; + + struct LLVMFunctions { llvm::Function *init_func; llvm::Function *plan_func; llvm::Function *tear_down_func; }; + using compiled_function_t = void (*)(FunctionArguments *); + + struct CompiledFunctions { + compiled_function_t init_func; + compiled_function_t plan_func; + compiled_function_t tear_down_func; + }; + /// This class cannot be copy or move-constructed DISALLOW_COPY_AND_MOVE(Query); @@ -61,7 +82,10 @@ class Query { * * @param funcs The compiled functions that implement the logic of the query */ - bool Prepare(const QueryFunctions &funcs); + void Prepare(const LLVMFunctions &funcs); + + // Compiles the function in this query to native code + void Compile(CompileStats *stats = nullptr); /** * @brief Executes the compiled query. @@ -94,6 +118,14 @@ class Query { /// Constructor. Private so callers use the QueryCompiler class. explicit Query(const planner::AbstractPlan &query_plan); + // Execute the query as native code (must already be compiled) + void ExecuteNative(FunctionArguments *function_arguments, + RuntimeStats *stats); + + // Execute the query using the interpreter + void ExecuteInterpreter(FunctionArguments *function_arguments, + RuntimeStats *stats); + private: // The query plan const planner::AbstractPlan &query_plan_; @@ -104,11 +136,14 @@ class Query { // The size of the parameter the functions take QueryState query_state_; - // The init(), plan() and tearDown() functions - typedef void (*compiled_function_t)(char *); - compiled_function_t init_func_; - compiled_function_t plan_func_; - compiled_function_t tear_down_func_; + // LLVM IR of the query functions + LLVMFunctions llvm_functions_; + + // Pointers to the compiled query functions + CompiledFunctions compiled_functions_; + + // Shows if the query has been compiled to native code + bool is_compiled_; }; } // namespace codegen diff --git a/src/include/codegen/query_compiler.h b/src/include/codegen/query_compiler.h index 377bafcada9..fd2c0b466fd 100644 --- a/src/include/codegen/query_compiler.h +++ b/src/include/codegen/query_compiler.h @@ -43,8 +43,8 @@ class QueryCompiler { // The time taken to generate all the IR for the plan double ir_gen_ms = 0.0; - // The time taken to perform JIT compilation - double jit_ms = 0.0; + // Time consumed by LLVM Optimizer + double optimize_ms = 0.0; }; // Constructor diff --git a/src/include/common/macros.h b/src/include/common/macros.h index 96aaf6ab0d2..593619bffb4 100644 --- a/src/include/common/macros.h +++ b/src/include/common/macros.h @@ -29,7 +29,19 @@ namespace peloton { //===--------------------------------------------------------------------===// #define NEVER_INLINE __attribute__((noinline)) + +#ifdef NDEBUG #define ALWAYS_INLINE __attribute__((always_inline)) +#else +#define ALWAYS_INLINE +#endif + +#ifdef __clang__ +#define NO_CLONE +#else +#define NO_CLONE __attribute__((noclone)) +#endif + #define UNUSED_ATTRIBUTE __attribute__((unused)) #define PACKED __attribute__((packed)) @@ -106,6 +118,19 @@ namespace peloton { #define GCC_AT_LEAST_6 0 #endif +#if __GNUC__ > 5 || (__GNUC__ == 5 && __GNUC_MINOR__ >= 1) +#define GCC_AT_LEAST_51 1 +#else +#define GCC_AT_LEAST_51 0 +#endif + +// g++-5.0 does not support overflow builtins +#if GCC_AT_LEAST_51 +#define GCC_OVERFLOW_BUILTINS_DEFINED 1 +#else +#define GCC_OVERFLOW_BUILTINS_DEFINED 0 +#endif + //===--------------------------------------------------------------------===// // Port to OSX //===--------------------------- diff --git a/src/include/common/overflow_builtins.h b/src/include/common/overflow_builtins.h new file mode 100644 index 00000000000..d77aae01002 --- /dev/null +++ b/src/include/common/overflow_builtins.h @@ -0,0 +1,78 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// overflow_builtins.h +// +// Identification: src/include/common/overflow_builtins.h +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "common/macros.h" + +#include + +//----------------------------------------------------------------------------// +// Fall back implementations if the gcc overflow builtins are not available. +// +// Documentation: +// https://gcc.gnu.org/onlinedocs/gcc/Integer-Overflow-Builtins.html +//----------------------------------------------------------------------------// + +namespace peloton { + +template +static inline bool builtin_add_overflow(type_t a, type_t b, type_t *res) { + *res = a + b; + + if (a >= 0 && b >= 0 && std::numeric_limits::max() - a < b) + return true; + else if (a < 0 && b < 0 && std::numeric_limits::min() - a > b) + return true; + + return false; +} + +template +static inline bool builtin_sub_overflow(type_t a, type_t b, type_t *res) { + *res = a - b; + + if (std::is_unsigned::value) + return b > a; + else + return ((((a ^ b)) & (*res ^ a)) & std::numeric_limits::min()) != 0; +} + +template +static inline bool builtin_mul_overflow(type_t a, type_t b, type_t *res) { + *res = a * b; + + if (a != 0 && *res / a != b) return true; + + return false; +} + +#if !GCC_OVERFLOW_BUILTINS_DEFINED + +template +static inline bool __builtin_add_overflow(type_t a, type_t b, type_t *res) { + return builtin_add_overflow(a, b, res); +} + +template +static inline bool __builtin_sub_overflow(type_t a, type_t b, type_t *res) { + return builtin_sub_overflow(a, b, res); +} + +template +static inline bool __builtin_mul_overflow(type_t a, type_t b, type_t *res) { + return builtin_mul_overflow(a, b, res); +} + +#endif + +} // namespace peloton diff --git a/src/include/settings/settings.h b/src/include/settings/settings.h index 757cc9043e6..a442fd151cf 100644 --- a/src/include/settings/settings.h +++ b/src/include/settings/settings.h @@ -200,6 +200,10 @@ SETTING_bool(codegen, true, true, true) +SETTING_bool(codegen_interpreter, + "Force interpretation of generated llvm code (default: false)", + false, true, true) + SETTING_bool(print_ir_stats, "Print statistics on generated IR (default: false)", false, diff --git a/src/include/util/math_util.h b/src/include/util/math_util.h new file mode 100644 index 00000000000..b4894421959 --- /dev/null +++ b/src/include/util/math_util.h @@ -0,0 +1,35 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// math_util.h +// +// Identification: src/include/util/math_util.h +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "common/macros.h" + +namespace peloton { + +/** + * Math Utility Functions + */ +class MathUtil { + public: + /** + * Performs a division of two integer values and rounds up the result. + * Calculation is made using a trick with integer division. + */ + static constexpr ALWAYS_INLINE inline size_t DivRoundUp(size_t numerator, + size_t denominator) { + // division must be integer division + return (numerator + denominator - 1) / denominator; + } +}; + +} // namespace peloton diff --git a/test/codegen/bloom_filter_test.cpp b/test/codegen/bloom_filter_test.cpp index 032a4ef2250..ef6bf168c05 100644 --- a/test/codegen/bloom_filter_test.cpp +++ b/test/codegen/bloom_filter_test.cpp @@ -165,7 +165,7 @@ TEST_F(BloomFilterCodegenTest, FalsePositiveRateTest) { func.ReturnAndFinish(); } - ASSERT_TRUE(code_context.Compile()); + code_context.Compile(); typedef void (*ftype)(codegen::util::BloomFilter * bloom_filter, int *, int, int *); @@ -312,6 +312,7 @@ double BloomFilterCodegenTest::ExecuteJoin(std::string query, *plan, executor_context.GetParams().GetQueryParametersMap(), consumer); // Run + compiled_query->Compile(); compiled_query->Execute(executor_context, consumer, &stats); LOG_INFO("Execution Time: %0.0f ms", stats.plan_ms); diff --git a/test/codegen/bytecode_interpreter_test.cpp b/test/codegen/bytecode_interpreter_test.cpp new file mode 100644 index 00000000000..9231a92c901 --- /dev/null +++ b/test/codegen/bytecode_interpreter_test.cpp @@ -0,0 +1,242 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// bytecode_interpreter_test.cpp +// +// Identification: test/codegen/bytecode_interpreter_test.cpp +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "codegen/interpreter/bytecode_interpreter.h" +#include "codegen/function_builder.h" +#include "codegen/interpreter/bytecode_builder.h" +#include "codegen/lang/loop.h" +#include "codegen/proxy/runtime_functions_proxy.h" +#include "common/harness.h" + +namespace peloton { +namespace test { + +class BytecodeInterpreterTest : public PelotonTest {}; + +TEST_F(BytecodeInterpreterTest, PHIResolveTest) { + // Create a loop that involves PHIs that have to be converted into move + // instructions. + + codegen::CodeContext code_context; + codegen::CodeGen cg{code_context}; + codegen::FunctionBuilder main{ + code_context, "main", cg.Int32Type(), {{"a", cg.Int32Type()}}}; + { + auto *a = main.GetArgumentByPosition(0); + auto *i = cg.Const32(0); + + codegen::lang::Loop loop{cg, cg.ConstBool(true), {{"i", i}, {"a", a}}}; + { + llvm::Value *i = loop.GetLoopVar(0); + llvm::Value *a = loop.GetLoopVar(1); + + a = cg->CreateSub(a, cg.Const32(1)); + i = cg->CreateAdd(i, cg.Const32(1)); + loop.LoopEnd(cg->CreateICmpULT(i, cg.Const32(10)), {i, a}); + } + + std::vector final; + loop.CollectFinalLoopVariables(final); + + auto *ret = final[1]; + main.ReturnAndFinish(ret); + } + + // create Bytecode + auto bytecode = codegen::interpreter::BytecodeBuilder::CreateBytecodeFunction( + code_context, main.GetFunction()); + + // run Bytecode + codegen::interpreter::value_t arg = 44; + codegen::interpreter::value_t ret = + codegen::interpreter::BytecodeInterpreter::ExecuteFunction(bytecode, + {arg}); + ASSERT_EQ(ret, arg - 10); +} + +TEST_F(BytecodeInterpreterTest, PHISwapProblemTest) { + // Produce the PHI swap problem, where additional moves have to be inserted + // in order to retrieve the correct result. + + codegen::CodeContext code_context; + codegen::CodeGen cg{code_context}; + codegen::FunctionBuilder main{ + code_context, "main", cg.Int32Type(), {{"a", cg.Int32Type()}}}; + { + auto *a = main.GetArgumentByPosition(0); + auto *b = cg.Const32(0); + auto *i = cg.Const32(0); + + codegen::lang::Loop loop{ + cg, cg.ConstBool(true), {{"i", i}, {"a", a}, {"b", b}}}; + { + llvm::Value *i = loop.GetLoopVar(0); + llvm::Value *a = loop.GetLoopVar(1); + llvm::Value *b = loop.GetLoopVar(2); + + i = cg->CreateAdd(i, cg.Const32(1)); + loop.LoopEnd(cg->CreateICmpULT(i, cg.Const32(2)), {i, b, a}); + } + + std::vector final; + loop.CollectFinalLoopVariables(final); + + auto *ret = final[1]; + main.ReturnAndFinish(ret); + } + + // create Bytecode + auto bytecode = codegen::interpreter::BytecodeBuilder::CreateBytecodeFunction( + code_context, main.GetFunction()); + + // run Bytecode + codegen::interpreter::value_t arg = 44; + codegen::interpreter::value_t ret = + codegen::interpreter::BytecodeInterpreter::ExecuteFunction(bytecode, + {arg}); + ASSERT_EQ(ret, arg); +} + +TEST_F(BytecodeInterpreterTest, OverflowIntrinsicsTest) { + // Use the overflow intrinsics and retrieve their output. During bytecode + // translation the extract instructions get omited and the values are written + // directly to their destination value slot. + + // We call the intrinsics several times and check the result statically + // right in the generated function. We merge all checks with AND and return + // it to the test case at the end. + + codegen::CodeContext code_context; + codegen::CodeGen cg{code_context}; + codegen::FunctionBuilder main{code_context, + "main", + cg.Int32Type(), + {{"a", cg.Int32Type()}, {"b", cg.Int32Type()}}}; + { + auto *a = main.GetArgumentByPosition(0); + auto *b = main.GetArgumentByPosition(1); + llvm::Value *add_overflow, *sub_overflow; + llvm::Value *ret = cg.ConstBool(true); + + auto *add_result = cg.CallAddWithOverflow(a, b, add_overflow); + auto *add_result_correct = cg->CreateICmp(llvm::CmpInst::Predicate::ICMP_EQ, + add_result, cg.Const32(10)); + ret = cg->CreateAnd(ret, add_result_correct); + auto *add_overflow_correct = cg->CreateNot(add_overflow); + ret = cg->CreateAnd(ret, add_overflow_correct); + + auto *sub_result = + cg.CallSubWithOverflow(cg.Const32(2147483648), b, sub_overflow); + auto *sub_result_correct = cg->CreateICmp( + llvm::CmpInst::Predicate::ICMP_EQ, sub_result, cg.Const32(2147483642)); + ret = cg->CreateAnd(ret, sub_result_correct); + ret = cg->CreateAnd(ret, sub_overflow); + + main.ReturnAndFinish(ret); + } + + // create Bytecode + auto bytecode = codegen::interpreter::BytecodeBuilder::CreateBytecodeFunction( + code_context, main.GetFunction()); + + // run Bytecode + codegen::interpreter::value_t ret = + codegen::interpreter::BytecodeInterpreter::ExecuteFunction(bytecode, + {4, 6}); + ASSERT_EQ(ret, 1); +} + +int f(int a, int b) { return a + b; } + +TEST_F(BytecodeInterpreterTest, ExternalCallTest) { + // Call an external function. + + codegen::CodeContext code_context; + codegen::CodeGen cg{code_context}; + + // create LLVM function declaration + auto *func_type = llvm::FunctionType::get( + cg.Int32Type(), {cg.Int32Type(), cg.Int32Type()}, false); + llvm::Function *func_decl = + llvm::Function::Create(func_type, llvm::Function::ExternalLinkage, "f", + &(cg.GetCodeContext().GetModule())); + code_context.RegisterExternalFunction(func_decl, (void *)f); + + codegen::FunctionBuilder main{code_context, + "main", + cg.Int32Type(), + {{"a", cg.Int32Type()}, {"b", cg.Int32Type()}}}; + { + auto *a = main.GetArgumentByPosition(0); + auto *b = main.GetArgumentByPosition(1); + + auto *ret = cg.CallFunc(func_decl, {a, b}); + + main.ReturnAndFinish(ret); + } + + // create Bytecode + auto bytecode = codegen::interpreter::BytecodeBuilder::CreateBytecodeFunction( + code_context, main.GetFunction()); + + // run Bytecode + codegen::interpreter::value_t ret = + codegen::interpreter::BytecodeInterpreter::ExecuteFunction(bytecode, + {4, 6}); + ASSERT_EQ(ret, 10); +} + +TEST_F(BytecodeInterpreterTest, InternalCallTest) { + // Call an internal function. + + codegen::CodeContext code_context; + codegen::CodeGen cg{code_context}; + + codegen::FunctionBuilder f{code_context, + "f", + cg.Int32Type(), + {{"a", cg.Int32Type()}, {"b", cg.Int32Type()}}}; + { + auto *a = f.GetArgumentByPosition(0); + auto *b = f.GetArgumentByPosition(1); + + auto *ret = cg->CreateAdd(a, b); + + f.ReturnAndFinish(ret); + } + + codegen::FunctionBuilder main{code_context, + "main", + cg.Int32Type(), + {{"a", cg.Int32Type()}, {"b", cg.Int32Type()}}}; + { + auto *a = main.GetArgumentByPosition(0); + auto *b = main.GetArgumentByPosition(1); + + auto *ret = cg.CallFunc(f.GetFunction(), {a, b}); + + main.ReturnAndFinish(ret); + } + + // create Bytecode + auto bytecode = codegen::interpreter::BytecodeBuilder::CreateBytecodeFunction( + code_context, main.GetFunction()); + + // run Bytecode + codegen::interpreter::value_t ret = + codegen::interpreter::BytecodeInterpreter::ExecuteFunction(bytecode, + {4, 6}); + ASSERT_EQ(ret, 10); +} + +} // namespace test +} // namespace peloton \ No newline at end of file diff --git a/test/codegen/function_builder_test.cpp b/test/codegen/function_builder_test.cpp index 1822e384f72..9235da51bb8 100644 --- a/test/codegen/function_builder_test.cpp +++ b/test/codegen/function_builder_test.cpp @@ -35,7 +35,7 @@ TEST_F(FunctionBuilderTest, ConstructSingleFunction) { func.ReturnAndFinish(cg.Const32(magic_num)); } - ASSERT_TRUE(code_context.Compile()); + code_context.Compile(); typedef int (*func_t)(void); func_t fn = (func_t) code_context.GetRawFunctionPointer(func.GetFunction()); @@ -80,7 +80,7 @@ TEST_F(FunctionBuilderTest, ConstructNestedFunction) { } // Make sure we can compile everything - ASSERT_TRUE(code_context.Compile()); + code_context.Compile(); typedef int (*func_t)(uint32_t); func_t fn = (func_t) code_context.GetRawFunctionPointer(main.GetFunction()); diff --git a/test/codegen/if_test.cpp b/test/codegen/if_test.cpp index 4c198cea2d3..6c00153269a 100644 --- a/test/codegen/if_test.cpp +++ b/test/codegen/if_test.cpp @@ -59,7 +59,7 @@ TEST_F(IfTest, TestIfOnly) { func.ReturnAndFinish(cond.BuildPHI(va, vb).GetValue()); } - ASSERT_TRUE(code_context.Compile()); + code_context.Compile(); typedef int (*ftype)(int); @@ -128,7 +128,7 @@ TEST_F(IfTest, TestIfInsideLoop) { func.ReturnAndFinish(final[1]); } - ASSERT_TRUE(code_context.Compile()); + code_context.Compile(); typedef int (*ftype)(int); @@ -174,7 +174,7 @@ TEST_F(IfTest, BreakTest) { func.ReturnAndFinish(final[0]); } - ASSERT_TRUE(code_context.Compile()); + code_context.Compile(); typedef int (*ftype)(int); @@ -235,7 +235,7 @@ TEST_F(IfTest, ComplexNestedIf) { func.ReturnAndFinish(cond.BuildPHI(vab, vc).GetValue()); } - ASSERT_TRUE(code_context.Compile()); + code_context.Compile(); typedef int (*ftype)(int); diff --git a/test/codegen/testing_codegen_util.cpp b/test/codegen/testing_codegen_util.cpp index a19598e33ed..b587aeac199 100644 --- a/test/codegen/testing_codegen_util.cpp +++ b/test/codegen/testing_codegen_util.cpp @@ -257,7 +257,7 @@ void PelotonCodeGenTest::CreateAndLoadTableWithLayout( txn_manager.CommitTransaction(txn); } -codegen::QueryCompiler::CompileStats PelotonCodeGenTest::CompileAndExecute( +PelotonCodeGenTest::CodeGenStats PelotonCodeGenTest::CompileAndExecute( planner::AbstractPlan &plan, codegen::ExecutionConsumer &consumer) { codegen::QueryParameters parameters(plan, {}); @@ -266,15 +266,18 @@ codegen::QueryCompiler::CompileStats PelotonCodeGenTest::CompileAndExecute( auto *txn = txn_manager.BeginTransaction(); // Compile the query. - codegen::QueryCompiler::CompileStats stats; + CodeGenStats stats; auto query = codegen::QueryCompiler().Compile( - plan, parameters.GetQueryParametersMap(), consumer, &stats); + plan, parameters.GetQueryParametersMap(), consumer, &stats.compile_stats); // Executor context executor::ExecutorContext exec_ctx{txn, std::move(parameters)}; - // Execute the query - query->Execute(exec_ctx, consumer); + // Compile Query to native code + query->Compile(); + + // Execute the quer + query->Execute(exec_ctx, consumer, &stats.runtime_stats); // Commit the transaction. txn_manager.CommitTransaction(txn); @@ -282,7 +285,7 @@ codegen::QueryCompiler::CompileStats PelotonCodeGenTest::CompileAndExecute( return stats; } -codegen::QueryCompiler::CompileStats PelotonCodeGenTest::CompileAndExecuteCache( +PelotonCodeGenTest::CodeGenStats PelotonCodeGenTest::CompileAndExecuteCache( std::shared_ptr plan, codegen::ExecutionConsumer &consumer, bool &cached, std::vector params) { @@ -294,19 +297,20 @@ codegen::QueryCompiler::CompileStats PelotonCodeGenTest::CompileAndExecuteCache( codegen::QueryParameters(*plan, params)}; // Compile - codegen::QueryCompiler::CompileStats stats; + CodeGenStats stats; codegen::Query *query = codegen::QueryCache::Instance().Find(plan); cached = (query != nullptr); if (query == nullptr) { codegen::QueryCompiler compiler; auto compiled_query = compiler.Compile( *plan, exec_ctx.GetParams().GetQueryParametersMap(), consumer); + compiled_query->Compile(); query = compiled_query.get(); codegen::QueryCache::Instance().Add(plan, std::move(compiled_query)); } // Execute the query. - query->Execute(exec_ctx, consumer); + query->Execute(exec_ctx, consumer, &stats.runtime_stats); // Commit the transaction. txn_manager.CommitTransaction(txn); diff --git a/test/codegen/value_integrity_test.cpp b/test/codegen/value_integrity_test.cpp index 0057721352b..97dfeca23e4 100644 --- a/test/codegen/value_integrity_test.cpp +++ b/test/codegen/value_integrity_test.cpp @@ -67,7 +67,7 @@ void DivideByZeroTest(const codegen::type::Type &data_type, ExpressionType op) { } // Should be able to compile - EXPECT_TRUE(code_context.Compile()); + code_context.Compile(); typedef void (*func)(CType); func f = (func)code_context.GetRawFunctionPointer(function.GetFunction()); @@ -134,7 +134,7 @@ void OverflowTest(const codegen::type::Type &data_type, ExpressionType op) { } // Should be able to compile - EXPECT_TRUE(code_context.Compile()); + code_context.Compile(); typedef void (*func)(CType); func f = (func)code_context.GetRawFunctionPointer(function.GetFunction()); diff --git a/test/common/overflow_builtins_test.cpp b/test/common/overflow_builtins_test.cpp new file mode 100644 index 00000000000..8abdd86fee9 --- /dev/null +++ b/test/common/overflow_builtins_test.cpp @@ -0,0 +1,212 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// overflow_builtins_test.cpp +// +// Identification: test/common/overflow_builtins_test.cpp +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "common/overflow_builtins.h" + +#include "common/harness.h" + +#include + +namespace peloton { +namespace test { + +//===--------------------------------------------------------------------===// +// Testing the fallback functions for overflow aware operations +//===--------------------------------------------------------------------===// + +class OverflowBuiltinsTest : public PelotonTest { + public: + using unsigned_t = unsigned int; + using signed_t = int; +}; + +TEST_F(OverflowBuiltinsTest, UnsignedAddTest) { + unsigned_t max = std::numeric_limits::max(); + unsigned_t min = std::numeric_limits::min(); + + unsigned_t c; + bool overflow; + + overflow = builtin_add_overflow(0, 3, &c); + EXPECT_EQ(c, 3); + EXPECT_EQ(overflow, false); + + overflow = builtin_add_overflow(0, 0, &c); + EXPECT_EQ(c, 0); + EXPECT_EQ(overflow, false); + + overflow = builtin_add_overflow(max - 12, 3, &c); + EXPECT_EQ(c, max - 9); + EXPECT_EQ(overflow, false); + + overflow = builtin_add_overflow(max - 12, 12, &c); + EXPECT_EQ(c, max); + EXPECT_EQ(overflow, false); + + overflow = builtin_add_overflow(max - 12, 13, &c); + EXPECT_EQ(c, min); + EXPECT_EQ(overflow, true); + + overflow = builtin_add_overflow(max - 12, 21, &c); + EXPECT_EQ(c, min + 8); + EXPECT_EQ(overflow, true); +} + +TEST_F(OverflowBuiltinsTest, SignedAddTest) { + signed_t max = std::numeric_limits::max(); + signed_t min = std::numeric_limits::min(); + + signed_t c; + bool overflow; + + overflow = builtin_add_overflow(min, 3, &c); + EXPECT_EQ(c, min + 3); + EXPECT_EQ(overflow, false); + + overflow = builtin_add_overflow(min, 0, &c); + EXPECT_EQ(c, min); + EXPECT_EQ(overflow, false); + + overflow = builtin_add_overflow(0, -12, &c); + EXPECT_EQ(c, -12); + EXPECT_EQ(overflow, false); + + overflow = builtin_add_overflow(min, -1, &c); + EXPECT_EQ(c, max); + EXPECT_EQ(overflow, true); + + overflow = builtin_add_overflow(max, 1, &c); + EXPECT_EQ(c, min); + EXPECT_EQ(overflow, true); + + overflow = builtin_add_overflow(0, -13, &c); + EXPECT_EQ(c, -13); + EXPECT_EQ(overflow, false); + + overflow = builtin_add_overflow(-12, 13, &c); + EXPECT_EQ(c, 1); + EXPECT_EQ(overflow, false); +} + +TEST_F(OverflowBuiltinsTest, UnsignedSubTest) { + unsigned_t max = std::numeric_limits::max(); + unsigned_t min = std::numeric_limits::min(); + + unsigned_t c; + bool overflow; + + overflow = builtin_sub_overflow(3, 3, &c); + EXPECT_EQ(c, 0); + EXPECT_EQ(overflow, false); + + overflow = builtin_sub_overflow(0, 0, &c); + EXPECT_EQ(c, 0); + EXPECT_EQ(overflow, false); + + overflow = builtin_sub_overflow(min + 12, 3, &c); + EXPECT_EQ(c, min + 9); + EXPECT_EQ(overflow, false); + + overflow = builtin_sub_overflow(min + 12, 12, &c); + EXPECT_EQ(c, min); + EXPECT_EQ(overflow, false); + + overflow = builtin_sub_overflow(min + 12, 13, &c); + EXPECT_EQ(c, max); + EXPECT_EQ(overflow, true); + + overflow = builtin_sub_overflow(min + 12, 15, &c); + EXPECT_EQ(c, max - 2); + EXPECT_EQ(overflow, true); +} + +TEST_F(OverflowBuiltinsTest, SignedSubTest) { + signed_t max = std::numeric_limits::max(); + signed_t min = std::numeric_limits::min(); + + signed_t c; + bool overflow; + + overflow = builtin_sub_overflow(min + 3, 3, &c); + EXPECT_EQ(c, min); + EXPECT_EQ(overflow, false); + + overflow = builtin_sub_overflow(min, 0, &c); + EXPECT_EQ(c, min); + EXPECT_EQ(overflow, false); + + overflow = builtin_sub_overflow(0, -12, &c); + EXPECT_EQ(c, 12); + EXPECT_EQ(overflow, false); + + overflow = builtin_sub_overflow(min, 1, &c); + EXPECT_EQ(c, max); + EXPECT_EQ(overflow, true); + + overflow = builtin_sub_overflow(max, -1, &c); + EXPECT_EQ(c, min); + EXPECT_EQ(overflow, true); + + overflow = builtin_sub_overflow(0, 13, &c); + EXPECT_EQ(c, -13); + EXPECT_EQ(overflow, false); + + overflow = builtin_sub_overflow(-12, -13, &c); + EXPECT_EQ(c, 1); + EXPECT_EQ(overflow, false); +} + +TEST_F(OverflowBuiltinsTest, UnsignedMulTest) { + unsigned_t max = std::numeric_limits::max(); + + unsigned_t c; + bool overflow; + + overflow = builtin_mul_overflow(3, 3, &c); + EXPECT_EQ(c, 9); + EXPECT_EQ(overflow, false); + + overflow = builtin_mul_overflow(0, 0, &c); + EXPECT_EQ(c, 0); + EXPECT_EQ(overflow, false); + + overflow = builtin_mul_overflow(max, 2, &c); + EXPECT_EQ(c, 4294967294); + EXPECT_EQ(overflow, true); +} + +TEST_F(OverflowBuiltinsTest, SignedMulTest) { + signed_t max = std::numeric_limits::max(); + // signed_t min = std::numeric_limits::min(); + + signed_t c; + bool overflow; + + overflow = builtin_mul_overflow(-1, 2, &c); + EXPECT_EQ(c, -2); + EXPECT_EQ(overflow, false); + + overflow = builtin_mul_overflow(2, -4, &c); + EXPECT_EQ(c, -8); + EXPECT_EQ(overflow, false); + + overflow = builtin_mul_overflow(-4, -4, &c); + EXPECT_EQ(c, 16); + EXPECT_EQ(overflow, false); + + overflow = builtin_mul_overflow(max, -2, &c); + EXPECT_EQ(c, 2); + EXPECT_EQ(overflow, true); +} + +} // namespace test +} // namespace peloton diff --git a/test/include/codegen/testing_codegen_util.h b/test/include/codegen/testing_codegen_util.h index c61a47e67c2..fa9fcf852cd 100644 --- a/test/include/codegen/testing_codegen_util.h +++ b/test/include/codegen/testing_codegen_util.h @@ -20,8 +20,8 @@ #include "codegen/execution_consumer.h" #include "codegen/value.h" #include "common/container_tuple.h" -#include "expression/constant_value_expression.h" #include "expression/comparison_expression.h" +#include "expression/constant_value_expression.h" #include "expression/tuple_value_expression.h" #include "planner/binding_context.h" #include "storage/data_table.h" @@ -68,6 +68,11 @@ class PelotonCodeGenTest : public PelotonTest { PelotonCodeGenTest(oid_t tuples_per_tilegroup = DEFAULT_TUPLES_PER_TILEGROUP, peloton::LayoutType layout_type = LayoutType::ROW); + struct CodeGenStats { + codegen::QueryCompiler::CompileStats compile_stats; + codegen::Query::RuntimeStats runtime_stats; + }; + virtual ~PelotonCodeGenTest(); // Get the test database @@ -103,10 +108,10 @@ class PelotonCodeGenTest : public PelotonTest { bool is_inlined); // Compile and execute the given plan - codegen::QueryCompiler::CompileStats CompileAndExecute( + CodeGenStats CompileAndExecute( planner::AbstractPlan &plan, codegen::ExecutionConsumer &consumer); - codegen::QueryCompiler::CompileStats CompileAndExecuteCache( + CodeGenStats CompileAndExecuteCache( std::shared_ptr plan, codegen::ExecutionConsumer &consumer, bool &cached, std::vector params = {}); From f83d53808dc99fa9188e4dee19c133381cb75388 Mon Sep 17 00:00:00 2001 From: Bowei Chen <411468452@qq.com> Date: Tue, 26 Jun 2018 10:42:30 -0700 Subject: [PATCH 08/19] Apply limit + order optimization in the optimizer (#1385) * Apply limit + order optimization in the optimizer --- src/include/optimizer/operators.h | 22 +++++++- src/include/planner/order_by_plan.h | 17 ++++++- src/optimizer/child_property_deriver.cpp | 14 ++++-- src/optimizer/input_column_deriver.cpp | 28 +++++++++-- src/optimizer/operators.cpp | 14 +++++- src/optimizer/plan_generator.cpp | 33 ++++++++++-- .../query_to_operator_transformer.cpp | 12 ++++- src/optimizer/rule_impls.cpp | 3 +- src/planner/order_by_plan.cpp | 50 ++++++++++--------- 9 files changed, 151 insertions(+), 42 deletions(-) diff --git a/src/include/optimizer/operators.h b/src/include/optimizer/operators.h index d51d66b01e8..fb5242ba8e3 100644 --- a/src/include/optimizer/operators.h +++ b/src/include/optimizer/operators.h @@ -299,9 +299,18 @@ class LogicalDistinct : public OperatorNode { //===--------------------------------------------------------------------===// class LogicalLimit : public OperatorNode { public: - static Operator make(int64_t offset, int64_t limit); + static Operator make( + int64_t offset, int64_t limit, + std::vector &&sort_exprs, + std::vector &&sort_ascending); int64_t offset; int64_t limit; + // When we get a query like "SELECT * FROM tab ORDER BY a LIMIT 5" + // We'll let the limit operator keep the order by clause's content as an + // internal order, then the limit operator will generate sort plan with + // limit as a optimization. + std::vector sort_exprs; + std::vector sort_ascending; }; //===--------------------------------------------------------------------===// @@ -470,9 +479,18 @@ class PhysicalOrderBy : public OperatorNode { //===--------------------------------------------------------------------===// class PhysicalLimit : public OperatorNode { public: - static Operator make(int64_t offset, int64_t limit); + static Operator make( + int64_t offset, int64_t limit, + std::vector sort_columns, + std::vector sort_ascending); int64_t offset; int64_t limit; + // When we get a query like "SELECT * FROM tab ORDER BY a LIMIT 5" + // We'll let the limit operator keep the order by clause's content as an + // internal order, then the limit operator will generate sort plan with + // limit as a optimization. + std::vector sort_exprs; + std::vector sort_acsending; }; //===--------------------------------------------------------------------===// diff --git a/src/include/planner/order_by_plan.h b/src/include/planner/order_by_plan.h index b5e354d391d..9bc0eb5cc67 100644 --- a/src/include/planner/order_by_plan.h +++ b/src/include/planner/order_by_plan.h @@ -28,6 +28,11 @@ class OrderByPlan : public AbstractPlan { const std::vector &descend_flags, const std::vector &output_column_ids); + OrderByPlan(const std::vector &sort_keys, + const std::vector &descend_flags, + const std::vector &output_column_ids, const uint64_t limit, + const uint64_t offset); + void PerformBinding(BindingContext &binding_context) override; //===--------------------------------------------------------------------===// @@ -51,13 +56,21 @@ class OrderByPlan : public AbstractPlan { return output_ais_; } - inline PlanNodeType GetPlanNodeType() const override { return PlanNodeType::ORDERBY; } + inline PlanNodeType GetPlanNodeType() const override { + return PlanNodeType::ORDERBY; + } void GetOutputColumns(std::vector &columns) const override { columns = GetOutputColumnIds(); } - const std::string GetInfo() const override { return "OrderBy"; } + const std::string GetInfo() const override { + return std::string("OrderBy") + + (limit_ + ? "(Limit : " + std::to_string(limit_number_) + ", Offset : " + + std::to_string(limit_offset_) + ")" + : ""); + } void SetUnderlyingOrder(bool same_order) { underling_ordered_ = same_order; } diff --git a/src/optimizer/child_property_deriver.cpp b/src/optimizer/child_property_deriver.cpp index 39ca06d811b..62b0c95288d 100644 --- a/src/optimizer/child_property_deriver.cpp +++ b/src/optimizer/child_property_deriver.cpp @@ -136,11 +136,17 @@ void ChildPropertyDeriver::Visit(const PhysicalAggregate *) { vector>{make_shared()})); } -void ChildPropertyDeriver::Visit(const PhysicalLimit *) { - // Let child fulfil all the required properties - vector> child_input_properties{requirements_}; +void ChildPropertyDeriver::Visit(const PhysicalLimit *op) { + // Limit fulfill the internal sort property + vector> child_input_properties{ + std::make_shared()}; + std::shared_ptr provided_prop(new PropertySet); + if (!op->sort_exprs.empty()) { + provided_prop->AddProperty( + std::make_shared(op->sort_exprs, op->sort_acsending)); + } - output_.push_back(make_pair(requirements_, move(child_input_properties))); + output_.push_back(make_pair(provided_prop, move(child_input_properties))); } void ChildPropertyDeriver::Visit(const PhysicalDistinct *) { diff --git a/src/optimizer/input_column_deriver.cpp b/src/optimizer/input_column_deriver.cpp index 019117ae68e..fdffb7e79a6 100644 --- a/src/optimizer/input_column_deriver.cpp +++ b/src/optimizer/input_column_deriver.cpp @@ -84,7 +84,28 @@ void InputColumnDeriver::Visit(const QueryDerivedScan *op) { output_cols, {input_cols}}; } -void InputColumnDeriver::Visit(const PhysicalLimit *) { Passdown(); } +void InputColumnDeriver::Visit(const PhysicalLimit *op) { + // All aggregate expressions and TVEs in the required columns and internal + // sort columns are needed by the child node + ExprSet input_cols_set; + for (auto expr : required_cols_) { + if (expression::ExpressionUtil::IsAggregateExpression(expr)) { + input_cols_set.insert(expr); + } else { + expression::ExpressionUtil::GetTupleValueExprs(input_cols_set, expr); + } + } + for (const auto& sort_column : op->sort_exprs) { + input_cols_set.insert(sort_column); + } + vector cols; + for (const auto &expr : input_cols_set) { + cols.push_back(expr); + } + output_input_cols_ = + pair, vector>>{ + cols, {cols}}; +} void InputColumnDeriver::Visit(const PhysicalOrderBy *) { // we need to pass down both required columns and sort columns @@ -92,10 +113,11 @@ void InputColumnDeriver::Visit(const PhysicalOrderBy *) { PELOTON_ASSERT(prop.get() != nullptr); ExprSet input_cols_set; for (auto expr : required_cols_) { - if (expression::ExpressionUtil::IsAggregateExpression(expr)) + if (expression::ExpressionUtil::IsAggregateExpression(expr)) { input_cols_set.insert(expr); - else + } else { expression::ExpressionUtil::GetTupleValueExprs(input_cols_set, expr); + } } auto sort_prop = prop->As(); size_t sort_col_size = sort_prop->GetSortColumnSize(); diff --git a/src/optimizer/operators.cpp b/src/optimizer/operators.cpp index 6457e769db2..c669125c191 100644 --- a/src/optimizer/operators.cpp +++ b/src/optimizer/operators.cpp @@ -443,10 +443,15 @@ Operator LogicalDistinct::make() { //===--------------------------------------------------------------------===// // Limit //===--------------------------------------------------------------------===// -Operator LogicalLimit::make(int64_t offset, int64_t limit) { +Operator LogicalLimit::make( + int64_t offset, int64_t limit, + std::vector &&sort_exprs, + std::vector &&sort_ascending) { LogicalLimit *limit_op = new LogicalLimit; limit_op->offset = offset; limit_op->limit = limit; + limit_op->sort_exprs = std::move(sort_exprs); + limit_op->sort_ascending = std::move(sort_ascending); return Operator(limit_op); } @@ -656,10 +661,15 @@ Operator PhysicalOrderBy::make() { //===--------------------------------------------------------------------===// // PhysicalLimit //===--------------------------------------------------------------------===// -Operator PhysicalLimit::make(int64_t offset, int64_t limit) { +Operator PhysicalLimit::make( + int64_t offset, int64_t limit, + std::vector sort_exprs, + std::vector sort_ascending) { PhysicalLimit *limit_op = new PhysicalLimit; limit_op->offset = offset; limit_op->limit = limit; + limit_op->sort_exprs = sort_exprs; + limit_op->sort_acsending = sort_ascending; return Operator(limit_op); } diff --git a/src/optimizer/plan_generator.cpp b/src/optimizer/plan_generator.cpp index 671ef94dabd..c73cd4721bc 100644 --- a/src/optimizer/plan_generator.cpp +++ b/src/optimizer/plan_generator.cpp @@ -156,10 +156,35 @@ void PlanGenerator::Visit(const QueryDerivedScan *) { } void PlanGenerator::Visit(const PhysicalLimit *op) { + // Generate order by + limit plan when there's internal sort order + output_plan_ = std::move(children_plans_[0]); + if (!op->sort_exprs.empty()) { + vector column_ids; + PELOTON_ASSERT(children_expr_map_.size() == 1); + auto &child_cols_map = children_expr_map_[0]; + for (size_t i = 0; i < output_cols_.size(); ++i) { + column_ids.push_back(child_cols_map[output_cols_[i]]); + } + + PELOTON_ASSERT(op->sort_exprs.size() == op->sort_acsending.size()); + auto sort_columns_size = op->sort_exprs.size(); + vector sort_col_ids; + vector sort_flags; + for (size_t i = 0; i < sort_columns_size; ++i) { + sort_col_ids.push_back(child_cols_map[op->sort_exprs[i]]); + // planner use desc flag + sort_flags.push_back(!op->sort_acsending[i]); + } + unique_ptr order_by_plan(new planner::OrderByPlan( + sort_col_ids, sort_flags, column_ids, op->limit, op->offset)); + order_by_plan->AddChild(std::move(output_plan_)); + output_plan_ = std::move(order_by_plan); + } + unique_ptr limit_plan( new planner::LimitPlan(op->limit, op->offset)); - limit_plan->AddChild(move(children_plans_[0])); - output_plan_ = move(limit_plan); + limit_plan->AddChild(move(output_plan_)); + output_plan_ = std::move(limit_plan); } void PlanGenerator::Visit(const PhysicalOrderBy *) { @@ -508,8 +533,8 @@ void PlanGenerator::BuildProjectionPlan() { void PlanGenerator::BuildAggregatePlan( AggregateType aggr_type, - const std::vector> * - groupby_cols, + const std::vector> + *groupby_cols, std::unique_ptr having_predicate) { vector aggr_terms; vector output_schema_columns; diff --git a/src/optimizer/query_to_operator_transformer.cpp b/src/optimizer/query_to_operator_transformer.cpp index 56925c3b117..97f1365b4c4 100644 --- a/src/optimizer/query_to_operator_transformer.cpp +++ b/src/optimizer/query_to_operator_transformer.cpp @@ -111,8 +111,18 @@ void QueryToOperatorTransformer::Visit(parser::SelectStatement *op) { } if (op->limit != nullptr) { + const auto &order_info = op->order; + std::vector sort_exprs; + std::vector sort_ascending; + for (auto &expr : order_info->exprs) { + sort_exprs.push_back(expr.get()); + } + for (auto &type : order_info->types) { + sort_ascending.push_back(type == parser::kOrderAsc); + } auto limit_expr = std::make_shared( - LogicalLimit::make(op->limit->offset, op->limit->limit)); + LogicalLimit::make(op->limit->offset, op->limit->limit, + std::move(sort_exprs), std::move(sort_ascending))); limit_expr->PushChild(output_expr_); output_expr_ = limit_expr; } diff --git a/src/optimizer/rule_impls.cpp b/src/optimizer/rule_impls.cpp index 33fb241df8d..6d5a1a1ebce 100644 --- a/src/optimizer/rule_impls.cpp +++ b/src/optimizer/rule_impls.cpp @@ -811,7 +811,8 @@ void ImplementLimit::Transform( const LogicalLimit *limit_op = input->Op().As(); auto result_plan = std::make_shared( - PhysicalLimit::make(limit_op->offset, limit_op->limit)); + PhysicalLimit::make(limit_op->offset, limit_op->limit, + limit_op->sort_exprs, limit_op->sort_ascending)); std::vector> children = input->Children(); PELOTON_ASSERT(children.size() == 1); diff --git a/src/planner/order_by_plan.cpp b/src/planner/order_by_plan.cpp index f6181b24bd8..05892eba995 100644 --- a/src/planner/order_by_plan.cpp +++ b/src/planner/order_by_plan.cpp @@ -9,7 +9,7 @@ // Copyright (c) 2015-17, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// - + #include #include @@ -21,24 +21,35 @@ namespace peloton { namespace planner { OrderByPlan::OrderByPlan(const std::vector &sort_keys, - const std::vector &descend_flags, - const std::vector &output_column_ids) - : sort_keys_(sort_keys), - descend_flags_(descend_flags), - output_column_ids_(output_column_ids) {} + const std::vector &descend_flags, + const std::vector &output_column_ids) + : sort_keys_(sort_keys), + descend_flags_(descend_flags), + output_column_ids_(output_column_ids) {} + +OrderByPlan::OrderByPlan(const std::vector &sort_keys, + const std::vector &descend_flags, + const std::vector &output_column_ids, + const uint64_t limit, const uint64_t offset) + : sort_keys_(sort_keys), + descend_flags_(descend_flags), + output_column_ids_(output_column_ids), + limit_(true), + limit_number_(limit), + limit_offset_(offset) {} void OrderByPlan::PerformBinding(BindingContext &binding_context) { // Let the child do its binding first AbstractPlan::PerformBinding(binding_context); for (const oid_t col_id : GetOutputColumnIds()) { - auto* ai = binding_context.Find(col_id); + auto *ai = binding_context.Find(col_id); PELOTON_ASSERT(ai != nullptr); output_ais_.push_back(ai); } for (const oid_t sort_key_col_id : GetSortKeys()) { - auto* ai = binding_context.Find(sort_key_col_id); + auto *ai = binding_context.Find(sort_key_col_id); PELOTON_ASSERT(ai != nullptr); sort_key_ais_.push_back(ai); } @@ -64,39 +75,32 @@ hash_t OrderByPlan::Hash() const { } bool OrderByPlan::operator==(const AbstractPlan &rhs) const { - if (GetPlanNodeType() != rhs.GetPlanNodeType()) - return false; + if (GetPlanNodeType() != rhs.GetPlanNodeType()) return false; auto &other = static_cast(rhs); - // Sort Keys + // Sort Keys size_t sort_keys_count = GetSortKeys().size(); - if (sort_keys_count != other.GetSortKeys().size()) - return false; + if (sort_keys_count != other.GetSortKeys().size()) return false; for (size_t i = 0; i < sort_keys_count; i++) { - if (GetSortKeys()[i] != other.GetSortKeys()[i]) - return false; + if (GetSortKeys()[i] != other.GetSortKeys()[i]) return false; } // Descend Flags size_t descend_flags_count = GetDescendFlags().size(); - if (descend_flags_count != other.GetDescendFlags().size()) - return false; + if (descend_flags_count != other.GetDescendFlags().size()) return false; for (size_t i = 0; i < descend_flags_count; i++) { - if (GetDescendFlags()[i] != other.GetDescendFlags()[i]) - return false; + if (GetDescendFlags()[i] != other.GetDescendFlags()[i]) return false; } // Output Column Ids size_t column_id_count = GetOutputColumnIds().size(); - if (column_id_count != other.GetOutputColumnIds().size()) - return false; + if (column_id_count != other.GetOutputColumnIds().size()) return false; for (size_t i = 0; i < column_id_count; i++) { - if (GetOutputColumnIds()[i] != other.GetOutputColumnIds()[i]) - return false; + if (GetOutputColumnIds()[i] != other.GetOutputColumnIds()[i]) return false; } return AbstractPlan::operator==(rhs); From d3e32898b20f7e0ee00c5377e2eeb53089199397 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Tue, 26 Jun 2018 15:14:49 -0400 Subject: [PATCH 09/19] Set LastReaderCommitId on new versions (#1429) * Set LastReaderCommitId on insert, update, and delete versions. Addresses parts 5 and 6 of #1420. --- src/concurrency/timestamp_ordering_transaction_manager.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index 3a39ccc8877..1deeccd0609 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -358,6 +358,8 @@ void TimestampOrderingTransactionManager::PerformInsert( PELOTON_ASSERT(tile_group_header->GetEndCommitId(tuple_id) == MAX_CID); tile_group_header->SetTransactionId(tuple_id, transaction_id); + tile_group_header->SetLastReaderCommitId(tuple_id, + current_txn->GetCommitId()); // no need to set next item pointer. @@ -412,6 +414,8 @@ void TimestampOrderingTransactionManager::PerformUpdate( new_tile_group_header->SetNextItemPointer(new_location.offset, old_location); new_tile_group_header->SetTransactionId(new_location.offset, transaction_id); + new_tile_group_header->SetLastReaderCommitId(new_location.offset, + current_txn->GetCommitId()); // we should guarantee that the newer version is all set before linking the // newer version to older version. @@ -515,6 +519,8 @@ void TimestampOrderingTransactionManager::PerformDelete( new_tile_group_header->SetNextItemPointer(new_location.offset, old_location); new_tile_group_header->SetTransactionId(new_location.offset, transaction_id); + new_tile_group_header->SetLastReaderCommitId(new_location.offset, + current_txn->GetCommitId()); new_tile_group_header->SetEndCommitId(new_location.offset, INVALID_CID); From 70f51954e7e4b79a71dc8a99727761c0f6256bb8 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 27 Jun 2018 12:19:56 -0400 Subject: [PATCH 10/19] fix #1386: PerformVectorizedRead optimization (#1434) * fix #1386 part 2: pass update boolean from scan plan to PerformRead correctly. * fix #1386 part 1: Perform predicate evaluation before recording reads with the transaction manager. * Rename new function RecordReads to PerformReads per PR feedback. --- .../operator/table_scan_translator.cpp | 29 ++++++++++++++++--- .../proxy/transaction_runtime_proxy.cpp | 1 + src/codegen/transaction_runtime.cpp | 23 ++++++++++----- .../codegen/proxy/transaction_runtime_proxy.h | 3 +- src/include/codegen/transaction_runtime.h | 11 +++++-- 5 files changed, 51 insertions(+), 16 deletions(-) diff --git a/src/codegen/operator/table_scan_translator.cpp b/src/codegen/operator/table_scan_translator.cpp index 3612e4f4cca..1c28b4e96df 100644 --- a/src/codegen/operator/table_scan_translator.cpp +++ b/src/codegen/operator/table_scan_translator.cpp @@ -105,6 +105,8 @@ class TableScanTranslator::ScanConsumer : public codegen::ScanCallback { llvm::Value *tid_end, Vector &selection_vector) const; + void PerformReads(CodeGen &codegen, Vector &selection_vector) const; + // Filter all the rows whose TIDs are in the range [tid_start, tid_end] and // store their TIDs in the output TID selection vector void FilterRowsByPredicate(CodeGen &codegen, @@ -290,14 +292,17 @@ void TableScanTranslator::ScanConsumer::ProcessTuples( selection_vector_); } - // 3. Setup the (filtered) row batch and setup attribute accessors + // 3. Record reads for all of the tuple that are visible and pass predicate + PerformReads(codegen, selection_vector_); + + // 4. Setup the (filtered) row batch and setup attribute accessors RowBatch batch{ctx_.GetCompilationContext(), tile_group_id_, tid_start, tid_end, selection_vector_, true}; std::vector attribute_accesses; SetupRowBatch(batch, tile_group_access, attribute_accesses); - // 4. Push the batch into the pipeline + // 5. Push the batch into the pipeline ctx_.Consume(batch); } @@ -333,9 +338,9 @@ void TableScanTranslator::ScanConsumer::FilterRowsByVisibility( llvm::Value *txn = ec.GetTransactionPtr(ctx_.GetCompilationContext()); llvm::Value *raw_sel_vec = selection_vector.GetVectorPtr(); - // Invoke TransactionRuntime::PerformRead(...) + // Invoke TransactionRuntime::PerformVisibilityCheck(...) llvm::Value *out_idx = - codegen.Call(TransactionRuntimeProxy::PerformVectorizedRead, + codegen.Call(TransactionRuntimeProxy::PerformVisibilityCheck, {txn, tile_group_ptr_, tid_start, tid_end, raw_sel_vec}); selection_vector.SetNumElements(out_idx); } @@ -379,5 +384,21 @@ void TableScanTranslator::ScanConsumer::FilterRowsByPredicate( }); } +void TableScanTranslator::ScanConsumer::PerformReads( + CodeGen &codegen, Vector &selection_vector) const { + ExecutionConsumer &ec = ctx_.GetCompilationContext().GetExecutionConsumer(); + llvm::Value *txn = ec.GetTransactionPtr(ctx_.GetCompilationContext()); + llvm::Value *raw_sel_vec = selection_vector.GetVectorPtr(); + + llvm::Value *is_for_update = codegen.ConstBool(plan_.IsForUpdate()); + llvm::Value *end_idx = selection_vector.GetNumElements(); + + // Invoke TransactionRuntime::PerformVectorizedRead(...) + llvm::Value *out_idx = + codegen.Call(TransactionRuntimeProxy::PerformVectorizedRead, + {txn, tile_group_ptr_, raw_sel_vec, end_idx, is_for_update}); + selection_vector.SetNumElements(out_idx); +} + } // namespace codegen } // namespace peloton diff --git a/src/codegen/proxy/transaction_runtime_proxy.cpp b/src/codegen/proxy/transaction_runtime_proxy.cpp index 10df0f7e024..ded04b52e6e 100644 --- a/src/codegen/proxy/transaction_runtime_proxy.cpp +++ b/src/codegen/proxy/transaction_runtime_proxy.cpp @@ -22,6 +22,7 @@ namespace peloton { namespace codegen { DEFINE_METHOD(peloton::codegen, TransactionRuntime, PerformVectorizedRead); +DEFINE_METHOD(peloton::codegen, TransactionRuntime, PerformVisibilityCheck); } // namespace codegen } // namespace peloton diff --git a/src/codegen/transaction_runtime.cpp b/src/codegen/transaction_runtime.cpp index 3aeb7745dd5..a01b2843341 100644 --- a/src/codegen/transaction_runtime.cpp +++ b/src/codegen/transaction_runtime.cpp @@ -22,10 +22,7 @@ namespace peloton { namespace codegen { -// Perform a read operation for all tuples in the tile group in the given range -// TODO: Right now, we split this check into two loops: a visibility check and -// the actual reading. Can this be merged? -uint32_t TransactionRuntime::PerformVectorizedRead( +uint32_t TransactionRuntime::PerformVisibilityCheck( concurrency::TransactionContext &txn, storage::TileGroup &tile_group, uint32_t tid_start, uint32_t tid_end, uint32_t *selection_vector) { // Get the transaction manager @@ -45,24 +42,34 @@ uint32_t TransactionRuntime::PerformVectorizedRead( selection_vector[out_idx] = i; out_idx += (visibility == VisibilityType::OK); } + return out_idx; +} + +uint32_t TransactionRuntime::PerformVectorizedRead( + concurrency::TransactionContext &txn, storage::TileGroup &tile_group, + uint32_t *selection_vector, uint32_t end_idx, bool is_for_update) { + // Get the transaction manager + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + + // Get the tile group header + auto tile_group_header = tile_group.GetHeader(); uint32_t tile_group_idx = tile_group.GetTileGroupId(); // Perform a read operation for every visible tuple we found - uint32_t end_idx = out_idx; - out_idx = 0; + uint32_t out_idx = 0; for (uint32_t idx = 0; idx < end_idx; idx++) { // Construct the item location ItemPointer location{tile_group_idx, selection_vector[idx]}; // Perform the read - bool can_read = txn_manager.PerformRead(&txn, location, tile_group_header, false); + bool can_read = txn_manager.PerformRead(&txn, location, tile_group_header, + is_for_update); // Update the selection vector and output position selection_vector[out_idx] = selection_vector[idx]; out_idx += static_cast(can_read); } - return out_idx; } diff --git a/src/include/codegen/proxy/transaction_runtime_proxy.h b/src/include/codegen/proxy/transaction_runtime_proxy.h index 57611071125..36cce2197a3 100644 --- a/src/include/codegen/proxy/transaction_runtime_proxy.h +++ b/src/include/codegen/proxy/transaction_runtime_proxy.h @@ -18,9 +18,8 @@ namespace peloton { namespace codegen { PROXY(TransactionRuntime) { - /// We only need to proxy PerformVectorizedRead() - /// in codegen::TransactionRuntime. DECLARE_METHOD(PerformVectorizedRead); + DECLARE_METHOD(PerformVisibilityCheck); }; } // namespace codegen diff --git a/src/include/codegen/transaction_runtime.h b/src/include/codegen/transaction_runtime.h index d3082505102..d548a5d3289 100644 --- a/src/include/codegen/transaction_runtime.h +++ b/src/include/codegen/transaction_runtime.h @@ -42,12 +42,19 @@ namespace codegen { //===----------------------------------------------------------------------===// class TransactionRuntime { public: + // Perform a visibility check for all tuples in the given tile group with IDs + // in the range [tid_start, tid_end) in the context of the given transaction + static uint32_t PerformVisibilityCheck(concurrency::TransactionContext &txn, + storage::TileGroup &tile_group, + uint32_t tid_start, uint32_t tid_end, + uint32_t *selection_vector); + // Perform a read operation for all tuples in the given tile group with IDs // in the range [tid_start, tid_end) in the context of the given transaction static uint32_t PerformVectorizedRead(concurrency::TransactionContext &txn, storage::TileGroup &tile_group, - uint32_t tid_start, uint32_t tid_end, - uint32_t *selection_vector); + uint32_t *selection_vector, + uint32_t end_idx, bool is_for_update); // Check Ownership static bool IsOwner(concurrency::TransactionContext &txn, storage::TileGroupHeader *tile_group_header, From d22bd241d3f3380ec9b9f3e87793914a953bd1a5 Mon Sep 17 00:00:00 2001 From: Tianyu Li Date: Wed, 27 Jun 2018 14:11:12 -0400 Subject: [PATCH 11/19] Catalog code cleanup (#1414) * Catalog code cleanup * Rename "XXXObject" to "CatalogEntry" * Rename AddPlpgsqlFunction --- src/binder/bind_node_visitor.cpp | 2 +- src/binder/binder_context.cpp | 16 +- src/brain/query_logger.cpp | 7 +- src/catalog/abstract_catalog.cpp | 70 +- src/catalog/catalog.cpp | 1438 ++++++++++------- src/catalog/catalog_cache.cpp | 58 +- src/catalog/column.cpp | 28 +- src/catalog/column_catalog.cpp | 134 +- src/catalog/column_stats_catalog.cpp | 109 +- src/catalog/constraint.cpp | 4 +- src/catalog/database_catalog.cpp | 206 +-- src/catalog/database_metrics_catalog.cpp | 33 +- src/catalog/index_catalog.cpp | 144 +- src/catalog/index_metrics_catalog.cpp | 42 +- src/catalog/language_catalog.cpp | 68 +- src/catalog/layout_catalog.cpp | 78 +- src/catalog/multi_constraint.cpp | 4 +- src/catalog/proc_catalog.cpp | 82 +- src/catalog/query_history_catalog.cpp | 24 +- src/catalog/query_metrics_catalog.cpp | 90 +- src/catalog/schema.cpp | 38 +- src/catalog/schema_catalog.cpp | 62 +- src/catalog/settings_catalog.cpp | 79 +- src/catalog/system_catalogs.cpp | 43 +- src/catalog/table_catalog.cpp | 434 ++--- src/catalog/table_metrics_catalog.cpp | 42 +- src/catalog/trigger_catalog.cpp | 125 +- src/catalog/zone_map_catalog.cpp | 71 +- src/common/init.cpp | 2 +- src/executor/create_executor.cpp | 73 +- src/executor/create_function_executor.cpp | 12 +- src/executor/drop_executor.cpp | 70 +- src/include/binder/binder_context.h | 8 +- src/include/catalog/abstract_catalog.h | 50 +- src/include/catalog/catalog.h | 182 ++- src/include/catalog/catalog_cache.h | 43 +- src/include/catalog/column.h | 50 +- src/include/catalog/column_catalog.h | 83 +- src/include/catalog/column_stats_catalog.h | 41 +- src/include/catalog/constraint.h | 56 +- src/include/catalog/database_catalog.h | 107 +- .../catalog/database_metrics_catalog.h | 16 +- src/include/catalog/foreign_key.h | 42 +- src/include/catalog/index_catalog.h | 90 +- src/include/catalog/index_metrics_catalog.h | 20 +- src/include/catalog/language_catalog.h | 23 +- src/include/catalog/layout_catalog.h | 27 +- src/include/catalog/manager.h | 4 +- src/include/catalog/multi_constraint.h | 18 +- src/include/catalog/proc_catalog.h | 33 +- src/include/catalog/query_history_catalog.h | 9 +- src/include/catalog/query_metrics_catalog.h | 34 +- src/include/catalog/schema.h | 64 +- src/include/catalog/schema_catalog.h | 44 +- src/include/catalog/settings_catalog.h | 28 +- src/include/catalog/system_catalogs.h | 9 +- src/include/catalog/table_catalog.h | 134 +- src/include/catalog/table_metrics_catalog.h | 20 +- src/include/catalog/trigger_catalog.h | 49 +- src/include/catalog/zone_map_catalog.h | 40 +- src/include/optimizer/operators.h | 46 +- src/include/optimizer/plan_generator.h | 4 +- src/main/tpch/tpch_database.cpp | 2 +- src/optimizer/child_property_deriver.cpp | 4 +- src/optimizer/operators.cpp | 22 +- src/optimizer/optimizer.cpp | 16 +- src/optimizer/plan_generator.cpp | 10 +- .../query_to_operator_transformer.cpp | 42 +- src/optimizer/rule_impls.cpp | 8 +- src/optimizer/stats/stats_storage.cpp | 34 +- src/optimizer/stats/tuple_samples_storage.cpp | 35 +- src/planner/analyze_plan.cpp | 19 +- src/planner/plan_util.cpp | 11 +- src/settings/settings_manager.cpp | 35 +- src/statistics/stats_aggregator.cpp | 53 +- src/storage/data_table.cpp | 6 +- src/storage/zone_map_manager.cpp | 26 +- src/traffic_cop/traffic_cop.cpp | 6 +- src/trigger/trigger.cpp | 9 +- src/tuning/index_tuner.cpp | 6 +- src/tuning/layout_tuner.cpp | 2 +- test/binder/binder_test.cpp | 21 +- test/catalog/catalog_test.cpp | 240 ++- test/catalog/constraints_test.cpp | 10 +- test/codegen/bloom_filter_test.cpp | 25 +- test/codegen/table_scan_translator_test.cpp | 40 +- test/codegen/testing_codegen_util.cpp | 59 +- test/executor/copy_test.cpp | 4 +- test/executor/create_index_test.cpp | 10 +- test/executor/create_test.cpp | 54 +- test/executor/drop_test.cpp | 141 +- test/executor/insert_test.cpp | 21 +- test/executor/testing_executor_util.cpp | 16 +- test/executor/update_test.cpp | 10 +- test/function/functions_test.cpp | 35 +- test/gc/garbage_collection_test.cpp | 5 +- test/gc/transaction_level_gc_manager_test.cpp | 7 +- .../catalog/testing_constraints_util.h | 10 +- test/optimizer/old_optimizer_test.cpp | 10 +- test/optimizer/operator_transformer_test.cpp | 4 +- test/optimizer/optimizer_test.cpp | 16 +- test/optimizer/selectivity_test.cpp | 24 +- test/optimizer/table_stats_collector_test.cpp | 20 +- test/optimizer/tuple_samples_storage_test.cpp | 8 +- test/planner/plan_util_test.cpp | 86 +- test/planner/planner_equality_test.cpp | 4 +- test/planner/planner_test.cpp | 76 +- test/settings/settings_manager_test.cpp | 18 +- test/sql/aggregate_groupby_sql_test.cpp | 16 +- test/sql/aggregate_sql_test.cpp | 4 +- test/sql/analyze_sql_test.cpp | 18 +- test/sql/case_sql_test.cpp | 20 +- test/sql/decimal_functions_sql_test.cpp | 12 +- test/sql/delete_sql_test.cpp | 4 +- test/sql/distinct_aggregate_sql_test.cpp | 40 +- test/sql/distinct_sql_test.cpp | 20 +- test/sql/drop_sql_test.cpp | 37 +- test/sql/foreign_key_sql_test.cpp | 8 +- test/sql/index_scan_sql_test.cpp | 12 +- test/sql/insert_sql_test.cpp | 44 +- test/sql/is_null_sql_test.cpp | 12 +- test/sql/optimizer_sql_test.cpp | 16 +- test/sql/order_by_sql_test.cpp | 72 +- test/sql/projection_sql_test.cpp | 4 +- test/sql/string_functions_sql_test.cpp | 4 +- test/sql/timestamp_functions_sql_test.cpp | 8 +- test/sql/type_sql_test.cpp | 4 +- test/sql/update_primary_index_sql_test.cpp | 4 +- test/sql/update_secondary_index_sql_test.cpp | 4 +- test/sql/update_sql_test.cpp | 28 +- test/statistics/stats_test.cpp | 19 +- test/storage/database_test.cpp | 6 +- test/trigger/trigger_test.cpp | 39 +- test/udf/udf_test.cpp | 16 +- 134 files changed, 3998 insertions(+), 2855 deletions(-) diff --git a/src/binder/bind_node_visitor.cpp b/src/binder/bind_node_visitor.cpp index eec8a03c091..2ccd85dcdb2 100644 --- a/src/binder/bind_node_visitor.cpp +++ b/src/binder/bind_node_visitor.cpp @@ -208,7 +208,7 @@ void BindNodeVisitor::Visit(parser::AnalyzeStatement *node) { void BindNodeVisitor::Visit(expression::TupleValueExpression *expr) { if (!expr->GetIsBound()) { std::tuple col_pos_tuple; - std::shared_ptr table_obj = nullptr; + std::shared_ptr table_obj = nullptr; type::TypeId value_type; int depth = -1; diff --git a/src/binder/binder_context.cpp b/src/binder/binder_context.cpp index 0413b488c37..4c518f5fa8d 100644 --- a/src/binder/binder_context.cpp +++ b/src/binder/binder_context.cpp @@ -38,8 +38,10 @@ void BinderContext::AddRegularTable(const std::string db_name, const std::string table_alias, concurrency::TransactionContext *txn) { // using catalog object to retrieve meta-data - auto table_object = catalog::Catalog::GetInstance()->GetTableObject( - db_name, schema_name, table_name, txn); + auto table_object = catalog::Catalog::GetInstance()->GetTableCatalogEntry(txn, + db_name, + schema_name, + table_name); if (regular_table_alias_map_.find(table_alias) != regular_table_alias_map_.end() || @@ -79,9 +81,9 @@ void BinderContext::AddNestedTable( bool BinderContext::GetColumnPosTuple( const std::string &col_name, - std::shared_ptr table_obj, + std::shared_ptr table_obj, std::tuple &col_pos_tuple, type::TypeId &value_type) { - auto column_object = table_obj->GetColumnObject(col_name); + auto column_object = table_obj->GetColumnCatalogEntry(col_name); if (column_object == nullptr) { return false; } @@ -138,7 +140,7 @@ bool BinderContext::GetColumnPosTuple( bool BinderContext::GetRegularTableObj( std::shared_ptr current_context, std::string &alias, - std::shared_ptr &table_obj, int &depth) { + std::shared_ptr &table_obj, int &depth) { while (current_context != nullptr) { auto iter = current_context->regular_table_alias_map_.find(alias); if (iter != current_context->regular_table_alias_map_.end()) { @@ -174,9 +176,9 @@ void BinderContext::GenerateAllColumnExpressions( std::vector> &exprs) { for (auto &entry : regular_table_alias_map_) { auto &table_obj = entry.second; - auto col_cnt = table_obj->GetColumnObjects().size(); + auto col_cnt = table_obj->GetColumnCatalogEntries().size(); for (size_t i = 0; i < col_cnt; i++) { - auto col_obj = table_obj->GetColumnObject(i); + auto col_obj = table_obj->GetColumnCatalogEntry(i); auto tv_expr = new expression::TupleValueExpression( std::string(col_obj->GetColumnName()), std::string(entry.first)); tv_expr->SetValueType(col_obj->GetColumnType()); diff --git a/src/brain/query_logger.cpp b/src/brain/query_logger.cpp index 6aa512e8e95..9d35d5e8cf0 100644 --- a/src/brain/query_logger.cpp +++ b/src/brain/query_logger.cpp @@ -41,8 +41,11 @@ void QueryLogger::LogQuery(std::string query_string, uint64_t timestamp) { // Log query + fingerprint auto &query_history_catalog = catalog::QueryHistoryCatalog::GetInstance(); - query_history_catalog.InsertQueryHistory( - query_string, fingerprint.GetFingerprint(), timestamp, nullptr, txn); + query_history_catalog.InsertQueryHistory(txn, + query_string, + fingerprint.GetFingerprint(), + timestamp, + nullptr); // We're done txn_manager.CommitTransaction(txn); diff --git a/src/catalog/abstract_catalog.cpp b/src/catalog/abstract_catalog.cpp index 9d9934a7c61..5ce2bceaedc 100644 --- a/src/catalog/abstract_catalog.cpp +++ b/src/catalog/abstract_catalog.cpp @@ -44,22 +44,22 @@ namespace peloton { namespace catalog { -AbstractCatalog::AbstractCatalog(oid_t catalog_table_oid, - std::string catalog_table_name, +AbstractCatalog::AbstractCatalog(storage::Database *pg_catalog, catalog::Schema *catalog_table_schema, - storage::Database *pg_catalog) { + oid_t catalog_table_oid, + std::string catalog_table_name) { // set database_oid - database_oid = pg_catalog->GetOid(); + database_oid_ = pg_catalog->GetOid(); // Create catalog_table_ catalog_table_ = storage::TableFactory::GetDataTable( - database_oid, catalog_table_oid, catalog_table_schema, catalog_table_name, + database_oid_, catalog_table_oid, catalog_table_schema, catalog_table_name, DEFAULT_TUPLES_PER_TILEGROUP, true, false, true); // Add catalog_table_ into pg_catalog database pg_catalog->AddTable(catalog_table_, true); } -AbstractCatalog::AbstractCatalog(const std::string &catalog_table_ddl, - concurrency::TransactionContext *txn) { +AbstractCatalog::AbstractCatalog(concurrency::TransactionContext *txn, + const std::string &catalog_table_ddl) { // get catalog table schema auto &peloton_parser = parser::PostgresParser::GetInstance(); auto create_plan = std::dynamic_pointer_cast( @@ -71,13 +71,19 @@ AbstractCatalog::AbstractCatalog(const std::string &catalog_table_ddl, auto catalog_database_name = create_plan->GetDatabaseName(); PELOTON_ASSERT(catalog_schema_name == std::string(CATALOG_SCHEMA_NAME)); // create catalog table - Catalog::GetInstance()->CreateTable( - catalog_database_name, catalog_schema_name, catalog_table_name, - std::unique_ptr(catalog_table_schema), txn, true); + Catalog::GetInstance()->CreateTable(txn, + catalog_database_name, + catalog_schema_name, + std::unique_ptr( + catalog_table_schema), + catalog_table_name, + true); // get catalog table oid - auto catalog_table_object = Catalog::GetInstance()->GetTableObject( - catalog_database_name, catalog_schema_name, catalog_table_name, txn); + auto catalog_table_object = Catalog::GetInstance()->GetTableCatalogEntry(txn, + catalog_database_name, + catalog_schema_name, + catalog_table_name); // set catalog_table_ try { @@ -85,7 +91,7 @@ AbstractCatalog::AbstractCatalog(const std::string &catalog_table_ddl, catalog_table_object->GetDatabaseOid(), catalog_table_object->GetTableOid()); // set database_oid - database_oid = catalog_table_object->GetDatabaseOid(); + database_oid_ = catalog_table_object->GetDatabaseOid(); } catch (CatalogException &e) { LOG_TRACE("Can't find table %d! Return false", catalog_table_object->GetTableOid()); @@ -97,8 +103,8 @@ AbstractCatalog::AbstractCatalog(const std::string &catalog_table_ddl, * @param txn TransactionContext * @return Whether insertion is Successful */ -bool AbstractCatalog::InsertTuple(std::unique_ptr tuple, - concurrency::TransactionContext *txn) { +bool AbstractCatalog::InsertTuple(concurrency::TransactionContext *txn, + std::unique_ptr tuple) { if (txn == nullptr) throw CatalogException("Insert tuple requires transaction"); @@ -137,9 +143,9 @@ bool AbstractCatalog::InsertTuple(std::unique_ptr tuple, * @param txn TransactionContext * @return Whether deletion is Successful */ -bool AbstractCatalog::DeleteWithIndexScan( - oid_t index_offset, std::vector values, - concurrency::TransactionContext *txn) { +bool AbstractCatalog::DeleteWithIndexScan(concurrency::TransactionContext *txn, + oid_t index_offset, + std::vector values) { if (txn == nullptr) throw CatalogException("Delete tuple requires transaction"); @@ -189,9 +195,10 @@ bool AbstractCatalog::DeleteWithIndexScan( */ std::unique_ptr>> AbstractCatalog::GetResultWithIndexScan( - std::vector column_offsets, oid_t index_offset, - std::vector values, - concurrency::TransactionContext *txn) const { + concurrency::TransactionContext *txn, + std::vector column_offsets, + oid_t index_offset, + std::vector values) const { if (txn == nullptr) throw CatalogException("Scan table requires transaction"); // Index scan @@ -238,9 +245,10 @@ AbstractCatalog::GetResultWithIndexScan( * @return Unique pointer of vector of logical tiles */ std::unique_ptr>> -AbstractCatalog::GetResultWithSeqScan(std::vector column_offsets, - expression::AbstractExpression *predicate, - concurrency::TransactionContext *txn) { +AbstractCatalog::GetResultWithSeqScan( + concurrency::TransactionContext *txn, + expression::AbstractExpression *predicate, + std::vector column_offsets) { if (txn == nullptr) throw CatalogException("Scan table requires transaction"); // Sequential scan @@ -272,8 +280,9 @@ AbstractCatalog::GetResultWithSeqScan(std::vector column_offsets, * Note: Use catalog::Catalog::CreateIndex() if you can, only ColumnCatalog and * IndexCatalog should need this */ -void AbstractCatalog::AddIndex(const std::vector &key_attrs, - oid_t index_oid, const std::string &index_name, +void AbstractCatalog::AddIndex(const std::string &index_name, + oid_t index_oid, + const std::vector &key_attrs, IndexConstraintType index_constraint) { auto schema = catalog_table_->GetSchema(); auto key_schema = catalog::Schema::CopySchema(schema, key_attrs); @@ -307,10 +316,11 @@ void AbstractCatalog::AddIndex(const std::vector &key_attrs, * @param index_offset Offset of index for scan * @return true if successfully executes */ -bool AbstractCatalog::UpdateWithIndexScan( - std::vector update_columns, std::vector update_values, - std::vector scan_values, oid_t index_offset, - concurrency::TransactionContext *txn) { +bool AbstractCatalog::UpdateWithIndexScan(concurrency::TransactionContext *txn, + oid_t index_offset, + std::vector scan_values, + std::vector update_columns, + std::vector update_values) { if (txn == nullptr) throw CatalogException("Scan table requires transaction"); std::unique_ptr context( diff --git a/src/catalog/catalog.cpp b/src/catalog/catalog.cpp index f2d30dd5ecf..b3dcd815ac3 100644 --- a/src/catalog/catalog.cpp +++ b/src/catalog/catalog.cpp @@ -65,12 +65,14 @@ Catalog::Catalog() : pool_(new type::EphemeralPool()) { storage_manager->AddDatabaseToStorageManager(peloton); // Create catalog tables - DatabaseCatalog::GetInstance(peloton, pool_.get(), txn); - BootstrapSystemCatalogs(peloton, txn); + DatabaseCatalog::GetInstance(txn, peloton, pool_.get()); + BootstrapSystemCatalogs(txn, peloton); // Insert peloton database into pg_database - DatabaseCatalog::GetInstance()->InsertDatabase( - CATALOG_DATABASE_OID, CATALOG_DATABASE_NAME, pool_.get(), txn); + DatabaseCatalog::GetInstance(txn)->InsertDatabase(txn, + CATALOG_DATABASE_OID, + CATALOG_DATABASE_NAME, + pool_.get()); // Commit transaction txn_manager.CommitTransaction(txn); @@ -83,115 +85,214 @@ Catalog::Catalog() : pool_(new type::EphemeralPool()) { * @param database database which this system catalogs belong to * @param txn transaction context */ -void Catalog::BootstrapSystemCatalogs(storage::Database *database, - concurrency::TransactionContext *txn) { +void Catalog::BootstrapSystemCatalogs(concurrency::TransactionContext *txn, + storage::Database *database) { oid_t database_oid = database->GetOid(); catalog_map_.emplace(database_oid, std::shared_ptr( - new SystemCatalogs(database, pool_.get(), txn))); + new SystemCatalogs(txn, database, pool_.get()))); auto system_catalogs = catalog_map_[database_oid]; // Create indexes on catalog tables, insert them into pg_index // actual index already added in - system_catalogs->GetIndexCatalog()->InsertIndex( - COLUMN_CATALOG_PKEY_OID, COLUMN_CATALOG_NAME "_pkey", COLUMN_CATALOG_OID, - CATALOG_SCHEMA_NAME, IndexType::BWTREE, IndexConstraintType::PRIMARY_KEY, - true, - {ColumnCatalog::ColumnId::TABLE_OID, - ColumnCatalog::ColumnId::COLUMN_NAME}, - pool_.get(), txn); - system_catalogs->GetIndexCatalog()->InsertIndex( - COLUMN_CATALOG_SKEY0_OID, COLUMN_CATALOG_NAME "_skey0", - COLUMN_CATALOG_OID, CATALOG_SCHEMA_NAME, IndexType::BWTREE, - IndexConstraintType::UNIQUE, true, - {ColumnCatalog::ColumnId::TABLE_OID, ColumnCatalog::ColumnId::COLUMN_ID}, - pool_.get(), txn); - system_catalogs->GetIndexCatalog()->InsertIndex( - COLUMN_CATALOG_SKEY1_OID, COLUMN_CATALOG_NAME "_skey1", - COLUMN_CATALOG_OID, CATALOG_SCHEMA_NAME, IndexType::BWTREE, - IndexConstraintType::DEFAULT, false, {ColumnCatalog::ColumnId::TABLE_OID}, - pool_.get(), txn); - - system_catalogs->GetIndexCatalog()->InsertIndex( - INDEX_CATALOG_PKEY_OID, INDEX_CATALOG_NAME "_pkey", INDEX_CATALOG_OID, - CATALOG_SCHEMA_NAME, IndexType::BWTREE, IndexConstraintType::PRIMARY_KEY, - true, {IndexCatalog::ColumnId::INDEX_OID}, pool_.get(), txn); - system_catalogs->GetIndexCatalog()->InsertIndex( - INDEX_CATALOG_SKEY0_OID, INDEX_CATALOG_NAME "_skey0", INDEX_CATALOG_OID, - CATALOG_SCHEMA_NAME, IndexType::BWTREE, IndexConstraintType::UNIQUE, true, - {IndexCatalog::ColumnId::INDEX_NAME}, pool_.get(), txn); - system_catalogs->GetIndexCatalog()->InsertIndex( - INDEX_CATALOG_SKEY1_OID, INDEX_CATALOG_NAME "_skey1", INDEX_CATALOG_OID, - CATALOG_SCHEMA_NAME, IndexType::BWTREE, IndexConstraintType::DEFAULT, - false, {IndexCatalog::ColumnId::TABLE_OID}, pool_.get(), txn); - - system_catalogs->GetIndexCatalog()->InsertIndex( - DATABASE_CATALOG_PKEY_OID, DATABASE_CATALOG_NAME "_pkey", - DATABASE_CATALOG_OID, CATALOG_SCHEMA_NAME, IndexType::BWTREE, - IndexConstraintType::PRIMARY_KEY, true, - {DatabaseCatalog::ColumnId::DATABASE_OID}, pool_.get(), txn); - system_catalogs->GetIndexCatalog()->InsertIndex( - DATABASE_CATALOG_SKEY0_OID, DATABASE_CATALOG_NAME "_skey0", - DATABASE_CATALOG_OID, CATALOG_SCHEMA_NAME, IndexType::BWTREE, - IndexConstraintType::UNIQUE, true, - {DatabaseCatalog::ColumnId::DATABASE_NAME}, pool_.get(), txn); - - system_catalogs->GetIndexCatalog()->InsertIndex( - SCHEMA_CATALOG_PKEY_OID, SCHEMA_CATALOG_NAME "_pkey", SCHEMA_CATALOG_OID, - CATALOG_SCHEMA_NAME, IndexType::BWTREE, IndexConstraintType::PRIMARY_KEY, - true, {SchemaCatalog::ColumnId::SCHEMA_OID}, pool_.get(), txn); - system_catalogs->GetIndexCatalog()->InsertIndex( - SCHEMA_CATALOG_SKEY0_OID, SCHEMA_CATALOG_NAME "_skey0", - SCHEMA_CATALOG_OID, CATALOG_SCHEMA_NAME, IndexType::BWTREE, - IndexConstraintType::UNIQUE, true, {SchemaCatalog::ColumnId::SCHEMA_NAME}, - pool_.get(), txn); - - system_catalogs->GetIndexCatalog()->InsertIndex( - TABLE_CATALOG_PKEY_OID, TABLE_CATALOG_NAME "_pkey", TABLE_CATALOG_OID, - CATALOG_SCHEMA_NAME, IndexType::BWTREE, IndexConstraintType::PRIMARY_KEY, - true, {TableCatalog::ColumnId::TABLE_OID}, pool_.get(), txn); - system_catalogs->GetIndexCatalog()->InsertIndex( - TABLE_CATALOG_SKEY0_OID, TABLE_CATALOG_NAME "_skey0", TABLE_CATALOG_OID, - CATALOG_SCHEMA_NAME, IndexType::BWTREE, IndexConstraintType::UNIQUE, true, - {TableCatalog::ColumnId::TABLE_NAME}, pool_.get(), txn); - system_catalogs->GetIndexCatalog()->InsertIndex( - TABLE_CATALOG_SKEY1_OID, TABLE_CATALOG_NAME "_skey1", TABLE_CATALOG_OID, - CATALOG_SCHEMA_NAME, IndexType::BWTREE, IndexConstraintType::DEFAULT, - false, {TableCatalog::ColumnId::DATABASE_OID}, pool_.get(), txn); + system_catalogs->GetIndexCatalog()->InsertIndex(txn, + CATALOG_SCHEMA_NAME, + COLUMN_CATALOG_OID, + COLUMN_CATALOG_PKEY_OID, + COLUMN_CATALOG_NAME "_pkey", + IndexType::BWTREE, + IndexConstraintType::PRIMARY_KEY, + true, + {ColumnCatalog::ColumnId::TABLE_OID, + ColumnCatalog::ColumnId::COLUMN_NAME}, + pool_.get()); + system_catalogs->GetIndexCatalog()->InsertIndex(txn, + CATALOG_SCHEMA_NAME, + COLUMN_CATALOG_OID, + COLUMN_CATALOG_SKEY0_OID, + COLUMN_CATALOG_NAME "_skey0", + IndexType::BWTREE, + IndexConstraintType::UNIQUE, + true, + {ColumnCatalog::ColumnId::TABLE_OID, + ColumnCatalog::ColumnId::COLUMN_ID}, + pool_.get()); + system_catalogs->GetIndexCatalog()->InsertIndex(txn, + CATALOG_SCHEMA_NAME, + COLUMN_CATALOG_OID, + COLUMN_CATALOG_SKEY1_OID, + COLUMN_CATALOG_NAME "_skey1", + IndexType::BWTREE, + IndexConstraintType::DEFAULT, + false, + {ColumnCatalog::ColumnId::TABLE_OID}, + pool_.get()); + + system_catalogs->GetIndexCatalog()->InsertIndex(txn, + CATALOG_SCHEMA_NAME, + INDEX_CATALOG_OID, + INDEX_CATALOG_PKEY_OID, + INDEX_CATALOG_NAME "_pkey", + IndexType::BWTREE, + IndexConstraintType::PRIMARY_KEY, + true, + {IndexCatalog::ColumnId::INDEX_OID}, + pool_.get()); + system_catalogs->GetIndexCatalog()->InsertIndex(txn, + CATALOG_SCHEMA_NAME, + INDEX_CATALOG_OID, + INDEX_CATALOG_SKEY0_OID, + INDEX_CATALOG_NAME "_skey0", + IndexType::BWTREE, + IndexConstraintType::UNIQUE, + true, + {IndexCatalog::ColumnId::INDEX_NAME}, + pool_.get()); + system_catalogs->GetIndexCatalog()->InsertIndex(txn, + CATALOG_SCHEMA_NAME, + INDEX_CATALOG_OID, + INDEX_CATALOG_SKEY1_OID, + INDEX_CATALOG_NAME "_skey1", + IndexType::BWTREE, + IndexConstraintType::DEFAULT, + false, + {IndexCatalog::ColumnId::TABLE_OID}, + pool_.get()); + + system_catalogs->GetIndexCatalog()->InsertIndex(txn, + CATALOG_SCHEMA_NAME, + DATABASE_CATALOG_OID, + DATABASE_CATALOG_PKEY_OID, + DATABASE_CATALOG_NAME "_pkey", + IndexType::BWTREE, + IndexConstraintType::PRIMARY_KEY, + true, + {DatabaseCatalog::ColumnId::DATABASE_OID}, + pool_.get()); + system_catalogs->GetIndexCatalog()->InsertIndex(txn, + CATALOG_SCHEMA_NAME, + DATABASE_CATALOG_OID, + DATABASE_CATALOG_SKEY0_OID, + DATABASE_CATALOG_NAME "_skey0", + IndexType::BWTREE, + IndexConstraintType::UNIQUE, + true, + {DatabaseCatalog::ColumnId::DATABASE_NAME}, + pool_.get()); + + system_catalogs->GetIndexCatalog()->InsertIndex(txn, + CATALOG_SCHEMA_NAME, + SCHEMA_CATALOG_OID, + SCHEMA_CATALOG_PKEY_OID, + SCHEMA_CATALOG_NAME "_pkey", + IndexType::BWTREE, + IndexConstraintType::PRIMARY_KEY, + true, + {SchemaCatalog::ColumnId::SCHEMA_OID}, + pool_.get()); + system_catalogs->GetIndexCatalog()->InsertIndex(txn, + CATALOG_SCHEMA_NAME, + SCHEMA_CATALOG_OID, + SCHEMA_CATALOG_SKEY0_OID, + SCHEMA_CATALOG_NAME "_skey0", + IndexType::BWTREE, + IndexConstraintType::UNIQUE, + true, + {SchemaCatalog::ColumnId::SCHEMA_NAME}, + pool_.get()); + + system_catalogs->GetIndexCatalog()->InsertIndex(txn, + CATALOG_SCHEMA_NAME, + TABLE_CATALOG_OID, + TABLE_CATALOG_PKEY_OID, + TABLE_CATALOG_NAME "_pkey", + IndexType::BWTREE, + IndexConstraintType::PRIMARY_KEY, + true, + {TableCatalog::ColumnId::TABLE_OID}, + pool_.get()); + system_catalogs->GetIndexCatalog()->InsertIndex(txn, + CATALOG_SCHEMA_NAME, + TABLE_CATALOG_OID, + TABLE_CATALOG_SKEY0_OID, + TABLE_CATALOG_NAME "_skey0", + IndexType::BWTREE, + IndexConstraintType::UNIQUE, + true, + {TableCatalog::ColumnId::TABLE_NAME}, + pool_.get()); + system_catalogs->GetIndexCatalog()->InsertIndex(txn, + CATALOG_SCHEMA_NAME, + TABLE_CATALOG_OID, + TABLE_CATALOG_SKEY1_OID, + TABLE_CATALOG_NAME "_skey1", + IndexType::BWTREE, + IndexConstraintType::DEFAULT, + false, + {TableCatalog::ColumnId::DATABASE_OID}, + pool_.get()); // Insert records(default + pg_catalog namespace) into pg_namespace - system_catalogs->GetSchemaCatalog()->InsertSchema( - CATALOG_SCHEMA_OID, CATALOG_SCHEMA_NAME, pool_.get(), txn); - system_catalogs->GetSchemaCatalog()->InsertSchema( - DEFAULT_SCHEMA_OID, DEFAULT_SCHEMA_NAME, pool_.get(), txn); + system_catalogs->GetSchemaCatalog()->InsertSchema(txn, + CATALOG_SCHEMA_OID, + CATALOG_SCHEMA_NAME, + pool_.get()); + system_catalogs->GetSchemaCatalog()->InsertSchema(txn, + DEFAULT_SCHEMA_OID, + DEFAULT_SCHEMA_NAME, + pool_.get()); // Insert catalog tables into pg_table // pg_database record is shared across different databases - system_catalogs->GetTableCatalog()->InsertTable( - DATABASE_CATALOG_OID, DATABASE_CATALOG_NAME, CATALOG_SCHEMA_NAME, - CATALOG_DATABASE_OID, ROW_STORE_LAYOUT_OID, pool_.get(), txn); - system_catalogs->GetTableCatalog()->InsertTable( - SCHEMA_CATALOG_OID, SCHEMA_CATALOG_NAME, CATALOG_SCHEMA_NAME, - database_oid, ROW_STORE_LAYOUT_OID, pool_.get(), txn); - system_catalogs->GetTableCatalog()->InsertTable( - TABLE_CATALOG_OID, TABLE_CATALOG_NAME, CATALOG_SCHEMA_NAME, database_oid, - ROW_STORE_LAYOUT_OID, pool_.get(), txn); - system_catalogs->GetTableCatalog()->InsertTable( - INDEX_CATALOG_OID, INDEX_CATALOG_NAME, CATALOG_SCHEMA_NAME, database_oid, - ROW_STORE_LAYOUT_OID, pool_.get(), txn); - system_catalogs->GetTableCatalog()->InsertTable( - COLUMN_CATALOG_OID, COLUMN_CATALOG_NAME, CATALOG_SCHEMA_NAME, - database_oid, ROW_STORE_LAYOUT_OID, pool_.get(), txn); - system_catalogs->GetTableCatalog()->InsertTable( - LAYOUT_CATALOG_OID, LAYOUT_CATALOG_NAME, CATALOG_SCHEMA_NAME, - database_oid, ROW_STORE_LAYOUT_OID, pool_.get(), txn); + system_catalogs->GetTableCatalog()->InsertTable(txn, + CATALOG_DATABASE_OID, + CATALOG_SCHEMA_NAME, + DATABASE_CATALOG_OID, + DATABASE_CATALOG_NAME, + ROW_STORE_LAYOUT_OID, + pool_.get()); + system_catalogs->GetTableCatalog()->InsertTable(txn, + database_oid, + CATALOG_SCHEMA_NAME, + SCHEMA_CATALOG_OID, + SCHEMA_CATALOG_NAME, + ROW_STORE_LAYOUT_OID, + pool_.get()); + system_catalogs->GetTableCatalog()->InsertTable(txn, + database_oid, + CATALOG_SCHEMA_NAME, + TABLE_CATALOG_OID, + TABLE_CATALOG_NAME, + ROW_STORE_LAYOUT_OID, + pool_.get()); + system_catalogs->GetTableCatalog()->InsertTable(txn, + database_oid, + CATALOG_SCHEMA_NAME, + INDEX_CATALOG_OID, + INDEX_CATALOG_NAME, + ROW_STORE_LAYOUT_OID, + pool_.get()); + system_catalogs->GetTableCatalog()->InsertTable(txn, + database_oid, + CATALOG_SCHEMA_NAME, + COLUMN_CATALOG_OID, + COLUMN_CATALOG_NAME, + ROW_STORE_LAYOUT_OID, + pool_.get()); + system_catalogs->GetTableCatalog()->InsertTable(txn, + database_oid, + CATALOG_SCHEMA_NAME, + LAYOUT_CATALOG_OID, + LAYOUT_CATALOG_NAME, + ROW_STORE_LAYOUT_OID, + pool_.get()); } void Catalog::Bootstrap() { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); // bootstrap pg_catalog database - catalog_map_[CATALOG_DATABASE_OID]->Bootstrap(CATALOG_DATABASE_NAME, txn); + catalog_map_[CATALOG_DATABASE_OID]->Bootstrap(txn, CATALOG_DATABASE_NAME); // bootstrap other global catalog tables DatabaseMetricsCatalog::GetInstance(txn); SettingsCatalog::GetInstance(txn); @@ -211,7 +312,9 @@ void Catalog::Bootstrap() { // Reset oid of each catalog to avoid collisions between catalog // values added by system and users when checkpoint recovery. - DatabaseCatalog::GetInstance()->UpdateOid(OID_FOR_USER_OFFSET); + DatabaseCatalog::GetInstance(nullptr, + nullptr, + nullptr)->UpdateOid(OID_FOR_USER_OFFSET); LanguageCatalog::GetInstance().UpdateOid(OID_FOR_USER_OFFSET); ProcCatalog::GetInstance().UpdateOid(OID_FOR_USER_OFFSET); } @@ -220,16 +323,17 @@ void Catalog::Bootstrap() { // CREATE FUNCTIONS //===----------------------------------------------------------------------===// -ResultType Catalog::CreateDatabase(const std::string &database_name, - concurrency::TransactionContext *txn) { +ResultType Catalog::CreateDatabase(concurrency::TransactionContext *txn, + const std::string &database_name) { if (txn == nullptr) throw CatalogException("Do not have transaction to create database " + - database_name); + database_name); - auto pg_database = DatabaseCatalog::GetInstance(); + auto pg_database = DatabaseCatalog::GetInstance(nullptr, nullptr, nullptr); auto storage_manager = storage::StorageManager::GetInstance(); // Check if a database with the same name exists - auto database_object = pg_database->GetDatabaseObject(database_name, txn); + auto database_object = + pg_database->GetDatabaseCatalogEntry(txn, database_name); if (database_object != nullptr) throw CatalogException("Database " + database_name + " already exists"); @@ -247,11 +351,11 @@ ResultType Catalog::CreateDatabase(const std::string &database_name, // put database object into rw_object_set txn->RecordCreate(database_oid, INVALID_OID, INVALID_OID); // Insert database record into pg_db - pg_database->InsertDatabase(database_oid, database_name, pool_.get(), txn); + pg_database->InsertDatabase(txn, database_oid, database_name, pool_.get()); // add core & non-core system catalog tables into database - BootstrapSystemCatalogs(database, txn); - catalog_map_[database_oid]->Bootstrap(database_name, txn); + BootstrapSystemCatalogs(txn, database); + catalog_map_[database_oid]->Bootstrap(txn, database_name); LOG_TRACE("Database %s created. Returning RESULT_SUCCESS.", database_name.c_str()); return ResultType::SUCCESS; @@ -263,30 +367,35 @@ ResultType Catalog::CreateDatabase(const std::string &database_name, * @param txn TransactionContext * @return TransactionContext ResultType(SUCCESS or FAILURE) */ -ResultType Catalog::CreateSchema(const std::string &database_name, - const std::string &schema_name, - concurrency::TransactionContext *txn) { +ResultType Catalog::CreateSchema(concurrency::TransactionContext *txn, + const std::string &database_name, + const std::string &schema_name) { if (txn == nullptr) throw CatalogException( "Do not have transaction to create schema(namespace) " + database_name); // check whether database exists from pg_database auto database_object = - DatabaseCatalog::GetInstance()->GetDatabaseObject(database_name, txn); + DatabaseCatalog::GetInstance(nullptr, + nullptr, + nullptr)->GetDatabaseCatalogEntry(txn, + database_name); if (database_object == nullptr) throw CatalogException("Can't find Database " + database_name + - " to create schema"); + " to create schema"); // check whether namespace exists from pg_namespace auto pg_namespace = catalog_map_[database_object->GetDatabaseOid()]->GetSchemaCatalog(); - auto schema_object = pg_namespace->GetSchemaObject(schema_name, txn); + auto schema_object = pg_namespace->GetSchemaCatalogEntry(txn, schema_name); if (schema_object != nullptr) throw CatalogException("Schema(namespace) " + schema_name + - " already exists"); + " already exists"); // Since there isn't physical class corresponds to schema(namespace), the only // thing needs to be done is inserting record into pg_namespace - pg_namespace->InsertSchema(pg_namespace->GetNextOid(), schema_name, - pool_.get(), txn); + pg_namespace->InsertSchema(txn, + pg_namespace->GetNextOid(), + schema_name, + pool_.get()); LOG_TRACE("Schema(namespace) %s created. Returning RESULT_SUCCESS.", schema_name.c_str()); @@ -301,38 +410,43 @@ ResultType Catalog::CreateSchema(const std::string &database_name, * @param txn TransactionContext * @return TransactionContext ResultType(SUCCESS or FAILURE) */ -ResultType Catalog::CreateTable(const std::string &database_name, +ResultType Catalog::CreateTable(concurrency::TransactionContext *txn, + const std::string &database_name, const std::string &schema_name, - const std::string &table_name, std::unique_ptr schema, - concurrency::TransactionContext *txn, - bool is_catalog, uint32_t tuples_per_tilegroup, - peloton::LayoutType layout_type) { + const std::string &table_name, + bool is_catalog, + uint32_t tuples_per_tilegroup, + LayoutType layout_type) { if (txn == nullptr) throw CatalogException("Do not have transaction to create table " + - table_name); + table_name); LOG_TRACE("Creating table %s in database %s", table_name.c_str(), database_name.c_str()); // check whether database exists from pg_database auto database_object = - DatabaseCatalog::GetInstance()->GetDatabaseObject(database_name, txn); + DatabaseCatalog::GetInstance(nullptr, + nullptr, + nullptr)->GetDatabaseCatalogEntry(txn, + database_name); if (database_object == nullptr) throw CatalogException("Can't find Database " + database_name + - " to create table"); + " to create table"); // check whether namespace exists from pg_namespace auto schema_object = catalog_map_[database_object->GetDatabaseOid()] - ->GetSchemaCatalog() - ->GetSchemaObject(schema_name, txn); + ->GetSchemaCatalog() + ->GetSchemaCatalogEntry(txn, schema_name); if (schema_object == nullptr) throw CatalogException("Can't find namespace " + schema_name + - " to create table"); + " to create table"); // get table oid from pg_table - auto table_object = database_object->GetTableObject(table_name, schema_name); + auto table_object = + database_object->GetTableCatalogEntry(table_name, schema_name); if (table_object != nullptr) throw CatalogException("Table: " + schema_name + "." + table_name + - " already exists"); + " already exists"); auto storage_manager = storage::StorageManager::GetInstance(); auto database = @@ -346,7 +460,7 @@ ResultType Catalog::CreateTable(const std::string &database_name, auto column_name = column.GetName(); if (column_names.count(column_name) == 1) throw CatalogException("Can't create table " + table_name + - " with duplicate column name"); + " with duplicate column name"); column_names.insert(column_name); } @@ -367,37 +481,55 @@ ResultType Catalog::CreateTable(const std::string &database_name, txn->RecordCreate(database_object->GetDatabaseOid(), table_oid, INVALID_OID); // Update pg_table with table info - pg_table->InsertTable(table_oid, table_name, schema_name, + pg_table->InsertTable(txn, database_object->GetDatabaseOid(), - table->GetDefaultLayout()->GetOid(), pool_.get(), txn); + schema_name, + table_oid, + table_name, + table->GetDefaultLayout()->GetOid(), + pool_.get()); oid_t column_id = 0; for (const auto &column : table->GetSchema()->GetColumns()) { - pg_attribute->InsertColumn(table_oid, column.GetName(), column_id, - column.GetOffset(), column.GetType(), - column.GetLength(), column.IsInlined(), - column.GetConstraints(), pool_.get(), txn); + pg_attribute->InsertColumn(txn, + table_oid, + column_id, + column.GetName(), + column.GetOffset(), + column.GetType(), + column.GetLength(), + column.GetConstraints(), + column.IsInlined(), + pool_.get()); // Create index on unique single column if (column.IsUnique()) { std::string col_name = column.GetName(); std::string index_name = table->GetName() + "_" + col_name + "_UNIQ"; - CreateIndex(database_name, schema_name, table_name, {column_id}, - index_name, true, IndexType::BWTREE, txn); + CreateIndex(txn, + database_name, + schema_name, + table_name, + index_name, + {column_id}, + true, + IndexType::BWTREE); LOG_DEBUG("Added a UNIQUE index on %s in %s.", col_name.c_str(), table_name.c_str()); } column_id++; } - CreatePrimaryIndex(database_object->GetDatabaseOid(), table_oid, schema_name, - txn); + CreatePrimaryIndex(txn, + database_object->GetDatabaseOid(), + schema_name, + table_oid); // Create layout as default layout auto pg_layout = catalog_map_[database_object->GetDatabaseOid()]->GetLayoutCatalog(); auto default_layout = table->GetDefaultLayout(); - if (!pg_layout->InsertLayout(table_oid, default_layout, pool_.get(), txn)) + if (!pg_layout->InsertLayout(txn, table_oid, default_layout, pool_.get())) throw CatalogException("Failed to create a new layout for table " - + table_name); + + table_name); return ResultType::SUCCESS; } @@ -411,9 +543,10 @@ ResultType Catalog::CreateTable(const std::string &database_name, * @param txn TransactionContext * @return TransactionContext ResultType(SUCCESS or FAILURE) */ -ResultType Catalog::CreatePrimaryIndex(oid_t database_oid, oid_t table_oid, +ResultType Catalog::CreatePrimaryIndex(concurrency::TransactionContext *txn, + oid_t database_oid, const std::string &schema_name, - concurrency::TransactionContext *txn) { + oid_t table_oid) { LOG_TRACE("Trying to create primary index for table %d", table_oid); auto storage_manager = storage::StorageManager::GetInstance(); @@ -460,9 +593,16 @@ ResultType Catalog::CreatePrimaryIndex(oid_t database_oid, oid_t table_oid, // put index object into rw_object_set txn->RecordCreate(database_oid, table_oid, index_oid); // insert index record into index_catalog(pg_index) table - pg_index->InsertIndex(index_oid, index_name, table_oid, schema_name, - IndexType::BWTREE, IndexConstraintType::PRIMARY_KEY, - unique_keys, key_attrs, pool_.get(), txn); + pg_index->InsertIndex(txn, + schema_name, + table_oid, + index_oid, + index_name, + IndexType::BWTREE, + IndexConstraintType::PRIMARY_KEY, + unique_keys, + key_attrs, + pool_.get()); LOG_TRACE("Successfully created primary key index '%s' for table '%s'", index_name.c_str(), table->GetName().c_str()); @@ -483,51 +623,68 @@ ResultType Catalog::CreatePrimaryIndex(oid_t database_oid, oid_t table_oid, * catalog table Initialization) * @return TransactionContext ResultType(SUCCESS or FAILURE) */ -ResultType Catalog::CreateIndex(const std::string &database_name, +ResultType Catalog::CreateIndex(concurrency::TransactionContext *txn, + const std::string &database_name, const std::string &schema_name, const std::string &table_name, + const std::string &index_name, const std::vector &key_attrs, - const std::string &index_name, bool unique_keys, - IndexType index_type, - concurrency::TransactionContext *txn) { + bool unique_keys, + IndexType index_type) { if (txn == nullptr) throw CatalogException("Do not have transaction to create database " + - index_name); + index_name); LOG_TRACE("Trying to create index %s for table %s", index_name.c_str(), table_name.c_str()); // check if database exists auto database_object = - DatabaseCatalog::GetInstance()->GetDatabaseObject(database_name, txn); + DatabaseCatalog::GetInstance(nullptr, + nullptr, + nullptr)->GetDatabaseCatalogEntry(txn, + database_name); if (database_object == nullptr) throw CatalogException("Can't find Database " + database_name + - " to create index"); + " to create index"); // check if table exists - auto table_object = database_object->GetTableObject(table_name, schema_name); + auto table_object = + database_object->GetTableCatalogEntry(table_name, schema_name); if (table_object == nullptr) throw CatalogException("Can't find table " + schema_name + "." + - table_name + " to create index"); + table_name + " to create index"); IndexConstraintType index_constraint = unique_keys ? IndexConstraintType::UNIQUE : IndexConstraintType::DEFAULT; - ResultType success = CreateIndex( - database_object->GetDatabaseOid(), table_object->GetTableOid(), key_attrs, - schema_name, index_name, index_type, index_constraint, unique_keys, txn); + ResultType success = CreateIndex(txn, + database_object->GetDatabaseOid(), + schema_name, + table_object->GetTableOid(), + false, + index_name, + key_attrs, + unique_keys, + index_type, + index_constraint); return success; } -ResultType Catalog::CreateIndex( - oid_t database_oid, oid_t table_oid, const std::vector &key_attrs, - const std::string &schema_name, const std::string &index_name, - IndexType index_type, IndexConstraintType index_constraint, - bool unique_keys, concurrency::TransactionContext *txn, bool is_catalog) { +ResultType Catalog::CreateIndex(concurrency::TransactionContext *txn, + oid_t database_oid, + const std::string &schema_name, + oid_t table_oid, + bool is_catalog, + const std::string &index_name, + const std::vector &key_attrs, + bool unique_keys, + IndexType index_type, + IndexConstraintType index_constraint) { if (txn == nullptr) throw CatalogException("Do not have transaction to create index " + - index_name); + index_name); LOG_TRACE("Trying to create index for table %d", table_oid); @@ -535,13 +692,16 @@ ResultType Catalog::CreateIndex( // check if table already has index with same name // only check when is_catalog flag == false auto database_object = - DatabaseCatalog::GetInstance()->GetDatabaseObject(database_oid, txn); - auto table_object = database_object->GetTableObject(table_oid); - auto index_object = table_object->GetIndexObject(index_name); + DatabaseCatalog::GetInstance(nullptr, + nullptr, + nullptr)->GetDatabaseCatalogEntry(txn, + database_oid); + auto table_object = database_object->GetTableCatalogEntry(table_oid); + auto index_object = table_object->GetIndexCatalogEntry(index_name); if (index_object != nullptr) throw CatalogException("Index " + index_name + " already exists in" + - database_object->GetDatabaseName()); + database_object->GetDatabaseName()); } auto storage_manager = storage::StorageManager::GetInstance(); auto database = storage_manager->GetDatabaseWithOid(database_oid); @@ -569,18 +729,26 @@ ResultType Catalog::CreateIndex( // Put index object into rw_object_set txn->RecordCreate(database_oid, table_oid, index_oid); // Insert index record into pg_index - pg_index->InsertIndex(index_oid, index_name, table_oid, schema_name, - index_type, index_constraint, unique_keys, key_attrs, - pool_.get(), txn); + pg_index->InsertIndex(txn, + schema_name, + table_oid, + index_oid, + index_name, + index_type, + index_constraint, + unique_keys, + key_attrs, + pool_.get()); LOG_TRACE("Successfully add index for table %s contains %d indexes", - table->GetName().c_str(), (int)table->GetValidIndexCount()); + table->GetName().c_str(), (int) table->GetValidIndexCount()); return ResultType::SUCCESS; } -std::shared_ptr Catalog::CreateLayout( - oid_t database_oid, oid_t table_oid, const column_map_type &column_map, - concurrency::TransactionContext *txn) { +std::shared_ptr Catalog::CreateLayout(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + const column_map_type &column_map) { auto storage_manager = storage::StorageManager::GetInstance(); auto database = storage_manager->GetDatabaseWithOid(database_oid); auto table = database->GetTableWithOid(table_oid); @@ -593,19 +761,20 @@ std::shared_ptr Catalog::CreateLayout( // Add the layout the pg_layout table auto pg_layout = catalog_map_[database_oid]->GetLayoutCatalog(); - if (pg_layout->GetLayoutWithOid(table_oid, new_layout->GetOid(), txn) + if (pg_layout->GetLayoutWithOid(txn, table_oid, new_layout->GetOid()) == nullptr && - !pg_layout->InsertLayout(table_oid, new_layout, pool_.get(), txn)) { - LOG_ERROR("Failed to create a new layout for table %u", table_oid); - return nullptr; + !pg_layout->InsertLayout(txn, table_oid, new_layout, pool_.get())) { + LOG_ERROR("Failed to create a new layout for table %u", table_oid); + return nullptr; } return new_layout; } -std::shared_ptr Catalog::CreateDefaultLayout( - oid_t database_oid, oid_t table_oid, const column_map_type &column_map, - concurrency::TransactionContext *txn) { - auto new_layout = CreateLayout(database_oid, table_oid, column_map, txn); +std::shared_ptr Catalog::CreateDefaultLayout(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + const column_map_type &column_map) { + auto new_layout = CreateLayout(txn, database_oid, table_oid, column_map); // If the layout creation was successful, set it as the default if (new_layout != nullptr) { auto storage_manager = storage::StorageManager::GetInstance(); @@ -615,7 +784,7 @@ std::shared_ptr Catalog::CreateDefaultLayout( // update table catalog catalog_map_[database_oid]->GetTableCatalog() - ->UpdateDefaultLayoutOid(new_layout->GetOid(), table_oid, txn); + ->UpdateDefaultLayoutOid(txn, table_oid, new_layout->GetOid()); } return new_layout; } @@ -624,39 +793,47 @@ std::shared_ptr Catalog::CreateDefaultLayout( // DROP FUNCTIONS //===----------------------------------------------------------------------===// -ResultType Catalog::DropDatabaseWithName(const std::string &database_name, - concurrency::TransactionContext *txn) { +ResultType Catalog::DropDatabaseWithName(concurrency::TransactionContext *txn, + const std::string &database_name) { if (txn == nullptr) throw CatalogException("Do not have transaction to drop database " + - database_name); + database_name); auto database_object = - DatabaseCatalog::GetInstance()->GetDatabaseObject(database_name, txn); + DatabaseCatalog::GetInstance(nullptr, + nullptr, + nullptr)->GetDatabaseCatalogEntry(txn, + database_name); if (database_object == nullptr) throw CatalogException("Drop Database: " + database_name + - " does not exist"); + " does not exist"); - return DropDatabaseWithOid(database_object->GetDatabaseOid(), txn); + return DropDatabaseWithOid(txn, database_object->GetDatabaseOid()); } -ResultType Catalog::DropDatabaseWithOid(oid_t database_oid, - concurrency::TransactionContext *txn) { +ResultType Catalog::DropDatabaseWithOid(concurrency::TransactionContext *txn, + oid_t database_oid) { if (txn == nullptr) throw CatalogException("Do not have transaction to drop database " + - std::to_string(database_oid)); + std::to_string(database_oid)); auto storage_manager = storage::StorageManager::GetInstance(); // Drop actual tables in the database auto database_object = - DatabaseCatalog::GetInstance()->GetDatabaseObject(database_oid, txn); - auto table_objects = database_object->GetTableObjects(); + DatabaseCatalog::GetInstance(nullptr, + nullptr, + nullptr)->GetDatabaseCatalogEntry(txn, + database_oid); + auto table_objects = database_object->GetTableCatalogEntries(); for (auto it : table_objects) { - DropTable(database_oid, it.second->GetTableOid(), txn); + DropTable(txn, database_oid, it.second->GetTableOid()); } // Drop database record in catalog - if (!DatabaseCatalog::GetInstance()->DeleteDatabase(database_oid, txn)) + if (!DatabaseCatalog::GetInstance(nullptr, + nullptr, + nullptr)->DeleteDatabase(txn, database_oid)) throw CatalogException("Database record: " + std::to_string(database_oid) + - " does not exist in pg_database"); + " does not exist in pg_database"); catalog_map_.erase(database_oid); // put database object into rw_object_set @@ -674,33 +851,34 @@ ResultType Catalog::DropDatabaseWithOid(oid_t database_oid, * @param txn TransactionContext * @return TransactionContext ResultType(SUCCESS or FAILURE) */ -ResultType Catalog::DropSchema(const std::string &database_name, - const std::string &schema_name, - concurrency::TransactionContext *txn) { +ResultType Catalog::DropSchema(concurrency::TransactionContext *txn, + const std::string &database_name, + const std::string &schema_name) { if (txn == nullptr) throw CatalogException("Do not have transaction to drop schema " + - schema_name); + schema_name); auto database_object = - DatabaseCatalog::GetInstance()->GetDatabaseObject(database_name, txn); + DatabaseCatalog::GetInstance(txn)->GetDatabaseCatalogEntry(txn, + database_name); if (database_object == nullptr) throw CatalogException("Drop Schema: database " + database_name + - " does not exist"); + " does not exist"); // check whether namespace exists from pg_namespace auto pg_namespace = catalog_map_[database_object->GetDatabaseOid()]->GetSchemaCatalog(); - auto schema_object = pg_namespace->GetSchemaObject(schema_name, txn); + auto schema_object = pg_namespace->GetSchemaCatalogEntry(txn, schema_name); if (schema_object == nullptr) throw CatalogException("Can't find namespace " + schema_name + " to drop"); - auto table_objects = database_object->GetTableObjects(schema_name); + auto table_objects = database_object->GetTableCatalogEntries(schema_name); for (auto it : table_objects) { - DropTable(it->GetDatabaseOid(), it->GetTableOid(), txn); + DropTable(txn, it->GetDatabaseOid(), it->GetTableOid()); } // remove record within pg_namespace - pg_namespace->DeleteSchema(schema_name, txn); + pg_namespace->DeleteSchema(txn, schema_name); return ResultType::SUCCESS; } @@ -718,29 +896,34 @@ ResultType Catalog::DropSchema(const std::string &database_name, * @param txn TransactionContext * @return TransactionContext ResultType(SUCCESS or FAILURE) */ -ResultType Catalog::DropTable(const std::string &database_name, +ResultType Catalog::DropTable(concurrency::TransactionContext *txn, + const std::string &database_name, const std::string &schema_name, - const std::string &table_name, - concurrency::TransactionContext *txn) { + const std::string &table_name) { if (txn == nullptr) throw CatalogException("Do not have transaction to drop table " + - table_name); + table_name); // Checking if statement is valid auto database_object = - DatabaseCatalog::GetInstance()->GetDatabaseObject(database_name, txn); + DatabaseCatalog::GetInstance(nullptr, + nullptr, + nullptr)->GetDatabaseCatalogEntry(txn, + database_name); if (database_object == nullptr) throw CatalogException("Drop Table: database " + database_name + - " does not exist"); + " does not exist"); // check if table exists - auto table_object = database_object->GetTableObject(table_name, schema_name); + auto table_object = + database_object->GetTableCatalogEntry(table_name, schema_name); if (table_object == nullptr) throw CatalogException("Drop Table: table " + schema_name + "." + - table_name + " does not exist"); + table_name + " does not exist"); - ResultType result = DropTable(database_object->GetDatabaseOid(), - table_object->GetTableOid(), txn); + ResultType result = DropTable(txn, + database_object->GetDatabaseOid(), + table_object->GetTableOid()); return result; } @@ -755,43 +938,49 @@ ResultType Catalog::DropTable(const std::string &database_name, * @param txn TransactionContext * @return TransactionContext ResultType(SUCCESS or FAILURE) */ -ResultType Catalog::DropTable(oid_t database_oid, oid_t table_oid, - concurrency::TransactionContext *txn) { +ResultType Catalog::DropTable(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid) { LOG_TRACE("Dropping table %d from database %d", database_oid, table_oid); auto storage_manager = storage::StorageManager::GetInstance(); auto database = storage_manager->GetDatabaseWithOid(database_oid); auto database_object = - DatabaseCatalog::GetInstance()->GetDatabaseObject(database_oid, txn); - auto table_object = database_object->GetTableObject(table_oid); - auto index_objects = table_object->GetIndexObjects(); - LOG_TRACE("dropping #%d indexes", (int)index_objects.size()); + DatabaseCatalog::GetInstance(nullptr, + nullptr, + nullptr)->GetDatabaseCatalogEntry(txn, + database_oid); + auto table_object = database_object->GetTableCatalogEntry(table_oid); + auto index_objects = table_object->GetIndexCatalogEntries(); + LOG_TRACE("dropping #%d indexes", (int) index_objects.size()); // delete trigger and records in pg_trigger auto pg_trigger = catalog_map_[database_object->GetDatabaseOid()]->GetTriggerCatalog(); std::unique_ptr trigger_lists = - pg_trigger->GetTriggers(table_oid, txn); + pg_trigger->GetTriggers(txn, table_oid); for (int i = 0; i < trigger_lists->GetTriggerListSize(); i++) - pg_trigger->DropTrigger(database_oid, table_oid, - trigger_lists->Get(i)->GetTriggerName(), txn); + pg_trigger->DropTrigger(txn, + database_oid, + table_oid, + trigger_lists->Get(i)->GetTriggerName()); // delete index and records pg_index for (auto it : index_objects) - DropIndex(database_oid, it.second->GetIndexOid(), txn); + DropIndex(txn, database_oid, it.second->GetIndexOid()); // delete record in pg_attribute auto pg_attribute = catalog_map_[database_object->GetDatabaseOid()]->GetColumnCatalog(); - pg_attribute->DeleteColumns(table_oid, txn); + pg_attribute->DeleteColumns(txn, table_oid); // delete record in pg_layout auto pg_layout = catalog_map_[database_object->GetDatabaseOid()]->GetLayoutCatalog(); - pg_layout->DeleteLayouts(table_oid, txn); + pg_layout->DeleteLayouts(txn, table_oid); // delete record in pg_table auto pg_table = catalog_map_[database_object->GetDatabaseOid()]->GetTableCatalog(); - pg_table->DeleteTable(table_oid, txn); + pg_table->DeleteTable(txn, table_oid); database->GetTableWithOid(table_oid); txn->RecordDrop(database_oid, table_oid, INVALID_OID); @@ -803,25 +992,27 @@ ResultType Catalog::DropTable(oid_t database_oid, oid_t table_oid, * @param txn TransactionContext * @return TransactionContext ResultType(SUCCESS or FAILURE) */ -ResultType Catalog::DropIndex(oid_t database_oid, oid_t index_oid, - concurrency::TransactionContext *txn) { +ResultType Catalog::DropIndex(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t index_oid) { if (txn == nullptr) throw CatalogException("Do not have transaction to drop index " + - std::to_string(index_oid)); + std::to_string(index_oid)); // find index catalog object by looking up pg_index or read from cache using // index_oid auto pg_index = catalog_map_[database_oid]->GetIndexCatalog(); - auto index_object = pg_index->GetIndexObject(database_oid, index_oid, txn); + auto index_object = + pg_index->GetIndexCatalogEntry(txn, database_oid, index_oid); if (index_object == nullptr) { throw CatalogException("Can't find index " + std::to_string(index_oid) + - " to drop"); + " to drop"); } auto storage_manager = storage::StorageManager::GetInstance(); auto table = storage_manager->GetTableWithOid(database_oid, index_object->GetTableOid()); // drop record in pg_index - pg_index->DeleteIndex(database_oid, index_oid, txn); + pg_index->DeleteIndex(txn, database_oid, index_oid); LOG_TRACE("Successfully drop index %d for table %s", index_oid, table->GetName().c_str()); @@ -832,9 +1023,10 @@ ResultType Catalog::DropIndex(oid_t database_oid, oid_t index_oid, return ResultType::SUCCESS; } -ResultType Catalog::DropLayout(oid_t database_oid, oid_t table_oid, - oid_t layout_oid, - concurrency::TransactionContext *txn) { +ResultType Catalog::DropLayout(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + oid_t layout_oid) { // Check if the default_layout of the table is the same. // If true reset it to a row store. auto storage_manager = storage::StorageManager::GetInstance(); @@ -843,7 +1035,7 @@ ResultType Catalog::DropLayout(oid_t database_oid, oid_t table_oid, auto default_layout = table->GetDefaultLayout(); auto pg_layout = catalog_map_[database_oid]->GetLayoutCatalog(); - if (!pg_layout->DeleteLayout(table_oid, layout_oid, txn)) { + if (!pg_layout->DeleteLayout(txn, table_oid, layout_oid)) { auto layout = table->GetDefaultLayout(); LOG_DEBUG("Layout delete failed. Default layout id: %u", layout->GetOid()); return ResultType::FAILURE; @@ -852,17 +1044,20 @@ ResultType Catalog::DropLayout(oid_t database_oid, oid_t table_oid, if (default_layout->GetOid() == layout_oid) { table->ResetDefaultLayout(); auto new_default_layout = table->GetDefaultLayout(); - if (pg_layout->GetLayoutWithOid(table_oid, new_default_layout->GetOid(), - txn) == nullptr && - !pg_layout->InsertLayout(table_oid, new_default_layout, - pool_.get(), txn)) { + if (pg_layout->GetLayoutWithOid(txn, + table_oid, + new_default_layout->GetOid()) == nullptr && + !pg_layout->InsertLayout(txn, + table_oid, + new_default_layout, + pool_.get())) { LOG_DEBUG("Failed to create a new layout for table %d", table_oid); return ResultType::FAILURE; } // update table catalog catalog_map_[database_oid]->GetTableCatalog() - ->UpdateDefaultLayoutOid(new_default_layout->GetOid(), table_oid, txn); + ->UpdateDefaultLayoutOid(txn, table_oid, new_default_layout->GetOid()); } return ResultType::SUCCESS; @@ -876,14 +1071,16 @@ ResultType Catalog::DropLayout(oid_t database_oid, oid_t table_oid, * get it from storage layer using database_oid, * throw exception and abort txn if not exists/invisible * */ -storage::Database *Catalog::GetDatabaseWithName( - const std::string &database_name, - concurrency::TransactionContext *txn) const { +storage::Database *Catalog::GetDatabaseWithName(concurrency::TransactionContext *txn, + const std::string &database_name) const { PELOTON_ASSERT(txn != nullptr); // Check in pg_database using txn auto database_object = - DatabaseCatalog::GetInstance()->GetDatabaseObject(database_name, txn); + DatabaseCatalog::GetInstance(nullptr, + nullptr, + nullptr)->GetDatabaseCatalogEntry(txn, + database_name); if (database_object == nullptr) { throw CatalogException("Database " + database_name + " is not found"); @@ -897,16 +1094,17 @@ storage::Database *Catalog::GetDatabaseWithName( * get it from storage layer using table_oid, * throw exception and abort txn if not exists/invisible * */ -storage::DataTable *Catalog::GetTableWithName( - const std::string &database_name, const std::string &schema_name, - const std::string &table_name, concurrency::TransactionContext *txn) { +storage::DataTable *Catalog::GetTableWithName(concurrency::TransactionContext *txn, + const std::string &database_name, + const std::string &schema_name, + const std::string &table_name) { PELOTON_ASSERT(txn != nullptr); LOG_TRACE("Looking for table %s in database %s", table_name.c_str(), database_name.c_str()); // Check in pg_table, throw exception and abort txn if not exists auto table_object = - GetTableObject(database_name, schema_name, table_name, txn); + GetTableCatalogEntry(txn, database_name, schema_name, table_name); // Get table from storage manager auto storage_manager = storage::StorageManager::GetInstance(); @@ -918,18 +1116,22 @@ storage::DataTable *Catalog::GetTableWithName( * get it from storage layer using table_oid, * throw exception and abort txn if not exists/invisible * */ -std::shared_ptr Catalog::GetDatabaseObject( - const std::string &database_name, concurrency::TransactionContext *txn) { +std::shared_ptr Catalog::GetDatabaseCatalogEntry( + concurrency::TransactionContext *txn, + const std::string &database_name) { if (txn == nullptr) { throw CatalogException("Do not have transaction to get table object " + - database_name); + database_name); } LOG_TRACE("Looking for database %s", database_name.c_str()); // Check in pg_database, throw exception and abort txn if not exists auto database_object = - DatabaseCatalog::GetInstance()->GetDatabaseObject(database_name, txn); + DatabaseCatalog::GetInstance(nullptr, + nullptr, + nullptr)->GetDatabaseCatalogEntry(txn, + database_name); if (!database_object || database_object->GetDatabaseOid() == INVALID_OID) { throw CatalogException("Database " + database_name + " is not found"); @@ -938,22 +1140,26 @@ std::shared_ptr Catalog::GetDatabaseObject( return database_object; } -std::shared_ptr Catalog::GetDatabaseObject( - oid_t database_oid, concurrency::TransactionContext *txn) { +std::shared_ptr Catalog::GetDatabaseCatalogEntry( + concurrency::TransactionContext *txn, + oid_t database_oid) { if (txn == nullptr) { throw CatalogException("Do not have transaction to get database object " + - std::to_string(database_oid)); + std::to_string(database_oid)); } LOG_TRACE("Looking for database %u", database_oid); // Check in pg_database, throw exception and abort txn if not exists auto database_object = - DatabaseCatalog::GetInstance()->GetDatabaseObject(database_oid, txn); + DatabaseCatalog::GetInstance(nullptr, + nullptr, + nullptr)->GetDatabaseCatalogEntry(txn, + database_oid); if (!database_object || database_object->GetDatabaseOid() == INVALID_OID) { throw CatalogException("Database " + std::to_string(database_oid) + - " is not found"); + " is not found"); } return database_object; @@ -963,12 +1169,13 @@ std::shared_ptr Catalog::GetDatabaseObject( * get it from storage layer using table_oid, * throw exception and abort txn if not exists/invisible * */ -std::shared_ptr Catalog::GetTableObject( - const std::string &database_name, const std::string &schema_name, - const std::string &table_name, concurrency::TransactionContext *txn) { +std::shared_ptr Catalog::GetTableCatalogEntry(concurrency::TransactionContext *txn, + const std::string &database_name, + const std::string &schema_name, + const std::string &table_name) { if (txn == nullptr) { throw CatalogException("Do not have transaction to get table object " + - database_name + "." + table_name); + database_name + "." + table_name); } LOG_TRACE("Looking for table %s in database %s", table_name.c_str(), @@ -976,50 +1183,58 @@ std::shared_ptr Catalog::GetTableObject( // Check in pg_database, throw exception and abort txn if not exists auto database_object = - DatabaseCatalog::GetInstance()->GetDatabaseObject(database_name, txn); + DatabaseCatalog::GetInstance(nullptr, + nullptr, + nullptr)->GetDatabaseCatalogEntry(txn, + database_name); if (!database_object || database_object->GetDatabaseOid() == INVALID_OID) { throw CatalogException("Database " + database_name + " is not found"); } // Check in pg_table using txn - auto table_object = database_object->GetTableObject(table_name, schema_name); + auto table_object = + database_object->GetTableCatalogEntry(table_name, schema_name); if (!table_object || table_object->GetTableOid() == INVALID_OID) { // throw table not found exception and explicitly abort txn throw CatalogException("Table " + schema_name + "." + table_name + - " is not found"); + " is not found"); } return table_object; } -std::shared_ptr Catalog::GetTableObject( - oid_t database_oid, oid_t table_oid, concurrency::TransactionContext *txn) { +std::shared_ptr Catalog::GetTableCatalogEntry(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid) { if (txn == nullptr) { throw CatalogException("Do not have transaction to get table object " + - std::to_string(database_oid) + "." + - std::to_string(table_oid)); + std::to_string(database_oid) + "." + + std::to_string(table_oid)); } LOG_TRACE("Looking for table %u in database %u", table_oid, database_oid); // Check in pg_database, throw exception and abort txn if not exists auto database_object = - DatabaseCatalog::GetInstance()->GetDatabaseObject(database_oid, txn); + DatabaseCatalog::GetInstance(nullptr, + nullptr, + nullptr)->GetDatabaseCatalogEntry(txn, + database_oid); if (!database_object || database_object->GetDatabaseOid() == INVALID_OID) { throw CatalogException("Database " + std::to_string(database_oid) + - " is not found"); + " is not found"); } // Check in pg_table using txn - auto table_object = database_object->GetTableObject(table_oid); + auto table_object = database_object->GetTableCatalogEntry(table_oid); if (!table_object || table_object->GetTableOid() == INVALID_OID) { // throw table not found exception and explicitly abort txn throw CatalogException("Table " + std::to_string(table_oid) + - " is not found"); + " is not found"); } return table_object; @@ -1029,7 +1244,7 @@ std::shared_ptr Catalog::GetSystemCatalogs( const oid_t database_oid) { if (catalog_map_.find(database_oid) == catalog_map_.end()) { throw CatalogException("Failed to find SystemCatalog for database_oid = " + - std::to_string(database_oid)); + std::to_string(database_oid)); } return catalog_map_[database_oid]; } @@ -1045,10 +1260,11 @@ void Catalog::AddDatabase(storage::Database *database) { storage_manager->AddDatabaseToStorageManager(database); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - BootstrapSystemCatalogs(database, txn); - DatabaseCatalog::GetInstance()->InsertDatabase( - database->GetOid(), database->GetDBName(), pool_.get(), - txn); // I guess this can pass tests + BootstrapSystemCatalogs(txn, database); + DatabaseCatalog::GetInstance(nullptr, nullptr, nullptr)->InsertDatabase(txn, + database->GetOid(), + database->GetDBName(), + pool_.get()); // I guess this can pass tests txn_manager.CommitTransaction(txn); } @@ -1074,13 +1290,20 @@ Catalog::~Catalog() { * @param func_name the function name in C++ source (should be unique) * @param func_ptr the pointer to the function */ -void Catalog::AddBuiltinFunction( - const std::string &name, const std::vector &argument_types, - const type::TypeId return_type, oid_t prolang, const std::string &func_name, - function::BuiltInFuncType func, concurrency::TransactionContext *txn) { - if (!ProcCatalog::GetInstance().InsertProc(name, return_type, argument_types, - prolang, func_name, pool_.get(), - txn)) { +void Catalog::AddBuiltinFunction(concurrency::TransactionContext *txn, + const std::string &name, + function::BuiltInFuncType func, + const std::string &func_name, + type::TypeId return_type, + const std::vector &argument_types, + oid_t prolang) { + if (!ProcCatalog::GetInstance().InsertProc(txn, + name, + return_type, + argument_types, + prolang, + func_name, + pool_.get())) { throw CatalogException("Failed to add function " + func_name); } function::BuiltInFunctions::AddFunction(func_name, func); @@ -1099,23 +1322,29 @@ void Catalog::AddBuiltinFunction( * @param code_context the code_context that holds the generated LLVM * query code */ -void Catalog::AddPlpgsqlFunction( - const std::string &name, const std::vector &argument_types, - const type::TypeId return_type, oid_t prolang, const std::string &func_src, - std::shared_ptr code_context, - concurrency::TransactionContext *txn) { +void Catalog::AddProcedure(concurrency::TransactionContext *txn, + const std::string &name, + type::TypeId return_type, + const std::vector &argument_types, + oid_t prolang, + std::shared_ptr code_context, + const std::string &func_src) { // Check if UDF already exists auto proc_catalog_obj = - ProcCatalog::GetInstance().GetProcByName(name, argument_types, txn); + ProcCatalog::GetInstance().GetProcByName(txn, name, argument_types); if (proc_catalog_obj == nullptr) { - if (!ProcCatalog::GetInstance().InsertProc(name, return_type, - argument_types, prolang, - func_src, pool_.get(), txn)) { + if (!ProcCatalog::GetInstance().InsertProc(txn, + name, + return_type, + argument_types, + prolang, + func_src, + pool_.get())) { throw CatalogException("Failed to add function " + name); } proc_catalog_obj = - ProcCatalog::GetInstance().GetProcByName(name, argument_types, txn); + ProcCatalog::GetInstance().GetProcByName(txn, name, argument_types); // Insert UDF into Catalog function::PlpgsqlFunctions::AddFunction(proc_catalog_obj->GetOid(), code_context); @@ -1129,7 +1358,7 @@ const FunctionData Catalog::GetFunction( // Lookup the function in pg_proc auto &proc_catalog = ProcCatalog::GetInstance(); - auto proc_catalog_obj = proc_catalog.GetProcByName(name, argument_types, txn); + auto proc_catalog_obj = proc_catalog.GetProcByName(txn, name, argument_types); if (proc_catalog_obj == nullptr) { txn_manager.AbortTransaction(txn); throw CatalogException("Failed to find function " + name); @@ -1139,12 +1368,12 @@ const FunctionData Catalog::GetFunction( auto lang_catalog_obj = proc_catalog_obj->GetLanguage(); if (lang_catalog_obj == nullptr || (lang_catalog_obj->GetName() != "internal" && - lang_catalog_obj->GetName() != "plpgsql")) { + lang_catalog_obj->GetName() != "plpgsql")) { txn_manager.AbortTransaction(txn); throw CatalogException( "Peloton currently only supports internal functions and plpgsql UDFs. \ Function " + - name + " has language '" + lang_catalog_obj->GetName() + "'"); + name + " has language '" + lang_catalog_obj->GetName() + "'"); } FunctionData result; @@ -1160,7 +1389,7 @@ const FunctionData Catalog::GetFunction( txn_manager.AbortTransaction(txn); throw CatalogException( "Function " + name + - " is internal, but doesn't have a function address"); + " is internal, but doesn't have a function address"); } } else if (lang_catalog_obj->GetName() == "plpgsql") { // If the function is a "plpgsql" udf, perform the lookup in the plpgsql @@ -1174,7 +1403,7 @@ const FunctionData Catalog::GetFunction( txn_manager.AbortTransaction(txn); throw CatalogException( "Function " + name + - " is plpgsql, but doesn't have a function address"); + " is plpgsql, but doesn't have a function address"); } } @@ -1188,14 +1417,16 @@ void Catalog::InitializeLanguages() { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); // add "internal" language - if (!LanguageCatalog::GetInstance().InsertLanguage("internal", pool_.get(), - txn)) { + if (!LanguageCatalog::GetInstance().InsertLanguage(txn, + "internal", + pool_.get())) { txn_manager.AbortTransaction(txn); throw CatalogException("Failed to add language 'internal'"); } // Add "plpgsql" language - if (!LanguageCatalog::GetInstance().InsertLanguage("plpgsql", pool_.get(), - txn)) { + if (!LanguageCatalog::GetInstance().InsertLanguage(txn, + "plpgsql", + pool_.get())) { txn_manager.AbortTransaction(txn); throw CatalogException("Failed to add language 'plpgsql'"); } @@ -1211,7 +1442,7 @@ void Catalog::InitializeFunctions() { auto txn = txn_manager.BeginTransaction(); auto lang_object = - LanguageCatalog::GetInstance().GetLanguageByName("internal", txn); + LanguageCatalog::GetInstance().GetLanguageByName(txn, "internal"); if (lang_object == nullptr) { throw CatalogException("Language 'internal' does not exist"); } @@ -1221,290 +1452,385 @@ void Catalog::InitializeFunctions() { /** * string functions */ - AddBuiltinFunction( - "ascii", {type::TypeId::VARCHAR}, type::TypeId::INTEGER, - internal_lang, "Ascii", - function::BuiltInFuncType{OperatorId::Ascii, - function::OldEngineStringFunctions::Ascii}, - txn); - AddBuiltinFunction( - "chr", {type::TypeId::INTEGER}, type::TypeId::VARCHAR, internal_lang, - "Chr", - function::BuiltInFuncType{OperatorId::Chr, - function::OldEngineStringFunctions::Chr}, - txn); - AddBuiltinFunction( - "concat", {type::TypeId::VARCHAR, type::TypeId::VARCHAR}, - type::TypeId::VARCHAR, internal_lang, "Concat", - function::BuiltInFuncType{OperatorId::Concat, - function::OldEngineStringFunctions::Concat}, - txn); - AddBuiltinFunction( - "substr", - {type::TypeId::VARCHAR, type::TypeId::INTEGER, type::TypeId::INTEGER}, - type::TypeId::VARCHAR, internal_lang, "Substr", - function::BuiltInFuncType{OperatorId::Substr, - function::OldEngineStringFunctions::Substr}, - txn); - AddBuiltinFunction("char_length", {type::TypeId::VARCHAR}, - type::TypeId::INTEGER, internal_lang, "CharLength", + AddBuiltinFunction(txn, + "ascii", + function::BuiltInFuncType{OperatorId::Ascii, + function::OldEngineStringFunctions::Ascii}, + "Ascii", + type::TypeId::INTEGER, + {type::TypeId::VARCHAR}, + internal_lang); + AddBuiltinFunction(txn, + "chr", + function::BuiltInFuncType{OperatorId::Chr, + function::OldEngineStringFunctions::Chr}, + "Chr", + type::TypeId::VARCHAR, + {type::TypeId::INTEGER}, + internal_lang); + AddBuiltinFunction(txn, + "concat", + function::BuiltInFuncType{OperatorId::Concat, + function::OldEngineStringFunctions::Concat}, + "Concat", + type::TypeId::VARCHAR, + {type::TypeId::VARCHAR, type::TypeId::VARCHAR}, + internal_lang); + AddBuiltinFunction(txn, + "substr", + function::BuiltInFuncType{OperatorId::Substr, + function::OldEngineStringFunctions::Substr}, + "Substr", + type::TypeId::VARCHAR, + {type::TypeId::VARCHAR, type::TypeId::INTEGER, + type::TypeId::INTEGER}, + internal_lang); + AddBuiltinFunction(txn, + "char_length", function::BuiltInFuncType{ OperatorId::CharLength, function::OldEngineStringFunctions::CharLength}, - txn); - AddBuiltinFunction("octet_length", {type::TypeId::VARCHAR}, - type::TypeId::INTEGER, internal_lang, "OctetLength", + "CharLength", + type::TypeId::INTEGER, + {type::TypeId::VARCHAR}, + internal_lang); + AddBuiltinFunction(txn, + "octet_length", function::BuiltInFuncType{ OperatorId::OctetLength, function::OldEngineStringFunctions::OctetLength}, - txn); - AddBuiltinFunction( - "length", {type::TypeId::VARCHAR}, type::TypeId::INTEGER, - internal_lang, "Length", - function::BuiltInFuncType{OperatorId::Length, - function::OldEngineStringFunctions::Length}, - txn); - AddBuiltinFunction( - "repeat", {type::TypeId::VARCHAR, type::TypeId::INTEGER}, - type::TypeId::VARCHAR, internal_lang, "Repeat", - function::BuiltInFuncType{OperatorId::Repeat, - function::OldEngineStringFunctions::Repeat}, - txn); - AddBuiltinFunction( - "replace", - {type::TypeId::VARCHAR, type::TypeId::VARCHAR, type::TypeId::VARCHAR}, - type::TypeId::VARCHAR, internal_lang, "Replace", - function::BuiltInFuncType{ - OperatorId::Replace, function::OldEngineStringFunctions::Replace}, - txn); - AddBuiltinFunction( - "ltrim", {type::TypeId::VARCHAR, type::TypeId::VARCHAR}, - type::TypeId::VARCHAR, internal_lang, "LTrim", - function::BuiltInFuncType{OperatorId::LTrim, - function::OldEngineStringFunctions::LTrim}, - txn); - AddBuiltinFunction( - "rtrim", {type::TypeId::VARCHAR, type::TypeId::VARCHAR}, - type::TypeId::VARCHAR, internal_lang, "RTrim", - function::BuiltInFuncType{OperatorId::RTrim, - function::OldEngineStringFunctions::RTrim}, - txn); - AddBuiltinFunction( - "btrim", {type::TypeId::VARCHAR, type::TypeId::VARCHAR}, - type::TypeId::VARCHAR, internal_lang, "btrim", - function::BuiltInFuncType{OperatorId::BTrim, - function::OldEngineStringFunctions::BTrim}, - txn); + "OctetLength", + type::TypeId::INTEGER, + {type::TypeId::VARCHAR}, + internal_lang); + AddBuiltinFunction(txn, + "length", + function::BuiltInFuncType{OperatorId::Length, + function::OldEngineStringFunctions::Length}, + "Length", + type::TypeId::INTEGER, + {type::TypeId::VARCHAR}, + internal_lang); + AddBuiltinFunction(txn, + "repeat", + function::BuiltInFuncType{OperatorId::Repeat, + function::OldEngineStringFunctions::Repeat}, + "Repeat", + type::TypeId::VARCHAR, + {type::TypeId::VARCHAR, type::TypeId::INTEGER}, + internal_lang); + AddBuiltinFunction(txn, + "replace", + function::BuiltInFuncType{ + OperatorId::Replace, + function::OldEngineStringFunctions::Replace}, + "Replace", + type::TypeId::VARCHAR, + {type::TypeId::VARCHAR, type::TypeId::VARCHAR, + type::TypeId::VARCHAR}, + internal_lang); + AddBuiltinFunction(txn, + "ltrim", + function::BuiltInFuncType{OperatorId::LTrim, + function::OldEngineStringFunctions::LTrim}, + "LTrim", + type::TypeId::VARCHAR, + {type::TypeId::VARCHAR, type::TypeId::VARCHAR}, + internal_lang); + AddBuiltinFunction(txn, + "rtrim", + function::BuiltInFuncType{OperatorId::RTrim, + function::OldEngineStringFunctions::RTrim}, + "RTrim", + type::TypeId::VARCHAR, + {type::TypeId::VARCHAR, type::TypeId::VARCHAR}, + internal_lang); + AddBuiltinFunction(txn, + "btrim", + function::BuiltInFuncType{OperatorId::BTrim, + function::OldEngineStringFunctions::BTrim}, + "btrim", + type::TypeId::VARCHAR, + {type::TypeId::VARCHAR, type::TypeId::VARCHAR}, + internal_lang); // Trim - AddBuiltinFunction( - "btrim", {type::TypeId::VARCHAR}, type::TypeId::VARCHAR, - internal_lang, "trim", - function::BuiltInFuncType{OperatorId::Trim, - function::OldEngineStringFunctions::Trim}, - txn); - AddBuiltinFunction( - "like", {type::TypeId::VARCHAR, type::TypeId::VARCHAR}, - type::TypeId::VARCHAR, internal_lang, "like", - function::BuiltInFuncType{OperatorId::Like, - function::OldEngineStringFunctions::Like}, - txn); + AddBuiltinFunction(txn, + "btrim", + function::BuiltInFuncType{OperatorId::Trim, + function::OldEngineStringFunctions::Trim}, + "trim", + type::TypeId::VARCHAR, + {type::TypeId::VARCHAR}, + internal_lang); + AddBuiltinFunction(txn, + "like", + function::BuiltInFuncType{OperatorId::Like, + function::OldEngineStringFunctions::Like}, + "like", + type::TypeId::VARCHAR, + {type::TypeId::VARCHAR, type::TypeId::VARCHAR}, + internal_lang); /** * decimal functions */ - AddBuiltinFunction("abs", {type::TypeId::DECIMAL}, type::TypeId::DECIMAL, - internal_lang, "Abs", + AddBuiltinFunction(txn, + "abs", function::BuiltInFuncType{ OperatorId::Abs, function::NumericFunctions::_Abs}, - txn); - AddBuiltinFunction( - "sqrt", {type::TypeId::TINYINT}, type::TypeId::DECIMAL, internal_lang, - "Sqrt", - function::BuiltInFuncType{OperatorId::Sqrt, - function::NumericFunctions::Sqrt}, - txn); - AddBuiltinFunction( - "sqrt", {type::TypeId::SMALLINT}, type::TypeId::DECIMAL, - internal_lang, "Sqrt", - function::BuiltInFuncType{OperatorId::Sqrt, - function::NumericFunctions::Sqrt}, - txn); - AddBuiltinFunction( - "sqrt", {type::TypeId::INTEGER}, type::TypeId::DECIMAL, internal_lang, - "Sqrt", - function::BuiltInFuncType{OperatorId::Sqrt, - function::NumericFunctions::Sqrt}, - txn); - AddBuiltinFunction( - "sqrt", {type::TypeId::BIGINT}, type::TypeId::DECIMAL, internal_lang, - "Sqrt", - function::BuiltInFuncType{OperatorId::Sqrt, - function::NumericFunctions::Sqrt}, - txn); - AddBuiltinFunction( - "sqrt", {type::TypeId::DECIMAL}, type::TypeId::DECIMAL, internal_lang, - "Sqrt", - function::BuiltInFuncType{OperatorId::Sqrt, - function::NumericFunctions::Sqrt}, - txn); - AddBuiltinFunction( - "floor", {type::TypeId::DECIMAL}, type::TypeId::DECIMAL, - internal_lang, "Floor", - function::BuiltInFuncType{OperatorId::Floor, - function::NumericFunctions::_Floor}, - txn); + "Abs", + type::TypeId::DECIMAL, + {type::TypeId::DECIMAL}, + internal_lang); + AddBuiltinFunction(txn, + "sqrt", + function::BuiltInFuncType{OperatorId::Sqrt, + function::NumericFunctions::Sqrt}, + "Sqrt", + type::TypeId::DECIMAL, + {type::TypeId::TINYINT}, + internal_lang); + AddBuiltinFunction(txn, + "sqrt", + function::BuiltInFuncType{OperatorId::Sqrt, + function::NumericFunctions::Sqrt}, + "Sqrt", + type::TypeId::DECIMAL, + {type::TypeId::SMALLINT}, + internal_lang); + AddBuiltinFunction(txn, + "sqrt", + function::BuiltInFuncType{OperatorId::Sqrt, + function::NumericFunctions::Sqrt}, + "Sqrt", + type::TypeId::DECIMAL, + {type::TypeId::INTEGER}, + internal_lang); + AddBuiltinFunction(txn, + "sqrt", + function::BuiltInFuncType{OperatorId::Sqrt, + function::NumericFunctions::Sqrt}, + "Sqrt", + type::TypeId::DECIMAL, + {type::TypeId::BIGINT}, + internal_lang); + AddBuiltinFunction(txn, + "sqrt", + function::BuiltInFuncType{OperatorId::Sqrt, + function::NumericFunctions::Sqrt}, + "Sqrt", + type::TypeId::DECIMAL, + {type::TypeId::DECIMAL}, + internal_lang); + AddBuiltinFunction(txn, + "floor", + function::BuiltInFuncType{OperatorId::Floor, + function::NumericFunctions::_Floor}, + "Floor", + type::TypeId::DECIMAL, + {type::TypeId::DECIMAL}, + internal_lang); /** * integer functions */ - AddBuiltinFunction("abs", {type::TypeId::TINYINT}, type::TypeId::TINYINT, - internal_lang, "Abs", + AddBuiltinFunction(txn, + "abs", function::BuiltInFuncType{ OperatorId::Abs, function::NumericFunctions::_Abs}, - txn); + "Abs", + type::TypeId::TINYINT, + {type::TypeId::TINYINT}, + internal_lang); - AddBuiltinFunction("abs", {type::TypeId::SMALLINT}, - type::TypeId::SMALLINT, internal_lang, "Abs", + AddBuiltinFunction(txn, + "abs", function::BuiltInFuncType{ OperatorId::Abs, function::NumericFunctions::_Abs}, - txn); + "Abs", + type::TypeId::SMALLINT, + {type::TypeId::SMALLINT}, + internal_lang); - AddBuiltinFunction("abs", {type::TypeId::INTEGER}, type::TypeId::INTEGER, - internal_lang, "Abs", + AddBuiltinFunction(txn, + "abs", function::BuiltInFuncType{ OperatorId::Abs, function::NumericFunctions::_Abs}, - txn); + "Abs", + type::TypeId::INTEGER, + {type::TypeId::INTEGER}, + internal_lang); - AddBuiltinFunction("abs", {type::TypeId::BIGINT}, type::TypeId::BIGINT, - internal_lang, "Abs", + AddBuiltinFunction(txn, + "abs", function::BuiltInFuncType{ OperatorId::Abs, function::NumericFunctions::_Abs}, - txn); - - AddBuiltinFunction( - "floor", {type::TypeId::INTEGER}, type::TypeId::DECIMAL, - internal_lang, "Floor", - function::BuiltInFuncType{OperatorId::Floor, - function::NumericFunctions::_Floor}, - txn); - AddBuiltinFunction( - "floor", {type::TypeId::BIGINT}, type::TypeId::DECIMAL, internal_lang, - "Floor", - function::BuiltInFuncType{OperatorId::Floor, - function::NumericFunctions::_Floor}, - txn); - AddBuiltinFunction( - "floor", {type::TypeId::TINYINT}, type::TypeId::DECIMAL, - internal_lang, "Floor", - function::BuiltInFuncType{OperatorId::Floor, - function::NumericFunctions::_Floor}, - txn); - AddBuiltinFunction( - "floor", {type::TypeId::SMALLINT}, type::TypeId::DECIMAL, - internal_lang, "Floor", - function::BuiltInFuncType{OperatorId::Floor, - function::NumericFunctions::_Floor}, - txn); - AddBuiltinFunction( - "round", {type::TypeId::DECIMAL}, type::TypeId::DECIMAL, - internal_lang, "Round", - function::BuiltInFuncType{OperatorId::Round, - function::NumericFunctions::_Round}, - txn); - - AddBuiltinFunction( - "ceil", {type::TypeId::DECIMAL}, type::TypeId::DECIMAL, internal_lang, - "Ceil", - function::BuiltInFuncType{OperatorId::Ceil, - function::NumericFunctions::_Ceil}, - txn); - - AddBuiltinFunction( - "ceil", {type::TypeId::TINYINT}, type::TypeId::DECIMAL, internal_lang, - "Ceil", - function::BuiltInFuncType{OperatorId::Ceil, - function::NumericFunctions::_Ceil}, - txn); - - AddBuiltinFunction( - "ceil", {type::TypeId::SMALLINT}, type::TypeId::DECIMAL, - internal_lang, "Ceil", - function::BuiltInFuncType{OperatorId::Ceil, - function::NumericFunctions::_Ceil}, - txn); - - AddBuiltinFunction( - "ceil", {type::TypeId::INTEGER}, type::TypeId::DECIMAL, internal_lang, - "Ceil", - function::BuiltInFuncType{OperatorId::Ceil, - function::NumericFunctions::_Ceil}, - txn); - - AddBuiltinFunction( - "ceil", {type::TypeId::BIGINT}, type::TypeId::DECIMAL, internal_lang, - "Ceil", - function::BuiltInFuncType{OperatorId::Ceil, - function::NumericFunctions::_Ceil}, - txn); - - AddBuiltinFunction( - "ceiling", {type::TypeId::DECIMAL}, type::TypeId::DECIMAL, - internal_lang, "Ceil", - function::BuiltInFuncType{OperatorId::Ceil, - function::NumericFunctions::_Ceil}, - txn); - - AddBuiltinFunction( - "ceiling", {type::TypeId::TINYINT}, type::TypeId::DECIMAL, - internal_lang, "Ceil", - function::BuiltInFuncType{OperatorId::Ceil, - function::NumericFunctions::_Ceil}, - txn); - - AddBuiltinFunction( - "ceiling", {type::TypeId::SMALLINT}, type::TypeId::DECIMAL, - internal_lang, "Ceil", - function::BuiltInFuncType{OperatorId::Ceil, - function::NumericFunctions::_Ceil}, - txn); - - AddBuiltinFunction( - "ceiling", {type::TypeId::INTEGER}, type::TypeId::DECIMAL, - internal_lang, "Ceil", - function::BuiltInFuncType{OperatorId::Ceil, - function::NumericFunctions::_Ceil}, - txn); - - AddBuiltinFunction( - "ceiling", {type::TypeId::BIGINT}, type::TypeId::DECIMAL, - internal_lang, "Ceil", - function::BuiltInFuncType{OperatorId::Ceil, - function::NumericFunctions::_Ceil}, - txn); + "Abs", + type::TypeId::BIGINT, + {type::TypeId::BIGINT}, + internal_lang); + + AddBuiltinFunction(txn, + "floor", + function::BuiltInFuncType{OperatorId::Floor, + function::NumericFunctions::_Floor}, + "Floor", + type::TypeId::DECIMAL, + {type::TypeId::INTEGER}, + internal_lang); + AddBuiltinFunction(txn, + "floor", + function::BuiltInFuncType{OperatorId::Floor, + function::NumericFunctions::_Floor}, + "Floor", + type::TypeId::DECIMAL, + {type::TypeId::BIGINT}, + internal_lang); + AddBuiltinFunction(txn, + "floor", + function::BuiltInFuncType{OperatorId::Floor, + function::NumericFunctions::_Floor}, + "Floor", + type::TypeId::DECIMAL, + {type::TypeId::TINYINT}, + internal_lang); + AddBuiltinFunction(txn, + "floor", + function::BuiltInFuncType{OperatorId::Floor, + function::NumericFunctions::_Floor}, + "Floor", + type::TypeId::DECIMAL, + {type::TypeId::SMALLINT}, + internal_lang); + AddBuiltinFunction(txn, + "round", + function::BuiltInFuncType{OperatorId::Round, + function::NumericFunctions::_Round}, + "Round", + type::TypeId::DECIMAL, + {type::TypeId::DECIMAL}, + internal_lang); + + AddBuiltinFunction(txn, + "ceil", + function::BuiltInFuncType{OperatorId::Ceil, + function::NumericFunctions::_Ceil}, + "Ceil", + type::TypeId::DECIMAL, + {type::TypeId::DECIMAL}, + internal_lang); + + AddBuiltinFunction(txn, + "ceil", + function::BuiltInFuncType{OperatorId::Ceil, + function::NumericFunctions::_Ceil}, + "Ceil", + type::TypeId::DECIMAL, + {type::TypeId::TINYINT}, + internal_lang); + + AddBuiltinFunction(txn, + "ceil", + function::BuiltInFuncType{OperatorId::Ceil, + function::NumericFunctions::_Ceil}, + "Ceil", + type::TypeId::DECIMAL, + {type::TypeId::SMALLINT}, + internal_lang); + + AddBuiltinFunction(txn, + "ceil", + function::BuiltInFuncType{OperatorId::Ceil, + function::NumericFunctions::_Ceil}, + "Ceil", + type::TypeId::DECIMAL, + {type::TypeId::INTEGER}, + internal_lang); + + AddBuiltinFunction(txn, + "ceil", + function::BuiltInFuncType{OperatorId::Ceil, + function::NumericFunctions::_Ceil}, + "Ceil", + type::TypeId::DECIMAL, + {type::TypeId::BIGINT}, + internal_lang); + + AddBuiltinFunction(txn, + "ceiling", + function::BuiltInFuncType{OperatorId::Ceil, + function::NumericFunctions::_Ceil}, + "Ceil", + type::TypeId::DECIMAL, + {type::TypeId::DECIMAL}, + internal_lang); + + AddBuiltinFunction(txn, + "ceiling", + function::BuiltInFuncType{OperatorId::Ceil, + function::NumericFunctions::_Ceil}, + "Ceil", + type::TypeId::DECIMAL, + {type::TypeId::TINYINT}, + internal_lang); + + AddBuiltinFunction(txn, + "ceiling", + function::BuiltInFuncType{OperatorId::Ceil, + function::NumericFunctions::_Ceil}, + "Ceil", + type::TypeId::DECIMAL, + {type::TypeId::SMALLINT}, + internal_lang); + + AddBuiltinFunction(txn, + "ceiling", + function::BuiltInFuncType{OperatorId::Ceil, + function::NumericFunctions::_Ceil}, + "Ceil", + type::TypeId::DECIMAL, + {type::TypeId::INTEGER}, + internal_lang); + + AddBuiltinFunction(txn, + "ceiling", + function::BuiltInFuncType{OperatorId::Ceil, + function::NumericFunctions::_Ceil}, + "Ceil", + type::TypeId::DECIMAL, + {type::TypeId::BIGINT}, + internal_lang); /** * date functions */ - AddBuiltinFunction( - "date_part", {type::TypeId::VARCHAR, type::TypeId::TIMESTAMP}, - type::TypeId::DECIMAL, internal_lang, "DatePart", - function::BuiltInFuncType{OperatorId::DatePart, - function::TimestampFunctions::_DatePart}, - txn); - - AddBuiltinFunction( - "date_trunc", {type::TypeId::VARCHAR, type::TypeId::TIMESTAMP}, - type::TypeId::TIMESTAMP, internal_lang, "DateTrunc", - function::BuiltInFuncType{OperatorId::DateTrunc, - function::TimestampFunctions::_DateTrunc}, - txn); + AddBuiltinFunction(txn, + "date_part", + function::BuiltInFuncType{OperatorId::DatePart, + function::TimestampFunctions::_DatePart}, + "DatePart", + type::TypeId::DECIMAL, + {type::TypeId::VARCHAR, type::TypeId::TIMESTAMP}, + internal_lang); + + AddBuiltinFunction(txn, + "date_trunc", + function::BuiltInFuncType{OperatorId::DateTrunc, + function::TimestampFunctions::_DateTrunc}, + "DateTrunc", + type::TypeId::TIMESTAMP, + {type::TypeId::VARCHAR, type::TypeId::TIMESTAMP}, + internal_lang); // add now() - AddBuiltinFunction("now", {}, type::TypeId::TIMESTAMP, internal_lang, - "Now", + AddBuiltinFunction(txn, + "now", function::BuiltInFuncType{ OperatorId::Now, function::DateFunctions::_Now}, - txn); + "Now", + type::TypeId::TIMESTAMP, + {}, + internal_lang); } catch (CatalogException &e) { txn_manager.AbortTransaction(txn); diff --git a/src/catalog/catalog_cache.cpp b/src/catalog/catalog_cache.cpp index b97a55b975a..69fdcbc486e 100644 --- a/src/catalog/catalog_cache.cpp +++ b/src/catalog/catalog_cache.cpp @@ -25,28 +25,28 @@ namespace catalog { * @return false only if database_oid already exists in cache */ bool CatalogCache::InsertDatabaseObject( - std::shared_ptr database_object) { + std::shared_ptr database_object) { if (!database_object || database_object->GetDatabaseOid() == INVALID_OID) { return false; // invalid object } // check if already in cache - if (database_objects_cache.find(database_object->GetDatabaseOid()) != - database_objects_cache.end()) { + if (database_objects_cache_.find(database_object->GetDatabaseOid()) != + database_objects_cache_.end()) { LOG_DEBUG("Database %u already exists in cache!", database_object->GetDatabaseOid()); return false; } - if (database_name_cache.find(database_object->GetDatabaseName()) != - database_name_cache.end()) { + if (database_name_cache_.find(database_object->GetDatabaseName()) != + database_name_cache_.end()) { LOG_DEBUG("Database %s already exists in cache!", database_object->GetDatabaseName().c_str()); return false; } - database_objects_cache.insert( + database_objects_cache_.insert( std::make_pair(database_object->GetDatabaseOid(), database_object)); - database_name_cache.insert( + database_name_cache_.insert( std::make_pair(database_object->GetDatabaseName(), database_object)); return true; } @@ -56,15 +56,15 @@ bool CatalogCache::InsertDatabaseObject( * @return true if database_oid is found and evicted; false if not found */ bool CatalogCache::EvictDatabaseObject(oid_t database_oid) { - auto it = database_objects_cache.find(database_oid); - if (it == database_objects_cache.end()) { + auto it = database_objects_cache_.find(database_oid); + if (it == database_objects_cache_.end()) { return false; // database oid not found in cache } auto database_object = it->second; PELOTON_ASSERT(database_object); - database_objects_cache.erase(it); - database_name_cache.erase(database_object->GetDatabaseName()); + database_objects_cache_.erase(it); + database_name_cache_.erase(database_object->GetDatabaseName()); return true; } @@ -73,15 +73,15 @@ bool CatalogCache::EvictDatabaseObject(oid_t database_oid) { * @return true if database_name is found and evicted; false if not found */ bool CatalogCache::EvictDatabaseObject(const std::string &database_name) { - auto it = database_name_cache.find(database_name); - if (it == database_name_cache.end()) { + auto it = database_name_cache_.find(database_name); + if (it == database_name_cache_.end()) { return false; // database name not found in cache } auto database_object = it->second; PELOTON_ASSERT(database_object); - database_name_cache.erase(it); - database_objects_cache.erase(database_object->GetDatabaseOid()); + database_name_cache_.erase(it); + database_objects_cache_.erase(database_object->GetDatabaseOid()); return true; } @@ -89,10 +89,10 @@ bool CatalogCache::EvictDatabaseObject(const std::string &database_name) { * @param database_oid * @return database catalog object; if not found return object with invalid oid */ -std::shared_ptr CatalogCache::GetDatabaseObject( +std::shared_ptr CatalogCache::GetDatabaseObject( oid_t database_oid) { - auto it = database_objects_cache.find(database_oid); - if (it == database_objects_cache.end()) { + auto it = database_objects_cache_.find(database_oid); + if (it == database_objects_cache_.end()) { return nullptr; } return it->second; @@ -102,10 +102,10 @@ std::shared_ptr CatalogCache::GetDatabaseObject( * @param database_name * @return database catalog object; if not found return null */ -std::shared_ptr CatalogCache::GetDatabaseObject( +std::shared_ptr CatalogCache::GetDatabaseObject( const std::string &database_name) { - auto it = database_name_cache.find(database_name); - if (it == database_name_cache.end()) { + auto it = database_name_cache_.find(database_name); + if (it == database_name_cache_.end()) { return nullptr; } return it->second; @@ -115,11 +115,11 @@ std::shared_ptr CatalogCache::GetDatabaseObject( * @param table_oid * @return table catalog object; if not found return null */ -std::shared_ptr CatalogCache::GetCachedTableObject( +std::shared_ptr CatalogCache::GetCachedTableObject( oid_t database_oid, oid_t table_oid) { auto database_object = GetDatabaseObject(database_oid); if (database_object == nullptr) return nullptr; - auto table_object = database_object->GetTableObject(table_oid, true); + auto table_object = database_object->GetTableCatalogEntry(table_oid, true); if (table_object) return table_object; return nullptr; } @@ -128,11 +128,11 @@ std::shared_ptr CatalogCache::GetCachedTableObject( * @param index_oid * @return index catalog object; if not found return null */ -std::shared_ptr CatalogCache::GetCachedIndexObject( +std::shared_ptr CatalogCache::GetCachedIndexObject( oid_t database_oid, oid_t index_oid) { auto database_object = GetDatabaseObject(database_oid); if (database_object == nullptr) return nullptr; - auto index_object = database_object->GetCachedIndexObject(index_oid); + auto index_object = database_object->GetCachedIndexCatalogEntry(index_oid); if (index_object) return index_object; return nullptr; } @@ -141,13 +141,13 @@ std::shared_ptr CatalogCache::GetCachedIndexObject( * @param index_name * @return index catalog object; if not found return null */ -std::shared_ptr CatalogCache::GetCachedIndexObject( - const std::string &database_name, const std::string &index_name, - const std::string &schema_name) { +std::shared_ptr CatalogCache::GetCachedIndexObject(const std::string &database_name, + const std::string &schema_name, + const std::string &index_name) { auto database_object = GetDatabaseObject(database_name); if (database_object == nullptr) return nullptr; auto index_object = - database_object->GetCachedIndexObject(index_name, schema_name); + database_object->GetCachedIndexCatalogEntry(index_name, schema_name); if (index_object) return index_object; return nullptr; } diff --git a/src/catalog/column.cpp b/src/catalog/column.cpp index 50406bef954..3195de231d4 100644 --- a/src/catalog/column.cpp +++ b/src/catalog/column.cpp @@ -21,23 +21,23 @@ namespace catalog { void Column::SetLength(size_t column_length) { // Set the column length based on whether it is inlined - if (is_inlined) { - fixed_length = column_length; - variable_length = 0; + if (is_inlined_) { + fixed_length_ = column_length; + variable_length_ = 0; } else { - fixed_length = sizeof(uintptr_t); - variable_length = column_length; + fixed_length_ = sizeof(uintptr_t); + variable_length_ = column_length; } } void Column::SetInlined() { - switch (column_type) { + switch (column_type_) { case type::TypeId::VARCHAR: case type::TypeId::VARBINARY: break; // No change of inlined setting default: - is_inlined = true; + is_inlined_ = true; break; } } @@ -45,19 +45,19 @@ void Column::SetInlined() { const std::string Column::GetInfo() const { std::ostringstream os; - os << "Column[" << column_name << ", " << TypeIdToString(column_type) << ", " - << "Offset:" << column_offset << ", "; + os << "Column[" << column_name << ", " << TypeIdToString(column_type_) << ", " + << "Offset:" << column_offset_ << ", "; - if (is_inlined) { - os << "FixedLength:" << fixed_length; + if (is_inlined_) { + os << "FixedLength:" << fixed_length_; } else { - os << "VarLength:" << variable_length; + os << "VarLength:" << variable_length_; } - if (constraints.empty() == false) { + if (constraints_.empty() == false) { os << ", {"; bool first = true; - for (auto constraint : constraints) { + for (auto constraint : constraints_) { if (first) { first = false; } else { diff --git a/src/catalog/column_catalog.cpp b/src/catalog/column_catalog.cpp index c28492696fd..6d4e7b717c1 100644 --- a/src/catalog/column_catalog.cpp +++ b/src/catalog/column_catalog.cpp @@ -23,50 +23,64 @@ namespace peloton { namespace catalog { -ColumnCatalogObject::ColumnCatalogObject(executor::LogicalTile *tile, +ColumnCatalogEntry::ColumnCatalogEntry(executor::LogicalTile *tile, int tupleId) - : table_oid(tile->GetValue(tupleId, ColumnCatalog::ColumnId::TABLE_OID) + : table_oid_(tile->GetValue(tupleId, ColumnCatalog::ColumnId::TABLE_OID) .GetAs()), - column_name(tile->GetValue(tupleId, ColumnCatalog::ColumnId::COLUMN_NAME) + column_name_(tile->GetValue(tupleId, ColumnCatalog::ColumnId::COLUMN_NAME) .ToString()), - column_id(tile->GetValue(tupleId, ColumnCatalog::ColumnId::COLUMN_ID) + column_id_(tile->GetValue(tupleId, ColumnCatalog::ColumnId::COLUMN_ID) .GetAs()), - column_offset( + column_offset_( tile->GetValue(tupleId, ColumnCatalog::ColumnId::COLUMN_OFFSET) .GetAs()), - column_type(StringToTypeId( + column_type_(StringToTypeId( tile->GetValue(tupleId, ColumnCatalog::ColumnId::COLUMN_TYPE) .ToString())), - column_length( + column_length_( tile->GetValue(tupleId, ColumnCatalog::ColumnId::COLUMN_LENGTH) .GetAs()), - is_inlined(tile->GetValue(tupleId, ColumnCatalog::ColumnId::IS_INLINED) + is_inlined_(tile->GetValue(tupleId, ColumnCatalog::ColumnId::IS_INLINED) .GetAs()), - is_primary(tile->GetValue(tupleId, ColumnCatalog::ColumnId::IS_PRIMARY) + is_primary_(tile->GetValue(tupleId, ColumnCatalog::ColumnId::IS_PRIMARY) .GetAs()), - is_not_null(tile->GetValue(tupleId, ColumnCatalog::ColumnId::IS_NOT_NULL) + is_not_null_(tile->GetValue(tupleId, ColumnCatalog::ColumnId::IS_NOT_NULL) .GetAs()) {} -ColumnCatalog::ColumnCatalog(storage::Database *pg_catalog, - type::AbstractPool *pool, - concurrency::TransactionContext *txn) - : AbstractCatalog(COLUMN_CATALOG_OID, COLUMN_CATALOG_NAME, - InitializeSchema().release(), pg_catalog) { +ColumnCatalog::ColumnCatalog(concurrency::TransactionContext *txn, + storage::Database *pg_catalog, + type::AbstractPool *pool) + : AbstractCatalog(pg_catalog, + InitializeSchema().release(), + COLUMN_CATALOG_OID, + COLUMN_CATALOG_NAME) { // Add indexes for pg_attribute - AddIndex({ColumnId::TABLE_OID, ColumnId::COLUMN_NAME}, - COLUMN_CATALOG_PKEY_OID, COLUMN_CATALOG_NAME "_pkey", + AddIndex(COLUMN_CATALOG_NAME "_pkey", + COLUMN_CATALOG_PKEY_OID, + {ColumnId::TABLE_OID, ColumnId::COLUMN_NAME}, IndexConstraintType::PRIMARY_KEY); - AddIndex({ColumnId::TABLE_OID, ColumnId::COLUMN_ID}, COLUMN_CATALOG_SKEY0_OID, - COLUMN_CATALOG_NAME "_skey0", IndexConstraintType::UNIQUE); - AddIndex({ColumnId::TABLE_OID}, COLUMN_CATALOG_SKEY1_OID, - COLUMN_CATALOG_NAME "_skey1", IndexConstraintType::DEFAULT); + AddIndex(COLUMN_CATALOG_NAME "_skey0", + COLUMN_CATALOG_SKEY0_OID, + {ColumnId::TABLE_OID, ColumnId::COLUMN_ID}, + IndexConstraintType::UNIQUE); + AddIndex(COLUMN_CATALOG_NAME "_skey1", + COLUMN_CATALOG_SKEY1_OID, + {ColumnId::TABLE_OID}, + IndexConstraintType::DEFAULT); // Insert columns of pg_attribute table into pg_attribute itself uint32_t column_id = 0; for (auto column : catalog_table_->GetSchema()->GetColumns()) { - InsertColumn(COLUMN_CATALOG_OID, column.GetName(), column_id, - column.GetOffset(), column.GetType(), column.GetLength(), - column.IsInlined(), column.GetConstraints(), pool, txn); + InsertColumn(txn, + COLUMN_CATALOG_OID, + column_id, + column.GetName(), + column.GetOffset(), + column.GetType(), + column.GetLength(), + column.GetConstraints(), + column.IsInlined(), + pool); column_id++; } } @@ -89,7 +103,7 @@ std::unique_ptr ColumnCatalog::InitializeSchema() { catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); auto column_name_column = catalog::Column( - type::TypeId::VARCHAR, max_name_size, "column_name", false); + type::TypeId::VARCHAR, max_name_size_, "column_name", false); column_name_column.AddConstraint(catalog::Constraint( ConstraintType::PRIMARY, primary_key_constraint_name)); column_name_column.AddConstraint( @@ -108,7 +122,7 @@ std::unique_ptr ColumnCatalog::InitializeSchema() { catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); auto column_type_column = catalog::Column( - type::TypeId::VARCHAR, max_name_size, "column_type", false); + type::TypeId::VARCHAR, max_name_size_, "column_type", false); column_type_column.AddConstraint( catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); @@ -144,14 +158,16 @@ std::unique_ptr ColumnCatalog::InitializeSchema() { return column_catalog_schema; } -bool ColumnCatalog::InsertColumn(oid_t table_oid, +bool ColumnCatalog::InsertColumn(concurrency::TransactionContext *txn, + oid_t table_oid, + oid_t column_id, const std::string &column_name, - oid_t column_id, oid_t column_offset, - type::TypeId column_type, size_t column_length, - bool is_inlined, + oid_t column_offset, + type::TypeId column_type, + size_t column_length, const std::vector &constraints, - type::AbstractPool *pool, - concurrency::TransactionContext *txn) { + bool is_inlined, + type::AbstractPool *pool) { // Create the tuple first std::unique_ptr tuple( new storage::Tuple(catalog_table_->GetSchema(), true)); @@ -187,12 +203,12 @@ bool ColumnCatalog::InsertColumn(oid_t table_oid, tuple->SetValue(ColumnId::IS_NOT_NULL, val8, pool); // Insert the tuple - return InsertTuple(std::move(tuple), txn); + return InsertTuple(txn, std::move(tuple)); } -bool ColumnCatalog::DeleteColumn(oid_t table_oid, - const std::string &column_name, - concurrency::TransactionContext *txn) { +bool ColumnCatalog::DeleteColumn(concurrency::TransactionContext *txn, + oid_t table_oid, + const std::string &column_name) { oid_t index_offset = IndexId::PRIMARY_KEY; // Index of table_oid & column_name std::vector values; @@ -202,12 +218,12 @@ bool ColumnCatalog::DeleteColumn(oid_t table_oid, // delete column from cache auto pg_table = Catalog::GetInstance() - ->GetSystemCatalogs(database_oid) + ->GetSystemCatalogs(database_oid_) ->GetTableCatalog(); - auto table_object = pg_table->GetTableObject(table_oid, txn); - table_object->EvictColumnObject(column_name); + auto table_object = pg_table->GetTableCatalogEntry(txn, table_oid); + table_object->EvictColumnCatalogEntry(column_name); - return DeleteWithIndexScan(index_offset, values, txn); + return DeleteWithIndexScan(txn, index_offset, values); } /* @brief delete all column records from the same table @@ -216,52 +232,56 @@ bool ColumnCatalog::DeleteColumn(oid_t table_oid, * @param txn TransactionContext * @return a vector of table oid */ -bool ColumnCatalog::DeleteColumns(oid_t table_oid, - concurrency::TransactionContext *txn) { +bool ColumnCatalog::DeleteColumns(concurrency::TransactionContext *txn, oid_t table_oid) { oid_t index_offset = IndexId::SKEY_TABLE_OID; // Index of table_oid std::vector values; values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); // delete columns from cache auto pg_table = Catalog::GetInstance() - ->GetSystemCatalogs(database_oid) + ->GetSystemCatalogs(database_oid_) ->GetTableCatalog(); - auto table_object = pg_table->GetTableObject(table_oid, txn); - table_object->EvictAllColumnObjects(); + auto table_object = pg_table->GetTableCatalogEntry(txn, table_oid); + table_object->EvictAllColumnCatalogEntries(); - return DeleteWithIndexScan(index_offset, values, txn); + return DeleteWithIndexScan(txn, index_offset, values); } -const std::unordered_map> -ColumnCatalog::GetColumnObjects(oid_t table_oid, - concurrency::TransactionContext *txn) { +const std::unordered_map> +ColumnCatalog::GetColumnCatalogEntries( + concurrency::TransactionContext *txn, + oid_t table_oid) { // try get from cache auto pg_table = Catalog::GetInstance() - ->GetSystemCatalogs(database_oid) + ->GetSystemCatalogs(database_oid_) ->GetTableCatalog(); - auto table_object = pg_table->GetTableObject(table_oid, txn); + auto table_object = pg_table->GetTableCatalogEntry(txn, table_oid); PELOTON_ASSERT(table_object && table_object->GetTableOid() == table_oid); - auto column_objects = table_object->GetColumnObjects(true); + auto column_objects = table_object->GetColumnCatalogEntries(true); if (column_objects.size() != 0) return column_objects; // cache miss, get from pg_attribute - std::vector column_ids(all_column_ids); + std::vector column_ids(all_column_ids_); oid_t index_offset = IndexId::SKEY_TABLE_OID; // Index of table_oid std::vector values; values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); for (auto &tile : (*result_tiles)) { for (auto tuple_id : *tile) { auto column_object = - std::make_shared(tile.get(), tuple_id); - table_object->InsertColumnObject(column_object); + std::make_shared(tile.get(), tuple_id); + table_object->InsertColumnCatalogEntry(column_object); } } - return table_object->GetColumnObjects(); + return table_object->GetColumnCatalogEntries(); } } // namespace catalog diff --git a/src/catalog/column_stats_catalog.cpp b/src/catalog/column_stats_catalog.cpp index bbe94340cdb..9aafff3bfb1 100644 --- a/src/catalog/column_stats_catalog.cpp +++ b/src/catalog/column_stats_catalog.cpp @@ -28,41 +28,55 @@ ColumnStatsCatalog *ColumnStatsCatalog::GetInstance( } ColumnStatsCatalog::ColumnStatsCatalog(concurrency::TransactionContext *txn) - : AbstractCatalog("CREATE TABLE " CATALOG_DATABASE_NAME - "." CATALOG_SCHEMA_NAME "." COLUMN_STATS_CATALOG_NAME - " (" - "database_id INT NOT NULL, " - "table_id INT NOT NULL, " - "column_id INT NOT NULL, " - "num_rows INT NOT NULL, " - "cardinality DECIMAL NOT NULL, " - "frac_null DECIMAL NOT NULL, " - "most_common_vals VARCHAR, " - "most_common_freqs VARCHAR, " - "histogram_bounds VARCHAR, " - "column_name VARCHAR, " - "has_index BOOLEAN);", - txn) { + : AbstractCatalog(txn, "CREATE TABLE " CATALOG_DATABASE_NAME + "." CATALOG_SCHEMA_NAME "." COLUMN_STATS_CATALOG_NAME + " (" + "database_id INT NOT NULL, " + "table_id INT NOT NULL, " + "column_id INT NOT NULL, " + "num_rows INT NOT NULL, " + "cardinality DECIMAL NOT NULL, " + "frac_null DECIMAL NOT NULL, " + "most_common_vals VARCHAR, " + "most_common_freqs VARCHAR, " + "histogram_bounds VARCHAR, " + "column_name VARCHAR, " + "has_index BOOLEAN);") { // unique key: (database_id, table_id, column_id) - Catalog::GetInstance()->CreateIndex( - CATALOG_DATABASE_NAME, CATALOG_SCHEMA_NAME, COLUMN_STATS_CATALOG_NAME, - {0, 1, 2}, COLUMN_STATS_CATALOG_NAME "_skey0", true, IndexType::BWTREE, - txn); + Catalog::GetInstance()->CreateIndex(txn, + CATALOG_DATABASE_NAME, + CATALOG_SCHEMA_NAME, + COLUMN_STATS_CATALOG_NAME, + COLUMN_STATS_CATALOG_NAME "_skey0", + {0, 1, 2}, + true, + IndexType::BWTREE); // non-unique key: (database_id, table_id) - Catalog::GetInstance()->CreateIndex( - CATALOG_DATABASE_NAME, CATALOG_SCHEMA_NAME, COLUMN_STATS_CATALOG_NAME, - {0, 1}, COLUMN_STATS_CATALOG_NAME "_skey1", false, IndexType::BWTREE, - txn); + Catalog::GetInstance()->CreateIndex(txn, + CATALOG_DATABASE_NAME, + CATALOG_SCHEMA_NAME, + COLUMN_STATS_CATALOG_NAME, + COLUMN_STATS_CATALOG_NAME "_skey1", + {0, 1}, + false, + IndexType::BWTREE); } ColumnStatsCatalog::~ColumnStatsCatalog() {} -bool ColumnStatsCatalog::InsertColumnStats( - oid_t database_id, oid_t table_id, oid_t column_id, int num_rows, - double cardinality, double frac_null, std::string most_common_vals, - std::string most_common_freqs, std::string histogram_bounds, - std::string column_name, bool has_index, type::AbstractPool *pool, - concurrency::TransactionContext *txn) { +bool ColumnStatsCatalog::InsertColumnStats(concurrency::TransactionContext *txn, + oid_t database_id, + oid_t table_id, + oid_t column_id, + std::string column_name, + int num_rows, + double frac_null, + std::string most_common_vals, + std::string most_common_freqs, + std::string histogram_bounds, + double cardinality, + bool has_index, + type::AbstractPool *pool) { std::unique_ptr tuple( new storage::Tuple(catalog_table_->GetSchema(), true)); @@ -109,12 +123,13 @@ bool ColumnStatsCatalog::InsertColumnStats( tuple->SetValue(ColumnId::HAS_INDEX, val_has_index, nullptr); // Insert the tuple into catalog table - return InsertTuple(std::move(tuple), txn); + return InsertTuple(txn, std::move(tuple)); } -bool ColumnStatsCatalog::DeleteColumnStats( - oid_t database_id, oid_t table_id, oid_t column_id, - concurrency::TransactionContext *txn) { +bool ColumnStatsCatalog::DeleteColumnStats(concurrency::TransactionContext *txn, + oid_t database_id, + oid_t table_id, + oid_t column_id) { oid_t index_offset = IndexId::SECONDARY_KEY_0; // Secondary key index std::vector values; @@ -122,12 +137,13 @@ bool ColumnStatsCatalog::DeleteColumnStats( values.push_back(type::ValueFactory::GetIntegerValue(table_id).Copy()); values.push_back(type::ValueFactory::GetIntegerValue(column_id).Copy()); - return DeleteWithIndexScan(index_offset, values, txn); + return DeleteWithIndexScan(txn, index_offset, values); } -std::unique_ptr> ColumnStatsCatalog::GetColumnStats( - oid_t database_id, oid_t table_id, oid_t column_id, - concurrency::TransactionContext *txn) { +std::unique_ptr> ColumnStatsCatalog::GetColumnStats(concurrency::TransactionContext *txn, + oid_t database_id, + oid_t table_id, + oid_t column_id) { std::vector column_ids( {ColumnId::NUM_ROWS, ColumnId::CARDINALITY, ColumnId::FRAC_NULL, ColumnId::MOST_COMMON_VALS, ColumnId::MOST_COMMON_FREQS, @@ -140,7 +156,10 @@ std::unique_ptr> ColumnStatsCatalog::GetColumnStats( values.push_back(type::ValueFactory::GetIntegerValue(column_id).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); PELOTON_ASSERT(result_tiles->size() <= 1); // unique if (result_tiles->size() == 0) { @@ -174,10 +193,11 @@ std::unique_ptr> ColumnStatsCatalog::GetColumnStats( } // Return value: number of column stats -size_t ColumnStatsCatalog::GetTableStats( - oid_t database_id, oid_t table_id, concurrency::TransactionContext *txn, - std::map>> - &column_stats_map) { +size_t ColumnStatsCatalog::GetTableStats(concurrency::TransactionContext *txn, + oid_t database_id, + oid_t table_id, + std::map>> &column_stats_map) { std::vector column_ids( {ColumnId::COLUMN_ID, ColumnId::NUM_ROWS, ColumnId::CARDINALITY, ColumnId::FRAC_NULL, ColumnId::MOST_COMMON_VALS, @@ -190,7 +210,10 @@ size_t ColumnStatsCatalog::GetTableStats( values.push_back(type::ValueFactory::GetIntegerValue(table_id).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); PELOTON_ASSERT(result_tiles->size() <= 1); // unique if (result_tiles->size() == 0) { diff --git a/src/catalog/constraint.cpp b/src/catalog/constraint.cpp index 01359e7f309..f9c2e025a9e 100644 --- a/src/catalog/constraint.cpp +++ b/src/catalog/constraint.cpp @@ -20,10 +20,10 @@ namespace catalog { const std::string Constraint::GetInfo() const { std::ostringstream os; os << "Constraint[" << GetName() << ", " - << ConstraintTypeToString(constraint_type); + << ConstraintTypeToString(constraint_type_); if (GetType() == ConstraintType::CHECK) { - os << ", " << exp.first << " " << exp.second.GetInfo(); + os << ", " << exp_.first << " " << exp_.second.GetInfo(); } os << "]"; return os.str(); diff --git a/src/catalog/database_catalog.cpp b/src/catalog/database_catalog.cpp index fc0b81c64d0..555e8c9d7d6 100644 --- a/src/catalog/database_catalog.cpp +++ b/src/catalog/database_catalog.cpp @@ -25,45 +25,45 @@ namespace peloton { namespace catalog { -DatabaseCatalogObject::DatabaseCatalogObject( - executor::LogicalTile *tile, concurrency::TransactionContext *txn) - : database_oid(tile->GetValue(0, DatabaseCatalog::ColumnId::DATABASE_OID) +DatabaseCatalogEntry::DatabaseCatalogEntry(concurrency::TransactionContext *txn, + executor::LogicalTile *tile) + : database_oid_(tile->GetValue(0, DatabaseCatalog::ColumnId::DATABASE_OID) .GetAs()), - database_name(tile->GetValue(0, DatabaseCatalog::ColumnId::DATABASE_NAME) + database_name_(tile->GetValue(0, DatabaseCatalog::ColumnId::DATABASE_NAME) .ToString()), - table_objects_cache(), - table_name_cache(), - valid_table_objects(false), - txn(txn) {} + table_catalog_entries_cache_(), + table_catalog_entries_cache_by_name(), + valid_table_catalog_entries(false), + txn_(txn) {} /* @brief insert table catalog object into cache * @param table_object * @return false if table_name already exists in cache */ -bool DatabaseCatalogObject::InsertTableObject( - std::shared_ptr table_object) { - if (!table_object || table_object->GetTableOid() == INVALID_OID) { +bool DatabaseCatalogEntry::InsertTableCatalogEntry( + std::shared_ptr table_catalog_entry) { + if (!table_catalog_entry || table_catalog_entry->GetTableOid() == INVALID_OID) { return false; // invalid object } // check if already in cache - if (table_objects_cache.find(table_object->GetTableOid()) != - table_objects_cache.end()) { - LOG_DEBUG("Table %u already exists in cache!", table_object->GetTableOid()); + if (table_catalog_entries_cache_.find(table_catalog_entry->GetTableOid()) != + table_catalog_entries_cache_.end()) { + LOG_DEBUG("Table %u already exists in cache!", table_catalog_entry->GetTableOid()); return false; } std::string key = - table_object->GetSchemaName() + "." + table_object->GetTableName(); - if (table_name_cache.find(key) != table_name_cache.end()) { + table_catalog_entry->GetSchemaName() + "." + table_catalog_entry->GetTableName(); + if (table_catalog_entries_cache_by_name.find(key) != table_catalog_entries_cache_by_name.end()) { LOG_DEBUG("Table %s already exists in cache!", - table_object->GetTableName().c_str()); + table_catalog_entry->GetTableName().c_str()); return false; } - table_objects_cache.insert( - std::make_pair(table_object->GetTableOid(), table_object)); - table_name_cache.insert(std::make_pair(key, table_object)); + table_catalog_entries_cache_.insert( + std::make_pair(table_catalog_entry->GetTableOid(), table_catalog_entry)); + table_catalog_entries_cache_by_name.insert(std::make_pair(key, table_catalog_entry)); return true; } @@ -71,20 +71,20 @@ bool DatabaseCatalogObject::InsertTableObject( * @param table_oid * @return true if table_oid is found and evicted; false if not found */ -bool DatabaseCatalogObject::EvictTableObject(oid_t table_oid) { +bool DatabaseCatalogEntry::EvictTableCatalogEntry(oid_t table_oid) { // find table name from table name cache - auto it = table_objects_cache.find(table_oid); - if (it == table_objects_cache.end()) { + auto it = table_catalog_entries_cache_.find(table_oid); + if (it == table_catalog_entries_cache_.end()) { return false; // table oid not found in cache } auto table_object = it->second; PELOTON_ASSERT(table_object); - table_objects_cache.erase(it); + table_catalog_entries_cache_.erase(it); // erase from table name cache std::string key = table_object->GetSchemaName() + "." + table_object->GetTableName(); - table_name_cache.erase(key); + table_catalog_entries_cache_by_name.erase(key); return true; } @@ -92,27 +92,27 @@ bool DatabaseCatalogObject::EvictTableObject(oid_t table_oid) { * @param table_name * @return true if table_name is found and evicted; false if not found */ -bool DatabaseCatalogObject::EvictTableObject(const std::string &table_name, - const std::string &schema_name) { +bool DatabaseCatalogEntry::EvictTableCatalogEntry(const std::string &table_name, + const std::string &schema_name) { std::string key = schema_name + "." + table_name; // find table name from table name cache - auto it = table_name_cache.find(key); - if (it == table_name_cache.end()) { + auto it = table_catalog_entries_cache_by_name.find(key); + if (it == table_catalog_entries_cache_by_name.end()) { return false; // table name not found in cache } auto table_object = it->second; PELOTON_ASSERT(table_object); - table_name_cache.erase(it); - table_objects_cache.erase(table_object->GetTableOid()); + table_catalog_entries_cache_by_name.erase(it); + table_catalog_entries_cache_.erase(table_object->GetTableOid()); return true; } /*@brief evict all table catalog objects in this database from cache */ -void DatabaseCatalogObject::EvictAllTableObjects() { - table_objects_cache.clear(); - table_name_cache.clear(); +void DatabaseCatalogEntry::EvictAllTableCatalogEntries() { + table_catalog_entries_cache_.clear(); + table_catalog_entries_cache_by_name.clear(); } /* @brief Get table catalog object from cache or all the way from storage @@ -120,10 +120,10 @@ void DatabaseCatalogObject::EvictAllTableObjects() { * @param cached_only if cached only, return nullptr on a cache miss * @return Shared pointer to the requested table catalog object */ -std::shared_ptr DatabaseCatalogObject::GetTableObject( +std::shared_ptr DatabaseCatalogEntry::GetTableCatalogEntry( oid_t table_oid, bool cached_only) { - auto it = table_objects_cache.find(table_oid); - if (it != table_objects_cache.end()) return it->second; + auto it = table_catalog_entries_cache_.find(table_oid); + if (it != table_catalog_entries_cache_.end()) return it->second; if (cached_only) { // cache miss return empty object @@ -131,9 +131,9 @@ std::shared_ptr DatabaseCatalogObject::GetTableObject( } else { // cache miss get from pg_table auto pg_table = Catalog::GetInstance() - ->GetSystemCatalogs(database_oid) + ->GetSystemCatalogs(database_oid_) ->GetTableCatalog(); - return pg_table->GetTableObject(table_oid, txn); + return pg_table->GetTableCatalogEntry(txn_, table_oid); } } @@ -144,12 +144,12 @@ std::shared_ptr DatabaseCatalogObject::GetTableObject( * @param cached_only if cached only, return nullptr on a cache miss * @return Shared pointer to the requested table catalog object */ -std::shared_ptr DatabaseCatalogObject::GetTableObject( +std::shared_ptr DatabaseCatalogEntry::GetTableCatalogEntry( const std::string &table_name, const std::string &schema_name, bool cached_only) { std::string key = schema_name + "." + table_name; - auto it = table_name_cache.find(key); - if (it != table_name_cache.end()) { + auto it = table_catalog_entries_cache_by_name.find(key); + if (it != table_catalog_entries_cache_by_name.end()) { return it->second; } @@ -159,9 +159,9 @@ std::shared_ptr DatabaseCatalogObject::GetTableObject( } else { // cache miss get from pg_table auto pg_table = Catalog::GetInstance() - ->GetSystemCatalogs(database_oid) + ->GetSystemCatalogs(database_oid_) ->GetTableCatalog(); - return pg_table->GetTableObject(table_name, schema_name, txn); + return pg_table->GetTableCatalogEntry(txn_, schema_name, table_name); } } @@ -170,20 +170,20 @@ std::shared_ptr DatabaseCatalogObject::GetTableObject( * @param schema_name * @return table catalog objects */ -std::vector> -DatabaseCatalogObject::GetTableObjects(const std::string &schema_name) { +std::vector> +DatabaseCatalogEntry::GetTableCatalogEntries(const std::string &schema_name) { // read directly from pg_table - if (!valid_table_objects) { + if (!valid_table_catalog_entries) { auto pg_table = Catalog::GetInstance() - ->GetSystemCatalogs(database_oid) + ->GetSystemCatalogs(database_oid_) ->GetTableCatalog(); // insert every table object into cache - pg_table->GetTableObjects(txn); + pg_table->GetTableCatalogEntries(txn_); } // make sure to check IsValidTableObjects() before getting table objects - PELOTON_ASSERT(valid_table_objects); - std::vector> result; - for (auto it : table_objects_cache) { + PELOTON_ASSERT(valid_table_catalog_entries); + std::vector> result; + for (auto it : table_catalog_entries_cache_) { if (it.second->GetSchemaName() == schema_name) { result.push_back(it.second); } @@ -197,30 +197,30 @@ DatabaseCatalogObject::GetTableObjects(const std::string &schema_name) { * @param cached_only if cached only, return nullptr on a cache miss * @return Shared pointer to the requested table catalog object */ -std::unordered_map> -DatabaseCatalogObject::GetTableObjects(bool cached_only) { - if (!cached_only && !valid_table_objects) { +std::unordered_map> +DatabaseCatalogEntry::GetTableCatalogEntries(bool cached_only) { + if (!cached_only && !valid_table_catalog_entries) { // cache miss get from pg_table auto pg_table = Catalog::GetInstance() - ->GetSystemCatalogs(database_oid) + ->GetSystemCatalogs(database_oid_) ->GetTableCatalog(); - return pg_table->GetTableObjects(txn); + return pg_table->GetTableCatalogEntries(txn_); } // make sure to check IsValidTableObjects() before getting table objects - PELOTON_ASSERT(valid_table_objects); - return table_objects_cache; + PELOTON_ASSERT(valid_table_catalog_entries); + return table_catalog_entries_cache_; } /*@brief search index catalog object from all cached database objects * @param index_oid * @return index catalog object; if not found return null */ -std::shared_ptr DatabaseCatalogObject::GetCachedIndexObject( +std::shared_ptr DatabaseCatalogEntry::GetCachedIndexCatalogEntry( oid_t index_oid) { - for (auto it = table_objects_cache.begin(); it != table_objects_cache.end(); + for (auto it = table_catalog_entries_cache_.begin(); it != table_catalog_entries_cache_.end(); ++it) { auto table_object = it->second; - auto index_object = table_object->GetIndexObject(index_oid, true); + auto index_object = table_object->GetIndexCatalogEntries(index_oid, true); if (index_object) return index_object; } return nullptr; @@ -230,37 +230,44 @@ std::shared_ptr DatabaseCatalogObject::GetCachedIndexObject( * @param index_name * @return index catalog object; if not found return null */ -std::shared_ptr DatabaseCatalogObject::GetCachedIndexObject( +std::shared_ptr DatabaseCatalogEntry::GetCachedIndexCatalogEntry( const std::string &index_name, const std::string &schema_name) { - for (auto it = table_objects_cache.begin(); it != table_objects_cache.end(); + for (auto it = table_catalog_entries_cache_.begin(); it != table_catalog_entries_cache_.end(); ++it) { auto table_object = it->second; if (table_object != nullptr && table_object->GetSchemaName() == schema_name) { - auto index_object = table_object->GetIndexObject(index_name, true); + auto index_object = table_object->GetIndexCatalogEntry(index_name, true); if (index_object) return index_object; } } return nullptr; } -DatabaseCatalog *DatabaseCatalog::GetInstance( - storage::Database *pg_catalog, type::AbstractPool *pool, - concurrency::TransactionContext *txn) { - static DatabaseCatalog database_catalog{pg_catalog, pool, txn}; +DatabaseCatalog *DatabaseCatalog::GetInstance(concurrency::TransactionContext *txn, + storage::Database *pg_catalog, + type::AbstractPool *pool) { + static DatabaseCatalog + database_catalog{txn, pg_catalog, pool}; return &database_catalog; } -DatabaseCatalog::DatabaseCatalog( - storage::Database *pg_catalog, UNUSED_ATTRIBUTE type::AbstractPool *pool, - UNUSED_ATTRIBUTE concurrency::TransactionContext *txn) - : AbstractCatalog(DATABASE_CATALOG_OID, DATABASE_CATALOG_NAME, - InitializeSchema().release(), pg_catalog) { +DatabaseCatalog::DatabaseCatalog(concurrency::TransactionContext *, + storage::Database *pg_catalog, + type::AbstractPool *) + : AbstractCatalog(pg_catalog, + InitializeSchema().release(), + DATABASE_CATALOG_OID, + DATABASE_CATALOG_NAME) { // Add indexes for pg_database - AddIndex({ColumnId::DATABASE_OID}, DATABASE_CATALOG_PKEY_OID, - DATABASE_CATALOG_NAME "_pkey", IndexConstraintType::PRIMARY_KEY); - AddIndex({ColumnId::DATABASE_NAME}, DATABASE_CATALOG_SKEY0_OID, - DATABASE_CATALOG_NAME "_skey0", IndexConstraintType::UNIQUE); + AddIndex(DATABASE_CATALOG_NAME "_pkey", + DATABASE_CATALOG_PKEY_OID, + {ColumnId::DATABASE_OID}, + IndexConstraintType::PRIMARY_KEY); + AddIndex(DATABASE_CATALOG_NAME "_skey0", + DATABASE_CATALOG_SKEY0_OID, + {ColumnId::DATABASE_NAME}, + IndexConstraintType::UNIQUE); } DatabaseCatalog::~DatabaseCatalog() {} @@ -281,7 +288,7 @@ std::unique_ptr DatabaseCatalog::InitializeSchema() { catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); auto database_name_column = catalog::Column( - type::TypeId::VARCHAR, max_name_size, "database_name", false); + type::TypeId::VARCHAR, max_name_size_, "database_name", false); database_name_column.AddConstraint( catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); @@ -290,10 +297,10 @@ std::unique_ptr DatabaseCatalog::InitializeSchema() { return database_catalog_schema; } -bool DatabaseCatalog::InsertDatabase(oid_t database_oid, +bool DatabaseCatalog::InsertDatabase(concurrency::TransactionContext *txn, + oid_t database_oid, const std::string &database_name, - type::AbstractPool *pool, - concurrency::TransactionContext *txn) { + type::AbstractPool *pool) { std::unique_ptr tuple( new storage::Tuple(catalog_table_->GetSchema(), true)); @@ -304,11 +311,10 @@ bool DatabaseCatalog::InsertDatabase(oid_t database_oid, tuple->SetValue(ColumnId::DATABASE_NAME, val1, pool); // Insert the tuple - return InsertTuple(std::move(tuple), txn); + return InsertTuple(txn, std::move(tuple)); } -bool DatabaseCatalog::DeleteDatabase(oid_t database_oid, - concurrency::TransactionContext *txn) { +bool DatabaseCatalog::DeleteDatabase(concurrency::TransactionContext *txn, oid_t database_oid) { oid_t index_offset = IndexId::PRIMARY_KEY; // Index of database_oid std::vector values; values.push_back(type::ValueFactory::GetIntegerValue(database_oid).Copy()); @@ -316,11 +322,12 @@ bool DatabaseCatalog::DeleteDatabase(oid_t database_oid, // evict cache txn->catalog_cache.EvictDatabaseObject(database_oid); - return DeleteWithIndexScan(index_offset, values, txn); + return DeleteWithIndexScan(txn, index_offset, values); } -std::shared_ptr DatabaseCatalog::GetDatabaseObject( - oid_t database_oid, concurrency::TransactionContext *txn) { +std::shared_ptr DatabaseCatalog::GetDatabaseCatalogEntry( + concurrency::TransactionContext *txn, + oid_t database_oid) { if (txn == nullptr) { throw CatalogException("Transaction is invalid!"); } @@ -329,17 +336,20 @@ std::shared_ptr DatabaseCatalog::GetDatabaseObject( if (database_object) return database_object; // cache miss, get from pg_database - std::vector column_ids(all_column_ids); + std::vector column_ids(all_column_ids_); oid_t index_offset = IndexId::PRIMARY_KEY; // Index of database_oid std::vector values; values.push_back(type::ValueFactory::GetIntegerValue(database_oid).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); if (result_tiles->size() == 1 && (*result_tiles)[0]->GetTupleCount() == 1) { auto database_object = - std::make_shared((*result_tiles)[0].get(), txn); + std::make_shared(txn, (*result_tiles)[0].get()); // insert into cache bool success = txn->catalog_cache.InsertDatabaseObject(database_object); PELOTON_ASSERT(success == true); @@ -358,8 +368,9 @@ std::shared_ptr DatabaseCatalog::GetDatabaseObject( * construct database object from pg_database, and insert into the * cache. */ -std::shared_ptr DatabaseCatalog::GetDatabaseObject( - const std::string &database_name, concurrency::TransactionContext *txn) { +std::shared_ptr DatabaseCatalog::GetDatabaseCatalogEntry( + concurrency::TransactionContext *txn, + const std::string &database_name) { if (txn == nullptr) { throw CatalogException("Transaction is invalid!"); } @@ -368,18 +379,21 @@ std::shared_ptr DatabaseCatalog::GetDatabaseObject( if (database_object) return database_object; // cache miss, get from pg_database - std::vector column_ids(all_column_ids); + std::vector column_ids(all_column_ids_); oid_t index_offset = IndexId::SKEY_DATABASE_NAME; // Index of database_name std::vector values; values.push_back( type::ValueFactory::GetVarcharValue(database_name, nullptr).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); if (result_tiles->size() == 1 && (*result_tiles)[0]->GetTupleCount() == 1) { auto database_object = - std::make_shared((*result_tiles)[0].get(), txn); + std::make_shared(txn, (*result_tiles)[0].get()); if (database_object) { // insert into cache bool success = txn->catalog_cache.InsertDatabaseObject(database_object); diff --git a/src/catalog/database_metrics_catalog.cpp b/src/catalog/database_metrics_catalog.cpp index 4fb04fbbb88..83ce60bedbf 100644 --- a/src/catalog/database_metrics_catalog.cpp +++ b/src/catalog/database_metrics_catalog.cpp @@ -27,23 +27,24 @@ DatabaseMetricsCatalog *DatabaseMetricsCatalog::GetInstance( DatabaseMetricsCatalog::DatabaseMetricsCatalog( concurrency::TransactionContext *txn) - : AbstractCatalog("CREATE TABLE " CATALOG_DATABASE_NAME - "." CATALOG_SCHEMA_NAME "." DATABASE_METRICS_CATALOG_NAME - " (" - "database_oid INT NOT NULL, " - "txn_committed INT NOT NULL, " - "txn_aborted INT NOT NULL, " - "time_stamp INT NOT NULL);", - txn) { + : AbstractCatalog(txn, "CREATE TABLE " CATALOG_DATABASE_NAME + "." CATALOG_SCHEMA_NAME "." DATABASE_METRICS_CATALOG_NAME + " (" + "database_oid INT NOT NULL, " + "txn_committed INT NOT NULL, " + "txn_aborted INT NOT NULL, " + "time_stamp INT NOT NULL);") { // Add secondary index here if necessary } DatabaseMetricsCatalog::~DatabaseMetricsCatalog() {} -bool DatabaseMetricsCatalog::InsertDatabaseMetrics( - oid_t database_oid, oid_t txn_committed, oid_t txn_aborted, - oid_t time_stamp, type::AbstractPool *pool, - concurrency::TransactionContext *txn) { +bool DatabaseMetricsCatalog::InsertDatabaseMetrics(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t txn_committed, + oid_t txn_aborted, + oid_t time_stamp, + type::AbstractPool *pool) { std::unique_ptr tuple( new storage::Tuple(catalog_table_->GetSchema(), true)); @@ -58,17 +59,17 @@ bool DatabaseMetricsCatalog::InsertDatabaseMetrics( tuple->SetValue(ColumnId::TIME_STAMP, val3, pool); // Insert the tuple into catalog table - return InsertTuple(std::move(tuple), txn); + return InsertTuple(txn, std::move(tuple)); } -bool DatabaseMetricsCatalog::DeleteDatabaseMetrics( - oid_t database_oid, concurrency::TransactionContext *txn) { +bool DatabaseMetricsCatalog::DeleteDatabaseMetrics(concurrency::TransactionContext *txn, + oid_t database_oid) { oid_t index_offset = IndexId::PRIMARY_KEY; // Primary key index std::vector values; values.push_back(type::ValueFactory::GetIntegerValue(database_oid).Copy()); - return DeleteWithIndexScan(index_offset, values, txn); + return DeleteWithIndexScan(txn, index_offset, values); } } // namespace catalog diff --git a/src/catalog/index_catalog.cpp b/src/catalog/index_catalog.cpp index 066590ac4d1..eaab5f3cfa7 100644 --- a/src/catalog/index_catalog.cpp +++ b/src/catalog/index_catalog.cpp @@ -26,21 +26,21 @@ namespace peloton { namespace catalog { -IndexCatalogObject::IndexCatalogObject(executor::LogicalTile *tile, int tupleId) - : index_oid(tile->GetValue(tupleId, IndexCatalog::ColumnId::INDEX_OID) +IndexCatalogEntry::IndexCatalogEntry(executor::LogicalTile *tile, int tupleId) + : index_oid_(tile->GetValue(tupleId, IndexCatalog::ColumnId::INDEX_OID) .GetAs()), - index_name(tile->GetValue(tupleId, IndexCatalog::ColumnId::INDEX_NAME) + index_name_(tile->GetValue(tupleId, IndexCatalog::ColumnId::INDEX_NAME) .ToString()), - table_oid(tile->GetValue(tupleId, IndexCatalog::ColumnId::TABLE_OID) + table_oid_(tile->GetValue(tupleId, IndexCatalog::ColumnId::TABLE_OID) .GetAs()), - schema_name(tile->GetValue(tupleId, IndexCatalog::ColumnId::SCHEMA_NAME) + schema_name_(tile->GetValue(tupleId, IndexCatalog::ColumnId::SCHEMA_NAME) .ToString()), - index_type(tile->GetValue(tupleId, IndexCatalog::ColumnId::INDEX_TYPE) + index_type_(tile->GetValue(tupleId, IndexCatalog::ColumnId::INDEX_TYPE) .GetAs()), - index_constraint( + index_constraint_( tile->GetValue(tupleId, IndexCatalog::ColumnId::INDEX_CONSTRAINT) .GetAs()), - unique_keys(tile->GetValue(tupleId, IndexCatalog::ColumnId::UNIQUE_KEYS) + unique_keys_(tile->GetValue(tupleId, IndexCatalog::ColumnId::UNIQUE_KEYS) .GetAs()) { std::string attr_str = tile->GetValue(tupleId, IndexCatalog::ColumnId::INDEXED_ATTRIBUTES) @@ -49,22 +49,30 @@ IndexCatalogObject::IndexCatalogObject(executor::LogicalTile *tile, int tupleId) std::string tok; while (std::getline(ss, tok, ' ')) { - key_attrs.push_back(std::stoi(tok)); + key_attrs_.push_back(std::stoi(tok)); } - LOG_TRACE("the size for indexed key is %lu", key_attrs.size()); + LOG_TRACE("the size for indexed key is %lu", key_attrs_.size()); } -IndexCatalog::IndexCatalog( - storage::Database *pg_catalog, UNUSED_ATTRIBUTE type::AbstractPool *pool, - UNUSED_ATTRIBUTE concurrency::TransactionContext *txn) - : AbstractCatalog(INDEX_CATALOG_OID, INDEX_CATALOG_NAME, - InitializeSchema().release(), pg_catalog) { +IndexCatalog::IndexCatalog(concurrency::TransactionContext *, + storage::Database *pg_catalog, + type::AbstractPool *) + : AbstractCatalog(pg_catalog, + InitializeSchema().release(), + INDEX_CATALOG_OID, + INDEX_CATALOG_NAME) { // Add indexes for pg_index - AddIndex({0}, INDEX_CATALOG_PKEY_OID, INDEX_CATALOG_NAME "_pkey", + AddIndex(INDEX_CATALOG_NAME "_pkey", + INDEX_CATALOG_PKEY_OID, + {0}, IndexConstraintType::PRIMARY_KEY); - AddIndex({1, 3}, INDEX_CATALOG_SKEY0_OID, INDEX_CATALOG_NAME "_skey0", + AddIndex(INDEX_CATALOG_NAME "_skey0", + INDEX_CATALOG_SKEY0_OID, + {1, 3}, IndexConstraintType::UNIQUE); - AddIndex({2}, INDEX_CATALOG_SKEY1_OID, INDEX_CATALOG_NAME "_skey1", + AddIndex(INDEX_CATALOG_NAME "_skey1", + INDEX_CATALOG_SKEY1_OID, + {2}, IndexConstraintType::DEFAULT); } @@ -85,7 +93,7 @@ std::unique_ptr IndexCatalog::InitializeSchema() { index_id_column.AddConstraint( catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); - auto index_name_column = catalog::Column(type::TypeId::VARCHAR, max_name_size, + auto index_name_column = catalog::Column(type::TypeId::VARCHAR, max_name_size_, "index_name", false); index_name_column.AddConstraint( catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); @@ -97,7 +105,7 @@ std::unique_ptr IndexCatalog::InitializeSchema() { catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); auto schema_name_column = catalog::Column( - type::TypeId::VARCHAR, max_name_size, "schema_name", false); + type::TypeId::VARCHAR, max_name_size_, "schema_name", false); schema_name_column.AddConstraint( catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); @@ -120,7 +128,7 @@ std::unique_ptr IndexCatalog::InitializeSchema() { catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); auto indexed_attributes_column = catalog::Column( - type::TypeId::VARCHAR, max_name_size, "indexed_attributes", false); + type::TypeId::VARCHAR, max_name_size_, "indexed_attributes", false); indexed_attributes_column.AddConstraint( catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); std::unique_ptr index_schema(new catalog::Schema( @@ -130,13 +138,16 @@ std::unique_ptr IndexCatalog::InitializeSchema() { return index_schema; } -bool IndexCatalog::InsertIndex(oid_t index_oid, const std::string &index_name, - oid_t table_oid, const std::string &schema_name, +bool IndexCatalog::InsertIndex(concurrency::TransactionContext *txn, + const std::string &schema_name, + oid_t table_oid, + oid_t index_oid, + const std::string &index_name, IndexType index_type, IndexConstraintType index_constraint, - bool unique_keys, std::vector indekeys, - type::AbstractPool *pool, - concurrency::TransactionContext *txn) { + bool unique_keys, + std::vector index_keys, + type::AbstractPool *pool) { // Create the tuple first std::unique_ptr tuple( new storage::Tuple(catalog_table_->GetSchema(), true)); @@ -151,7 +162,7 @@ bool IndexCatalog::InsertIndex(oid_t index_oid, const std::string &index_name, auto val6 = type::ValueFactory::GetBooleanValue(unique_keys); std::stringstream os; - for (oid_t indkey : indekeys) os << std::to_string(indkey) << " "; + for (oid_t indkey : index_keys) os << std::to_string(indkey) << " "; auto val7 = type::ValueFactory::GetVarcharValue(os.str(), nullptr); tuple->SetValue(IndexCatalog::ColumnId::INDEX_OID, val0, pool); @@ -164,11 +175,12 @@ bool IndexCatalog::InsertIndex(oid_t index_oid, const std::string &index_name, tuple->SetValue(IndexCatalog::ColumnId::INDEXED_ATTRIBUTES, val7, pool); // Insert the tuple - return InsertTuple(std::move(tuple), txn); + return InsertTuple(txn, std::move(tuple)); } -bool IndexCatalog::DeleteIndex(oid_t database_oid, oid_t index_oid, - concurrency::TransactionContext *txn) { +bool IndexCatalog::DeleteIndex(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t index_oid) { oid_t index_offset = IndexId::PRIMARY_KEY; // Index of index_oid std::vector values; values.push_back(type::ValueFactory::GetIntegerValue(index_oid).Copy()); @@ -179,14 +191,16 @@ bool IndexCatalog::DeleteIndex(oid_t database_oid, oid_t index_oid, auto table_object = txn->catalog_cache.GetCachedTableObject(database_oid, index_object->GetTableOid()); - table_object->EvictAllIndexObjects(); + table_object->EvictAllIndexCatalogEntries(); } - return DeleteWithIndexScan(index_offset, values, txn); + return DeleteWithIndexScan(txn, index_offset, values); } -std::shared_ptr IndexCatalog::GetIndexObject( - oid_t database_oid, oid_t index_oid, concurrency::TransactionContext *txn) { +std::shared_ptr IndexCatalog::GetIndexCatalogEntry( + concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t index_oid) { if (txn == nullptr) { throw CatalogException("Transaction is invalid!"); } @@ -204,20 +218,23 @@ std::shared_ptr IndexCatalog::GetIndexObject( values.push_back(type::ValueFactory::GetIntegerValue(index_oid).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); if (result_tiles->size() == 1 && (*result_tiles)[0]->GetTupleCount() == 1) { auto index_object = - std::make_shared((*result_tiles)[0].get()); + std::make_shared((*result_tiles)[0].get()); // fetch all indexes into table object (cannot use the above index object) auto pg_table = Catalog::GetInstance() ->GetSystemCatalogs(database_oid) ->GetTableCatalog(); auto table_object = - pg_table->GetTableObject(index_object->GetTableOid(), txn); + pg_table->GetTableCatalogEntry(txn, index_object->GetTableOid()); PELOTON_ASSERT(table_object && table_object->GetTableOid() == index_object->GetTableOid()); - return table_object->GetIndexObject(index_oid); + return table_object->GetIndexCatalogEntries(index_oid); } else { LOG_DEBUG("Found %lu index with oid %u", result_tiles->size(), index_oid); } @@ -226,16 +243,19 @@ std::shared_ptr IndexCatalog::GetIndexObject( return nullptr; } -std::shared_ptr IndexCatalog::GetIndexObject( - const std::string &database_name, const std::string &index_name, - const std::string &schema_name, concurrency::TransactionContext *txn) { +std::shared_ptr IndexCatalog::GetIndexCatalogEntry( + concurrency::TransactionContext *txn, + const std::string &database_name, + const std::string &schema_name, + const std::string &index_name) { if (txn == nullptr) { throw CatalogException("Transaction is invalid!"); } // try get from cache auto index_object = - txn->catalog_cache.GetCachedIndexObject(database_name, index_name, - schema_name); + txn->catalog_cache.GetCachedIndexObject(database_name, + schema_name, + index_name); if (index_object) { return index_object; } @@ -251,20 +271,23 @@ std::shared_ptr IndexCatalog::GetIndexObject( type::ValueFactory::GetVarcharValue(schema_name, nullptr).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); if (result_tiles->size() == 1 && (*result_tiles)[0]->GetTupleCount() == 1) { auto index_object = - std::make_shared((*result_tiles)[0].get()); + std::make_shared((*result_tiles)[0].get()); // fetch all indexes into table object (cannot use the above index object) auto pg_table = Catalog::GetInstance() - ->GetSystemCatalogs(database_oid) + ->GetSystemCatalogs(database_oid_) ->GetTableCatalog(); auto table_object = - pg_table->GetTableObject(index_object->GetTableOid(), txn); + pg_table->GetTableCatalogEntry(txn, index_object->GetTableOid()); PELOTON_ASSERT(table_object && table_object->GetTableOid() == index_object->GetTableOid()); - return table_object->GetIndexObject(index_name); + return table_object->GetIndexCatalogEntry(index_name); } else { LOG_DEBUG("Found %lu index with name %s", result_tiles->size(), index_name.c_str()); @@ -280,19 +303,21 @@ std::shared_ptr IndexCatalog::GetIndexObject( * @param txn TransactionContext * @return a vector of index catalog objects */ -const std::unordered_map> -IndexCatalog::GetIndexObjects(oid_t table_oid, - concurrency::TransactionContext *txn) { +const std::unordered_map> +IndexCatalog::GetIndexCatalogEntries( + concurrency::TransactionContext *txn, + oid_t table_oid) { if (txn == nullptr) { throw CatalogException("Transaction is invalid!"); } // try get from cache auto pg_table = Catalog::GetInstance() - ->GetSystemCatalogs(database_oid) + ->GetSystemCatalogs(database_oid_) ->GetTableCatalog(); - auto table_object = pg_table->GetTableObject(table_oid, txn); + auto table_object = pg_table->GetTableCatalogEntry(txn, table_oid); PELOTON_ASSERT(table_object && table_object->GetTableOid() == table_oid); - auto index_objects = table_object->GetIndexObjects(true); + auto index_objects = table_object->GetIndexCatalogEntries(true); if (index_objects.empty() == false) return index_objects; // cache miss, get from pg_index @@ -302,17 +327,20 @@ IndexCatalog::GetIndexObjects(oid_t table_oid, values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); for (auto &tile : (*result_tiles)) { for (auto tuple_id : *tile) { auto index_object = - std::make_shared(tile.get(), tuple_id); - table_object->InsertIndexObject(index_object); + std::make_shared(tile.get(), tuple_id); + table_object->InsertIndexCatalogEntry(index_object); } } - return table_object->GetIndexObjects(); + return table_object->GetIndexCatalogEntries(); } } // namespace catalog diff --git a/src/catalog/index_metrics_catalog.cpp b/src/catalog/index_metrics_catalog.cpp index e744765ef96..3d7c730f31e 100644 --- a/src/catalog/index_metrics_catalog.cpp +++ b/src/catalog/index_metrics_catalog.cpp @@ -19,27 +19,30 @@ namespace peloton { namespace catalog { -IndexMetricsCatalog::IndexMetricsCatalog(const std::string &database_name, - concurrency::TransactionContext *txn) - : AbstractCatalog("CREATE TABLE " + database_name + - "." CATALOG_SCHEMA_NAME "." INDEX_METRICS_CATALOG_NAME - " (" - "table_oid INT NOT NULL, " - "index_oid INT NOT NULL, " - "reads INT NOT NULL, " - "deletes INT NOT NULL, " - "inserts INT NOT NULL, " - "time_stamp INT NOT NULL);", - txn) { +IndexMetricsCatalog::IndexMetricsCatalog(concurrency::TransactionContext *txn, + const std::string &database_name) + : AbstractCatalog(txn, "CREATE TABLE " + database_name + + "." CATALOG_SCHEMA_NAME "." INDEX_METRICS_CATALOG_NAME + " (" + "table_oid INT NOT NULL, " + "index_oid INT NOT NULL, " + "reads INT NOT NULL, " + "deletes INT NOT NULL, " + "inserts INT NOT NULL, " + "time_stamp INT NOT NULL);") { // Add secondary index here if necessary } IndexMetricsCatalog::~IndexMetricsCatalog() {} -bool IndexMetricsCatalog::InsertIndexMetrics( - oid_t table_oid, oid_t index_oid, int64_t reads, int64_t deletes, - int64_t inserts, int64_t time_stamp, type::AbstractPool *pool, - concurrency::TransactionContext *txn) { +bool IndexMetricsCatalog::InsertIndexMetrics(concurrency::TransactionContext *txn, + oid_t table_oid, + oid_t index_oid, + int64_t reads, + int64_t deletes, + int64_t inserts, + int64_t time_stamp, + type::AbstractPool *pool) { std::unique_ptr tuple( new storage::Tuple(catalog_table_->GetSchema(), true)); @@ -58,17 +61,16 @@ bool IndexMetricsCatalog::InsertIndexMetrics( tuple->SetValue(ColumnId::TIME_STAMP, val6, pool); // Insert the tuple - return InsertTuple(std::move(tuple), txn); + return InsertTuple(txn, std::move(tuple)); } -bool IndexMetricsCatalog::DeleteIndexMetrics( - oid_t index_oid, concurrency::TransactionContext *txn) { +bool IndexMetricsCatalog::DeleteIndexMetrics(concurrency::TransactionContext *txn, oid_t index_oid) { oid_t index_offset = IndexId::PRIMARY_KEY; // Primary key index std::vector values; values.push_back(type::ValueFactory::GetIntegerValue(index_oid).Copy()); - return DeleteWithIndexScan(index_offset, values, txn); + return DeleteWithIndexScan(txn, index_offset, values); } } // namespace catalog diff --git a/src/catalog/language_catalog.cpp b/src/catalog/language_catalog.cpp index ddcceb6d89d..e3a7ce1924d 100644 --- a/src/catalog/language_catalog.cpp +++ b/src/catalog/language_catalog.cpp @@ -20,7 +20,7 @@ namespace peloton { namespace catalog { -LanguageCatalogObject::LanguageCatalogObject(executor::LogicalTile *tuple) +LanguageCatalogEntry::LanguageCatalogEntry(executor::LogicalTile *tuple) : lang_oid_(tuple->GetValue(0, 0).GetAs()), lang_name_(tuple->GetValue(0, 1).GetAs()) {} @@ -33,21 +33,25 @@ LanguageCatalog &LanguageCatalog::GetInstance( LanguageCatalog::~LanguageCatalog(){}; LanguageCatalog::LanguageCatalog(concurrency::TransactionContext *txn) - : AbstractCatalog("CREATE TABLE " CATALOG_DATABASE_NAME - "." CATALOG_SCHEMA_NAME "." LANGUAGE_CATALOG_NAME - " (" - "language_oid INT NOT NULL PRIMARY KEY, " - "lanname VARCHAR NOT NULL);", - txn) { - Catalog::GetInstance()->CreateIndex( - CATALOG_DATABASE_NAME, CATALOG_SCHEMA_NAME, LANGUAGE_CATALOG_NAME, {1}, - LANGUAGE_CATALOG_NAME "_skey0", false, IndexType::BWTREE, txn); + : AbstractCatalog(txn, "CREATE TABLE " CATALOG_DATABASE_NAME + "." CATALOG_SCHEMA_NAME "." LANGUAGE_CATALOG_NAME + " (" + "language_oid INT NOT NULL PRIMARY KEY, " + "lanname VARCHAR NOT NULL);") { + Catalog::GetInstance()->CreateIndex(txn, + CATALOG_DATABASE_NAME, + CATALOG_SCHEMA_NAME, + LANGUAGE_CATALOG_NAME, + LANGUAGE_CATALOG_NAME "_skey0", + {1}, + false, + IndexType::BWTREE); } // insert a new language by name -bool LanguageCatalog::InsertLanguage(const std::string &lanname, - type::AbstractPool *pool, - concurrency::TransactionContext *txn) { +bool LanguageCatalog::InsertLanguage(concurrency::TransactionContext *txn, + const std::string &lanname, + type::AbstractPool *pool) { std::unique_ptr tuple( new storage::Tuple(catalog_table_->GetSchema(), true)); @@ -59,56 +63,62 @@ bool LanguageCatalog::InsertLanguage(const std::string &lanname, tuple->SetValue(ColumnId::LANNAME, val1, pool); // Insert the tuple - return InsertTuple(std::move(tuple), txn); + return InsertTuple(txn, std::move(tuple)); } // delete a language by name -bool LanguageCatalog::DeleteLanguage(const std::string &lanname, - concurrency::TransactionContext *txn) { +bool LanguageCatalog::DeleteLanguage(concurrency::TransactionContext *txn, + const std::string &lanname) { oid_t index_offset = IndexId::SECONDARY_KEY_0; std::vector values; values.push_back( type::ValueFactory::GetVarcharValue(lanname, nullptr).Copy()); - return DeleteWithIndexScan(index_offset, values, txn); + return DeleteWithIndexScan(txn, index_offset, values); } -std::unique_ptr LanguageCatalog::GetLanguageByOid( - oid_t lang_oid, concurrency::TransactionContext *txn) const { - std::vector column_ids(all_column_ids); +std::unique_ptr LanguageCatalog::GetLanguageByOid(concurrency::TransactionContext *txn, + oid_t lang_oid) const { + std::vector column_ids(all_column_ids_); oid_t index_offset = IndexId::PRIMARY_KEY; std::vector values; values.push_back(type::ValueFactory::GetIntegerValue(lang_oid).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); PELOTON_ASSERT(result_tiles->size() <= 1); - std::unique_ptr ret; + std::unique_ptr ret; if (result_tiles->size() == 1) { PELOTON_ASSERT((*result_tiles)[0]->GetTupleCount() <= 1); - ret.reset(new LanguageCatalogObject((*result_tiles)[0].get())); + ret.reset(new LanguageCatalogEntry((*result_tiles)[0].get())); } return ret; } -std::unique_ptr LanguageCatalog::GetLanguageByName( - const std::string &lang_name, concurrency::TransactionContext *txn) const { - std::vector column_ids(all_column_ids); +std::unique_ptr LanguageCatalog::GetLanguageByName(concurrency::TransactionContext *txn, + const std::string &lang_name) const { + std::vector column_ids(all_column_ids_); oid_t index_offset = IndexId::SECONDARY_KEY_0; std::vector values; values.push_back(type::ValueFactory::GetVarcharValue(lang_name).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); PELOTON_ASSERT(result_tiles->size() <= 1); - std::unique_ptr ret; + std::unique_ptr ret; if (result_tiles->size() == 1) { PELOTON_ASSERT((*result_tiles)[0]->GetTupleCount() <= 1); - ret.reset(new LanguageCatalogObject((*result_tiles)[0].get())); + ret.reset(new LanguageCatalogEntry((*result_tiles)[0].get())); } return ret; diff --git a/src/catalog/layout_catalog.cpp b/src/catalog/layout_catalog.cpp index 2189426fb0d..8db442a03e0 100644 --- a/src/catalog/layout_catalog.cpp +++ b/src/catalog/layout_catalog.cpp @@ -26,16 +26,22 @@ namespace catalog { /** @brief Constructor invoked by the SystemsCatalog constructor. * @param pg_catalog The database to which this pg_layout belongs. */ -LayoutCatalog::LayoutCatalog( - storage::Database *pg_catalog, UNUSED_ATTRIBUTE type::AbstractPool *pool, - UNUSED_ATTRIBUTE concurrency::TransactionContext *txn) - : AbstractCatalog(LAYOUT_CATALOG_OID, LAYOUT_CATALOG_NAME, - InitializeSchema().release(), pg_catalog) { +LayoutCatalog::LayoutCatalog(concurrency::TransactionContext *, + storage::Database *pg_catalog, + type::AbstractPool *) + : AbstractCatalog(pg_catalog, + InitializeSchema().release(), + LAYOUT_CATALOG_OID, + LAYOUT_CATALOG_NAME) { // Add indexes for pg_attribute - AddIndex({ColumnId::TABLE_OID, ColumnId::LAYOUT_OID}, LAYOUT_CATALOG_PKEY_OID, - LAYOUT_CATALOG_NAME "_pkey", IndexConstraintType::PRIMARY_KEY); - AddIndex({ColumnId::TABLE_OID}, LAYOUT_CATALOG_SKEY0_OID, - LAYOUT_CATALOG_NAME "_skey0", IndexConstraintType::DEFAULT); + AddIndex(LAYOUT_CATALOG_NAME "_pkey", + LAYOUT_CATALOG_PKEY_OID, + {ColumnId::TABLE_OID, ColumnId::LAYOUT_OID}, + IndexConstraintType::PRIMARY_KEY); + AddIndex(LAYOUT_CATALOG_NAME "_skey0", + LAYOUT_CATALOG_SKEY0_OID, + {ColumnId::TABLE_OID}, + IndexConstraintType::DEFAULT); } /** @brief Destructor. Do nothing. Layouts will be dropped by DropTable. */ LayoutCatalog::~LayoutCatalog() {} @@ -89,10 +95,10 @@ std::unique_ptr LayoutCatalog::InitializeSchema() { * @param txn TransactionContext for adding the layout. * @return true on success. */ -bool LayoutCatalog::InsertLayout(oid_t table_oid, +bool LayoutCatalog::InsertLayout(concurrency::TransactionContext *txn, + oid_t table_oid, std::shared_ptr layout, - type::AbstractPool *pool, - concurrency::TransactionContext *txn) { + type::AbstractPool *pool) { // Create the tuple first std::unique_ptr tuple( new storage::Tuple(catalog_table_->GetSchema(), true)); @@ -109,7 +115,7 @@ bool LayoutCatalog::InsertLayout(oid_t table_oid, tuple->SetValue(LayoutCatalog::ColumnId::COLUMN_MAP, val3, pool); // Insert the tuple - return InsertTuple(std::move(tuple), txn); + return InsertTuple(txn, std::move(tuple)); } /** @brief Delete a layout from the pg_layout table. @@ -118,8 +124,9 @@ bool LayoutCatalog::InsertLayout(oid_t table_oid, * @param txn TransactionContext for deleting the layout. * @return true on success. */ -bool LayoutCatalog::DeleteLayout(oid_t table_oid, oid_t layout_oid, - concurrency::TransactionContext *txn) { +bool LayoutCatalog::DeleteLayout(concurrency::TransactionContext *txn, + oid_t table_oid, + oid_t layout_oid) { oid_t index_offset = IndexId::PRIMARY_KEY; // Index of table_oid & layout_oid std::vector values; @@ -127,14 +134,14 @@ bool LayoutCatalog::DeleteLayout(oid_t table_oid, oid_t layout_oid, values.push_back(type::ValueFactory::GetIntegerValue(layout_oid).Copy()); auto pg_table = Catalog::GetInstance() - ->GetSystemCatalogs(database_oid) + ->GetSystemCatalogs(database_oid_) ->GetTableCatalog(); // delete column from cache - auto table_object = pg_table->GetTableObject(table_oid, txn); + auto table_object = pg_table->GetTableCatalogEntry(txn, table_oid); table_object->EvictLayout(layout_oid); - return DeleteWithIndexScan(index_offset, values, txn); + return DeleteWithIndexScan(txn, index_offset, values); } /** @brief Delete all layouts correponding to a table from the pg_layout. @@ -142,20 +149,19 @@ bool LayoutCatalog::DeleteLayout(oid_t table_oid, oid_t layout_oid, * @param txn TransactionContext for deleting the layouts. * @return true on success. */ -bool LayoutCatalog::DeleteLayouts(oid_t table_oid, - concurrency::TransactionContext *txn) { +bool LayoutCatalog::DeleteLayouts(concurrency::TransactionContext *txn, oid_t table_oid) { oid_t index_offset = IndexId::SKEY_TABLE_OID; // Index of table_oid std::vector values; values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); // delete layouts from cache auto pg_table = Catalog::GetInstance() - ->GetSystemCatalogs(database_oid) + ->GetSystemCatalogs(database_oid_) ->GetTableCatalog(); - auto table_object = pg_table->GetTableObject(table_oid, txn); + auto table_object = pg_table->GetTableCatalogEntry(txn, table_oid); table_object->EvictAllLayouts(); - return DeleteWithIndexScan(index_offset, values, txn); + return DeleteWithIndexScan(txn, index_offset, values); } /** @brief Get all layouts correponding to a table from the pg_layout. @@ -163,14 +169,16 @@ bool LayoutCatalog::DeleteLayouts(oid_t table_oid, * @param txn TransactionContext for getting the layouts. * @return unordered_map containing a layout_oid -> layout mapping. */ -const std::unordered_map> -LayoutCatalog::GetLayouts(oid_t table_oid, - concurrency::TransactionContext *txn) { +const std::unordered_map> +LayoutCatalog::GetLayouts( +concurrency::TransactionContext *txn, +oid_t table_oid) { // Try to find the layouts in the cache auto pg_table = Catalog::GetInstance() - ->GetSystemCatalogs(database_oid) + ->GetSystemCatalogs(database_oid_) ->GetTableCatalog(); - auto table_object = pg_table->GetTableObject(table_oid, txn); + auto table_object = pg_table->GetTableCatalogEntry(txn, table_oid); PELOTON_ASSERT(table_object && table_object->GetTableOid() == table_oid); auto layout_objects = table_object->GetLayouts(true); if (layout_objects.size() != 0) { @@ -178,13 +186,16 @@ LayoutCatalog::GetLayouts(oid_t table_oid, } // Cache miss, get from pg_catalog - std::vector column_ids(all_column_ids); + std::vector column_ids(all_column_ids_); oid_t index_offset = IndexId::SKEY_TABLE_OID; // Index of table_oid std::vector values; values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); for (auto &tile : (*result_tiles)) { // Iterate through the result_tiles for (auto tuple_id : *tile) { @@ -220,9 +231,10 @@ LayoutCatalog::GetLayouts(oid_t table_oid, * @return shared_ptr corresponding to the layout_oid if found. * nullptr otherwise. */ -std::shared_ptr LayoutCatalog::GetLayoutWithOid( - oid_t table_oid, oid_t layout_oid, concurrency::TransactionContext *txn) { - auto table_layouts = GetLayouts(table_oid, txn); +std::shared_ptr LayoutCatalog::GetLayoutWithOid(concurrency::TransactionContext *txn, + oid_t table_oid, + oid_t layout_oid) { + auto table_layouts = GetLayouts(txn, table_oid); for (const auto &layout_entry : table_layouts) { if (layout_entry.second->GetOid() == layout_oid) { return layout_entry.second; diff --git a/src/catalog/multi_constraint.cpp b/src/catalog/multi_constraint.cpp index 0261bccc3c2..d1e4e3d9817 100644 --- a/src/catalog/multi_constraint.cpp +++ b/src/catalog/multi_constraint.cpp @@ -21,9 +21,9 @@ namespace catalog { const std::string MultiConstraint::GetInfo() const { std::ostringstream os; os << "Constraint[" << GetName() << ", " - << ConstraintTypeToString(constraint_type) << " , related columns: ("; + << ConstraintTypeToString(constraint_type_) << " , related columns: ("; bool first = true; - for (auto id : column_ids) { + for (auto id : column_ids_) { if (first) { os << id; first = false; diff --git a/src/catalog/proc_catalog.cpp b/src/catalog/proc_catalog.cpp index 6da75db67fc..7c3fe1db77e 100644 --- a/src/catalog/proc_catalog.cpp +++ b/src/catalog/proc_catalog.cpp @@ -23,8 +23,8 @@ namespace catalog { #define PROC_CATALOG_NAME "pg_proc" -ProcCatalogObject::ProcCatalogObject(executor::LogicalTile *tile, - concurrency::TransactionContext *txn) +ProcCatalogEntry::ProcCatalogEntry(concurrency::TransactionContext *txn, + executor::LogicalTile *tile) : oid_(tile->GetValue(0, 0).GetAs()), name_(tile->GetValue(0, 1).GetAs()), ret_type_(tile->GetValue(0, 2).GetAs()), @@ -33,8 +33,8 @@ ProcCatalogObject::ProcCatalogObject(executor::LogicalTile *tile, src_(tile->GetValue(0, 5).GetAs()), txn_(txn) {} -std::unique_ptr ProcCatalogObject::GetLanguage() const { - return LanguageCatalog::GetInstance().GetLanguageByOid(GetLangOid(), txn_); +std::unique_ptr ProcCatalogEntry::GetLanguage() const { + return LanguageCatalog::GetInstance().GetLanguageByOid(txn_, GetLangOid()); } ProcCatalog &ProcCatalog::GetInstance(concurrency::TransactionContext *txn) { @@ -45,27 +45,32 @@ ProcCatalog &ProcCatalog::GetInstance(concurrency::TransactionContext *txn) { ProcCatalog::~ProcCatalog(){}; ProcCatalog::ProcCatalog(concurrency::TransactionContext *txn) - : AbstractCatalog("CREATE TABLE " CATALOG_DATABASE_NAME - "." CATALOG_SCHEMA_NAME "." PROC_CATALOG_NAME - " (" - "proc_oid INT NOT NULL PRIMARY KEY, " - "proname VARCHAR NOT NULL, " - "prorettype INT NOT NULL, " - "proargtypes VARCHAR NOT NULL, " - "prolang INT NOT NULL, " - "prosrc VARCHAR NOT NULL);", - txn) { - Catalog::GetInstance()->CreateIndex( - CATALOG_DATABASE_NAME, CATALOG_SCHEMA_NAME, PROC_CATALOG_NAME, {1, 3}, - PROC_CATALOG_NAME "_skey0", false, IndexType::BWTREE, txn); + : AbstractCatalog(txn, "CREATE TABLE " CATALOG_DATABASE_NAME + "." CATALOG_SCHEMA_NAME "." PROC_CATALOG_NAME + " (" + "proc_oid INT NOT NULL PRIMARY KEY, " + "proname VARCHAR NOT NULL, " + "prorettype INT NOT NULL, " + "proargtypes VARCHAR NOT NULL, " + "prolang INT NOT NULL, " + "prosrc VARCHAR NOT NULL);") { + Catalog::GetInstance()->CreateIndex(txn, + CATALOG_DATABASE_NAME, + CATALOG_SCHEMA_NAME, + PROC_CATALOG_NAME, + PROC_CATALOG_NAME "_skey0", + {1, 3}, + false, + IndexType::BWTREE); } -bool ProcCatalog::InsertProc(const std::string &proname, +bool ProcCatalog::InsertProc(concurrency::TransactionContext *txn, + const std::string &proname, type::TypeId prorettype, const std::vector &proargtypes, - oid_t prolang, const std::string &prosrc, - type::AbstractPool *pool, - concurrency::TransactionContext *txn) { + oid_t prolang, + const std::string &prosrc, + type::AbstractPool *pool) { std::unique_ptr tuple( new storage::Tuple(catalog_table_->GetSchema(), true)); @@ -86,34 +91,36 @@ bool ProcCatalog::InsertProc(const std::string &proname, tuple->SetValue(ColumnId::PROSRC, val5, pool); // Insert the tuple - return InsertTuple(std::move(tuple), txn); + return InsertTuple(txn, std::move(tuple)); } -std::unique_ptr ProcCatalog::GetProcByOid( - oid_t proc_oid, concurrency::TransactionContext *txn) const { - std::vector column_ids(all_column_ids); +std::unique_ptr ProcCatalog::GetProcByOid(concurrency::TransactionContext *txn, + oid_t proc_oid) const { + std::vector column_ids(all_column_ids_); oid_t index_offset = IndexId::PRIMARY_KEY; std::vector values; values.push_back(type::ValueFactory::GetIntegerValue(proc_oid).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); PELOTON_ASSERT(result_tiles->size() <= 1); - std::unique_ptr ret; + std::unique_ptr ret; if (result_tiles->size() == 1) { PELOTON_ASSERT((*result_tiles)[0]->GetTupleCount() <= 1); - ret.reset(new ProcCatalogObject((*result_tiles)[0].get(), txn)); + ret.reset(new ProcCatalogEntry(txn, (*result_tiles)[0].get())); } return ret; } -std::unique_ptr ProcCatalog::GetProcByName( - const std::string &proc_name, - const std::vector &proc_arg_types, - concurrency::TransactionContext *txn) const { - std::vector column_ids(all_column_ids); +std::unique_ptr ProcCatalog::GetProcByName(concurrency::TransactionContext *txn, + const std::string &proc_name, + const std::vector &proc_arg_types) const { + std::vector column_ids(all_column_ids_); oid_t index_offset = IndexId::SECONDARY_KEY_0; std::vector values; values.push_back(type::ValueFactory::GetVarcharValue(proc_name).Copy()); @@ -122,13 +129,16 @@ std::unique_ptr ProcCatalog::GetProcByName( .Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); PELOTON_ASSERT(result_tiles->size() <= 1); - std::unique_ptr ret; + std::unique_ptr ret; if (result_tiles->size() == 1) { PELOTON_ASSERT((*result_tiles)[0]->GetTupleCount() <= 1); - ret.reset(new ProcCatalogObject((*result_tiles)[0].get(), txn)); + ret.reset(new ProcCatalogEntry(txn, (*result_tiles)[0].get())); } return ret; diff --git a/src/catalog/query_history_catalog.cpp b/src/catalog/query_history_catalog.cpp index 4433197ba28..51f5690d3d2 100644 --- a/src/catalog/query_history_catalog.cpp +++ b/src/catalog/query_history_catalog.cpp @@ -26,20 +26,20 @@ QueryHistoryCatalog &QueryHistoryCatalog::GetInstance( } QueryHistoryCatalog::QueryHistoryCatalog(concurrency::TransactionContext *txn) - : AbstractCatalog("CREATE TABLE " CATALOG_DATABASE_NAME - "." CATALOG_SCHEMA_NAME "." QUERY_HISTORY_CATALOG_NAME - " (" - "query_string VARCHAR NOT NULL, " - "fingerprint VARCHAR NOT NULL, " - "timestamp TIMESTAMP NOT NULL);", - txn) {} + : AbstractCatalog(txn, "CREATE TABLE " CATALOG_DATABASE_NAME + "." CATALOG_SCHEMA_NAME "." QUERY_HISTORY_CATALOG_NAME + " (" + "query_string VARCHAR NOT NULL, " + "fingerprint VARCHAR NOT NULL, " + "timestamp TIMESTAMP NOT NULL);") {} QueryHistoryCatalog::~QueryHistoryCatalog() = default; -bool QueryHistoryCatalog::InsertQueryHistory( - const std::string &query_string, const std::string &fingerprint, - uint64_t timestamp, type::AbstractPool *pool, - concurrency::TransactionContext *txn) { +bool QueryHistoryCatalog::InsertQueryHistory(concurrency::TransactionContext *txn, + const std::string &query_string, + const std::string &fingerprint, + uint64_t timestamp, + type::AbstractPool *pool) { std::unique_ptr tuple( new storage::Tuple(catalog_table_->GetSchema(), true)); @@ -53,7 +53,7 @@ bool QueryHistoryCatalog::InsertQueryHistory( tuple->SetValue(ColumnId::TIMESTAMP, val2, pool != nullptr ? pool : &pool_); // Insert the tuple - return InsertTuple(std::move(tuple), txn); + return InsertTuple(txn, std::move(tuple)); } } // namespace catalog diff --git a/src/catalog/query_metrics_catalog.cpp b/src/catalog/query_metrics_catalog.cpp index 0fd6b7c44d7..62661d98d75 100644 --- a/src/catalog/query_metrics_catalog.cpp +++ b/src/catalog/query_metrics_catalog.cpp @@ -20,38 +20,44 @@ namespace peloton { namespace catalog { -QueryMetricsCatalog::QueryMetricsCatalog(const std::string &database_name, - concurrency::TransactionContext *txn) - : AbstractCatalog("CREATE TABLE " + database_name + - "." CATALOG_SCHEMA_NAME "." QUERY_METRICS_CATALOG_NAME - " (" - "query_name VARCHAR NOT NULL PRIMARY KEY, " - "database_oid INT NOT NULL PRIMARY KEY, " - "num_params INT NOT NULL, " - "param_types VARBINARY, " - "param_formats VARBINARY, " - "param_values VARBINARY, " - "reads INT NOT NULL, " - "updates INT NOT NULL, " - "deletes INT NOT NULL, " - "inserts INT NOT NULL, " - "latency INT NOT NULL, " - "cpu_time INT NOT NULL, " - "time_stamp INT NOT NULL);", - txn) { +QueryMetricsCatalog::QueryMetricsCatalog(concurrency::TransactionContext *txn, + const std::string &database_name) + : AbstractCatalog(txn, "CREATE TABLE " + database_name + + "." CATALOG_SCHEMA_NAME "." QUERY_METRICS_CATALOG_NAME + " (" + "query_name VARCHAR NOT NULL PRIMARY KEY, " + "database_oid INT NOT NULL PRIMARY KEY, " + "num_params INT NOT NULL, " + "param_types VARBINARY, " + "param_formats VARBINARY, " + "param_values VARBINARY, " + "reads INT NOT NULL, " + "updates INT NOT NULL, " + "deletes INT NOT NULL, " + "inserts INT NOT NULL, " + "latency INT NOT NULL, " + "cpu_time INT NOT NULL, " + "time_stamp INT NOT NULL);") { // Add secondary index here if necessary } QueryMetricsCatalog::~QueryMetricsCatalog() {} -bool QueryMetricsCatalog::InsertQueryMetrics( - const std::string &name, oid_t database_oid, int64_t num_params, - const stats::QueryMetric::QueryParamBuf &type_buf, - const stats::QueryMetric::QueryParamBuf &format_buf, - const stats::QueryMetric::QueryParamBuf &value_buf, int64_t reads, - int64_t updates, int64_t deletes, int64_t inserts, int64_t latency, - int64_t cpu_time, int64_t time_stamp, type::AbstractPool *pool, - concurrency::TransactionContext *txn) { +bool QueryMetricsCatalog::InsertQueryMetrics(concurrency::TransactionContext *txn, + const std::string &name, + oid_t database_oid, + int64_t num_params, + const stats::QueryMetric::QueryParamBuf &type_buf, + const stats::QueryMetric::QueryParamBuf &format_buf, + const stats::QueryMetric::QueryParamBuf &value_buf, + int64_t reads, + int64_t updates, + int64_t deletes, + int64_t inserts, + int64_t latency, + int64_t cpu_time, + int64_t time_stamp, + type::AbstractPool *pool) { std::unique_ptr tuple( new storage::Tuple(catalog_table_->GetSchema(), true)); @@ -95,30 +101,33 @@ bool QueryMetricsCatalog::InsertQueryMetrics( tuple->SetValue(ColumnId::TIME_STAMP, val12, pool); // Insert the tuple - return InsertTuple(std::move(tuple), txn); + return InsertTuple(txn, std::move(tuple)); } -bool QueryMetricsCatalog::DeleteQueryMetrics( - const std::string &name, concurrency::TransactionContext *txn) { +bool QueryMetricsCatalog::DeleteQueryMetrics(concurrency::TransactionContext *txn, + const std::string &name) { oid_t index_offset = IndexId::PRIMARY_KEY; // Primary key index std::vector values; values.push_back(type::ValueFactory::GetVarcharValue(name, nullptr).Copy()); - values.push_back(type::ValueFactory::GetIntegerValue(database_oid).Copy()); + values.push_back(type::ValueFactory::GetIntegerValue(database_oid_).Copy()); - return DeleteWithIndexScan(index_offset, values, txn); + return DeleteWithIndexScan(txn, index_offset, values); } -stats::QueryMetric::QueryParamBuf QueryMetricsCatalog::GetParamTypes( - const std::string &name, concurrency::TransactionContext *txn) { +stats::QueryMetric::QueryParamBuf QueryMetricsCatalog::GetParamTypes(concurrency::TransactionContext *txn, + const std::string &name) { std::vector column_ids({ColumnId::PARAM_TYPES}); // param_types oid_t index_offset = IndexId::PRIMARY_KEY; // Primary key index std::vector values; values.push_back(type::ValueFactory::GetVarcharValue(name, nullptr).Copy()); - values.push_back(type::ValueFactory::GetIntegerValue(database_oid).Copy()); + values.push_back(type::ValueFactory::GetIntegerValue(database_oid_).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); stats::QueryMetric::QueryParamBuf param_types; PELOTON_ASSERT(result_tiles->size() <= 1); // unique @@ -135,15 +144,18 @@ stats::QueryMetric::QueryParamBuf QueryMetricsCatalog::GetParamTypes( return param_types; } -int64_t QueryMetricsCatalog::GetNumParams( - const std::string &name, concurrency::TransactionContext *txn) { +int64_t QueryMetricsCatalog::GetNumParams(concurrency::TransactionContext *txn, + const std::string &name) { std::vector column_ids({ColumnId::NUM_PARAMS}); // num_params oid_t index_offset = IndexId::PRIMARY_KEY; // Primary key index std::vector values; values.push_back(type::ValueFactory::GetVarcharValue(name, nullptr).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); int64_t num_params = 0; PELOTON_ASSERT(result_tiles->size() <= 1); // unique diff --git a/src/catalog/schema.cpp b/src/catalog/schema.cpp index ab38f8c8501..66f557f2532 100644 --- a/src/catalog/schema.cpp +++ b/src/catalog/schema.cpp @@ -37,24 +37,24 @@ void Schema::CreateTupleSchema( column_offset += column.GetFixedLength(); - columns.push_back(std::move(column)); + columns_.push_back(std::move(column)); if (is_inlined[column_itr] == false) { tup_is_inlined = false; - uninlined_columns.push_back(column_itr); + uninlined_columns_.push_back(column_itr); } } - length = column_offset; - tuple_is_inlined = tup_is_inlined; + length_ = column_offset; + tuple_is_inlined_ = tup_is_inlined; - column_count = columns.size(); - uninlined_column_count = uninlined_columns.size(); + column_count_ = columns_.size(); + uninlined_column_count_ = uninlined_columns_.size(); } // Construct schema from vector of Column Schema::Schema(const std::vector &columns) - : length(0), tuple_is_inlined(false) { + : length_(0), tuple_is_inlined_(false) { oid_t column_count = columns.size(); std::vector column_types; @@ -105,7 +105,7 @@ std::shared_ptr Schema::CopySchema( for (oid_t column_itr = 0; column_itr < column_count; column_itr++) { // If column exists in set if (std::find(set.begin(), set.end(), column_itr) != set.end()) { - columns.push_back(schema->columns[column_itr]); + columns.push_back(schema->columns_[column_itr]); } } @@ -153,9 +153,9 @@ Schema *Schema::CopySchema(const Schema *schema, // For each column index, push the column for (oid_t column_index : index_list) { // Make sure the index does not refer to invalid element - PELOTON_ASSERT(column_index < schema->columns.size()); + PELOTON_ASSERT(column_index < schema->columns_.size()); - column_list.push_back(schema->columns[column_index]); + column_list.push_back(schema->columns_[column_index]); } Schema *ret_schema = new Schema(column_list); @@ -195,7 +195,7 @@ Schema *Schema::FilterSchema(const Schema *schema, for (oid_t column_itr = 0; column_itr < column_count; column_itr++) { // If column exists in set if (std::find(set.begin(), set.end(), column_itr) != set.end()) { - columns.push_back(schema->columns[column_itr]); + columns.push_back(schema->columns_[column_itr]); } } @@ -261,7 +261,7 @@ Schema *Schema::AppendSchemaPtrList( for (oid_t column_itr = 0; column_itr < column_count; column_itr++) { // If column exists in set. if (std::find(subset.begin(), subset.end(), column_itr) != subset.end()) { - columns.push_back(schema->columns[column_itr]); + columns.push_back(schema->columns_[column_itr]); } } } @@ -275,20 +275,20 @@ const std::string Schema::GetInfo() const { std::ostringstream os; os << "Schema[" - << "NumColumns:" << column_count << ", " - << "IsInlined:" << tuple_is_inlined << ", " - << "Length:" << length << ", " - << "UninlinedCount:" << uninlined_column_count << "]"; + << "NumColumns:" << column_count_ << ", " + << "IsInlined:" << tuple_is_inlined_ << ", " + << "Length:" << length_ << ", " + << "UninlinedCount:" << uninlined_column_count_ << "]"; bool first = true; os << " :: ("; - for (oid_t i = 0; i < column_count; i++) { + for (oid_t i = 0; i < column_count_; i++) { if (first) { first = false; } else { os << ", "; } - os << columns[i].GetInfo(); + os << columns_[i].GetInfo(); } os << ")"; @@ -305,7 +305,7 @@ hash_t Schema::Hash() const { auto is_inlined = IsInlined(); hash = HashUtil::CombineHashes(hash, HashUtil::Hash(&is_inlined)); - for (const auto &column : columns) { + for (const auto &column : columns_) { hash = HashUtil::CombineHashes(hash, column.Hash()); } return hash; diff --git a/src/catalog/schema_catalog.cpp b/src/catalog/schema_catalog.cpp index 4761bb0e776..815c6e467a6 100644 --- a/src/catalog/schema_catalog.cpp +++ b/src/catalog/schema_catalog.cpp @@ -24,23 +24,29 @@ namespace peloton { namespace catalog { -SchemaCatalogObject::SchemaCatalogObject(executor::LogicalTile *tile, - concurrency::TransactionContext *txn) - : schema_oid(tile->GetValue(0, SchemaCatalog::ColumnId::SCHEMA_OID) +SchemaCatalogEntry::SchemaCatalogEntry(concurrency::TransactionContext *txn, + executor::LogicalTile *tile) + : schema_oid_(tile->GetValue(0, SchemaCatalog::ColumnId::SCHEMA_OID) .GetAs()), - schema_name( + schema_name_( tile->GetValue(0, SchemaCatalog::ColumnId::SCHEMA_NAME).ToString()), - txn(txn) {} - -SchemaCatalog::SchemaCatalog( - storage::Database *database, UNUSED_ATTRIBUTE type::AbstractPool *pool, - UNUSED_ATTRIBUTE concurrency::TransactionContext *txn) - : AbstractCatalog(SCHEMA_CATALOG_OID, SCHEMA_CATALOG_NAME, - InitializeSchema().release(), database) { + txn_(txn) {} + +SchemaCatalog::SchemaCatalog(concurrency::TransactionContext *, + storage::Database *database, + type::AbstractPool *) + : AbstractCatalog(database, + InitializeSchema().release(), + SCHEMA_CATALOG_OID, + SCHEMA_CATALOG_NAME) { // Add indexes for pg_namespace - AddIndex({0}, SCHEMA_CATALOG_PKEY_OID, SCHEMA_CATALOG_NAME "_pkey", + AddIndex(SCHEMA_CATALOG_NAME "_pkey", + SCHEMA_CATALOG_PKEY_OID, + {0}, IndexConstraintType::PRIMARY_KEY); - AddIndex({1}, SCHEMA_CATALOG_SKEY0_OID, SCHEMA_CATALOG_NAME "_skey0", + AddIndex(SCHEMA_CATALOG_NAME "_skey0", + SCHEMA_CATALOG_SKEY0_OID, + {1}, IndexConstraintType::UNIQUE); } @@ -62,7 +68,7 @@ std::unique_ptr SchemaCatalog::InitializeSchema() { catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); auto schema_name_column = catalog::Column( - type::TypeId::VARCHAR, max_name_size, "schema_name", false); + type::TypeId::VARCHAR, max_name_size_, "schema_name", false); schema_name_column.AddConstraint( catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); @@ -71,10 +77,10 @@ std::unique_ptr SchemaCatalog::InitializeSchema() { return schema; } -bool SchemaCatalog::InsertSchema(oid_t schema_oid, +bool SchemaCatalog::InsertSchema(concurrency::TransactionContext *txn, + oid_t schema_oid, const std::string &schema_name, - type::AbstractPool *pool, - concurrency::TransactionContext *txn) { + type::AbstractPool *pool) { // Create the tuple first std::unique_ptr tuple( new storage::Tuple(catalog_table_->GetSchema(), true)); @@ -86,37 +92,41 @@ bool SchemaCatalog::InsertSchema(oid_t schema_oid, tuple->SetValue(SchemaCatalog::ColumnId::SCHEMA_NAME, val1, pool); // Insert the tuple - return InsertTuple(std::move(tuple), txn); + return InsertTuple(txn, std::move(tuple)); } -bool SchemaCatalog::DeleteSchema(const std::string &schema_name, - concurrency::TransactionContext *txn) { +bool SchemaCatalog::DeleteSchema(concurrency::TransactionContext *txn, + const std::string &schema_name) { oid_t index_offset = IndexId::SKEY_SCHEMA_NAME; // Index of schema_name std::vector values; values.push_back( type::ValueFactory::GetVarcharValue(schema_name, nullptr).Copy()); - return DeleteWithIndexScan(index_offset, values, txn); + return DeleteWithIndexScan(txn, index_offset, values); } -std::shared_ptr SchemaCatalog::GetSchemaObject( - const std::string &schema_name, concurrency::TransactionContext *txn) { +std::shared_ptr SchemaCatalog::GetSchemaCatalogEntry( + concurrency::TransactionContext *txn, + const std::string &schema_name) { if (txn == nullptr) { throw CatalogException("Transaction is invalid!"); } // get from pg_namespace, index scan - std::vector column_ids(all_column_ids); + std::vector column_ids(all_column_ids_); oid_t index_offset = IndexId::SKEY_SCHEMA_NAME; // Index of database_name std::vector values; values.push_back( type::ValueFactory::GetVarcharValue(schema_name, nullptr).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); if (result_tiles->size() == 1 && (*result_tiles)[0]->GetTupleCount() == 1) { auto schema_object = - std::make_shared((*result_tiles)[0].get(), txn); + std::make_shared(txn, (*result_tiles)[0].get()); // TODO: we don't have cache for schema object right now return schema_object; } diff --git a/src/catalog/settings_catalog.cpp b/src/catalog/settings_catalog.cpp index bcdc518af24..de483314d4b 100644 --- a/src/catalog/settings_catalog.cpp +++ b/src/catalog/settings_catalog.cpp @@ -28,33 +28,42 @@ SettingsCatalog &SettingsCatalog::GetInstance( } SettingsCatalog::SettingsCatalog(concurrency::TransactionContext *txn) - : AbstractCatalog("CREATE TABLE " CATALOG_DATABASE_NAME - "." CATALOG_SCHEMA_NAME "." SETTINGS_CATALOG_NAME - " (" - "name VARCHAR NOT NULL, " - "value VARCHAR NOT NULL, " - "value_type VARCHAR NOT NULL, " - "description VARCHAR, " - "min_value VARCHAR, " - "max_value VARCHAR, " - "default_value VARCHAR NOT NULL, " - "is_mutable BOOL NOT NULL, " - "is_persistent BOOL NOT NULL);", - txn) { + : AbstractCatalog(txn, "CREATE TABLE " CATALOG_DATABASE_NAME + "." CATALOG_SCHEMA_NAME "." SETTINGS_CATALOG_NAME + " (" + "name VARCHAR NOT NULL, " + "value VARCHAR NOT NULL, " + "value_type VARCHAR NOT NULL, " + "description VARCHAR, " + "min_value VARCHAR, " + "max_value VARCHAR, " + "default_value VARCHAR NOT NULL, " + "is_mutable BOOL NOT NULL, " + "is_persistent BOOL NOT NULL);") { // Add secondary index here if necessary - Catalog::GetInstance()->CreateIndex( - CATALOG_DATABASE_NAME, CATALOG_SCHEMA_NAME, SETTINGS_CATALOG_NAME, {0}, - SETTINGS_CATALOG_NAME "_skey0", false, IndexType::BWTREE, txn); + Catalog::GetInstance()->CreateIndex(txn, + CATALOG_DATABASE_NAME, + CATALOG_SCHEMA_NAME, + SETTINGS_CATALOG_NAME, + SETTINGS_CATALOG_NAME "_skey0", + {0}, + false, + IndexType::BWTREE); } SettingsCatalog::~SettingsCatalog() {} -bool SettingsCatalog::InsertSetting( - const std::string &name, const std::string &value, type::TypeId value_type, - const std::string &description, const std::string &min_value, - const std::string &max_value, const std::string &default_value, - bool is_mutable, bool is_persistent, type::AbstractPool *pool, - concurrency::TransactionContext *txn) { +bool SettingsCatalog::InsertSetting(concurrency::TransactionContext *txn, + const std::string &name, + const std::string &value, + type::TypeId value_type, + const std::string &description, + const std::string &min_value, + const std::string &max_value, + const std::string &default_value, + bool is_mutable, + bool is_persistent, + type::AbstractPool *pool) { // Create the tuple first std::unique_ptr tuple( new storage::Tuple(catalog_table_->GetSchema(), true)); @@ -81,27 +90,30 @@ bool SettingsCatalog::InsertSetting( tuple->SetValue(static_cast(ColumnId::IS_PERSISTENT), val8, pool); // Insert the tuple - return InsertTuple(std::move(tuple), txn); + return InsertTuple(txn, std::move(tuple)); } -bool SettingsCatalog::DeleteSetting(const std::string &name, - concurrency::TransactionContext *txn) { +bool SettingsCatalog::DeleteSetting(concurrency::TransactionContext *txn, + const std::string &name) { oid_t index_offset = 0; std::vector values; values.push_back(type::ValueFactory::GetVarcharValue(name, nullptr).Copy()); - return DeleteWithIndexScan(index_offset, values, txn); + return DeleteWithIndexScan(txn, index_offset, values); } -std::string SettingsCatalog::GetSettingValue( - const std::string &name, concurrency::TransactionContext *txn) { +std::string SettingsCatalog::GetSettingValue(concurrency::TransactionContext *txn, + const std::string &name) { std::vector column_ids({static_cast(ColumnId::VALUE)}); oid_t index_offset = static_cast(IndexId::SECONDARY_KEY_0); std::vector values; values.push_back(type::ValueFactory::GetVarcharValue(name, nullptr).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); std::string config_value = ""; PELOTON_ASSERT(result_tiles->size() <= 1); @@ -114,15 +126,18 @@ std::string SettingsCatalog::GetSettingValue( return config_value; } -std::string SettingsCatalog::GetDefaultValue( - const std::string &name, concurrency::TransactionContext *txn) { +std::string SettingsCatalog::GetDefaultValue(concurrency::TransactionContext *txn, + const std::string &name) { std::vector column_ids({static_cast(ColumnId::VALUE)}); oid_t index_offset = static_cast(IndexId::SECONDARY_KEY_0); std::vector values; values.push_back(type::ValueFactory::GetVarcharValue(name, nullptr).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); std::string config_value = ""; PELOTON_ASSERT(result_tiles->size() <= 1); diff --git a/src/catalog/system_catalogs.cpp b/src/catalog/system_catalogs.cpp index 611f6562c42..7e92755c001 100644 --- a/src/catalog/system_catalogs.cpp +++ b/src/catalog/system_catalogs.cpp @@ -27,19 +27,19 @@ namespace catalog { * @param database the database which the catalog tables belongs to * @param txn TransactionContext */ -SystemCatalogs::SystemCatalogs(storage::Database *database, - type::AbstractPool *pool, - concurrency::TransactionContext *txn) +SystemCatalogs::SystemCatalogs(concurrency::TransactionContext *txn, + storage::Database *database, + type::AbstractPool *pool) : pg_trigger_(nullptr), pg_table_metrics_(nullptr), pg_index_metrics_(nullptr), pg_query_metrics_(nullptr) { oid_t database_oid = database->GetOid(); - pg_attribute_ = new ColumnCatalog(database, pool, txn); - pg_namespace_ = new SchemaCatalog(database, pool, txn); - pg_table_ = new TableCatalog(database, pool, txn); - pg_index_ = new IndexCatalog(database, pool, txn); - pg_layout_ = new LayoutCatalog(database, pool, txn); + pg_attribute_ = new ColumnCatalog(txn, database, pool); + pg_namespace_ = new SchemaCatalog(txn, database, pool); + pg_table_ = new TableCatalog(txn, database, pool); + pg_index_ = new IndexCatalog(txn, database, pool); + pg_layout_ = new LayoutCatalog(txn, database, pool); // TODO: can we move this to BootstrapSystemCatalogs()? // insert column information into pg_attribute @@ -57,11 +57,16 @@ SystemCatalogs::SystemCatalogs(storage::Database *database, ->GetTableWithOid(shared_tables[i].first, shared_tables[i].second) ->GetSchema() ->GetColumns()) { - pg_attribute_->InsertColumn(shared_tables[i].second, column.GetName(), - column_id, column.GetOffset(), - column.GetType(), column.GetLength(), - column.IsInlined(), column.GetConstraints(), - pool, txn); + pg_attribute_->InsertColumn(txn, + shared_tables[i].second, + column_id, + column.GetName(), + column.GetOffset(), + column.GetType(), + column.GetLength(), + column.GetConstraints(), + column.IsInlined(), + pool); column_id++; } } @@ -84,12 +89,12 @@ SystemCatalogs::~SystemCatalogs() { * @param database_name the database which the namespace belongs to * @param txn TransactionContext */ -void SystemCatalogs::Bootstrap(const std::string &database_name, - concurrency::TransactionContext *txn) { +void SystemCatalogs::Bootstrap(concurrency::TransactionContext *txn, + const std::string &database_name) { LOG_DEBUG("Bootstrapping database: %s", database_name.c_str()); if (!pg_trigger_) { - pg_trigger_ = new TriggerCatalog(database_name, txn); + pg_trigger_ = new TriggerCatalog(txn, database_name); } // if (!pg_proc) { @@ -97,15 +102,15 @@ void SystemCatalogs::Bootstrap(const std::string &database_name, // } if (!pg_table_metrics_) { - pg_table_metrics_ = new TableMetricsCatalog(database_name, txn); + pg_table_metrics_ = new TableMetricsCatalog(txn, database_name); } if (!pg_index_metrics_) { - pg_index_metrics_ = new IndexMetricsCatalog(database_name, txn); + pg_index_metrics_ = new IndexMetricsCatalog(txn, database_name); } if (!pg_query_metrics_) { - pg_query_metrics_ = new QueryMetricsCatalog(database_name, txn); + pg_query_metrics_ = new QueryMetricsCatalog(txn, database_name); } // Reset oid of each catalog to avoid collisions between catalog diff --git a/src/catalog/table_catalog.cpp b/src/catalog/table_catalog.cpp index 478bb5f2d79..8031fdd961e 100644 --- a/src/catalog/table_catalog.cpp +++ b/src/catalog/table_catalog.cpp @@ -28,8 +28,8 @@ namespace peloton { namespace catalog { -TableCatalogObject::TableCatalogObject(executor::LogicalTile *tile, - concurrency::TransactionContext *txn, +TableCatalogEntry::TableCatalogEntry(concurrency::TransactionContext *txn, + executor::LogicalTile *tile, int tupleId) : table_oid(tile->GetValue(tupleId, TableCatalog::ColumnId::TABLE_OID) .GetAs()), @@ -42,43 +42,43 @@ TableCatalogObject::TableCatalogObject(executor::LogicalTile *tile, version_id(tile->GetValue(tupleId, TableCatalog::ColumnId::VERSION_ID) .GetAs()), default_layout_oid(tile->GetValue(tupleId, - TableCatalog::ColumnId::DEFAULT_LAYOUT_OID).GetAs()), - index_objects(), - index_names(), - valid_index_objects(false), - column_objects(), - column_names(), - valid_column_objects(false), - valid_layout_objects_(false), - txn(txn) {} + TableCatalog::ColumnId::DEFAULT_LAYOUT_OID).GetAs()), + index_catalog_entries(), + index_catalog_entries_by_name_(), + valid_index_catalog_entries_(false), + column_catalog_entries_(), + column_names_(), + valid_column_catalog_entries_(false), + valid_layout_catalog_entries_(false), + txn_(txn) {} /* @brief insert index catalog object into cache * @param index_object * @return false if index_name already exists in cache */ -bool TableCatalogObject::InsertIndexObject( - std::shared_ptr index_object) { - if (!index_object || index_object->GetIndexOid() == INVALID_OID) { +bool TableCatalogEntry::InsertIndexCatalogEntry( + std::shared_ptr index_catalog_entry) { + if (!index_catalog_entry || index_catalog_entry->GetIndexOid() == INVALID_OID) { return false; // invalid object } // check if already in cache - if (index_objects.find(index_object->GetIndexOid()) != index_objects.end()) { - LOG_DEBUG("Index %u already exists in cache!", index_object->GetIndexOid()); + if (index_catalog_entries.find(index_catalog_entry->GetIndexOid()) != index_catalog_entries.end()) { + LOG_DEBUG("Index %u already exists in cache!", index_catalog_entry->GetIndexOid()); return false; } - if (index_names.find(index_object->GetIndexName()) != index_names.end()) { + if (index_catalog_entries_by_name_.find(index_catalog_entry->GetIndexName()) != index_catalog_entries_by_name_.end()) { LOG_DEBUG("Index %s already exists in cache!", - index_object->GetIndexName().c_str()); + index_catalog_entry->GetIndexName().c_str()); return false; } - valid_index_objects = true; - index_objects.insert( - std::make_pair(index_object->GetIndexOid(), index_object)); - index_names.insert( - std::make_pair(index_object->GetIndexName(), index_object)); + valid_index_catalog_entries_ = true; + index_catalog_entries.insert( + std::make_pair(index_catalog_entry->GetIndexOid(), index_catalog_entry)); + index_catalog_entries_by_name_.insert( + std::make_pair(index_catalog_entry->GetIndexName(), index_catalog_entry)); return true; } @@ -86,19 +86,19 @@ bool TableCatalogObject::InsertIndexObject( * @param index_oid * @return true if index_oid is found and evicted; false if not found */ -bool TableCatalogObject::EvictIndexObject(oid_t index_oid) { - if (!valid_index_objects) return false; +bool TableCatalogEntry::EvictIndexCatalogEntry(oid_t index_oid) { + if (!valid_index_catalog_entries_) return false; // find index name from index name cache - auto it = index_objects.find(index_oid); - if (it == index_objects.end()) { + auto it = index_catalog_entries.find(index_oid); + if (it == index_catalog_entries.end()) { return false; // index oid not found in cache } auto index_object = it->second; PELOTON_ASSERT(index_object); - index_objects.erase(it); - index_names.erase(index_object->GetIndexName()); + index_catalog_entries.erase(it); + index_catalog_entries_by_name_.erase(index_object->GetIndexName()); return true; } @@ -106,44 +106,44 @@ bool TableCatalogObject::EvictIndexObject(oid_t index_oid) { * @param index_name * @return true if index_name is found and evicted; false if not found */ -bool TableCatalogObject::EvictIndexObject(const std::string &index_name) { - if (!valid_index_objects) return false; +bool TableCatalogEntry::EvictIndexCatalogEntry(const std::string &index_name) { + if (!valid_index_catalog_entries_) return false; // find index name from index name cache - auto it = index_names.find(index_name); - if (it == index_names.end()) { + auto it = index_catalog_entries_by_name_.find(index_name); + if (it == index_catalog_entries_by_name_.end()) { return false; // index name not found in cache } auto index_object = it->second; PELOTON_ASSERT(index_object); - index_names.erase(it); - index_objects.erase(index_object->GetIndexOid()); + index_catalog_entries_by_name_.erase(it); + index_catalog_entries.erase(index_object->GetIndexOid()); return true; } /* @brief evict all index catalog objects from cache */ -void TableCatalogObject::EvictAllIndexObjects() { - index_objects.clear(); - index_names.clear(); - valid_index_objects = false; +void TableCatalogEntry::EvictAllIndexCatalogEntries() { + index_catalog_entries.clear(); + index_catalog_entries_by_name_.clear(); + valid_index_catalog_entries_ = false; } /* @brief get all index objects of this table into cache * @return map from index oid to cached index object */ -std::unordered_map> -TableCatalogObject::GetIndexObjects(bool cached_only) { - if (!valid_index_objects && !cached_only) { +std::unordered_map> +TableCatalogEntry::GetIndexCatalogEntries(bool cached_only) { + if (!valid_index_catalog_entries_ && !cached_only) { // get index catalog objects from pg_index - valid_index_objects = true; + valid_index_catalog_entries_ = true; auto pg_index = Catalog::GetInstance() ->GetSystemCatalogs(database_oid) ->GetIndexCatalog(); - index_objects = pg_index->GetIndexObjects(table_oid, txn); + index_catalog_entries = pg_index->GetIndexCatalogEntries(txn_, table_oid); } - return index_objects; + return index_catalog_entries; } /* @brief get index object with index oid from cache @@ -151,11 +151,11 @@ TableCatalogObject::GetIndexObjects(bool cached_only) { * @param cached_only if cached only, return nullptr on a cache miss * @return shared pointer to the cached index object, nullptr if not found */ -std::shared_ptr TableCatalogObject::GetIndexObject( +std::shared_ptr TableCatalogEntry::GetIndexCatalogEntries( oid_t index_oid, bool cached_only) { - GetIndexObjects(cached_only); // fetch index objects in case we have not - auto it = index_objects.find(index_oid); - if (it != index_objects.end()) { + GetIndexCatalogEntries(cached_only); // fetch index objects in case we have not + auto it = index_catalog_entries.find(index_oid); + if (it != index_catalog_entries.end()) { return it->second; } return nullptr; @@ -166,11 +166,11 @@ std::shared_ptr TableCatalogObject::GetIndexObject( * @param cached_only if cached only, return nullptr on a cache miss * @return shared pointer to the cached index object, nullptr if not found */ -std::shared_ptr TableCatalogObject::GetIndexObject( +std::shared_ptr TableCatalogEntry::GetIndexCatalogEntry( const std::string &index_name, bool cached_only) { - GetIndexObjects(cached_only); // fetch index objects in case we have not - auto it = index_names.find(index_name); - if (it != index_names.end()) { + GetIndexCatalogEntries(cached_only); // fetch index objects in case we have not + auto it = index_catalog_entries_by_name_.find(index_name); + if (it != index_catalog_entries_by_name_.end()) { return it->second; } return nullptr; @@ -180,31 +180,31 @@ std::shared_ptr TableCatalogObject::GetIndexObject( * @param column_object * @return false if column_name already exists in cache */ -bool TableCatalogObject::InsertColumnObject( - std::shared_ptr column_object) { - if (!column_object || column_object->GetTableOid() == INVALID_OID) { +bool TableCatalogEntry::InsertColumnCatalogEntry( + std::shared_ptr column_catalog_entry) { + if (!column_catalog_entry || column_catalog_entry->GetTableOid() == INVALID_OID) { return false; // invalid object } // check if already in cache - if (column_objects.find(column_object->GetColumnId()) != - column_objects.end()) { + if (column_catalog_entries_.find(column_catalog_entry->GetColumnId()) != + column_catalog_entries_.end()) { LOG_DEBUG("Column %u already exists in cache!", - column_object->GetColumnId()); + column_catalog_entry->GetColumnId()); return false; } - if (column_names.find(column_object->GetColumnName()) != column_names.end()) { + if (column_names_.find(column_catalog_entry->GetColumnName()) != column_names_.end()) { LOG_DEBUG("Column %s already exists in cache!", - column_object->GetColumnName().c_str()); + column_catalog_entry->GetColumnName().c_str()); return false; } - valid_column_objects = true; - column_objects.insert( - std::make_pair(column_object->GetColumnId(), column_object)); - column_names.insert( - std::make_pair(column_object->GetColumnName(), column_object)); + valid_column_catalog_entries_ = true; + column_catalog_entries_.insert( + std::make_pair(column_catalog_entry->GetColumnId(), column_catalog_entry)); + column_names_.insert( + std::make_pair(column_catalog_entry->GetColumnName(), column_catalog_entry)); return true; } @@ -212,19 +212,19 @@ bool TableCatalogObject::InsertColumnObject( * @param column_id * @return true if column_id is found and evicted; false if not found */ -bool TableCatalogObject::EvictColumnObject(oid_t column_id) { - if (!valid_column_objects) return false; +bool TableCatalogEntry::EvictColumnCatalogEntry(oid_t column_id) { + if (!valid_column_catalog_entries_) return false; // find column name from column name cache - auto it = column_objects.find(column_id); - if (it == column_objects.end()) { + auto it = column_catalog_entries_.find(column_id); + if (it == column_catalog_entries_.end()) { return false; // column id not found in cache } auto column_object = it->second; PELOTON_ASSERT(column_object); - column_objects.erase(it); - column_names.erase(column_object->GetColumnName()); + column_catalog_entries_.erase(it); + column_names_.erase(column_object->GetColumnName()); return true; } @@ -232,62 +232,62 @@ bool TableCatalogObject::EvictColumnObject(oid_t column_id) { * @param column_name * @return true if column_name is found and evicted; false if not found */ -bool TableCatalogObject::EvictColumnObject(const std::string &column_name) { - if (!valid_column_objects) return false; +bool TableCatalogEntry::EvictColumnCatalogEntry(const std::string &column_name) { + if (!valid_column_catalog_entries_) return false; // find column name from column name cache - auto it = column_names.find(column_name); - if (it == column_names.end()) { + auto it = column_names_.find(column_name); + if (it == column_names_.end()) { return false; // column name not found in cache } auto column_object = it->second; PELOTON_ASSERT(column_object); - column_names.erase(it); - column_objects.erase(column_object->GetColumnId()); + column_names_.erase(it); + column_catalog_entries_.erase(column_object->GetColumnId()); return true; } /* @brief evict all column catalog objects from cache * @return true if column_name is found and evicted; false if not found */ -void TableCatalogObject::EvictAllColumnObjects() { - column_objects.clear(); - column_names.clear(); - valid_column_objects = false; +void TableCatalogEntry::EvictAllColumnCatalogEntries() { + column_catalog_entries_.clear(); + column_names_.clear(); + valid_column_catalog_entries_ = false; } /* @brief get all column objects of this table into cache * @return map from column id to cached column object */ -std::unordered_map> -TableCatalogObject::GetColumnObjects(bool cached_only) { - if (!valid_column_objects && !cached_only) { +std::unordered_map> +TableCatalogEntry::GetColumnCatalogEntries(bool cached_only) { + if (!valid_column_catalog_entries_ && !cached_only) { // get column catalog objects from pg_column auto pg_attribute = Catalog::GetInstance() ->GetSystemCatalogs(database_oid) ->GetColumnCatalog(); - pg_attribute->GetColumnObjects(table_oid, txn); - valid_column_objects = true; + pg_attribute->GetColumnCatalogEntries(txn_, table_oid); + valid_column_catalog_entries_ = true; } - return column_objects; + return column_catalog_entries_; } /* @brief get all column objects of this table into cache * @return map from column name to cached column object */ -std::unordered_map> -TableCatalogObject::GetColumnNames(bool cached_only) { - if (!valid_column_objects && !cached_only) { - auto column_objects = GetColumnObjects(); - std::unordered_map> +std::unordered_map> +TableCatalogEntry::GetColumnCatalogEntriesByName(bool cached_only) { + if (!valid_column_catalog_entries_ && !cached_only) { + auto column_objects = GetColumnCatalogEntries(); + std::unordered_map> column_names; for (auto &pair : column_objects) { auto column = pair.second; column_names[column->GetColumnName()] = column; } } - return column_names; + return column_names_; } /* @brief get column object with column id from cache @@ -295,11 +295,11 @@ TableCatalogObject::GetColumnNames(bool cached_only) { * @param cached_only if cached only, return nullptr on a cache miss * @return shared pointer to the cached column object, nullptr if not found */ -std::shared_ptr TableCatalogObject::GetColumnObject( +std::shared_ptr TableCatalogEntry::GetColumnCatalogEntry( oid_t column_id, bool cached_only) { - GetColumnObjects(cached_only); // fetch column objects in case we have not - auto it = column_objects.find(column_id); - if (it != column_objects.end()) { + GetColumnCatalogEntries(cached_only); // fetch column objects in case we have not + auto it = column_catalog_entries_.find(column_id); + if (it != column_catalog_entries_.end()) { return it->second; } return nullptr; @@ -310,27 +310,35 @@ std::shared_ptr TableCatalogObject::GetColumnObject( * @param cached_only if cached only, return nullptr on a cache miss * @return shared pointer to the cached column object, nullptr if not found */ -std::shared_ptr TableCatalogObject::GetColumnObject( +std::shared_ptr TableCatalogEntry::GetColumnCatalogEntry( const std::string &column_name, bool cached_only) { - GetColumnObjects(cached_only); // fetch column objects in case we have not - auto it = column_names.find(column_name); - if (it != column_names.end()) { + GetColumnCatalogEntries(cached_only); // fetch column objects in case we have not + auto it = column_names_.find(column_name); + if (it != column_names_.end()) { return it->second; } return nullptr; } -TableCatalog::TableCatalog( - storage::Database *database, UNUSED_ATTRIBUTE type::AbstractPool *pool, - UNUSED_ATTRIBUTE concurrency::TransactionContext *txn) - : AbstractCatalog(TABLE_CATALOG_OID, TABLE_CATALOG_NAME, - InitializeSchema().release(), database) { +TableCatalog::TableCatalog(concurrency::TransactionContext *, + storage::Database *database, + type::AbstractPool *) + : AbstractCatalog(database, + InitializeSchema().release(), + TABLE_CATALOG_OID, + TABLE_CATALOG_NAME) { // Add indexes for pg_namespace - AddIndex({0}, TABLE_CATALOG_PKEY_OID, TABLE_CATALOG_NAME "_pkey", + AddIndex(TABLE_CATALOG_NAME "_pkey", + TABLE_CATALOG_PKEY_OID, + {0}, IndexConstraintType::PRIMARY_KEY); - AddIndex({1, 2}, TABLE_CATALOG_SKEY0_OID, TABLE_CATALOG_NAME "_skey0", + AddIndex(TABLE_CATALOG_NAME "_skey0", + TABLE_CATALOG_SKEY0_OID, + {1, 2}, IndexConstraintType::UNIQUE); - AddIndex({3}, TABLE_CATALOG_SKEY1_OID, TABLE_CATALOG_NAME "_skey1", + AddIndex(TABLE_CATALOG_NAME "_skey1", + TABLE_CATALOG_SKEY1_OID, + {3}, IndexConstraintType::DEFAULT); } @@ -338,7 +346,7 @@ TableCatalog::TableCatalog( * @param layout Layout object to be inserted * @return false if layout already exists in cache */ -bool TableCatalogObject::InsertLayout( +bool TableCatalogEntry::InsertLayout( std::shared_ptr layout) { // Invalid object if (!layout || (layout->GetOid() == INVALID_OID)) { @@ -347,19 +355,19 @@ bool TableCatalogObject::InsertLayout( oid_t layout_id = layout->GetOid(); // layout is already present in the cache. - if (layout_objects_.find(layout_id) != layout_objects_.end()) { + if (layout_catalog_entries_.find(layout_id) != layout_catalog_entries_.end()) { LOG_DEBUG("Layout %u already exists in cache!", layout_id); return false; } - layout_objects_.insert(std::make_pair(layout_id, layout)); + layout_catalog_entries_.insert(std::make_pair(layout_id, layout)); return true; } /** @brief evict all layout objects from cache. */ -void TableCatalogObject::EvictAllLayouts() { - layout_objects_.clear(); - valid_layout_objects_ = false; +void TableCatalogEntry::EvictAllLayouts() { + layout_catalog_entries_.clear(); + valid_layout_catalog_entries_ = false; } /** @brief Get all layout objects of this table. @@ -368,16 +376,16 @@ void TableCatalogObject::EvictAllLayouts() { * @return Map from layout_oid to cached layout object. */ std::unordered_map> -TableCatalogObject::GetLayouts(bool cached_only) { - if (!valid_layout_objects_ && !cached_only) { +TableCatalogEntry::GetLayouts(bool cached_only) { + if (!valid_layout_catalog_entries_ && !cached_only) { // get layout catalog objects from pg_layout auto pg_layout = Catalog::GetInstance() ->GetSystemCatalogs(database_oid) ->GetLayoutCatalog(); - pg_layout->GetLayouts(table_oid, txn); - valid_column_objects = true; + pg_layout->GetLayouts(txn_, table_oid); + valid_column_catalog_entries_ = true; } - return layout_objects_; + return layout_catalog_entries_; } /** @brief Get the layout object of the given layout_id. @@ -385,12 +393,12 @@ TableCatalogObject::GetLayouts(bool cached_only) { * @param cached_only If set to true, don't fetch the layout objects. * @return Layout object of corresponding to the layout_id if present. */ -std::shared_ptr TableCatalogObject::GetLayout( +std::shared_ptr TableCatalogEntry::GetLayout( oid_t layout_id, bool cached_entry) { // fetch layout objects in case we have not GetLayouts(cached_entry); - auto it = layout_objects_.find(layout_id); - if (it != layout_objects_.end()) { + auto it = layout_catalog_entries_.find(layout_id); + if (it != layout_catalog_entries_.end()) { return it->second; } return nullptr; @@ -400,18 +408,18 @@ std::shared_ptr TableCatalogObject::GetLayout( * @param layout_id Id of the layout to be deleted. * @return true if layout_id is found and evicted; false if not found. */ -bool TableCatalogObject::EvictLayout(oid_t layout_id) { - if (!valid_layout_objects_) return false; +bool TableCatalogEntry::EvictLayout(oid_t layout_id) { + if (!valid_layout_catalog_entries_) return false; // find layout from the cache - auto it = layout_objects_.find(layout_id); - if (it == layout_objects_.end()) { + auto it = layout_catalog_entries_.find(layout_id); + if (it == layout_catalog_entries_.end()) { return false; // layout_id not found in cache } auto layout = it->second; PELOTON_ASSERT(layout); - layout_objects_.erase(it); + layout_catalog_entries_.erase(it); return true; } @@ -432,13 +440,13 @@ std::unique_ptr TableCatalog::InitializeSchema() { table_id_column.AddConstraint( catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); - auto table_name_column = catalog::Column(type::TypeId::VARCHAR, max_name_size, + auto table_name_column = catalog::Column(type::TypeId::VARCHAR, max_name_size_, "table_name", false); table_name_column.AddConstraint( catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); auto schema_name_column = catalog::Column( - type::TypeId::VARCHAR, max_name_size, "schema_name", false); + type::TypeId::VARCHAR, max_name_size_, "schema_name", false); schema_name_column.AddConstraint( catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); @@ -474,11 +482,13 @@ std::unique_ptr TableCatalog::InitializeSchema() { * @param txn TransactionContext * @return Whether insertion is Successful */ -bool TableCatalog::InsertTable(oid_t table_oid, const std::string &table_name, +bool TableCatalog::InsertTable(concurrency::TransactionContext *txn, + oid_t database_oid, const std::string &schema_name, - oid_t database_oid, oid_t layout_oid, - type::AbstractPool *pool, - concurrency::TransactionContext *txn) { + oid_t table_oid, + const std::string &table_name, + oid_t layout_oid, + type::AbstractPool *pool) { // Create the tuple first std::unique_ptr tuple( new storage::Tuple(catalog_table_->GetSchema(), true)); @@ -498,7 +508,7 @@ bool TableCatalog::InsertTable(oid_t table_oid, const std::string &table_name, tuple->SetValue(TableCatalog::ColumnId::DEFAULT_LAYOUT_OID, val5, pool); // Insert the tuple - return InsertTuple(std::move(tuple), txn); + return InsertTuple(txn, std::move(tuple)); } /*@brief delete a tuple about table info from pg_table(using index scan) @@ -506,22 +516,24 @@ bool TableCatalog::InsertTable(oid_t table_oid, const std::string &table_name, * @param txn TransactionContext * @return Whether deletion is successful */ -bool TableCatalog::DeleteTable(oid_t table_oid, - concurrency::TransactionContext *txn) { +bool TableCatalog::DeleteTable(concurrency::TransactionContext *txn, oid_t table_oid) { oid_t index_offset = IndexId::PRIMARY_KEY; // Index of table_oid std::vector values; values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); // evict from cache - auto table_object = txn->catalog_cache.GetCachedTableObject(database_oid, - table_oid); + auto table_object = txn->catalog_cache.GetCachedTableObject(database_oid_, + table_oid); if (table_object) { auto database_object = - DatabaseCatalog::GetInstance()->GetDatabaseObject(database_oid, txn); - database_object->EvictTableObject(table_oid); + DatabaseCatalog::GetInstance(nullptr, + nullptr, + nullptr)->GetDatabaseCatalogEntry(txn, + database_oid_); + database_object->EvictTableCatalogEntry(table_oid); } - return DeleteWithIndexScan(index_offset, values, txn); + return DeleteWithIndexScan(txn, index_offset, values); } /*@brief read table catalog object from pg_table using table oid @@ -529,33 +541,40 @@ bool TableCatalog::DeleteTable(oid_t table_oid, * @param txn TransactionContext * @return table catalog object */ -std::shared_ptr TableCatalog::GetTableObject( - oid_t table_oid, concurrency::TransactionContext *txn) { +std::shared_ptr TableCatalog::GetTableCatalogEntry( + concurrency::TransactionContext *txn, + oid_t table_oid) { if (txn == nullptr) { throw CatalogException("Transaction is invalid!"); } // try get from cache - auto table_object = txn->catalog_cache.GetCachedTableObject(database_oid, - table_oid); + auto table_object = txn->catalog_cache.GetCachedTableObject(database_oid_, + table_oid); if (table_object) return table_object; // cache miss, get from pg_table - std::vector column_ids(all_column_ids); + std::vector column_ids(all_column_ids_); oid_t index_offset = IndexId::PRIMARY_KEY; // Index of table_oid std::vector values; values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); if (result_tiles->size() == 1 && (*result_tiles)[0]->GetTupleCount() == 1) { auto table_object = - std::make_shared((*result_tiles)[0].get(), txn); + std::make_shared(txn, (*result_tiles)[0].get()); // insert into cache auto database_object = - DatabaseCatalog::GetInstance()->GetDatabaseObject(database_oid, txn); + DatabaseCatalog::GetInstance(nullptr, + nullptr, + nullptr)->GetDatabaseCatalogEntry(txn, + database_oid_); PELOTON_ASSERT(database_object); - bool success = database_object->InsertTableObject(table_object); + bool success = database_object->InsertTableCatalogEntry(table_object); PELOTON_ASSERT(success == true); (void)success; return table_object; @@ -575,22 +594,23 @@ std::shared_ptr TableCatalog::GetTableObject( * @param txn TransactionContext * @return table catalog object */ -std::shared_ptr TableCatalog::GetTableObject( - const std::string &table_name, const std::string &schema_name, - concurrency::TransactionContext *txn) { +std::shared_ptr TableCatalog::GetTableCatalogEntry( + concurrency::TransactionContext *txn, + const std::string &schema_name, + const std::string &table_name) { if (txn == nullptr) { throw CatalogException("Transaction is invalid!"); } // try get from cache - auto database_object = txn->catalog_cache.GetDatabaseObject(database_oid); + auto database_object = txn->catalog_cache.GetDatabaseObject(database_oid_); if (database_object) { auto table_object = - database_object->GetTableObject(table_name, schema_name, true); + database_object->GetTableCatalogEntry(table_name, schema_name, true); if (table_object) return table_object; } // cache miss, get from pg_table - std::vector column_ids(all_column_ids); + std::vector column_ids(all_column_ids_); oid_t index_offset = IndexId::SKEY_TABLE_NAME; // Index of table_name std::vector values; values.push_back( @@ -599,16 +619,22 @@ std::shared_ptr TableCatalog::GetTableObject( type::ValueFactory::GetVarcharValue(schema_name, nullptr).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); if (result_tiles->size() == 1 && (*result_tiles)[0]->GetTupleCount() == 1) { auto table_object = - std::make_shared((*result_tiles)[0].get(), txn); + std::make_shared(txn, (*result_tiles)[0].get()); // insert into cache auto database_object = - DatabaseCatalog::GetInstance()->GetDatabaseObject(database_oid, txn); + DatabaseCatalog::GetInstance(nullptr, + nullptr, + nullptr)->GetDatabaseCatalogEntry(txn, + database_oid_); PELOTON_ASSERT(database_object); - bool success = database_object->InsertTableObject(table_object); + bool success = database_object->InsertTableCatalogEntry(table_object); PELOTON_ASSERT(success == true); (void)success; return table_object; @@ -623,38 +649,44 @@ std::shared_ptr TableCatalog::GetTableObject( * @param txn TransactionContext * @return table catalog objects */ -std::unordered_map> -TableCatalog::GetTableObjects(concurrency::TransactionContext *txn) { +std::unordered_map> +TableCatalog::GetTableCatalogEntries(concurrency::TransactionContext *txn) { if (txn == nullptr) { throw CatalogException("Transaction is invalid!"); } // try get from cache auto database_object = - DatabaseCatalog::GetInstance()->GetDatabaseObject(database_oid, txn); + DatabaseCatalog::GetInstance(nullptr, + nullptr, + nullptr)->GetDatabaseCatalogEntry(txn, + database_oid_); PELOTON_ASSERT(database_object != nullptr); - if (database_object->IsValidTableObjects()) { - return database_object->GetTableObjects(true); + if (database_object->IsValidTableCatalogEntries()) { + return database_object->GetTableCatalogEntries(true); } // cache miss, get from pg_table - std::vector column_ids(all_column_ids); + std::vector column_ids(all_column_ids_); oid_t index_offset = IndexId::SKEY_DATABASE_OID; // Index of database_oid std::vector values; - values.push_back(type::ValueFactory::GetIntegerValue(database_oid).Copy()); + values.push_back(type::ValueFactory::GetIntegerValue(database_oid_).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); for (auto &tile : (*result_tiles)) { for (auto tuple_id : *tile) { auto table_object = - std::make_shared(tile.get(), txn, tuple_id); - database_object->InsertTableObject(table_object); + std::make_shared(txn, tile.get(), tuple_id); + database_object->InsertTableCatalogEntry(table_object); } } - database_object->SetValidTableObjects(true); - return database_object->GetTableObjects(); + database_object->SetValidTableCatalogEntries(true); + return database_object->GetTableCatalogEntries(); } /*@brief update version id column within pg_table @@ -663,8 +695,9 @@ TableCatalog::GetTableObjects(concurrency::TransactionContext *txn) { * @param txn TransactionContext * @return Whether update is successful */ -bool TableCatalog::UpdateVersionId(oid_t update_val, oid_t table_oid, - concurrency::TransactionContext *txn) { +bool TableCatalog::UpdateVersionId(concurrency::TransactionContext *txn, + oid_t table_oid, + oid_t update_val) { std::vector update_columns({ColumnId::VERSION_ID}); // version_id oid_t index_offset = IndexId::PRIMARY_KEY; // Index of table_oid // values to execute index scan @@ -676,16 +709,22 @@ bool TableCatalog::UpdateVersionId(oid_t update_val, oid_t table_oid, type::ValueFactory::GetIntegerValue(update_val).Copy()); // get table object, then evict table object - auto table_object = txn->catalog_cache.GetCachedTableObject(database_oid, - table_oid); + auto table_object = txn->catalog_cache.GetCachedTableObject(database_oid_, + table_oid); if (table_object) { auto database_object = - DatabaseCatalog::GetInstance()->GetDatabaseObject(database_oid, txn); - database_object->EvictTableObject(table_oid); - } - - return UpdateWithIndexScan(update_columns, update_values, scan_values, - index_offset, txn); + DatabaseCatalog::GetInstance(nullptr, + nullptr, + nullptr)->GetDatabaseCatalogEntry(txn, + database_oid_); + database_object->EvictTableCatalogEntry(table_oid); + } + + return UpdateWithIndexScan(txn, + index_offset, + scan_values, + update_columns, + update_values); } /*@brief update default layout oid column within pg_table @@ -694,8 +733,9 @@ bool TableCatalog::UpdateVersionId(oid_t update_val, oid_t table_oid, * @param txn TransactionContext * @return Whether update is successful */ -bool TableCatalog::UpdateDefaultLayoutOid(oid_t update_val, oid_t table_oid, - concurrency::TransactionContext *txn) { +bool TableCatalog::UpdateDefaultLayoutOid(concurrency::TransactionContext *txn, + oid_t table_oid, + oid_t update_val) { std::vector update_columns({ColumnId::DEFAULT_LAYOUT_OID}); // defalut_layout_oid oid_t index_offset = IndexId::PRIMARY_KEY; // Index of table_oid // values to execute index scan @@ -707,16 +747,22 @@ bool TableCatalog::UpdateDefaultLayoutOid(oid_t update_val, oid_t table_oid, type::ValueFactory::GetIntegerValue(update_val).Copy()); // get table object, then evict table object - auto table_object = txn->catalog_cache.GetCachedTableObject(database_oid, - table_oid); + auto table_object = txn->catalog_cache.GetCachedTableObject(database_oid_, + table_oid); if (table_object) { auto database_object = - DatabaseCatalog::GetInstance()->GetDatabaseObject(database_oid, txn); - database_object->EvictTableObject(table_oid); - } - - return UpdateWithIndexScan(update_columns, update_values, scan_values, - index_offset, txn); + DatabaseCatalog::GetInstance(nullptr, + nullptr, + nullptr)->GetDatabaseCatalogEntry(txn, + database_oid_); + database_object->EvictTableCatalogEntry(table_oid); + } + + return UpdateWithIndexScan(txn, + index_offset, + scan_values, + update_columns, + update_values); } diff --git a/src/catalog/table_metrics_catalog.cpp b/src/catalog/table_metrics_catalog.cpp index f1f91f46474..a7960b966ca 100644 --- a/src/catalog/table_metrics_catalog.cpp +++ b/src/catalog/table_metrics_catalog.cpp @@ -19,27 +19,30 @@ namespace peloton { namespace catalog { -TableMetricsCatalog::TableMetricsCatalog(const std::string &database_name, - concurrency::TransactionContext *txn) - : AbstractCatalog("CREATE TABLE " + database_name + - "." CATALOG_SCHEMA_NAME "." TABLE_METRICS_CATALOG_NAME - " (" - "table_oid INT NOT NULL, " - "reads INT NOT NULL, " - "updates INT NOT NULL, " - "deletes INT NOT NULL, " - "inserts INT NOT NULL, " - "time_stamp INT NOT NULL);", - txn) { +TableMetricsCatalog::TableMetricsCatalog(concurrency::TransactionContext *txn, + const std::string &database_name) + : AbstractCatalog(txn, "CREATE TABLE " + database_name + + "." CATALOG_SCHEMA_NAME "." TABLE_METRICS_CATALOG_NAME + " (" + "table_oid INT NOT NULL, " + "reads INT NOT NULL, " + "updates INT NOT NULL, " + "deletes INT NOT NULL, " + "inserts INT NOT NULL, " + "time_stamp INT NOT NULL);") { // Add secondary index here if necessary } TableMetricsCatalog::~TableMetricsCatalog() {} -bool TableMetricsCatalog::InsertTableMetrics( - oid_t table_oid, int64_t reads, int64_t updates, int64_t deletes, - int64_t inserts, int64_t time_stamp, type::AbstractPool *pool, - concurrency::TransactionContext *txn) { +bool TableMetricsCatalog::InsertTableMetrics(concurrency::TransactionContext *txn, + oid_t table_oid, + int64_t reads, + int64_t updates, + int64_t deletes, + int64_t inserts, + int64_t time_stamp, + type::AbstractPool *pool) { std::unique_ptr tuple( new storage::Tuple(catalog_table_->GetSchema(), true)); @@ -58,17 +61,16 @@ bool TableMetricsCatalog::InsertTableMetrics( tuple->SetValue(ColumnId::TIME_STAMP, val6, pool); // Insert the tuple - return InsertTuple(std::move(tuple), txn); + return InsertTuple(txn, std::move(tuple)); } -bool TableMetricsCatalog::DeleteTableMetrics( - oid_t table_oid, concurrency::TransactionContext *txn) { +bool TableMetricsCatalog::DeleteTableMetrics(concurrency::TransactionContext *txn, oid_t table_oid) { oid_t index_offset = IndexId::PRIMARY_KEY; // Primary key index std::vector values; values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); - return DeleteWithIndexScan(index_offset, values, txn); + return DeleteWithIndexScan(txn, index_offset, values); } } // namespace catalog diff --git a/src/catalog/trigger_catalog.cpp b/src/catalog/trigger_catalog.cpp index 1474dd20d4c..7c95b705f00 100644 --- a/src/catalog/trigger_catalog.cpp +++ b/src/catalog/trigger_catalog.cpp @@ -22,46 +22,61 @@ namespace peloton { namespace catalog { -TriggerCatalog::TriggerCatalog(const std::string &database_name, - concurrency::TransactionContext *txn) - : AbstractCatalog("CREATE TABLE " + database_name + - "." CATALOG_SCHEMA_NAME "." TRIGGER_CATALOG_NAME - " (" - "oid INT NOT NULL PRIMARY KEY, " - "tgrelid INT NOT NULL, " - "tgname VARCHAR NOT NULL, " - "tgfoid VARCHAR, " - "tgtype INT NOT NULL, " - "tgargs VARCHAR, " - "tgqual VARBINARY, " - "timestamp TIMESTAMP NOT NULL);", - txn) { +TriggerCatalog::TriggerCatalog(concurrency::TransactionContext *txn, + const std::string &database_name) + : AbstractCatalog(txn, "CREATE TABLE " + database_name + + "." CATALOG_SCHEMA_NAME "." TRIGGER_CATALOG_NAME + " (" + "oid INT NOT NULL PRIMARY KEY, " + "tgrelid INT NOT NULL, " + "tgname VARCHAR NOT NULL, " + "tgfoid VARCHAR, " + "tgtype INT NOT NULL, " + "tgargs VARCHAR, " + "tgqual VARBINARY, " + "timestamp TIMESTAMP NOT NULL);") { // Add secondary index here if necessary - Catalog::GetInstance()->CreateIndex( - database_name, CATALOG_SCHEMA_NAME, TRIGGER_CATALOG_NAME, - {ColumnId::TABLE_OID, ColumnId::TRIGGER_TYPE}, - TRIGGER_CATALOG_NAME "_skey0", false, IndexType::BWTREE, txn); - - Catalog::GetInstance()->CreateIndex( - database_name, CATALOG_SCHEMA_NAME, TRIGGER_CATALOG_NAME, - {ColumnId::TABLE_OID}, TRIGGER_CATALOG_NAME "_skey1", false, - IndexType::BWTREE, txn); - - Catalog::GetInstance()->CreateIndex( - database_name, CATALOG_SCHEMA_NAME, TRIGGER_CATALOG_NAME, - {ColumnId::TRIGGER_NAME, ColumnId::TABLE_OID}, - TRIGGER_CATALOG_NAME "_skey2", false, IndexType::BWTREE, txn); + Catalog::GetInstance()->CreateIndex(txn, + database_name, + CATALOG_SCHEMA_NAME, + TRIGGER_CATALOG_NAME, + TRIGGER_CATALOG_NAME "_skey0", + {ColumnId::TABLE_OID, + ColumnId::TRIGGER_TYPE}, + false, + IndexType::BWTREE); + + Catalog::GetInstance()->CreateIndex(txn, + database_name, + CATALOG_SCHEMA_NAME, + TRIGGER_CATALOG_NAME, + TRIGGER_CATALOG_NAME "_skey1", + {ColumnId::TABLE_OID}, + false, + IndexType::BWTREE); + + Catalog::GetInstance()->CreateIndex(txn, + database_name, + CATALOG_SCHEMA_NAME, + TRIGGER_CATALOG_NAME, + TRIGGER_CATALOG_NAME "_skey2", + {ColumnId::TRIGGER_NAME, + ColumnId::TABLE_OID}, + false, + IndexType::BWTREE); } TriggerCatalog::~TriggerCatalog() {} -bool TriggerCatalog::InsertTrigger(oid_t table_oid, std::string trigger_name, - int16_t trigger_type, std::string proc_oid, +bool TriggerCatalog::InsertTrigger(concurrency::TransactionContext *txn, + oid_t table_oid, + std::string trigger_name, + int16_t trigger_type, + std::string proc_oid, std::string function_arguments, type::Value fire_condition, type::Value timestamp, - type::AbstractPool *pool, - concurrency::TransactionContext *txn) { + type::AbstractPool *pool) { std::unique_ptr tuple( new storage::Tuple(catalog_table_->GetSchema(), true)); @@ -86,14 +101,14 @@ bool TriggerCatalog::InsertTrigger(oid_t table_oid, std::string trigger_name, tuple->SetValue(ColumnId::TIMESTAMP, val7, pool); // Insert the tuple - return InsertTuple(std::move(tuple), txn); + return InsertTuple(txn, std::move(tuple)); } -ResultType TriggerCatalog::DropTrigger(const oid_t database_oid, +ResultType TriggerCatalog::DropTrigger(concurrency::TransactionContext *txn, + const oid_t database_oid, const oid_t table_oid, - const std::string &trigger_name, - concurrency::TransactionContext *txn) { - bool delete_success = DeleteTriggerByName(trigger_name, table_oid, txn); + const std::string &trigger_name) { + bool delete_success = DeleteTriggerByName(txn, table_oid, trigger_name); if (delete_success) { LOG_TRACE("Delete trigger successfully"); // ask target table to update its trigger list variable @@ -107,8 +122,9 @@ ResultType TriggerCatalog::DropTrigger(const oid_t database_oid, return ResultType::FAILURE; } -oid_t TriggerCatalog::GetTriggerOid(std::string trigger_name, oid_t table_oid, - concurrency::TransactionContext *txn) { +oid_t TriggerCatalog::GetTriggerOid(concurrency::TransactionContext *txn, + oid_t table_oid, + std::string trigger_name) { std::vector column_ids({ColumnId::TRIGGER_OID}); oid_t index_offset = IndexId::NAME_TABLE_KEY_2; std::vector values; @@ -116,7 +132,10 @@ oid_t TriggerCatalog::GetTriggerOid(std::string trigger_name, oid_t table_oid, values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); oid_t trigger_oid = INVALID_OID; if (result_tiles->size() == 0) { @@ -131,20 +150,20 @@ oid_t TriggerCatalog::GetTriggerOid(std::string trigger_name, oid_t table_oid, return trigger_oid; } -bool TriggerCatalog::DeleteTriggerByName(const std::string &trigger_name, +bool TriggerCatalog::DeleteTriggerByName(concurrency::TransactionContext *txn, oid_t table_oid, - concurrency::TransactionContext *txn) { + const std::string &trigger_name) { oid_t index_offset = IndexId::NAME_TABLE_KEY_2; std::vector values; values.push_back(type::ValueFactory::GetVarcharValue(trigger_name).Copy()); values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); - return DeleteWithIndexScan(index_offset, values, txn); + return DeleteWithIndexScan(txn, index_offset, values); } -std::unique_ptr TriggerCatalog::GetTriggersByType( - oid_t table_oid, int16_t trigger_type, - concurrency::TransactionContext *txn) { +std::unique_ptr TriggerCatalog::GetTriggersByType(concurrency::TransactionContext *txn, + oid_t table_oid, + int16_t trigger_type) { LOG_INFO("Get triggers for table %d", table_oid); // select trigger_name, fire condition, function_name, function_args std::vector column_ids( @@ -157,7 +176,10 @@ std::unique_ptr TriggerCatalog::GetTriggersByType( // the result is a vector of executor::LogicalTile auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); // carefull! the result tile could be null! if (result_tiles == nullptr) { LOG_INFO("no trigger on table %d", table_oid); @@ -185,8 +207,8 @@ std::unique_ptr TriggerCatalog::GetTriggersByType( return new_trigger_list; } -std::unique_ptr TriggerCatalog::GetTriggers( - oid_t table_oid, concurrency::TransactionContext *txn) { +std::unique_ptr TriggerCatalog::GetTriggers(concurrency::TransactionContext *txn, + oid_t table_oid) { // LOG_DEBUG("Get triggers for table %d", table_oid); // select trigger_name, fire condition, function_name, function_args std::vector column_ids( @@ -200,7 +222,10 @@ std::unique_ptr TriggerCatalog::GetTriggers( // the result is a vector of executor::LogicalTile auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); // carefull! the result tile could be null! // if (result_tiles == nullptr) { // LOG_INFO("no trigger on table %d", table_oid); diff --git a/src/catalog/zone_map_catalog.cpp b/src/catalog/zone_map_catalog.cpp index ec59dd24d82..b38e73da8db 100644 --- a/src/catalog/zone_map_catalog.cpp +++ b/src/catalog/zone_map_catalog.cpp @@ -33,29 +33,37 @@ ZoneMapCatalog *ZoneMapCatalog::GetInstance( } ZoneMapCatalog::ZoneMapCatalog(concurrency::TransactionContext *txn) - : AbstractCatalog("CREATE TABLE " CATALOG_DATABASE_NAME - "." CATALOG_SCHEMA_NAME "." ZONE_MAP_CATALOG_NAME - " (" - "database_id INT NOT NULL, " - "table_id INT NOT NULL, " - "tile_group_id INT NOT NULL, " - "column_id INT NOT NULL, " - "minimum VARCHAR, " - "maximum VARCHAR, " - "type VARCHAR);", - txn) { - Catalog::GetInstance()->CreateIndex( - CATALOG_DATABASE_NAME, CATALOG_SCHEMA_NAME, ZONE_MAP_CATALOG_NAME, - {0, 1, 2, 3}, ZONE_MAP_CATALOG_NAME "_skey0", true, IndexType::BWTREE, - txn); + : AbstractCatalog(txn, "CREATE TABLE " CATALOG_DATABASE_NAME + "." CATALOG_SCHEMA_NAME "." ZONE_MAP_CATALOG_NAME + " (" + "database_id INT NOT NULL, " + "table_id INT NOT NULL, " + "tile_group_id INT NOT NULL, " + "column_id INT NOT NULL, " + "minimum VARCHAR, " + "maximum VARCHAR, " + "type VARCHAR);") { + Catalog::GetInstance()->CreateIndex(txn, + CATALOG_DATABASE_NAME, + CATALOG_SCHEMA_NAME, + ZONE_MAP_CATALOG_NAME, + ZONE_MAP_CATALOG_NAME "_skey0", + {0, 1, 2, 3}, + true, + IndexType::BWTREE); } ZoneMapCatalog::~ZoneMapCatalog() {} -bool ZoneMapCatalog::InsertColumnStatistics( - oid_t database_id, oid_t table_id, oid_t tile_group_id, oid_t column_id, - std::string minimum, std::string maximum, std::string type, - type::AbstractPool *pool, concurrency::TransactionContext *txn) { +bool ZoneMapCatalog::InsertColumnStatistics(concurrency::TransactionContext *txn, + oid_t database_id, + oid_t table_id, + oid_t tile_group_id, + oid_t column_id, + std::string minimum, + std::string maximum, + std::string type, + type::AbstractPool *pool) { std::unique_ptr tuple( new storage::Tuple(catalog_table_->GetSchema(), true)); @@ -77,25 +85,29 @@ bool ZoneMapCatalog::InsertColumnStatistics( tuple->SetValue(static_cast(ColumnId::MAXIMUM), val_maximum, pool); tuple->SetValue(static_cast(ColumnId::TYPE), val_type, pool); - bool return_val = InsertTuple(std::move(tuple), txn); + bool return_val = InsertTuple(txn, std::move(tuple)); return return_val; } -bool ZoneMapCatalog::DeleteColumnStatistics( - oid_t database_id, oid_t table_id, oid_t tile_group_id, oid_t column_id, - concurrency::TransactionContext *txn) { +bool ZoneMapCatalog::DeleteColumnStatistics(concurrency::TransactionContext *txn, + oid_t database_id, + oid_t table_id, + oid_t tile_group_id, + oid_t column_id) { oid_t index_offset = static_cast(IndexId::SECONDARY_KEY_0); std::vector values( {type::ValueFactory::GetIntegerValue(database_id), type::ValueFactory::GetIntegerValue(table_id), type::ValueFactory::GetIntegerValue(tile_group_id), type::ValueFactory::GetIntegerValue(column_id)}); - return DeleteWithIndexScan(index_offset, values, txn); + return DeleteWithIndexScan(txn, index_offset, values); } -std::unique_ptr> ZoneMapCatalog::GetColumnStatistics( - oid_t database_id, oid_t table_id, oid_t tile_group_id, oid_t column_id, - concurrency::TransactionContext *txn) { +std::unique_ptr> ZoneMapCatalog::GetColumnStatistics(concurrency::TransactionContext *txn, + oid_t database_id, + oid_t table_id, + oid_t tile_group_id, + oid_t column_id) { std::vector column_ids({static_cast(ColumnId::MINIMUM), static_cast(ColumnId::MAXIMUM), static_cast(ColumnId::TYPE)}); @@ -109,7 +121,10 @@ std::unique_ptr> ZoneMapCatalog::GetColumnStatistics( type::ValueFactory::GetIntegerValue(column_id)}); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(txn, + column_ids, + index_offset, + values); PELOTON_ASSERT(result_tiles->size() <= 1); // unique if (result_tiles->size() == 0) { diff --git a/src/common/init.cpp b/src/common/init.cpp index fdc085e6ce3..c8b87133211 100644 --- a/src/common/init.cpp +++ b/src/common/init.cpp @@ -87,7 +87,7 @@ void PelotonInit::Initialize() { // initialize the catalog and add the default database, so we don't do this on // the first query - pg_catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + pg_catalog->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); diff --git a/src/executor/create_executor.cpp b/src/executor/create_executor.cpp index 83e85c92c48..bbd3e773d43 100644 --- a/src/executor/create_executor.cpp +++ b/src/executor/create_executor.cpp @@ -94,7 +94,7 @@ bool CreateExecutor::CreateDatabase(const planner::CreatePlan &node) { auto database_name = node.GetDatabaseName(); // invoke logic within catalog.cpp ResultType result = - catalog::Catalog::GetInstance()->CreateDatabase(database_name, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, database_name); txn->SetResult(result); LOG_TRACE("Result is: %s", ResultTypeToString(txn->GetResult()).c_str()); return (true); @@ -105,8 +105,9 @@ bool CreateExecutor::CreateSchema(const planner::CreatePlan &node) { auto database_name = node.GetDatabaseName(); auto schema_name = node.GetSchemaName(); // invoke logic within catalog.cpp - ResultType result = catalog::Catalog::GetInstance()->CreateSchema( - database_name, schema_name, txn); + ResultType result = catalog::Catalog::GetInstance()->CreateSchema(txn, + database_name, + schema_name); txn->SetResult(result); LOG_TRACE("Result is: %s", ResultTypeToString(txn->GetResult()).c_str()); return (true); @@ -119,8 +120,12 @@ bool CreateExecutor::CreateTable(const planner::CreatePlan &node) { std::string database_name = node.GetDatabaseName(); std::unique_ptr schema(node.GetSchema()); - ResultType result = catalog::Catalog::GetInstance()->CreateTable( - database_name, schema_name, table_name, std::move(schema), current_txn); + ResultType result = catalog::Catalog::GetInstance()->CreateTable(current_txn, + database_name, + schema_name, + std::move(schema), + table_name, + false); current_txn->SetResult(result); if (current_txn->GetResult() == ResultType::SUCCESS) { @@ -130,12 +135,16 @@ bool CreateExecutor::CreateTable(const planner::CreatePlan &node) { if (node.GetForeignKeys().empty() == false) { int count = 1; auto catalog = catalog::Catalog::GetInstance(); - auto source_table = catalog->GetTableWithName(database_name, schema_name, - table_name, current_txn); + auto source_table = catalog->GetTableWithName(current_txn, + database_name, + schema_name, + table_name); for (auto fk : node.GetForeignKeys()) { - auto sink_table = catalog->GetTableWithName( - database_name, schema_name, fk.sink_table_name, current_txn); + auto sink_table = catalog->GetTableWithName(current_txn, + database_name, + schema_name, + fk.sink_table_name); // Source Column Offsets std::vector source_col_ids; for (auto col_name : fk.foreign_key_sources) { @@ -182,9 +191,14 @@ bool CreateExecutor::CreateTable(const planner::CreatePlan &node) { std::vector source_col_names = fk.foreign_key_sources; std::string index_name = table_name + "_FK_" + sink_table->GetName() + "_" + std::to_string(count); - catalog->CreateIndex(database_name, schema_name, table_name, - source_col_ids, index_name, false, - IndexType::BWTREE, current_txn); + catalog->CreateIndex(current_txn, + database_name, + schema_name, + table_name, + index_name, + source_col_ids, + false, + IndexType::BWTREE); count++; #ifdef LOG_DEBUG_ENABLED @@ -220,9 +234,14 @@ bool CreateExecutor::CreateIndex(const planner::CreatePlan &node) { auto key_attrs = node.GetKeyAttrs(); - ResultType result = catalog::Catalog::GetInstance()->CreateIndex( - database_name, schema_name, table_name, key_attrs, index_name, - unique_flag, index_type, txn); + ResultType result = catalog::Catalog::GetInstance()->CreateIndex(txn, + database_name, + schema_name, + table_name, + index_name, + key_attrs, + unique_flag, + index_type); txn->SetResult(result); if (txn->GetResult() == ResultType::SUCCESS) { @@ -243,8 +262,10 @@ bool CreateExecutor::CreateTrigger(const planner::CreatePlan &node) { std::string trigger_name = node.GetTriggerName(); trigger::Trigger newTrigger(node); - auto table_object = catalog::Catalog::GetInstance()->GetTableObject( - database_name, schema_name, table_name, txn); + auto table_object = catalog::Catalog::GetInstance()->GetTableCatalogEntry(txn, + database_name, + schema_name, + table_name); // durable trigger: insert the information of this trigger in the trigger // catalog table @@ -262,13 +283,21 @@ bool CreateExecutor::CreateTrigger(const planner::CreatePlan &node) { catalog::Catalog::GetInstance() ->GetSystemCatalogs(table_object->GetDatabaseOid()) ->GetTriggerCatalog() - ->InsertTrigger(table_object->GetTableOid(), trigger_name, - newTrigger.GetTriggerType(), newTrigger.GetFuncname(), - newTrigger.GetArgs(), when, time_stamp, pool_.get(), txn); + ->InsertTrigger(txn, + table_object->GetTableOid(), + trigger_name, + newTrigger.GetTriggerType(), + newTrigger.GetFuncname(), + newTrigger.GetArgs(), + when, + time_stamp, + pool_.get()); // ask target table to update its trigger list variable storage::DataTable *target_table = - catalog::Catalog::GetInstance()->GetTableWithName( - database_name, schema_name, table_name, txn); + catalog::Catalog::GetInstance()->GetTableWithName(txn, + database_name, + schema_name, + table_name); target_table->UpdateTriggerListFromCatalog(txn); // hardcode SUCCESS result for txn diff --git a/src/executor/create_function_executor.cpp b/src/executor/create_function_executor.cpp index 11ceafb0c17..cd85d9197d4 100644 --- a/src/executor/create_function_executor.cpp +++ b/src/executor/create_function_executor.cpp @@ -40,7 +40,7 @@ bool CreateFunctionExecutor::DExecute() { auto proname = node.GetFunctionName(); oid_t prolang = catalog::LanguageCatalog::GetInstance() - .GetLanguageByName("plpgsql", current_txn) + .GetLanguageByName(current_txn, "plpgsql") ->GetOid(); auto prorettype = node.GetReturnType(); auto proargtypes = node.GetFunctionParameterTypes(); @@ -63,9 +63,13 @@ bool CreateFunctionExecutor::DExecute() { auto func_ptr = code_context->GetUDF(); if (func_ptr != nullptr) { // Insert into catalog - catalog::Catalog::GetInstance()->AddPlpgsqlFunction( - proname, proargtypes, prorettype, prolang, prosrc, code_context, - current_txn); + catalog::Catalog::GetInstance()->AddProcedure(current_txn, + proname, + prorettype, + proargtypes, + prolang, + code_context, + prosrc); result = ResultType::SUCCESS; } else { result = ResultType::FAILURE; diff --git a/src/executor/drop_executor.cpp b/src/executor/drop_executor.cpp index cf4073ce9d2..66fcf3800d7 100644 --- a/src/executor/drop_executor.cpp +++ b/src/executor/drop_executor.cpp @@ -80,8 +80,9 @@ bool DropExecutor::DropDatabase(const planner::DropPlan &node, if (node.IsMissing()) { try { - auto database_object = catalog::Catalog::GetInstance()->GetDatabaseObject( - database_name, txn); + auto database_object = + catalog::Catalog::GetInstance()->GetDatabaseCatalogEntry(txn, + database_name); } catch (CatalogException &e) { LOG_TRACE("Database %s does not exist.", database_name.c_str()); return false; @@ -89,10 +90,11 @@ bool DropExecutor::DropDatabase(const planner::DropPlan &node, } auto database_object = - catalog::Catalog::GetInstance()->GetDatabaseObject(database_name, txn); + catalog::Catalog::GetInstance()->GetDatabaseCatalogEntry(txn, + database_name); ResultType result = - catalog::Catalog::GetInstance()->DropDatabaseWithName(database_name, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, database_name); txn->SetResult(result); if (txn->GetResult() == ResultType::SUCCESS) { @@ -100,7 +102,7 @@ bool DropExecutor::DropDatabase(const planner::DropPlan &node, if (StatementCacheManager::GetStmtCacheManager().get()) { std::set table_ids; - auto table_objects = database_object->GetTableObjects(false); + auto table_objects = database_object->GetTableCatalogEntries(false); for (auto it : table_objects) { table_ids.insert(it.second->GetTableOid()); } @@ -118,8 +120,9 @@ bool DropExecutor::DropSchema(const planner::DropPlan &node, std::string database_name = node.GetDatabaseName(); std::string schema_name = node.GetSchemaName(); - ResultType result = catalog::Catalog::GetInstance()->DropSchema( - database_name, schema_name, txn); + ResultType result = catalog::Catalog::GetInstance()->DropSchema(txn, + database_name, + schema_name); txn->SetResult(result); if (txn->GetResult() == ResultType::SUCCESS) { @@ -127,9 +130,10 @@ bool DropExecutor::DropSchema(const planner::DropPlan &node, // add dropped table into StatementCacheManager if (StatementCacheManager::GetStmtCacheManager().get()) { std::set table_ids; - auto database_object = catalog::Catalog::GetInstance()->GetDatabaseObject( - database_name, txn); - auto table_objects = database_object->GetTableObjects(schema_name); + auto database_object = + catalog::Catalog::GetInstance()->GetDatabaseCatalogEntry(txn, + database_name); + auto table_objects = database_object->GetTableCatalogEntries(schema_name); for (int i = 0; i < (int)table_objects.size(); i++) { table_ids.insert(table_objects[i]->GetTableOid()); } @@ -150,16 +154,21 @@ bool DropExecutor::DropTable(const planner::DropPlan &node, if (node.IsMissing()) { try { - auto table_object = catalog::Catalog::GetInstance()->GetTableObject( - database_name, schema_name, table_name, txn); + auto table_object = + catalog::Catalog::GetInstance()->GetTableCatalogEntry(txn, + database_name, + schema_name, + table_name); } catch (CatalogException &e) { LOG_TRACE("Table %s does not exist.", table_name.c_str()); return false; } } - ResultType result = catalog::Catalog::GetInstance()->DropTable( - database_name, schema_name, table_name, txn); + ResultType result = catalog::Catalog::GetInstance()->DropTable(txn, + database_name, + schema_name, + table_name); txn->SetResult(result); if (txn->GetResult() == ResultType::SUCCESS) { @@ -168,7 +177,10 @@ bool DropExecutor::DropTable(const planner::DropPlan &node, if (StatementCacheManager::GetStmtCacheManager().get()) { oid_t table_id = catalog::Catalog::GetInstance() - ->GetTableObject(database_name, schema_name, table_name, txn) + ->GetTableCatalogEntry(txn, + database_name, + schema_name, + table_name) ->GetTableOid(); StatementCacheManager::GetStmtCacheManager()->InvalidateTableOid( table_id); @@ -186,15 +198,19 @@ bool DropExecutor::DropTrigger(const planner::DropPlan &node, std::string table_name = node.GetTableName(); std::string trigger_name = node.GetTriggerName(); - auto table_object = catalog::Catalog::GetInstance()->GetTableObject( - database_name, schema_name, table_name, txn); + auto table_object = catalog::Catalog::GetInstance()->GetTableCatalogEntry(txn, + database_name, + schema_name, + table_name); // drop trigger ResultType result = catalog::Catalog::GetInstance() ->GetSystemCatalogs(table_object->GetDatabaseOid()) ->GetTriggerCatalog() - ->DropTrigger(table_object->GetDatabaseOid(), - table_object->GetTableOid(), trigger_name, txn); + ->DropTrigger(txn, + table_object->GetDatabaseOid(), + table_object->GetTableOid(), + trigger_name); txn->SetResult(result); if (txn->GetResult() == ResultType::SUCCESS) { LOG_DEBUG("Dropping trigger succeeded!"); @@ -219,8 +235,9 @@ bool DropExecutor::DropIndex(const planner::DropPlan &node, concurrency::TransactionContext *txn) { std::string index_name = node.GetIndexName(); std::string schema_name = node.GetSchemaName(); - auto database_object = catalog::Catalog::GetInstance()->GetDatabaseObject( - node.GetDatabaseName(), txn); + auto database_object = + catalog::Catalog::GetInstance()->GetDatabaseCatalogEntry(txn, + node.GetDatabaseName()); if (database_object == nullptr) { throw CatalogException("Index name " + index_name + " cannot be found"); } @@ -228,15 +245,18 @@ bool DropExecutor::DropIndex(const planner::DropPlan &node, auto pg_index = catalog::Catalog::GetInstance() ->GetSystemCatalogs(database_object->GetDatabaseOid()) ->GetIndexCatalog(); - auto index_object = pg_index->GetIndexObject(database_object->GetDatabaseName(), - index_name, schema_name, txn); + auto index_object = pg_index->GetIndexCatalogEntry(txn, + database_object->GetDatabaseName(), + schema_name, + index_name); if (index_object == nullptr) { throw CatalogException("Can't find index " + schema_name + "." + index_name + " to drop"); } // invoke directly using oid - ResultType result = catalog::Catalog::GetInstance()->DropIndex( - database_object->GetDatabaseOid(), index_object->GetIndexOid(), txn); + ResultType result = catalog::Catalog::GetInstance()->DropIndex(txn, + database_object->GetDatabaseOid(), + index_object->GetIndexOid()); txn->SetResult(result); if (txn->GetResult() == ResultType::SUCCESS) { diff --git a/src/include/binder/binder_context.h b/src/include/binder/binder_context.h index 93aca140539..299dbe75c3e 100644 --- a/src/include/binder/binder_context.h +++ b/src/include/binder/binder_context.h @@ -27,7 +27,7 @@ class TransactionContext; } namespace catalog { -class TableCatalogObject; +class TableCatalogEntry; } namespace binder { @@ -96,7 +96,7 @@ class BinderContext { */ static bool GetColumnPosTuple( const std::string &col_name, - std::shared_ptr table_obj, + std::shared_ptr table_obj, std::tuple &col_pos_tuple, type::TypeId &value_type); /** @@ -133,7 +133,7 @@ class BinderContext { */ static bool GetRegularTableObj( std::shared_ptr current_context, std::string &alias, - std::shared_ptr &table_obj, int &depth); + std::shared_ptr &table_obj, int &depth); static bool CheckNestedTableColumn( std::shared_ptr current_context, std::string &alias, @@ -154,7 +154,7 @@ class BinderContext { private: /** @brief Map table alias to table obj */ - std::unordered_map> + std::unordered_map> regular_table_alias_map_; std::unordered_map> nested_table_alias_map_; diff --git a/src/include/catalog/abstract_catalog.h b/src/include/catalog/abstract_catalog.h index e0c8d81df53..0d180975309 100644 --- a/src/include/catalog/abstract_catalog.h +++ b/src/include/catalog/abstract_catalog.h @@ -45,41 +45,47 @@ class AbstractCatalog { protected: /* For pg_database, pg_table, pg_index, pg_column */ - AbstractCatalog(oid_t catalog_table_oid, std::string catalog_table_name, + AbstractCatalog(storage::Database *pg_catalog, catalog::Schema *catalog_table_schema, - storage::Database *pg_catalog); + oid_t catalog_table_oid, + std::string catalog_table_name); /* For other catalogs */ - AbstractCatalog(const std::string &catalog_table_ddl, - concurrency::TransactionContext *txn); + AbstractCatalog(concurrency::TransactionContext *txn, + const std::string &catalog_table_ddl); //===--------------------------------------------------------------------===// // Helper Functions //===--------------------------------------------------------------------===// - bool InsertTuple(std::unique_ptr tuple, - concurrency::TransactionContext *txn); + bool InsertTuple(concurrency::TransactionContext *txn, + std::unique_ptr tuple); - bool DeleteWithIndexScan(oid_t index_offset, std::vector values, - concurrency::TransactionContext *txn); + bool DeleteWithIndexScan(concurrency::TransactionContext *txn, + oid_t index_offset, + std::vector values); std::unique_ptr>> - GetResultWithIndexScan(std::vector column_offsets, oid_t index_offset, - std::vector values, - concurrency::TransactionContext *txn) const; + GetResultWithIndexScan( + concurrency::TransactionContext *txn, + std::vector column_offsets, + oid_t index_offset, + std::vector values) const; std::unique_ptr>> - GetResultWithSeqScan(std::vector column_offsets, - expression::AbstractExpression *predicate, - concurrency::TransactionContext *txn); + GetResultWithSeqScan( + concurrency::TransactionContext *txn, + expression::AbstractExpression *predicate, + std::vector column_offsets); - bool UpdateWithIndexScan(std::vector update_columns, - std::vector update_values, - std::vector scan_values, + bool UpdateWithIndexScan(concurrency::TransactionContext *txn, oid_t index_offset, - concurrency::TransactionContext *txn); + std::vector scan_values, + std::vector update_columns, + std::vector update_values); - void AddIndex(const std::vector &key_attrs, oid_t index_oid, - const std::string &index_name, + void AddIndex(const std::string &index_name, + oid_t index_oid, + const std::vector &key_attrs, IndexConstraintType index_constraint); //===--------------------------------------------------------------------===// @@ -87,9 +93,9 @@ class AbstractCatalog { //===--------------------------------------------------------------------===// // Maximum column name size for catalog schemas - static const size_t max_name_size = 64; + static const size_t max_name_size_ = 64; // which database catalog table is stored int - oid_t database_oid; + oid_t database_oid_; // Local oid (without catalog type mask) starts from START_OID + OID_OFFSET std::atomic oid_ = ATOMIC_VAR_INIT(START_OID + OID_OFFSET); diff --git a/src/include/catalog/catalog.h b/src/include/catalog/catalog.h index 5b9d4fd2ba1..5940f162864 100644 --- a/src/include/catalog/catalog.h +++ b/src/include/catalog/catalog.h @@ -21,9 +21,9 @@ namespace peloton { namespace catalog { class Schema; -class DatabaseCatalogObject; -class TableCatalogObject; -class IndexCatalogObject; +class DatabaseCatalogEntry; +class TableCatalogEntry; +class IndexCatalogEntry; class SystemCatalogs; } // namespace catalog @@ -91,38 +91,44 @@ class Catalog { // CREATE FUNCTIONS //===--------------------------------------------------------------------===// // Create a database - ResultType CreateDatabase(const std::string &database_name, - concurrency::TransactionContext *txn); + ResultType CreateDatabase(concurrency::TransactionContext *txn, + const std::string &database_name); // Create a schema(namespace) - ResultType CreateSchema(const std::string &database_name, - const std::string &schema_name, - concurrency::TransactionContext *txn); + ResultType CreateSchema(concurrency::TransactionContext *txn, + const std::string &database_name, + const std::string &schema_name); // Create a table in a database - ResultType CreateTable( - const std::string &database_name, const std::string &schema_name, - const std::string &table_name, std::unique_ptr, - concurrency::TransactionContext *txn, bool is_catalog = false, - uint32_t tuples_per_tilegroup = DEFAULT_TUPLES_PER_TILEGROUP, - peloton::LayoutType layout_type = LayoutType::ROW); + ResultType CreateTable(concurrency::TransactionContext *txn, + const std::string &database_name, + const std::string &schema_name, + std::unique_ptr schema, + const std::string &table_name, + bool is_catalog, + uint32_t tuples_per_tilegroup = DEFAULT_TUPLES_PER_TILEGROUP, + LayoutType layout_type = LayoutType::ROW); // Create index for a table - ResultType CreateIndex(const std::string &database_name, + ResultType CreateIndex(concurrency::TransactionContext *txn, + const std::string &database_name, const std::string &schema_name, const std::string &table_name, + const std::string &index_name, const std::vector &key_attrs, - const std::string &index_name, bool unique_keys, - IndexType index_type, - concurrency::TransactionContext *txn); + bool unique_keys, + IndexType index_type); - ResultType CreateIndex(oid_t database_oid, oid_t table_oid, - const std::vector &key_attrs, + ResultType CreateIndex(concurrency::TransactionContext *txn, + oid_t database_oid, const std::string &schema_name, - const std::string &index_name, IndexType index_type, - IndexConstraintType index_constraint, bool unique_keys, - concurrency::TransactionContext *txn, - bool is_catalog = false); + oid_t table_oid, + bool is_catalog, + const std::string &index_name, + const std::vector &key_attrs, + bool unique_keys, + IndexType index_type, + IndexConstraintType index_constraint); /** * @brief create a new layout for a table @@ -133,9 +139,10 @@ class Catalog { * @return shared_ptr shared_ptr to the newly created layout in case of * success. nullptr in case of failure. */ - std::shared_ptr CreateLayout( - oid_t database_oid, oid_t table_oid, const column_map_type &column_map, - concurrency::TransactionContext *txn); + std::shared_ptr CreateLayout(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + const column_map_type &column_map); /** * @brief create a new layout for a table and make it the default if @@ -147,37 +154,42 @@ class Catalog { * @return shared_ptr shared_ptr to the newly created layout in case of * success. nullptr in case of failure. */ - std::shared_ptr CreateDefaultLayout( - oid_t database_oid, oid_t table_oid, const column_map_type &column_map, - concurrency::TransactionContext *txn); + std::shared_ptr CreateDefaultLayout(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + const column_map_type &column_map); //===--------------------------------------------------------------------===// // DROP FUNCTIONS //===--------------------------------------------------------------------===// // Drop a database with its name - ResultType DropDatabaseWithName(const std::string &database_name, - concurrency::TransactionContext *txn); + ResultType DropDatabaseWithName(concurrency::TransactionContext *txn, + const std::string &database_name); // Drop a database with its oid - ResultType DropDatabaseWithOid(oid_t database_oid, - concurrency::TransactionContext *txn); + ResultType DropDatabaseWithOid(concurrency::TransactionContext *txn, + oid_t database_oid); // Drop a schema(namespace) using schema name - ResultType DropSchema(const std::string &database_name, - const std::string &schema_name, - concurrency::TransactionContext *txn); + ResultType DropSchema(concurrency::TransactionContext *txn, + const std::string &database_name, + const std::string &schema_name); // Drop a table using table name - ResultType DropTable(const std::string &database_name, + ResultType DropTable(concurrency::TransactionContext *txn, + const std::string &database_name, const std::string &schema_name, - const std::string &table_name, - concurrency::TransactionContext *txn); + const std::string &table_name); + // Drop a table, use this one in the future - ResultType DropTable(oid_t database_oid, oid_t table_oid, - concurrency::TransactionContext *txn); + ResultType DropTable(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid); + // Drop an index, using its index_oid - ResultType DropIndex(oid_t database_oid, oid_t index_oid, - concurrency::TransactionContext *txn); + ResultType DropIndex(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t index_oid); /** @brief Drop layout * tile_groups @@ -187,8 +199,10 @@ class Catalog { * @param txn TransactionContext * @return ResultType(SUCCESS or FAILURE) */ - ResultType DropLayout(oid_t database_oid, oid_t table_oid, oid_t layout_oid, - concurrency::TransactionContext *txn); + ResultType DropLayout(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + oid_t layout_oid); //===--------------------------------------------------------------------===// // GET WITH NAME - CHECK FROM CATALOG TABLES, USING TRANSACTION //===--------------------------------------------------------------------===// @@ -197,42 +211,45 @@ class Catalog { * get it from storage layer using database_oid, * throw exception and abort txn if not exists/invisible * */ - storage::Database *GetDatabaseWithName( - const std::string &db_name, concurrency::TransactionContext *txn) const; + storage::Database *GetDatabaseWithName(concurrency::TransactionContext *txn, + const std::string &db_name) const; /* Check table from pg_table with table_name & schema_name using txn, * get it from storage layer using table_oid, * throw exception and abort txn if not exists/invisible * */ - storage::DataTable *GetTableWithName(const std::string &database_name, + storage::DataTable *GetTableWithName(concurrency::TransactionContext *txn, + const std::string &database_name, const std::string &schema_name, - const std::string &table_name, - concurrency::TransactionContext *txn); + const std::string &table_name); /* Check table from pg_database with database_name using txn, * get it from storage layer using table_oid, * throw exception and abort txn if not exists/invisible * */ - std::shared_ptr GetDatabaseObject( - const std::string &database_name, concurrency::TransactionContext *txn); - std::shared_ptr GetDatabaseObject( - oid_t database_oid, concurrency::TransactionContext *txn); + std::shared_ptr GetDatabaseCatalogEntry(concurrency::TransactionContext *txn, + const std::string &database_name); + + std::shared_ptr GetDatabaseCatalogEntry(concurrency::TransactionContext *txn, + oid_t database_oid); /* Check table from pg_table with table_name using txn, * get it from storage layer using table_oid, * throw exception and abort txn if not exists/invisible * */ - std::shared_ptr GetTableObject( - const std::string &database_name, const std::string &schema_name, - const std::string &table_name, concurrency::TransactionContext *txn); - std::shared_ptr GetTableObject( - oid_t database_oid, oid_t table_oid, - concurrency::TransactionContext *txn); + std::shared_ptr GetTableCatalogEntry(concurrency::TransactionContext *txn, + const std::string &database_name, + const std::string &schema_name, + const std::string &table_name); + + std::shared_ptr GetTableCatalogEntry(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid); /* * Using database oid to get system catalog object */ - std::shared_ptr GetSystemCatalogs(const oid_t database_oid); + std::shared_ptr GetSystemCatalogs(oid_t database_oid); //===--------------------------------------------------------------------===// // DEPRECATED FUNCTIONS //===--------------------------------------------------------------------===// @@ -252,34 +269,39 @@ class Catalog { void InitializeFunctions(); - void AddPlpgsqlFunction( - const std::string &name, const std::vector &argument_types, - const type::TypeId return_type, oid_t prolang, - const std::string &func_src, - std::shared_ptr code_context, - concurrency::TransactionContext *txn); - - void AddBuiltinFunction(const std::string &name, - const std::vector &argument_types, - const type::TypeId return_type, oid_t prolang, - const std::string &func_name, + void AddProcedure(concurrency::TransactionContext *txn, + const std::string &name, + type::TypeId return_type, + const std::vector &argument_types, + oid_t prolang, + std::shared_ptr code_context, + const std::string &func_src); + + // TODO(Tianyu): Somebody should comment on what the difference between name + // and func_name is. I am confused. + void AddBuiltinFunction(concurrency::TransactionContext *txn, + const std::string &name, function::BuiltInFuncType func, - concurrency::TransactionContext *txn); + const std::string &func_name, + type::TypeId return_type, + const std::vector &argument_types, + oid_t prolang); - const FunctionData GetFunction( - const std::string &name, const std::vector &argument_types); + const FunctionData GetFunction(const std::string &name, + const std::vector &argument_types); private: Catalog(); - void BootstrapSystemCatalogs(storage::Database *database, - concurrency::TransactionContext *txn); + void BootstrapSystemCatalogs(concurrency::TransactionContext *txn, + storage::Database *database); // Create the primary key index for a table, don't call this function outside // catalog.cpp - ResultType CreatePrimaryIndex(oid_t database_oid, oid_t table_oid, + ResultType CreatePrimaryIndex(concurrency::TransactionContext *txn, + oid_t database_oid, const std::string &schema_name, - concurrency::TransactionContext *txn); + oid_t table_oid); // The pool for new varlen tuple fields std::unique_ptr pool_; diff --git a/src/include/catalog/catalog_cache.h b/src/include/catalog/catalog_cache.h index 28ff9edacd0..3f00559a8db 100644 --- a/src/include/catalog/catalog_cache.h +++ b/src/include/catalog/catalog_cache.h @@ -25,49 +25,48 @@ class PlanUtil; namespace catalog { -class DatabaseCatalogObject; -class TableCatalogObject; -class IndexCatalogObject; +class DatabaseCatalogEntry; +class TableCatalogEntry; +class IndexCatalogEntry; class CatalogCache { friend class Transaction; friend class DatabaseCatalog; friend class TableCatalog; friend class IndexCatalog; - friend class DatabaseCatalogObject; - friend class TableCatalogObject; - friend class IndexCatalogObject; + friend class DatabaseCatalogEntry; + friend class TableCatalogEntry; + friend class IndexCatalogEntry; friend class planner::PlanUtil; public: CatalogCache() {} - CatalogCache(CatalogCache const &) = delete; - CatalogCache &operator=(CatalogCache const &) = delete; + DISALLOW_COPY(CatalogCache) private: - std::shared_ptr GetDatabaseObject(oid_t database_oid); - std::shared_ptr GetDatabaseObject( + std::shared_ptr GetDatabaseObject(oid_t database_oid); + std::shared_ptr GetDatabaseObject( const std::string &name); - std::shared_ptr GetCachedTableObject(oid_t database_oid, - oid_t table_oid); - std::shared_ptr GetCachedIndexObject(oid_t database_oid, - oid_t index_oid); - std::shared_ptr GetCachedIndexObject( - const std::string &database_name, const std::string &index_name, - const std::string &schema_name); + std::shared_ptr GetCachedTableObject(oid_t database_oid, + oid_t table_oid); + std::shared_ptr GetCachedIndexObject(oid_t database_oid, + oid_t index_oid); + std::shared_ptr GetCachedIndexObject(const std::string &database_name, + const std::string &schema_name, + const std::string &index_name); // database catalog cache interface bool InsertDatabaseObject( - std::shared_ptr database_object); + std::shared_ptr database_object); bool EvictDatabaseObject(oid_t database_oid); bool EvictDatabaseObject(const std::string &database_name); // cache for database catalog object - std::unordered_map> - database_objects_cache; - std::unordered_map> - database_name_cache; + std::unordered_map> + database_objects_cache_; + std::unordered_map> + database_name_cache_; }; } // namespace catalog diff --git a/src/include/catalog/column.h b/src/include/catalog/column.h index f351f72c5b9..a8741247648 100644 --- a/src/include/catalog/column.h +++ b/src/include/catalog/column.h @@ -29,7 +29,7 @@ class Column : public Printable { friend class Constraint; public: - Column() : column_type(type::TypeId::INVALID), fixed_length(INVALID_OID) { + Column() : column_type_(type::TypeId::INVALID), fixed_length_(INVALID_OID) { // Nothing to see... } @@ -37,10 +37,10 @@ class Column : public Printable { std::string column_name, bool is_inlined = false, oid_t column_offset = INVALID_OID) : column_name(column_name), - column_type(value_type), - fixed_length(INVALID_OID), - is_inlined(is_inlined), - column_offset(column_offset) { + column_type_(value_type), + fixed_length_(INVALID_OID), + is_inlined_(is_inlined), + column_offset_(column_offset) { SetInlined(); // We should not have an inline value of length 0 @@ -61,24 +61,24 @@ class Column : public Printable { // Set the appropriate column length void SetLength(size_t column_length); - oid_t GetOffset() const { return column_offset; } + oid_t GetOffset() const { return column_offset_; } std::string GetName() const { return column_name; } size_t GetLength() const { - if (is_inlined) - return fixed_length; + if (is_inlined_) + return fixed_length_; else - return variable_length; + return variable_length_; } - size_t GetFixedLength() const { return fixed_length; } + size_t GetFixedLength() const { return fixed_length_; } - size_t GetVariableLength() const { return variable_length; } + size_t GetVariableLength() const { return variable_length_; } - inline type::TypeId GetType() const { return column_type; } + inline type::TypeId GetType() const { return column_type_; } - inline bool IsInlined() const { return is_inlined; } + inline bool IsInlined() const { return is_inlined_; } inline bool IsPrimary() const { return is_primary_; } @@ -88,9 +88,9 @@ class Column : public Printable { void AddConstraint(const catalog::Constraint &constraint) { if (constraint.GetType() == ConstraintType::DEFAULT) { // Add the default constraint to the front - constraints.insert(constraints.begin(), constraint); + constraints_.insert(constraints_.begin(), constraint); } else { - constraints.push_back(constraint); + constraints_.push_back(constraint); } if (constraint.GetType() == ConstraintType::PRIMARY) { @@ -101,16 +101,16 @@ class Column : public Printable { } } - const std::vector &GetConstraints() const { return constraints; } + const std::vector &GetConstraints() const { return constraints_; } hash_t Hash() const { - hash_t hash = HashUtil::Hash(&column_type); - return HashUtil::CombineHashes(hash, HashUtil::Hash(&is_inlined)); + hash_t hash = HashUtil::Hash(&column_type_); + return HashUtil::CombineHashes(hash, HashUtil::Hash(&is_inlined_)); } // Compare two column objects bool operator==(const Column &other) const { - if (other.column_type != column_type || other.is_inlined != is_inlined) { + if (other.column_type_ != column_type_ || other.is_inlined_ != is_inlined_) { return false; } return true; @@ -130,18 +130,18 @@ class Column : public Printable { private: // value type of column - type::TypeId column_type; // = type::TypeId::INVALID; + type::TypeId column_type_; // = type::TypeId::INVALID; // if the column is not inlined, this is set to pointer size // else, it is set to length of the fixed length column - size_t fixed_length; // = INVALID_OID; + size_t fixed_length_; // = INVALID_OID; // if the column is inlined, this is set to 0 // else, it is set to length of the variable length column - size_t variable_length = 0; + size_t variable_length_ = 0; // is the column inlined ? - bool is_inlined = false; + bool is_inlined_ = false; // is the column contained the primary key? bool is_primary_ = false; @@ -150,10 +150,10 @@ class Column : public Printable { bool is_unique_ = false; // offset of column in tuple - oid_t column_offset = INVALID_OID; + oid_t column_offset_ = INVALID_OID; // Constraints - std::vector constraints; + std::vector constraints_; }; } // namespace catalog diff --git a/src/include/catalog/column_catalog.h b/src/include/catalog/column_catalog.h index 6a9173029fb..490855822f1 100644 --- a/src/include/catalog/column_catalog.h +++ b/src/include/catalog/column_catalog.h @@ -38,41 +38,42 @@ namespace peloton { namespace catalog { -class ColumnCatalogObject { +class ColumnCatalogEntry { public: - ColumnCatalogObject(executor::LogicalTile *tile, int tupleId = 0); - - inline oid_t GetTableOid() { return table_oid; } - inline const std::string &GetColumnName() { return column_name; } - inline oid_t GetColumnId() { return column_id; } - inline oid_t GetColumnOffset() { return column_offset; } - inline type::TypeId GetColumnType() { return column_type; } - inline size_t GetColumnLength() { return column_length; } - inline bool IsInlined() { return is_inlined; } - inline bool IsPrimary() { return is_primary; } - inline bool IsNotNull() { return is_not_null; } + ColumnCatalogEntry(executor::LogicalTile *tile, int tupleId = 0); + + inline oid_t GetTableOid() { return table_oid_; } + inline const std::string &GetColumnName() { return column_name_; } + inline oid_t GetColumnId() { return column_id_; } + inline oid_t GetColumnOffset() { return column_offset_; } + inline type::TypeId GetColumnType() { return column_type_; } + inline size_t GetColumnLength() { return column_length_; } + inline bool IsInlined() { return is_inlined_; } + inline bool IsPrimary() { return is_primary_; } + inline bool IsNotNull() { return is_not_null_; } private: // member variables - oid_t table_oid; - std::string column_name; - oid_t column_id; - oid_t column_offset; - type::TypeId column_type; - size_t column_length; - bool is_inlined; - bool is_primary; - bool is_not_null; + oid_t table_oid_; + std::string column_name_; + oid_t column_id_; + oid_t column_offset_; + type::TypeId column_type_; + size_t column_length_; + bool is_inlined_; + bool is_primary_; + bool is_not_null_; }; class ColumnCatalog : public AbstractCatalog { - friend class ColumnCatalogObject; - friend class TableCatalogObject; + friend class ColumnCatalogEntry; + friend class TableCatalogEntry; friend class Catalog; public: - ColumnCatalog(storage::Database *pg_catalog, type::AbstractPool *pool, - concurrency::TransactionContext *txn); + ColumnCatalog(concurrency::TransactionContext *txn, + storage::Database *pg_catalog, + type::AbstractPool *pool); ~ColumnCatalog(); @@ -84,22 +85,30 @@ class ColumnCatalog : public AbstractCatalog { //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// - bool InsertColumn(oid_t table_oid, const std::string &column_name, - oid_t column_id, oid_t column_offset, - type::TypeId column_type, size_t column_length, - bool is_inlined, const std::vector &constraints, - type::AbstractPool *pool, - concurrency::TransactionContext *txn); - bool DeleteColumn(oid_t table_oid, const std::string &column_name, - concurrency::TransactionContext *txn); - bool DeleteColumns(oid_t table_oid, concurrency::TransactionContext *txn); + bool InsertColumn(concurrency::TransactionContext *txn, + oid_t table_oid, + oid_t column_id, + const std::string &column_name, + oid_t column_offset, + type::TypeId column_type, + size_t column_length, + const std::vector &constraints, + bool is_inlined, + type::AbstractPool *pool); + + bool DeleteColumn(concurrency::TransactionContext *txn, + oid_t table_oid, + const std::string &column_name); + + bool DeleteColumns(concurrency::TransactionContext *txn, oid_t table_oid); private: //===--------------------------------------------------------------------===// // Read Related API(only called within table catalog object) //===--------------------------------------------------------------------===// - const std::unordered_map> - GetColumnObjects(oid_t table_oid, concurrency::TransactionContext *txn); + const std::unordered_map> + GetColumnCatalogEntries(concurrency::TransactionContext *txn, + oid_t table_oid); std::unique_ptr InitializeSchema(); @@ -115,7 +124,7 @@ class ColumnCatalog : public AbstractCatalog { IS_NOT_NULL = 8, // Add new columns here in creation order }; - std::vector all_column_ids = {0, 1, 2, 3, 4, 5, 6, 7, 8}; + std::vector all_column_ids_ = {0, 1, 2, 3, 4, 5, 6, 7, 8}; enum IndexId { PRIMARY_KEY = 0, diff --git a/src/include/catalog/column_stats_catalog.h b/src/include/catalog/column_stats_catalog.h index d409a9da338..fd643400db6 100644 --- a/src/include/catalog/column_stats_catalog.h +++ b/src/include/catalog/column_stats_catalog.h @@ -56,27 +56,38 @@ class ColumnStatsCatalog : public AbstractCatalog { //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// - bool InsertColumnStats(oid_t database_id, oid_t table_id, oid_t column_id, - int num_rows, double cardinality, double frac_null, + bool InsertColumnStats(concurrency::TransactionContext *txn, + oid_t database_id, + oid_t table_id, + oid_t column_id, + std::string column_name, + int num_rows, + double frac_null, std::string most_common_vals, std::string most_common_freqs, - std::string histogram_bounds, std::string column_name, - bool has_index, type::AbstractPool *pool, - concurrency::TransactionContext *txn); - bool DeleteColumnStats(oid_t database_id, oid_t table_id, oid_t column_id, - concurrency::TransactionContext *txn); + std::string histogram_bounds, + double cardinality, + bool has_index, + type::AbstractPool *pool); + + bool DeleteColumnStats(concurrency::TransactionContext *txn, + oid_t database_id, + oid_t table_id, + oid_t column_id); //===--------------------------------------------------------------------===// // Read-only Related API //===--------------------------------------------------------------------===// - std::unique_ptr> GetColumnStats( - oid_t database_id, oid_t table_id, oid_t column_id, - concurrency::TransactionContext *txn); - - size_t GetTableStats( - oid_t database_id, oid_t table_id, concurrency::TransactionContext *txn, - std::map>> & - column_stats_map); + std::unique_ptr> GetColumnStats(concurrency::TransactionContext *txn, + oid_t database_id, + oid_t table_id, + oid_t column_id); + + size_t GetTableStats(concurrency::TransactionContext *txn, + oid_t database_id, + oid_t table_id, + std::map>> + &column_stats_map); // TODO: add more if needed enum ColumnId { diff --git a/src/include/catalog/constraint.h b/src/include/catalog/constraint.h index c63b881c727..7f67b1f78b6 100644 --- a/src/include/catalog/constraint.h +++ b/src/include/catalog/constraint.h @@ -14,6 +14,7 @@ #pragma once #include +#include #include #include "common/printable.h" @@ -30,56 +31,55 @@ namespace catalog { class Constraint : public Printable { public: Constraint(ConstraintType type, std::string constraint_name) - : constraint_type(type), constraint_name(constraint_name) {} + : constraint_type_(type), constraint_name_(std::move(constraint_name)) {} Constraint(ConstraintType type, std::string constraint_name, std::string check_cmd) - : constraint_type(type), - constraint_name(constraint_name), - check_cmd(check_cmd) {} + : constraint_type_(type), + constraint_name_(std::move(constraint_name)), + check_cmd_(std::move(check_cmd)) {} //===--------------------------------------------------------------------===// // ACCESSORS //===--------------------------------------------------------------------===// - ConstraintType GetType() const { return constraint_type; } + ConstraintType GetType() const { return constraint_type_; } - std::pair GetCheckExpression() { return exp; } + std::pair GetCheckExpression() { return exp_; } // Offset into the list of "reference tables" in the Table. - void SetForeignKeyListOffset(oid_t offset) { fk_list_offset = offset; } + void SetForeignKeyListOffset(oid_t offset) { fk_list_offset_ = offset; } // Offset into the list of "unique indices" in the Table. - void SetUniqueIndexOffset(oid_t offset) { unique_index_list_offset = offset; } + void SetUniqueIndexOffset(oid_t offset) { + unique_index_list_offset_ = offset; + } // Get the offset - oid_t GetForeignKeyListOffset() const { return fk_list_offset; } + oid_t GetForeignKeyListOffset() const { return fk_list_offset_; } // Get the offset - oid_t GetUniqueIndexOffset() const { return unique_index_list_offset; } + oid_t GetUniqueIndexOffset() const { return unique_index_list_offset_; } - std::string GetName() const { return constraint_name; } + std::string GetName() const { return constraint_name_; } // Get a string representation for debugging - const std::string GetInfo() const; + const std::string GetInfo() const override; // Todo: default union data structure, // For default constraint void addDefaultValue(const type::Value &value) { - if (constraint_type != ConstraintType::DEFAULT || default_value.get() != nullptr) { - return; - } - - default_value.reset(new peloton::type::Value(value)); + if (constraint_type_ != ConstraintType::DEFAULT + || default_value_.get() != nullptr) return; + default_value_.reset(new peloton::type::Value(value)); } - type::Value* getDefaultValue() { - return default_value.get(); + type::Value *getDefaultValue() { + return default_value_.get(); } // Add check constrain void AddCheck(ExpressionType op, peloton::type::Value val) { - exp = std::pair(op, val); - return; + exp_ = std::pair(op, val); }; private: @@ -88,21 +88,21 @@ class Constraint : public Printable { //===--------------------------------------------------------------------===// // The type of constraint - ConstraintType constraint_type = ConstraintType::INVALID; + ConstraintType constraint_type_ = ConstraintType::INVALID; // Offsets into the Unique index and reference table lists in Table - oid_t fk_list_offset = INVALID_OID; + oid_t fk_list_offset_ = INVALID_OID; - oid_t unique_index_list_offset = INVALID_OID; + oid_t unique_index_list_offset_ = INVALID_OID; - std::string constraint_name; + std::string constraint_name_; - std::shared_ptr default_value; + std::shared_ptr default_value_; - std::string check_cmd = ""; + std::string check_cmd_ = ""; // key string is column name - std::pair exp; + std::pair exp_; }; } // namespace catalog diff --git a/src/include/catalog/database_catalog.h b/src/include/catalog/database_catalog.h index fc94b1944bb..91267c3d995 100644 --- a/src/include/catalog/database_catalog.h +++ b/src/include/catalog/database_catalog.h @@ -33,69 +33,75 @@ namespace peloton { namespace catalog { -class TableCatalogObject; -class IndexCatalogObject; +class TableCatalogEntry; +class IndexCatalogEntry; -class DatabaseCatalogObject { +class DatabaseCatalogEntry { friend class DatabaseCatalog; friend class TableCatalog; friend class CatalogCache; public: - DatabaseCatalogObject(executor::LogicalTile *tile, - concurrency::TransactionContext *txn); + DatabaseCatalogEntry(concurrency::TransactionContext *txn, + executor::LogicalTile *tile); - void EvictAllTableObjects(); - std::shared_ptr GetTableObject(oid_t table_oid, - bool cached_only = false); - std::shared_ptr GetTableObject( + void EvictAllTableCatalogEntries(); + + std::shared_ptr GetTableCatalogEntry(oid_t table_oid, + bool cached_only = false); + std::shared_ptr GetTableCatalogEntry( const std::string &table_name, const std::string &schema_name, bool cached_only = false); - bool IsValidTableObjects() { - // return true if this database object contains all table - // objects within the database - return valid_table_objects; + bool IsValidTableCatalogEntries() { + // return true if this database catalog entries contains all table + // catalog entries within the database + return valid_table_catalog_entries; } - std::vector> GetTableObjects( + std::vector> GetTableCatalogEntries( const std::string &schema_name); - std::unordered_map> - GetTableObjects(bool cached_only = false); + std::unordered_map> + GetTableCatalogEntries(bool cached_only = false); + + inline oid_t GetDatabaseOid() { return database_oid_; } - inline oid_t GetDatabaseOid() { return database_oid; } - inline const std::string &GetDatabaseName() { return database_name; } + inline const std::string &GetDatabaseName() { return database_name_; } private: // member variables - oid_t database_oid; - std::string database_name; + oid_t database_oid_; + std::string database_name_; + + bool InsertTableCatalogEntry(std::shared_ptr table_catalog_entry); + + bool EvictTableCatalogEntry(oid_t table_oid); - bool InsertTableObject(std::shared_ptr table_object); - bool EvictTableObject(oid_t table_oid); - bool EvictTableObject(const std::string &table_name, - const std::string &schema_name); - void SetValidTableObjects(bool valid = true) { valid_table_objects = valid; } + bool EvictTableCatalogEntry(const std::string &table_name, + const std::string &schema_name); - std::shared_ptr GetCachedIndexObject(oid_t index_oid); - std::shared_ptr GetCachedIndexObject( + void SetValidTableCatalogEntries(bool valid = true) { valid_table_catalog_entries = valid; } + + std::shared_ptr GetCachedIndexCatalogEntry(oid_t index_oid); + + std::shared_ptr GetCachedIndexCatalogEntry( const std::string &index_name, const std::string &schema_name); // cache for table name to oid translation - std::unordered_map> - table_objects_cache; - std::unordered_map> - table_name_cache; - bool valid_table_objects; + std::unordered_map> + table_catalog_entries_cache_; + std::unordered_map> + table_catalog_entries_cache_by_name; + bool valid_table_catalog_entries; // Pointer to its corresponding transaction // This object is only visible during this transaction - concurrency::TransactionContext *txn; + concurrency::TransactionContext *txn_; }; class DatabaseCatalog : public AbstractCatalog { - friend class DatabaseCatalogObject; + friend class DatabaseCatalogEntry; friend class TableCatalog; friend class CatalogCache; friend class Catalog; @@ -104,10 +110,9 @@ class DatabaseCatalog : public AbstractCatalog { ~DatabaseCatalog(); // Global Singleton, only the first call requires passing parameters. - static DatabaseCatalog *GetInstance( - storage::Database *pg_catalog = nullptr, - type::AbstractPool *pool = nullptr, - concurrency::TransactionContext *txn = nullptr); + static DatabaseCatalog *GetInstance(concurrency::TransactionContext *txn = nullptr, + storage::Database *pg_catalog = nullptr, + type::AbstractPool *pool = nullptr); inline oid_t GetNextOid() { return oid_++ | DATABASE_OID_MASK; } @@ -116,22 +121,26 @@ class DatabaseCatalog : public AbstractCatalog { //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// - bool InsertDatabase(oid_t database_oid, const std::string &database_name, - type::AbstractPool *pool, - concurrency::TransactionContext *txn); - bool DeleteDatabase(oid_t database_oid, concurrency::TransactionContext *txn); + bool InsertDatabase(concurrency::TransactionContext *txn, + oid_t database_oid, + const std::string &database_name, + type::AbstractPool *pool); + + bool DeleteDatabase(concurrency::TransactionContext *txn, oid_t database_oid); private: //===--------------------------------------------------------------------===// // Read Related API //===--------------------------------------------------------------------===// - std::shared_ptr GetDatabaseObject( - oid_t database_oid, concurrency::TransactionContext *txn); - std::shared_ptr GetDatabaseObject( - const std::string &database_name, concurrency::TransactionContext *txn); + std::shared_ptr GetDatabaseCatalogEntry(concurrency::TransactionContext *txn, + oid_t database_oid); + + std::shared_ptr GetDatabaseCatalogEntry(concurrency::TransactionContext *txn, + const std::string &database_name); - DatabaseCatalog(storage::Database *pg_catalog, type::AbstractPool *pool, - concurrency::TransactionContext *txn); + DatabaseCatalog(concurrency::TransactionContext *txn, + storage::Database *pg_catalog, + type::AbstractPool *pool); std::unique_ptr InitializeSchema(); @@ -140,7 +149,7 @@ class DatabaseCatalog : public AbstractCatalog { DATABASE_NAME = 1, // Add new columns here in creation order }; - std::vector all_column_ids = {0, 1}; + std::vector all_column_ids_ = {0, 1}; enum IndexId { PRIMARY_KEY = 0, diff --git a/src/include/catalog/database_metrics_catalog.h b/src/include/catalog/database_metrics_catalog.h index a035063a1a5..bb72bc540dc 100644 --- a/src/include/catalog/database_metrics_catalog.h +++ b/src/include/catalog/database_metrics_catalog.h @@ -45,16 +45,20 @@ class DatabaseMetricsCatalog : public AbstractCatalog { //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// - bool InsertDatabaseMetrics(oid_t database_oid, oid_t txn_committed, - oid_t txn_aborted, oid_t time_stamp, - type::AbstractPool *pool, - concurrency::TransactionContext *txn); - bool DeleteDatabaseMetrics(oid_t database_oid, concurrency::TransactionContext *txn); + bool InsertDatabaseMetrics(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t txn_committed, + oid_t txn_aborted, + oid_t time_stamp, + type::AbstractPool *pool); + + bool DeleteDatabaseMetrics(concurrency::TransactionContext *txn, + oid_t database_oid); //===--------------------------------------------------------------------===// // Read-only Related API //===--------------------------------------------------------------------===// - oid_t GetTimeStamp(oid_t database_oid, concurrency::TransactionContext *txn); + oid_t GetTimestamp(concurrency::TransactionContext *txn, oid_t database_oid); // TODO: add more if needed enum ColumnId { diff --git a/src/include/catalog/foreign_key.h b/src/include/catalog/foreign_key.h index 02b28d7abf6..1fb745560e7 100644 --- a/src/include/catalog/foreign_key.h +++ b/src/include/catalog/foreign_key.h @@ -36,40 +36,40 @@ class ForeignKey { FKConstrActionType delete_action, std::string constraint_name) - : source_table_id(source_table_id), - sink_table_id(sink_table_id), - sink_col_ids(sink_col_ids), - source_col_ids(source_col_ids), - update_action(update_action), - delete_action(delete_action), - fk_name(constraint_name) {} + : source_table_id_(source_table_id), + sink_table_id_(sink_table_id), + sink_col_ids_(sink_col_ids), + source_col_ids_(source_col_ids), + update_action_(update_action), + delete_action_(delete_action), + fk_name_(constraint_name) {} - oid_t GetSourceTableOid() const { return source_table_id; } - oid_t GetSinkTableOid() const { return sink_table_id; } + oid_t GetSourceTableOid() const { return source_table_id_; } + oid_t GetSinkTableOid() const { return sink_table_id_; } - std::vector GetSinkColumnIds() const { return sink_col_ids; } - std::vector GetSourceColumnIds() const { return source_col_ids; } + std::vector GetSinkColumnIds() const { return sink_col_ids_; } + std::vector GetSourceColumnIds() const { return source_col_ids_; } - FKConstrActionType GetUpdateAction() const { return update_action; } - FKConstrActionType GetDeleteAction() const { return delete_action; } - std::string &GetConstraintName() { return fk_name; } + FKConstrActionType GetUpdateAction() const { return update_action_; } + FKConstrActionType GetDeleteAction() const { return delete_action_; } + std::string &GetConstraintName() { return fk_name_; } private: - oid_t source_table_id = INVALID_OID; - oid_t sink_table_id = INVALID_OID; + oid_t source_table_id_ = INVALID_OID; + oid_t sink_table_id_ = INVALID_OID; // Columns in the reference table (sink) - std::vector sink_col_ids; + std::vector sink_col_ids_; // Columns in the current table (source) // Can be a single column or multiple columns depending // on the constraint - std::vector source_col_ids; + std::vector source_col_ids_; - FKConstrActionType update_action; - FKConstrActionType delete_action; + FKConstrActionType update_action_; + FKConstrActionType delete_action_; - std::string fk_name; + std::string fk_name_; }; } // namespace catalog diff --git a/src/include/catalog/index_catalog.h b/src/include/catalog/index_catalog.h index 68e4d754f4c..dcd15ea285a 100644 --- a/src/include/catalog/index_catalog.h +++ b/src/include/catalog/index_catalog.h @@ -40,41 +40,42 @@ namespace peloton { namespace catalog { -class IndexCatalogObject { - friend class TableCatalogObject; +class IndexCatalogEntry { + friend class TableCatalogEntry; public: - IndexCatalogObject(executor::LogicalTile *tile, int tupleId = 0); + IndexCatalogEntry(executor::LogicalTile *tile, int tupleId = 0); - inline oid_t GetIndexOid() { return index_oid; } - inline const std::string &GetIndexName() { return index_name; } - inline oid_t GetTableOid() { return table_oid; } - inline const std::string &GetSchemaName() { return schema_name; } - inline IndexType GetIndexType() { return index_type; } - inline IndexConstraintType GetIndexConstraint() { return index_constraint; } - inline bool HasUniqueKeys() { return unique_keys; } - inline const std::vector &GetKeyAttrs() { return key_attrs; } + inline oid_t GetIndexOid() { return index_oid_; } + inline const std::string &GetIndexName() { return index_name_; } + inline oid_t GetTableOid() { return table_oid_; } + inline const std::string &GetSchemaName() { return schema_name_; } + inline IndexType GetIndexType() { return index_type_; } + inline IndexConstraintType GetIndexConstraint() { return index_constraint_; } + inline bool HasUniqueKeys() { return unique_keys_; } + inline const std::vector &GetKeyAttrs() { return key_attrs_; } private: // member variables - oid_t index_oid; - std::string index_name; - oid_t table_oid; - std::string schema_name; - IndexType index_type; - IndexConstraintType index_constraint; - bool unique_keys; - std::vector key_attrs; + oid_t index_oid_; + std::string index_name_; + oid_t table_oid_; + std::string schema_name_; + IndexType index_type_; + IndexConstraintType index_constraint_; + bool unique_keys_; + std::vector key_attrs_; }; class IndexCatalog : public AbstractCatalog { - friend class IndexCatalogObject; - friend class TableCatalogObject; + friend class IndexCatalogEntry; + friend class TableCatalogEntry; friend class Catalog; public: - IndexCatalog(storage::Database *pg_catalog, type::AbstractPool *pool, - concurrency::TransactionContext *txn); + IndexCatalog(concurrency::TransactionContext *txn, + storage::Database *pg_catalog, + type::AbstractPool *pool); ~IndexCatalog(); @@ -83,27 +84,36 @@ class IndexCatalog : public AbstractCatalog { void UpdateOid(oid_t add_value) { oid_ += add_value; } /** Write Related API */ - bool InsertIndex(oid_t index_oid, const std::string &index_name, - oid_t table_oid, const std::string &schema_name, - IndexType index_type, IndexConstraintType index_constraint, - bool unique_keys, std::vector indekeys, - type::AbstractPool *pool, - concurrency::TransactionContext *txn); - bool DeleteIndex(oid_t database_oid, oid_t index_oid, - concurrency::TransactionContext *txn); + bool InsertIndex(concurrency::TransactionContext *txn, + const std::string &schema_name, + oid_t table_oid, + oid_t index_oid, + const std::string &index_name, + IndexType index_type, + IndexConstraintType index_constraint, + bool unique_keys, + std::vector index_keys, + type::AbstractPool *pool); + + bool DeleteIndex(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t index_oid); /** Read Related API */ - std::shared_ptr GetIndexObject( - const std::string &database_name, const std::string &index_name, - const std::string &schema_name, concurrency::TransactionContext *txn); + std::shared_ptr GetIndexCatalogEntry(concurrency::TransactionContext *txn, + const std::string &database_name, + const std::string &schema_name, + const std::string &index_name); private: - std::shared_ptr GetIndexObject( - oid_t database_oid, oid_t index_oid, - concurrency::TransactionContext *txn); - - const std::unordered_map> - GetIndexObjects(oid_t table_oid, concurrency::TransactionContext *txn); + std::shared_ptr GetIndexCatalogEntry(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t index_oid); + + const std::unordered_map> + GetIndexCatalogEntries( + concurrency::TransactionContext *txn, + oid_t table_oid); std::unique_ptr InitializeSchema(); diff --git a/src/include/catalog/index_metrics_catalog.h b/src/include/catalog/index_metrics_catalog.h index 7b806623565..6047bfe063f 100644 --- a/src/include/catalog/index_metrics_catalog.h +++ b/src/include/catalog/index_metrics_catalog.h @@ -38,19 +38,23 @@ namespace catalog { class IndexMetricsCatalog : public AbstractCatalog { public: - IndexMetricsCatalog(const std::string &database_name, - concurrency::TransactionContext *txn); + IndexMetricsCatalog(concurrency::TransactionContext *txn, + const std::string &database_name); ~IndexMetricsCatalog(); //===--------------------------------------------------------------------===// // Write Related API //===--------------------------------------------------------------------===// - bool InsertIndexMetrics(oid_t table_oid, oid_t index_oid, int64_t reads, - int64_t deletes, int64_t inserts, int64_t time_stamp, - type::AbstractPool *pool, - concurrency::TransactionContext *txn); - bool DeleteIndexMetrics(oid_t index_oid, - concurrency::TransactionContext *txn); + bool InsertIndexMetrics(concurrency::TransactionContext *txn, + oid_t table_oid, + oid_t index_oid, + int64_t reads, + int64_t deletes, + int64_t inserts, + int64_t time_stamp, + type::AbstractPool *pool); + + bool DeleteIndexMetrics(concurrency::TransactionContext *txn, oid_t index_oid); //===--------------------------------------------------------------------===// // Read-only Related API diff --git a/src/include/catalog/language_catalog.h b/src/include/catalog/language_catalog.h index 68249dcbe48..7a6e906d422 100644 --- a/src/include/catalog/language_catalog.h +++ b/src/include/catalog/language_catalog.h @@ -36,9 +36,9 @@ class LogicalTile; namespace catalog { -class LanguageCatalogObject { +class LanguageCatalogEntry { public: - LanguageCatalogObject(executor::LogicalTile *tuple); + LanguageCatalogEntry(executor::LogicalTile *tuple); oid_t GetOid() const { return lang_oid_; } @@ -63,28 +63,29 @@ class LanguageCatalog : public AbstractCatalog { //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// - bool InsertLanguage(const std::string &lanname, type::AbstractPool *pool, - concurrency::TransactionContext *txn); + bool InsertLanguage(concurrency::TransactionContext *txn, + const std::string &lanname, + type::AbstractPool *pool); - bool DeleteLanguage(const std::string &lanname, - concurrency::TransactionContext *txn); + bool DeleteLanguage(concurrency::TransactionContext *txn, + const std::string &lanname); //===--------------------------------------------------------------------===// // Read-only Related API //===--------------------------------------------------------------------===// - std::unique_ptr GetLanguageByOid( - oid_t lang_oid, concurrency::TransactionContext *txn) const; + std::unique_ptr GetLanguageByOid(concurrency::TransactionContext *txn, + oid_t lang_oid) const; - std::unique_ptr GetLanguageByName( - const std::string &lang_name, concurrency::TransactionContext *txn) const; + std::unique_ptr GetLanguageByName(concurrency::TransactionContext *txn, + const std::string &lang_name) const; enum ColumnId { OID = 0, LANNAME = 1, // Add new columns here in creation order }; - std::vector all_column_ids = {0, 1}; + std::vector all_column_ids_ = {0, 1}; private: LanguageCatalog(concurrency::TransactionContext *txn); diff --git a/src/include/catalog/layout_catalog.h b/src/include/catalog/layout_catalog.h index 0fc224f094d..babdb346583 100644 --- a/src/include/catalog/layout_catalog.h +++ b/src/include/catalog/layout_catalog.h @@ -25,32 +25,35 @@ namespace catalog { class LayoutCatalog : public AbstractCatalog { public: - LayoutCatalog(storage::Database *pg_catalog, type::AbstractPool *pool, - concurrency::TransactionContext *txn); + LayoutCatalog(concurrency::TransactionContext *txn, + storage::Database *pg_catalog, + type::AbstractPool *pool); ~LayoutCatalog(); //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// - bool InsertLayout(oid_t table_oid, + bool InsertLayout(concurrency::TransactionContext *txn, + oid_t table_oid, std::shared_ptr layout, - type::AbstractPool *pool, - concurrency::TransactionContext *txn); + type::AbstractPool *pool); - bool DeleteLayout(oid_t table_oid, oid_t layout_oid, - concurrency::TransactionContext *txn); + bool DeleteLayout(concurrency::TransactionContext *txn, + oid_t table_oid, + oid_t layout_oid); - bool DeleteLayouts(oid_t table_oid, concurrency::TransactionContext *txn); + bool DeleteLayouts(concurrency::TransactionContext *txn, oid_t table_oid); //===--------------------------------------------------------------------===// // Read Related API //===--------------------------------------------------------------------===// const std::unordered_map> - GetLayouts(oid_t table_oid, concurrency::TransactionContext *txn); + GetLayouts(concurrency::TransactionContext *txn, oid_t table_oid); - std::shared_ptr GetLayoutWithOid( - oid_t table_oid, oid_t layout_oid, concurrency::TransactionContext *txn); + std::shared_ptr GetLayoutWithOid(concurrency::TransactionContext *txn, + oid_t table_oid, + oid_t layout_oid); private: std::unique_ptr InitializeSchema(); @@ -62,7 +65,7 @@ class LayoutCatalog : public AbstractCatalog { COLUMN_MAP = 3, // Add new columns here in creation order }; - std::vector all_column_ids = {0, 1, 2, 3}; + std::vector all_column_ids_ = {0, 1, 2, 3}; enum IndexId { PRIMARY_KEY = 0, diff --git a/src/include/catalog/manager.h b/src/include/catalog/manager.h index 87aee3fcc2e..40ff5079cda 100644 --- a/src/include/catalog/manager.h +++ b/src/include/catalog/manager.h @@ -57,9 +57,9 @@ class Manager { void DropIndirectionArray(const oid_t oid); - void ClearIndirectionArray(void); + void ClearIndirectionArray(); - Manager(Manager const &) = delete; + DISALLOW_COPY(Manager) private: diff --git a/src/include/catalog/multi_constraint.h b/src/include/catalog/multi_constraint.h index 9a5e6e13f9d..46b254ed90e 100644 --- a/src/include/catalog/multi_constraint.h +++ b/src/include/catalog/multi_constraint.h @@ -30,26 +30,26 @@ namespace catalog { class MultiConstraint : public Printable { public: MultiConstraint(ConstraintType type, std::string constraint_name) - : constraint_type(type), constraint_name(constraint_name){}; + : constraint_type_(type), constraint_name_(constraint_name){}; MultiConstraint(ConstraintType type, std::string constraint_name, std::vector column_ids) - : constraint_type(type), constraint_name(constraint_name) { - this->column_ids = column_ids; + : constraint_type_(type), constraint_name_(constraint_name) { + this->column_ids_ = column_ids; }; //===--------------------------------------------------------------------===// // ACCESSORS //===--------------------------------------------------------------------===// - ConstraintType GetType() const { return constraint_type; } + ConstraintType GetType() const { return constraint_type_; } - std::string GetName() const { return constraint_name; } + std::string GetName() const { return constraint_name_; } // Get a string representation for debugging const std::string GetInfo() const; - std::vector GetCols() const { return column_ids; } + std::vector GetCols() const { return column_ids_; } private: //===--------------------------------------------------------------------===// @@ -57,13 +57,13 @@ class MultiConstraint : public Printable { //===--------------------------------------------------------------------===// // The type of constraint - ConstraintType constraint_type = ConstraintType::INVALID; + ConstraintType constraint_type_ = ConstraintType::INVALID; // constraints on column set - std::vector column_ids; + std::vector column_ids_; // we do not allow duplicate constraint name in single table - std::string constraint_name; + std::string constraint_name_; }; } // namespace catalog diff --git a/src/include/catalog/proc_catalog.h b/src/include/catalog/proc_catalog.h index 75c1a666123..19fc39fbd43 100644 --- a/src/include/catalog/proc_catalog.h +++ b/src/include/catalog/proc_catalog.h @@ -34,14 +34,15 @@ namespace peloton { namespace catalog { // Forward declare -class LanguageCatalogObject; +class LanguageCatalogEntry; //===----------------------------------------------------------------------===// // In-memory representation of a row from the pg_proc table. //===----------------------------------------------------------------------===// -class ProcCatalogObject { +class ProcCatalogEntry { public: - ProcCatalogObject(executor::LogicalTile *tile, concurrency::TransactionContext *txn); + ProcCatalogEntry(concurrency::TransactionContext *txn, + executor::LogicalTile *tile); // Accessors @@ -55,7 +56,7 @@ class ProcCatalogObject { oid_t GetLangOid() const { return lang_oid_; } - std::unique_ptr GetLanguage() const; + std::unique_ptr GetLanguage() const; const std::string &GetSrc() const { return src_; } @@ -89,22 +90,24 @@ class ProcCatalog : public AbstractCatalog { //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// - bool InsertProc(const std::string &proname, type::TypeId prorettype, - const std::vector &proargtypes, oid_t prolang, - const std::string &prosrc, type::AbstractPool *pool, - concurrency::TransactionContext *txn); + bool InsertProc(concurrency::TransactionContext *txn, + const std::string &proname, + type::TypeId prorettype, + const std::vector &proargtypes, + oid_t prolang, + const std::string &prosrc, + type::AbstractPool *pool); //===--------------------------------------------------------------------===// // Read-only Related API //===--------------------------------------------------------------------===// - std::unique_ptr GetProcByOid( - oid_t proc_oid, concurrency::TransactionContext *txn) const; + std::unique_ptr GetProcByOid(concurrency::TransactionContext *txn, + oid_t proc_oid) const; - std::unique_ptr GetProcByName( - const std::string &proc_name, - const std::vector &proc_arg_types, - concurrency::TransactionContext *txn) const; + std::unique_ptr GetProcByName(concurrency::TransactionContext *txn, + const std::string &proc_name, + const std::vector &proc_arg_types) const; enum ColumnId { OID = 0, @@ -115,7 +118,7 @@ class ProcCatalog : public AbstractCatalog { PROSRC = 5, // Add new columns here in creation order }; - std::vector all_column_ids = {0, 1, 2, 3, 4, 5}; + std::vector all_column_ids_ = {0, 1, 2, 3, 4, 5}; private: ProcCatalog(concurrency::TransactionContext *txn); diff --git a/src/include/catalog/query_history_catalog.h b/src/include/catalog/query_history_catalog.h index 3f004508d02..0c69fe9c54e 100644 --- a/src/include/catalog/query_history_catalog.h +++ b/src/include/catalog/query_history_catalog.h @@ -41,10 +41,11 @@ class QueryHistoryCatalog : public AbstractCatalog { //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// - bool InsertQueryHistory(const std::string &query_string, - const std::string &fingerprint, uint64_t timestamp, - type::AbstractPool *pool, - concurrency::TransactionContext *txn); + bool InsertQueryHistory(concurrency::TransactionContext *txn, + const std::string &query_string, + const std::string &fingerprint, + uint64_t timestamp, + type::AbstractPool *pool); enum ColumnId { QUERY_STRING = 0, diff --git a/src/include/catalog/query_metrics_catalog.h b/src/include/catalog/query_metrics_catalog.h index a8045fd94e1..30e44ccbc43 100644 --- a/src/include/catalog/query_metrics_catalog.h +++ b/src/include/catalog/query_metrics_catalog.h @@ -43,32 +43,40 @@ namespace catalog { class QueryMetricsCatalog : public AbstractCatalog { public: - QueryMetricsCatalog(const std::string &database_name, - concurrency::TransactionContext *txn); + QueryMetricsCatalog(concurrency::TransactionContext *txn, + const std::string &database_name); ~QueryMetricsCatalog(); //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// - bool InsertQueryMetrics(const std::string &name, oid_t database_oid, + bool InsertQueryMetrics(concurrency::TransactionContext *txn, + const std::string &name, + oid_t database_oid, int64_t num_params, const stats::QueryMetric::QueryParamBuf &type_buf, const stats::QueryMetric::QueryParamBuf &format_buf, const stats::QueryMetric::QueryParamBuf &value_buf, - int64_t reads, int64_t updates, int64_t deletes, - int64_t inserts, int64_t latency, int64_t cpu_time, - int64_t time_stamp, type::AbstractPool *pool, - concurrency::TransactionContext *txn); - bool DeleteQueryMetrics(const std::string &name, - concurrency::TransactionContext *txn); + int64_t reads, + int64_t updates, + int64_t deletes, + int64_t inserts, + int64_t latency, + int64_t cpu_time, + int64_t time_stamp, + type::AbstractPool *pool); + + bool DeleteQueryMetrics(concurrency::TransactionContext *txn, + const std::string &name); //===--------------------------------------------------------------------===// // Read-only Related API //===--------------------------------------------------------------------===// - stats::QueryMetric::QueryParamBuf GetParamTypes( - const std::string &name, concurrency::TransactionContext *txn); - int64_t GetNumParams(const std::string &name, - concurrency::TransactionContext *txn); + stats::QueryMetric::QueryParamBuf GetParamTypes(concurrency::TransactionContext *txn, + const std::string &name); + + int64_t GetNumParams(concurrency::TransactionContext *txn, + const std::string &name); // TODO: In theory, we don't need database_oid // but now we store all the query metrics under default database "peloton" enum ColumnId { diff --git a/src/include/catalog/schema.h b/src/include/catalog/schema.h index 2f6875b453b..e70c7636d5d 100644 --- a/src/include/catalog/schema.h +++ b/src/include/catalog/schema.h @@ -92,16 +92,16 @@ class Schema : public Printable { //===--------------------------------------------------------------------===// inline size_t GetOffset(const oid_t column_id) const { - return columns[column_id].GetOffset(); + return columns_[column_id].GetOffset(); } inline type::TypeId GetType(const oid_t column_id) const { - return columns[column_id].GetType(); + return columns_[column_id].GetType(); } // Return appropriate length based on whether column is inlined inline size_t GetAppropriateLength(const oid_t column_id) const { - auto is_inlined = columns[column_id].IsInlined(); + auto is_inlined = columns_[column_id].IsInlined(); size_t column_length; if (is_inlined) { @@ -115,19 +115,19 @@ class Schema : public Printable { // Returns fixed length inline size_t GetLength(const oid_t column_id) const { - return columns[column_id].GetLength(); + return columns_[column_id].GetLength(); } inline size_t GetVariableLength(const oid_t column_id) const { - return columns[column_id].GetVariableLength(); + return columns_[column_id].GetVariableLength(); } inline bool IsInlined(const oid_t column_id) const { - return columns[column_id].IsInlined(); + return columns_[column_id].IsInlined(); } inline const Column &GetColumn(const oid_t column_id) const { - return columns[column_id]; + return columns_[column_id]; } /** @@ -138,8 +138,8 @@ class Schema : public Printable { */ inline oid_t GetColumnID(std::string col_name) const { oid_t index = INVALID_OID; - for (oid_t i = 0, cnt = columns.size(); i < cnt; ++i) { - if (columns[i].GetName() == col_name) { + for (oid_t i = 0, cnt = columns_.size(); i < cnt; ++i) { + if (columns_[i].GetName() == col_name) { index = i; break; } @@ -148,23 +148,23 @@ class Schema : public Printable { } inline oid_t GetUninlinedColumn(const oid_t column_id) const { - return uninlined_columns[column_id]; + return uninlined_columns_[column_id]; } - inline const std::vector &GetColumns() const { return columns; } + inline const std::vector &GetColumns() const { return columns_; } // Return the number of columns in the schema for the tuple. - inline size_t GetColumnCount() const { return column_count; } + inline size_t GetColumnCount() const { return column_count_; } inline oid_t GetUninlinedColumnCount() const { - return uninlined_column_count; + return uninlined_column_count_; } // Return the number of bytes used by one tuple. - inline oid_t GetLength() const { return length; } + inline oid_t GetLength() const { return length_; } // Returns a flag indicating whether all columns are inlined - inline bool IsInlined() const { return tuple_is_inlined; } + inline bool IsInlined() const { return tuple_is_inlined_; } inline void SetIndexedColumns(const std::vector &indexed_columns) { indexed_columns_ = indexed_columns; @@ -176,7 +176,7 @@ class Schema : public Printable { // Get the nullability of the column at a given index. inline bool AllowNull(const oid_t column_id) const { - for (auto constraint : columns[column_id].GetConstraints()) { + for (auto constraint : columns_[column_id].GetConstraints()) { if (constraint.GetType() == ConstraintType::NOTNULL) return false; } return true; @@ -184,7 +184,7 @@ class Schema : public Printable { // For single column default inline bool AllowDefault(const oid_t column_id) const { - for (auto constraint : columns[column_id].GetConstraints()) { + for (auto constraint : columns_[column_id].GetConstraints()) { if (constraint.GetType() == ConstraintType::DEFAULT) { return true; } @@ -194,8 +194,8 @@ class Schema : public Printable { } // Get the default value for the column - inline type::Value* GetDefaultValue(const oid_t column_id) const { - for (auto constraint : columns[column_id].GetConstraints()) { + inline type::Value *GetDefaultValue(const oid_t column_id) const { + for (auto constraint : columns_[column_id].GetConstraints()) { if (constraint.GetType() == ConstraintType::DEFAULT) { return constraint.getDefaultValue(); } @@ -207,25 +207,25 @@ class Schema : public Printable { // Add constraint for column by id inline void AddConstraint(oid_t column_id, const catalog::Constraint &constraint) { - columns[column_id].AddConstraint(constraint); + columns_[column_id].AddConstraint(constraint); } // Add constraint for column by name inline void AddConstraint(std::string column_name, const catalog::Constraint &constraint) { - for (size_t column_itr = 0; column_itr < columns.size(); column_itr++) { - if (columns[column_itr].GetName() == column_name) { - columns[column_itr].AddConstraint(constraint); + for (size_t column_itr = 0; column_itr < columns_.size(); column_itr++) { + if (columns_[column_itr].GetName() == column_name) { + columns_[column_itr].AddConstraint(constraint); } } } inline void AddMultiConstraints(const catalog::MultiConstraint &mc) { - multi_constraints.push_back(mc); + multi_constraints_.push_back(mc); } inline std::vector GetMultiConstraints() { - return multi_constraints; + return multi_constraints_; } // Get a string representation for debugging @@ -233,24 +233,24 @@ class Schema : public Printable { private: // size of fixed length columns - size_t length; + size_t length_; // all inlined and uninlined columns in the tuple - std::vector columns; + std::vector columns_; // keeps track of unlined columns - std::vector uninlined_columns; + std::vector uninlined_columns_; // keeps multi_constraints - std::vector multi_constraints; + std::vector multi_constraints_; // keep these in sync with the vectors above - oid_t column_count = INVALID_OID; + oid_t column_count_ = INVALID_OID; - oid_t uninlined_column_count = INVALID_OID; + oid_t uninlined_column_count_ = INVALID_OID; // are all columns inlined - bool tuple_is_inlined; + bool tuple_is_inlined_; // keeps track of indexed columns in original table std::vector indexed_columns_; diff --git a/src/include/catalog/schema_catalog.h b/src/include/catalog/schema_catalog.h index 38a3f3cbb96..357afd911e6 100644 --- a/src/include/catalog/schema_catalog.h +++ b/src/include/catalog/schema_catalog.h @@ -31,32 +31,34 @@ namespace peloton { namespace catalog { -class SchemaCatalogObject { - friend class DatabaseCatalogObject; +class SchemaCatalogEntry { + friend class DatabaseCatalogEntry; public: - SchemaCatalogObject(executor::LogicalTile *tile, - concurrency::TransactionContext *txn); + SchemaCatalogEntry(concurrency::TransactionContext *txn, + executor::LogicalTile *tile); - inline oid_t GetSchemaOid() { return schema_oid; } - inline const std::string &GetSchemaName() { return schema_name; } + inline oid_t GetSchemaOid() { return schema_oid_; } + + inline const std::string &GetSchemaName() { return schema_name_; } private: // member variables - oid_t schema_oid; - std::string schema_name; + oid_t schema_oid_; + std::string schema_name_; // Pointer to its corresponding transaction // This object is only visible during this transaction - concurrency::TransactionContext *txn; + concurrency::TransactionContext *txn_; }; class SchemaCatalog : public AbstractCatalog { - friend class SchemaCatalogObject; + friend class SchemaCatalogEntry; friend class Catalog; public: - SchemaCatalog(storage::Database *peloton, type::AbstractPool *pool, - concurrency::TransactionContext *txn); + SchemaCatalog(concurrency::TransactionContext *txn, + storage::Database *peloton, + type::AbstractPool *pool); ~SchemaCatalog(); @@ -67,17 +69,19 @@ class SchemaCatalog : public AbstractCatalog { //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// - bool InsertSchema(oid_t schema_oid, const std::string &schema_name, - type::AbstractPool *pool, - concurrency::TransactionContext *txn); - bool DeleteSchema(const std::string &schema_name, - concurrency::TransactionContext *txn); + bool InsertSchema(concurrency::TransactionContext *txn, + oid_t schema_oid, + const std::string &schema_name, + type::AbstractPool *pool); + + bool DeleteSchema(concurrency::TransactionContext *txn, + const std::string &schema_name); //===--------------------------------------------------------------------===// // Read Related API //===--------------------------------------------------------------------===// - std::shared_ptr GetSchemaObject( - const std::string &schema_name, concurrency::TransactionContext *txn); + std::shared_ptr GetSchemaCatalogEntry(concurrency::TransactionContext *txn, + const std::string &schema_name); private: std::unique_ptr InitializeSchema(); @@ -87,7 +91,7 @@ class SchemaCatalog : public AbstractCatalog { SCHEMA_NAME = 1, // Add new columns here in creation order }; - std::vector all_column_ids = {0, 1}; + std::vector all_column_ids_ = {0, 1}; enum IndexId { PRIMARY_KEY = 0, diff --git a/src/include/catalog/settings_catalog.h b/src/include/catalog/settings_catalog.h index a3c8b1bf6df..3441658ef0a 100644 --- a/src/include/catalog/settings_catalog.h +++ b/src/include/catalog/settings_catalog.h @@ -27,23 +27,29 @@ class SettingsCatalog : public AbstractCatalog { //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// - bool InsertSetting(const std::string &name, const std::string &value, - type::TypeId value_type, const std::string &description, - const std::string &min_value, const std::string &max_value, - const std::string &default_value, bool is_mutable, - bool is_persistent, type::AbstractPool *pool, - concurrency::TransactionContext *txn); + bool InsertSetting(concurrency::TransactionContext *txn, + const std::string &name, + const std::string &value, + type::TypeId value_type, + const std::string &description, + const std::string &min_value, + const std::string &max_value, + const std::string &default_value, + bool is_mutable, + bool is_persistent, + type::AbstractPool *pool); - bool DeleteSetting(const std::string &name, concurrency::TransactionContext *txn); + bool DeleteSetting(concurrency::TransactionContext *txn, + const std::string &name); //===--------------------------------------------------------------------===// // Read-only Related API //===--------------------------------------------------------------------===// - std::string GetSettingValue(const std::string &name, - concurrency::TransactionContext *txn); + std::string GetSettingValue(concurrency::TransactionContext *txn, + const std::string &name); - std::string GetDefaultValue(const std::string &name, - concurrency::TransactionContext *txn); + std::string GetDefaultValue(concurrency::TransactionContext *txn, + const std::string &name); enum class ColumnId { NAME = 0, diff --git a/src/include/catalog/system_catalogs.h b/src/include/catalog/system_catalogs.h index 9792d180f9d..1d3e8c081dc 100644 --- a/src/include/catalog/system_catalogs.h +++ b/src/include/catalog/system_catalogs.h @@ -40,13 +40,14 @@ class SystemCatalogs { public: SystemCatalogs() = delete; - SystemCatalogs(storage::Database *database, type::AbstractPool *pool, - concurrency::TransactionContext *txn); + SystemCatalogs(concurrency::TransactionContext *txn, + storage::Database *database, + type::AbstractPool *pool); ~SystemCatalogs(); - void Bootstrap(const std::string &database_name, - concurrency::TransactionContext *txn); + void Bootstrap(concurrency::TransactionContext *txn, + const std::string &database_name); //===--------------------------------------------------------------------===// // GET FUNCTIONS diff --git a/src/include/catalog/table_catalog.h b/src/include/catalog/table_catalog.h index c4698f90a1c..e22af23b262 100644 --- a/src/include/catalog/table_catalog.h +++ b/src/include/catalog/table_catalog.h @@ -43,40 +43,47 @@ class Layout; namespace catalog { -class IndexCatalogObject; -class ColumnCatalogObject; +class IndexCatalogEntry; +class ColumnCatalogEntry; -class TableCatalogObject { +class TableCatalogEntry { friend class TableCatalog; friend class IndexCatalog; friend class ColumnCatalog; friend class LayoutCatalog; public: - TableCatalogObject(executor::LogicalTile *tile, - concurrency::TransactionContext *txn, int tupleId = 0); + TableCatalogEntry(concurrency::TransactionContext *txn, + executor::LogicalTile *tile, + int tupleId = 0); public: // Get indexes - void EvictAllIndexObjects(); - std::unordered_map> - GetIndexObjects(bool cached_only = false); - std::unordered_map> - GetIndexNames(bool cached_only = false); - std::shared_ptr GetIndexObject(oid_t index_oid, - bool cached_only = false); - std::shared_ptr GetIndexObject( + void EvictAllIndexCatalogEntries(); + + std::unordered_map> + GetIndexCatalogEntries(bool cached_only = false); + + + std::shared_ptr GetIndexCatalogEntries(oid_t index_oid, + bool cached_only = false); + + std::shared_ptr GetIndexCatalogEntry( const std::string &index_name, bool cached_only = false); // Get columns - void EvictAllColumnObjects(); - std::unordered_map> - GetColumnObjects(bool cached_only = false); - std::unordered_map> - GetColumnNames(bool cached_only = false); - std::shared_ptr GetColumnObject( + void EvictAllColumnCatalogEntries(); + + std::unordered_map> + GetColumnCatalogEntries(bool cached_only = false); + + std::unordered_map> + GetColumnCatalogEntriesByName(bool cached_only = false); + + std::shared_ptr GetColumnCatalogEntry( oid_t column_id, bool cached_only = false); - std::shared_ptr GetColumnObject( + + std::shared_ptr GetColumnCatalogEntry( const std::string &column_name, bool cached_only = false); // Evict all layouts from the cache @@ -105,14 +112,14 @@ class TableCatalogObject { oid_t default_layout_oid; // Get index objects - bool InsertIndexObject(std::shared_ptr index_object); - bool EvictIndexObject(oid_t index_oid); - bool EvictIndexObject(const std::string &index_name); + bool InsertIndexCatalogEntry(std::shared_ptr index_catalog_entry); + bool EvictIndexCatalogEntry(oid_t index_oid); + bool EvictIndexCatalogEntry(const std::string &index_name); // Get column objects - bool InsertColumnObject(std::shared_ptr column_object); - bool EvictColumnObject(oid_t column_id); - bool EvictColumnObject(const std::string &column_name); + bool InsertColumnCatalogEntry(std::shared_ptr column_catalog_entry); + bool EvictColumnCatalogEntry(oid_t column_id); + bool EvictColumnCatalogEntry(const std::string &column_name); // Insert layout into table object bool InsertLayout(std::shared_ptr layout); @@ -120,38 +127,39 @@ class TableCatalogObject { bool EvictLayout(oid_t layout_id); // cache for *all* index catalog objects in this table - std::unordered_map> index_objects; - std::unordered_map> - index_names; - bool valid_index_objects; + std::unordered_map> index_catalog_entries; + std::unordered_map> + index_catalog_entries_by_name_; + bool valid_index_catalog_entries_; // cache for *all* column catalog objects in this table - std::unordered_map> - column_objects; - std::unordered_map> - column_names; - bool valid_column_objects; + std::unordered_map> + column_catalog_entries_; + std::unordered_map> + column_names_; + bool valid_column_catalog_entries_; // cache for *all* layout objects in the table std::unordered_map> - layout_objects_; - bool valid_layout_objects_; + layout_catalog_entries_; + bool valid_layout_catalog_entries_; // Pointer to its corresponding transaction - concurrency::TransactionContext *txn; + concurrency::TransactionContext *txn_; }; class TableCatalog : public AbstractCatalog { - friend class TableCatalogObject; - friend class DatabaseCatalogObject; + friend class TableCatalogEntry; + friend class DatabaseCatalogEntry; friend class ColumnCatalog; friend class IndexCatalog; friend class LayoutCatalog; friend class Catalog; public: - TableCatalog(storage::Database *pg_catalog, type::AbstractPool *pool, - concurrency::TransactionContext *txn); + TableCatalog(concurrency::TransactionContext *txn, + storage::Database *pg_catalog, + type::AbstractPool *pool); ~TableCatalog(); @@ -162,29 +170,37 @@ class TableCatalog : public AbstractCatalog { //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// - bool InsertTable(oid_t table_oid, const std::string &table_name, - const std::string &schema_name, oid_t database_oid, - oid_t layout_oid, type::AbstractPool *pool, - concurrency::TransactionContext *txn); - bool DeleteTable(oid_t table_oid, concurrency::TransactionContext *txn); + bool InsertTable(concurrency::TransactionContext *txn, + oid_t database_oid, + const std::string &schema_name, + oid_t table_oid, + const std::string &table_name, + oid_t layout_oid, + type::AbstractPool *pool); + + bool DeleteTable(concurrency::TransactionContext *txn, oid_t table_oid); - bool UpdateVersionId(oid_t update_val, oid_t table_oid, - concurrency::TransactionContext *txn); + bool UpdateVersionId(concurrency::TransactionContext *txn, + oid_t table_oid, + oid_t update_val); - bool UpdateDefaultLayoutOid(oid_t update_val, oid_t table_oid, - concurrency::TransactionContext *txn); + bool UpdateDefaultLayoutOid(concurrency::TransactionContext *txn, + oid_t table_oid, + oid_t update_val); //===--------------------------------------------------------------------===// // Read Related API //===--------------------------------------------------------------------===// private: - std::shared_ptr GetTableObject( - oid_t table_oid, concurrency::TransactionContext *txn); - std::shared_ptr GetTableObject( - const std::string &table_name, const std::string &schema_name, - concurrency::TransactionContext *txn); - std::unordered_map> - GetTableObjects(concurrency::TransactionContext *txn); + std::shared_ptr GetTableCatalogEntry(concurrency::TransactionContext *txn, + oid_t table_oid); + + std::shared_ptr GetTableCatalogEntry(concurrency::TransactionContext *txn, + const std::string &schema_name, + const std::string &table_name); + + std::unordered_map> + GetTableCatalogEntries(concurrency::TransactionContext *txn); std::unique_ptr InitializeSchema(); @@ -197,7 +213,7 @@ class TableCatalog : public AbstractCatalog { DEFAULT_LAYOUT_OID = 5, // Add new columns here in creation order }; - std::vector all_column_ids = {0, 1, 2, 3, 4, 5}; + std::vector all_column_ids_ = {0, 1, 2, 3, 4, 5}; enum IndexId { PRIMARY_KEY = 0, diff --git a/src/include/catalog/table_metrics_catalog.h b/src/include/catalog/table_metrics_catalog.h index 38c59a76857..ba2e8d1c5b1 100644 --- a/src/include/catalog/table_metrics_catalog.h +++ b/src/include/catalog/table_metrics_catalog.h @@ -38,19 +38,23 @@ namespace catalog { class TableMetricsCatalog : public AbstractCatalog { public: - TableMetricsCatalog(const std::string &database_name, - concurrency::TransactionContext *txn); + TableMetricsCatalog(concurrency::TransactionContext *txn, + const std::string &database_name); ~TableMetricsCatalog(); //===--------------------------------------------------------------------===// // Write Related API //===--------------------------------------------------------------------===// - bool InsertTableMetrics(oid_t table_oid, int64_t reads, int64_t updates, - int64_t deletes, int64_t inserts, int64_t time_stamp, - type::AbstractPool *pool, - concurrency::TransactionContext *txn); - bool DeleteTableMetrics(oid_t table_oid, - concurrency::TransactionContext *txn); + bool InsertTableMetrics(concurrency::TransactionContext *txn, + oid_t table_oid, + int64_t reads, + int64_t updates, + int64_t deletes, + int64_t inserts, + int64_t time_stamp, + type::AbstractPool *pool); + + bool DeleteTableMetrics(concurrency::TransactionContext *txn, oid_t table_oid); //===--------------------------------------------------------------------===// // Read-only Related API diff --git a/src/include/catalog/trigger_catalog.h b/src/include/catalog/trigger_catalog.h index 18b2a774868..83088727aa8 100644 --- a/src/include/catalog/trigger_catalog.h +++ b/src/include/catalog/trigger_catalog.h @@ -47,8 +47,8 @@ namespace catalog { class TriggerCatalog : public AbstractCatalog { public: - TriggerCatalog(const std::string &database_name, - concurrency::TransactionContext *txn); + TriggerCatalog(concurrency::TransactionContext *txn, + const std::string &database_name); ~TriggerCatalog(); oid_t GetNextOid() { return oid_++ | TRIGGER_OID_MASK; } @@ -58,35 +58,42 @@ class TriggerCatalog : public AbstractCatalog { //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// - bool InsertTrigger(oid_t table_oid, std::string trigger_name, - int16_t trigger_type, std::string proc_oid, - std::string function_arguments, type::Value fire_condition, - type::Value timestamp, type::AbstractPool *pool, - concurrency::TransactionContext *txn); - - ResultType DropTrigger(const oid_t database_oid, const oid_t table_oid, - const std::string &trigger_name, - concurrency::TransactionContext *txn); - - bool DeleteTriggerByName(const std::string &trigger_name, oid_t table_oid, - concurrency::TransactionContext *txn); + bool InsertTrigger(concurrency::TransactionContext *txn, + oid_t table_oid, + std::string trigger_name, + int16_t trigger_type, + std::string proc_oid, + std::string function_arguments, + type::Value fire_condition, + type::Value timestamp, + type::AbstractPool *pool); + + ResultType DropTrigger(concurrency::TransactionContext *txn, + const oid_t database_oid, + const oid_t table_oid, + const std::string &trigger_name); + + bool DeleteTriggerByName(concurrency::TransactionContext *txn, + oid_t table_oid, + const std::string &trigger_name); //===--------------------------------------------------------------------===// // get triggers for a specific table; one table may have multiple triggers // of the same type //===--------------------------------------------------------------------===// - std::unique_ptr GetTriggersByType( - oid_t table_oid, int16_t trigger_type, - concurrency::TransactionContext *txn); + std::unique_ptr GetTriggersByType(concurrency::TransactionContext *txn, + oid_t table_oid, + int16_t trigger_type); //===--------------------------------------------------------------------===// // get all types of triggers for a specific table //===--------------------------------------------------------------------===// - std::unique_ptr GetTriggers( - oid_t table_oid, concurrency::TransactionContext *txn); + std::unique_ptr GetTriggers(concurrency::TransactionContext *txn, + oid_t table_oid); - oid_t GetTriggerOid(std::string trigger_name, oid_t table_oid, - concurrency::TransactionContext *txn); + oid_t GetTriggerOid(concurrency::TransactionContext *txn, + oid_t table_oid, + std::string trigger_name); enum ColumnId { TRIGGER_OID = 0, diff --git a/src/include/catalog/zone_map_catalog.h b/src/include/catalog/zone_map_catalog.h index a38d26d0597..9933652b1b8 100644 --- a/src/include/catalog/zone_map_catalog.h +++ b/src/include/catalog/zone_map_catalog.h @@ -36,23 +36,31 @@ class ZoneMapCatalog : public AbstractCatalog { //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// - bool InsertColumnStatistics(oid_t database_id, oid_t table_id, - oid_t tile_group_id, oid_t column_id, - std::string minimum, std::string maximum, - std::string type, type::AbstractPool *pool, - concurrency::TransactionContext *txn); - - bool DeleteColumnStatistics(oid_t database_id, oid_t table_id, - oid_t tile_group_id, oid_t column_id, - concurrency::TransactionContext *txn); + bool InsertColumnStatistics(concurrency::TransactionContext *txn, + oid_t database_id, + oid_t table_id, + oid_t tile_group_id, + oid_t column_id, + std::string minimum, + std::string maximum, + std::string type, + type::AbstractPool *pool); + + bool DeleteColumnStatistics(concurrency::TransactionContext *txn, + oid_t database_id, + oid_t table_id, + oid_t tile_group_id, + oid_t column_id); //===--------------------------------------------------------------------===// // Read-only Related API //===--------------------------------------------------------------------===// - std::unique_ptr> GetColumnStatistics( - oid_t database_id, oid_t table_id, oid_t tile_group_id, oid_t column_id, - concurrency::TransactionContext *txn); + std::unique_ptr> GetColumnStatistics(concurrency::TransactionContext *txn, + oid_t database_id, + oid_t table_id, + oid_t tile_group_id, + oid_t column_id); enum class ColumnId { DATABASE_ID = 0, @@ -64,10 +72,10 @@ class ZoneMapCatalog : public AbstractCatalog { TYPE = 6 }; - enum class ZoneMapOffset { - MINIMUM_OFF = 0, - MAXIMUM_OFF = 1, - TYPE_OFF = 2 + enum class ZoneMapOffset { + MINIMUM_OFF = 0, + MAXIMUM_OFF = 1, + TYPE_OFF = 2 }; private: diff --git a/src/include/optimizer/operators.h b/src/include/optimizer/operators.h index fb5242ba8e3..c8a8483d242 100644 --- a/src/include/optimizer/operators.h +++ b/src/include/optimizer/operators.h @@ -30,7 +30,7 @@ class UpdateClause; } namespace catalog { -class TableCatalogObject; +class TableCatalogEntry; } namespace optimizer { @@ -52,7 +52,7 @@ class LogicalGet : public OperatorNode { public: static Operator make( oid_t get_id = 0, std::vector predicates = {}, - std::shared_ptr table = nullptr, + std::shared_ptr table = nullptr, std::string alias = "", bool update = false); bool operator==(const BaseOperatorNode &r) override; @@ -62,7 +62,7 @@ class LogicalGet : public OperatorNode { // identifier for all get operators oid_t get_id; std::vector predicates; - std::shared_ptr table; + std::shared_ptr table; std::string table_alias; bool is_for_update; }; @@ -267,12 +267,12 @@ class LogicalAggregateAndGroupBy class LogicalInsert : public OperatorNode { public: static Operator make( - std::shared_ptr target_table, + std::shared_ptr target_table, const std::vector *columns, const std::vector>> *values); - std::shared_ptr target_table; + std::shared_ptr target_table; const std::vector *columns; const std::vector< std::vector>> *values; @@ -281,9 +281,9 @@ class LogicalInsert : public OperatorNode { class LogicalInsertSelect : public OperatorNode { public: static Operator make( - std::shared_ptr target_table); + std::shared_ptr target_table); - std::shared_ptr target_table; + std::shared_ptr target_table; }; //===--------------------------------------------------------------------===// @@ -319,9 +319,9 @@ class LogicalLimit : public OperatorNode { class LogicalDelete : public OperatorNode { public: static Operator make( - std::shared_ptr target_table); + std::shared_ptr target_table); - std::shared_ptr target_table; + std::shared_ptr target_table; }; //===--------------------------------------------------------------------===// @@ -330,10 +330,10 @@ class LogicalDelete : public OperatorNode { class LogicalUpdate : public OperatorNode { public: static Operator make( - std::shared_ptr target_table, + std::shared_ptr target_table, const std::vector> *updates); - std::shared_ptr target_table; + std::shared_ptr target_table; const std::vector> *updates; }; @@ -371,7 +371,7 @@ class DummyScan : public OperatorNode { class PhysicalSeqScan : public OperatorNode { public: static Operator make(oid_t get_id, - std::shared_ptr table, + std::shared_ptr table, std::string alias, std::vector predicates, bool update); @@ -385,7 +385,7 @@ class PhysicalSeqScan : public OperatorNode { std::vector predicates; std::string table_alias; bool is_for_update; - std::shared_ptr table_; + std::shared_ptr table_; }; //===--------------------------------------------------------------------===// @@ -394,7 +394,7 @@ class PhysicalSeqScan : public OperatorNode { class PhysicalIndexScan : public OperatorNode { public: static Operator make(oid_t get_id, - std::shared_ptr table, + std::shared_ptr table, std::string alias, std::vector predicates, bool update, oid_t index_id, std::vector key_column_id_list, @@ -410,7 +410,7 @@ class PhysicalIndexScan : public OperatorNode { std::vector predicates; std::string table_alias; bool is_for_update; - std::shared_ptr table_; + std::shared_ptr table_; // Index info. // Match planner::IndexScanPlan::IndexScanDesc index_scan_desc( @@ -599,12 +599,12 @@ class PhysicalOuterHashJoin : public OperatorNode { class PhysicalInsert : public OperatorNode { public: static Operator make( - std::shared_ptr target_table, + std::shared_ptr target_table, const std::vector *columns, const std::vector>> *values); - std::shared_ptr target_table; + std::shared_ptr target_table; const std::vector *columns; const std::vector< std::vector>> *values; @@ -613,9 +613,9 @@ class PhysicalInsert : public OperatorNode { class PhysicalInsertSelect : public OperatorNode { public: static Operator make( - std::shared_ptr target_table); + std::shared_ptr target_table); - std::shared_ptr target_table; + std::shared_ptr target_table; }; //===--------------------------------------------------------------------===// @@ -624,8 +624,8 @@ class PhysicalInsertSelect : public OperatorNode { class PhysicalDelete : public OperatorNode { public: static Operator make( - std::shared_ptr target_table); - std::shared_ptr target_table; + std::shared_ptr target_table); + std::shared_ptr target_table; }; //===--------------------------------------------------------------------===// @@ -634,10 +634,10 @@ class PhysicalDelete : public OperatorNode { class PhysicalUpdate : public OperatorNode { public: static Operator make( - std::shared_ptr target_table, + std::shared_ptr target_table, const std::vector> *updates); - std::shared_ptr target_table; + std::shared_ptr target_table; const std::vector> *updates; }; diff --git a/src/include/optimizer/plan_generator.h b/src/include/optimizer/plan_generator.h index 9fba272d4a8..6c020a38783 100644 --- a/src/include/optimizer/plan_generator.h +++ b/src/include/optimizer/plan_generator.h @@ -109,7 +109,7 @@ class PlanGenerator : public OperatorVisitor { */ std::vector> GenerateTableTVExprs(const std::string &alias, - std::shared_ptr table); + std::shared_ptr table); /** * @brief Generate the column oids vector for a scan plan @@ -131,7 +131,7 @@ class PlanGenerator : public OperatorVisitor { std::unique_ptr GeneratePredicateForScan( const std::shared_ptr predicate_expr, const std::string &alias, - std::shared_ptr table); + std::shared_ptr table); /** * @brief Generate projection info and projection schema for join diff --git a/src/main/tpch/tpch_database.cpp b/src/main/tpch/tpch_database.cpp index 4c93c91c4dd..5252104cc88 100644 --- a/src/main/tpch/tpch_database.cpp +++ b/src/main/tpch/tpch_database.cpp @@ -104,7 +104,7 @@ TPCHDatabase::TPCHDatabase(const Configuration &c) : config_(c) { TPCHDatabase::~TPCHDatabase() { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto *txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithOid(kTPCHDatabaseId, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithOid(txn, kTPCHDatabaseId); txn_manager.CommitTransaction(txn); } diff --git a/src/optimizer/child_property_deriver.cpp b/src/optimizer/child_property_deriver.cpp index 62b0c95288d..b432067fae1 100644 --- a/src/optimizer/child_property_deriver.cpp +++ b/src/optimizer/child_property_deriver.cpp @@ -50,7 +50,7 @@ void ChildPropertyDeriver::Visit(const PhysicalSeqScan *) { void ChildPropertyDeriver::Visit(const PhysicalIndexScan *op) { auto provided_prop = make_shared(); - std::shared_ptr target_table = op->table_; + std::shared_ptr target_table = op->table_; for (auto prop : requirements_->Properties()) { if (prop->Type() == PropertyType::SORT) { // Walk through all indices in the table, check if any of the index could @@ -69,7 +69,7 @@ void ChildPropertyDeriver::Visit(const PhysicalIndexScan *op) { } } if (!can_fulfill) break; - for (auto &index : target_table->GetIndexObjects()) { + for (auto &index : target_table->GetIndexCatalogEntries()) { auto key_oids = index.second->GetKeyAttrs(); // If the sort column size is larger, then can't be fulfill by the index if (sort_col_size > key_oids.size()) { diff --git a/src/optimizer/operators.cpp b/src/optimizer/operators.cpp index c669125c191..52cf83f9a8c 100644 --- a/src/optimizer/operators.cpp +++ b/src/optimizer/operators.cpp @@ -32,7 +32,7 @@ Operator LeafOperator::make(GroupID group) { //===--------------------------------------------------------------------===// Operator LogicalGet::make(oid_t get_id, std::vector predicates, - std::shared_ptr table, + std::shared_ptr table, std::string alias, bool update) { LogicalGet *get = new LogicalGet; get->table = table; @@ -391,7 +391,7 @@ hash_t LogicalAggregateAndGroupBy::Hash() const { // Insert //===--------------------------------------------------------------------===// Operator LogicalInsert::make( - std::shared_ptr target_table, + std::shared_ptr target_table, const std::vector *columns, const std::vector>> *values) { @@ -403,7 +403,7 @@ Operator LogicalInsert::make( } Operator LogicalInsertSelect::make( - std::shared_ptr target_table) { + std::shared_ptr target_table) { LogicalInsertSelect *insert_op = new LogicalInsertSelect; insert_op->target_table = target_table; return Operator(insert_op); @@ -413,7 +413,7 @@ Operator LogicalInsertSelect::make( // Delete //===--------------------------------------------------------------------===// Operator LogicalDelete::make( - std::shared_ptr target_table) { + std::shared_ptr target_table) { LogicalDelete *delete_op = new LogicalDelete; delete_op->target_table = target_table; return Operator(delete_op); @@ -423,7 +423,7 @@ Operator LogicalDelete::make( // Update //===--------------------------------------------------------------------===// Operator LogicalUpdate::make( - std::shared_ptr target_table, + std::shared_ptr target_table, const std::vector> * updates) { LogicalUpdate *update_op = new LogicalUpdate; @@ -502,7 +502,7 @@ Operator DummyScan::make() { // SeqScan //===--------------------------------------------------------------------===// Operator PhysicalSeqScan::make( - oid_t get_id, std::shared_ptr table, + oid_t get_id, std::shared_ptr table, std::string alias, std::vector predicates, bool update) { PELOTON_ASSERT(table != nullptr); @@ -539,7 +539,7 @@ hash_t PhysicalSeqScan::Hash() const { // IndexScan //===--------------------------------------------------------------------===// Operator PhysicalIndexScan::make( - oid_t get_id, std::shared_ptr table, + oid_t get_id, std::shared_ptr table, std::string alias, std::vector predicates, bool update, oid_t index_id, std::vector key_column_id_list, std::vector expr_type_list, @@ -832,7 +832,7 @@ Operator PhysicalOuterHashJoin::make( // PhysicalInsert //===--------------------------------------------------------------------===// Operator PhysicalInsert::make( - std::shared_ptr target_table, + std::shared_ptr target_table, const std::vector *columns, const std::vector>> *values) { @@ -847,7 +847,7 @@ Operator PhysicalInsert::make( // PhysicalInsertSelect //===--------------------------------------------------------------------===// Operator PhysicalInsertSelect::make( - std::shared_ptr target_table) { + std::shared_ptr target_table) { PhysicalInsertSelect *insert_op = new PhysicalInsertSelect; insert_op->target_table = target_table; return Operator(insert_op); @@ -857,7 +857,7 @@ Operator PhysicalInsertSelect::make( // PhysicalDelete //===--------------------------------------------------------------------===// Operator PhysicalDelete::make( - std::shared_ptr target_table) { + std::shared_ptr target_table) { PhysicalDelete *delete_op = new PhysicalDelete; delete_op->target_table = target_table; return Operator(delete_op); @@ -867,7 +867,7 @@ Operator PhysicalDelete::make( // PhysicalUpdate //===--------------------------------------------------------------------===// Operator PhysicalUpdate::make( - std::shared_ptr target_table, + std::shared_ptr target_table, const std::vector> * updates) { PhysicalUpdate *update = new PhysicalUpdate; diff --git a/src/optimizer/optimizer.cpp b/src/optimizer/optimizer.cpp index 2525915fcc1..9087f826dae 100644 --- a/src/optimizer/optimizer.cpp +++ b/src/optimizer/optimizer.cpp @@ -165,15 +165,19 @@ unique_ptr Optimizer::HandleDDLStatement( if (create_plan->GetCreateType() == peloton::CreateType::INDEX) { auto create_stmt = (parser::CreateStatement *)tree; auto target_table = catalog::Catalog::GetInstance()->GetTableWithName( - create_stmt->GetDatabaseName(), create_stmt->GetSchemaName(), - create_stmt->GetTableName(), txn); + txn, + create_stmt->GetDatabaseName(), + create_stmt->GetSchemaName(), + create_stmt->GetTableName()); std::vector column_ids; // use catalog object instead of schema to acquire metadata - auto table_object = catalog::Catalog::GetInstance()->GetTableObject( - create_stmt->GetDatabaseName(), create_stmt->GetSchemaName(), - create_stmt->GetTableName(), txn); + auto table_object = + catalog::Catalog::GetInstance()->GetTableCatalogEntry(txn, + create_stmt->GetDatabaseName(), + create_stmt->GetSchemaName(), + create_stmt->GetTableName()); for (auto column_name : create_plan->GetIndexAttributes()) { - auto column_object = table_object->GetColumnObject(column_name); + auto column_object = table_object->GetColumnCatalogEntry(column_name); // Check if column is missing if (column_object == nullptr) throw CatalogException( diff --git a/src/optimizer/plan_generator.cpp b/src/optimizer/plan_generator.cpp index c73cd4721bc..fd1824360e3 100644 --- a/src/optimizer/plan_generator.cpp +++ b/src/optimizer/plan_generator.cpp @@ -383,7 +383,7 @@ void PlanGenerator::Visit(const PhysicalUpdate *op) { // Evaluate update expression and add to target list for (auto &update : *(op->updates)) { auto column_name = update->column; - auto col_id = op->target_table->GetColumnObject(column_name)->GetColumnId(); + auto col_id = op->target_table->GetColumnCatalogEntry(column_name)->GetColumnId(); if (update_col_ids.find(col_id) != update_col_ids.end()) throw SyntaxException("Multiple assignments to same column " + column_name); @@ -395,7 +395,7 @@ void PlanGenerator::Visit(const PhysicalUpdate *op) { } // Add other columns to direct map - for (auto &column_id_obj_pair : op->target_table->GetColumnObjects()) { + for (auto &column_id_obj_pair : op->target_table->GetColumnCatalogEntries()) { auto &col_id = column_id_obj_pair.first; if (update_col_ids.find(col_id) == update_col_ids.end()) dml.emplace_back(col_id, std::pair(0, col_id)); @@ -423,12 +423,12 @@ void PlanGenerator::Visit(const PhysicalExportExternalFile *op) { /************************* Private Functions *******************************/ vector> PlanGenerator::GenerateTableTVExprs( - const std::string &alias, shared_ptr table) { + const std::string &alias, shared_ptr table) { // TODO(boweic): we seems to provide all columns here, in case where there are // a lot of attributes and we're only visiting a few this is not efficient oid_t db_id = table->GetDatabaseOid(); oid_t table_id = table->GetTableOid(); - auto column_objects = table->GetColumnObjects(); + auto column_objects = table->GetColumnCatalogEntries(); vector> exprs( column_objects.size()); for (auto &column_id_object_pair : column_objects) { @@ -466,7 +466,7 @@ vector PlanGenerator::GenerateColumnsForScan() { std::unique_ptr PlanGenerator::GeneratePredicateForScan( const std::shared_ptr predicate_expr, - const std::string &alias, shared_ptr table) { + const std::string &alias, shared_ptr table) { if (predicate_expr == nullptr) { return nullptr; } diff --git a/src/optimizer/query_to_operator_transformer.cpp b/src/optimizer/query_to_operator_transformer.cpp index 97f1365b4c4..c539826ae53 100644 --- a/src/optimizer/query_to_operator_transformer.cpp +++ b/src/optimizer/query_to_operator_transformer.cpp @@ -223,10 +223,11 @@ void QueryToOperatorTransformer::Visit(parser::TableRef *node) { // Single table else { if (node->list.size() == 1) node = node->list.at(0).get(); - std::shared_ptr target_table = - catalog::Catalog::GetInstance()->GetTableObject( - node->GetDatabaseName(), node->GetSchemaName(), - node->GetTableName(), txn_); + std::shared_ptr target_table = + catalog::Catalog::GetInstance()->GetTableCatalogEntry(txn_, + node->GetDatabaseName(), + node->GetSchemaName(), + node->GetTableName()); std::string table_alias = StringUtil::Lower(std::string(node->GetTableAlias())); output_expr_ = std::make_shared(LogicalGet::make( @@ -242,9 +243,11 @@ void QueryToOperatorTransformer::Visit(parser::CreateFunctionStatement *) {} void QueryToOperatorTransformer::Visit( UNUSED_ATTRIBUTE parser::CreateStatement *op) {} void QueryToOperatorTransformer::Visit(parser::InsertStatement *op) { - std::shared_ptr target_table = - catalog::Catalog::GetInstance()->GetTableObject( - op->GetDatabaseName(), op->GetSchemaName(), op->GetTableName(), txn_); + std::shared_ptr target_table = + catalog::Catalog::GetInstance()->GetTableCatalogEntry(txn_, + op->GetDatabaseName(), + op->GetSchemaName(), + op->GetTableName()); if (op->type == InsertType::SELECT) { auto insert_expr = std::make_shared( @@ -256,7 +259,7 @@ void QueryToOperatorTransformer::Visit(parser::InsertStatement *op) { } // column_objects represents the columns for the current table as defined in // its schema - auto column_objects = target_table->GetColumnObjects(); + auto column_objects = target_table->GetColumnCatalogEntries(); // INSERT INTO table_name VALUES (val1, val2, ...), (val_a, val_b, ...), ... if (op->columns.empty()) { for (const auto &values : op->insert_values) { @@ -292,7 +295,7 @@ void QueryToOperatorTransformer::Visit(parser::InsertStatement *op) { // set below contains names of columns mentioned in the insert statement std::unordered_set specified; - auto column_names = target_table->GetColumnNames(); + auto column_names = target_table->GetColumnCatalogEntriesByName(); for (const auto col : op->columns) { if (column_names.find(col) == column_names.end()) { @@ -320,8 +323,11 @@ void QueryToOperatorTransformer::Visit(parser::InsertStatement *op) { } void QueryToOperatorTransformer::Visit(parser::DeleteStatement *op) { - auto target_table = catalog::Catalog::GetInstance()->GetTableObject( - op->GetDatabaseName(), op->GetSchemaName(), op->GetTableName(), txn_); + auto target_table = + catalog::Catalog::GetInstance()->GetTableCatalogEntry(txn_, + op->GetDatabaseName(), + op->GetSchemaName(), + op->GetTableName()); std::shared_ptr table_scan; if (op->expr != nullptr) { std::vector predicates = @@ -346,9 +352,11 @@ void QueryToOperatorTransformer::Visit( void QueryToOperatorTransformer::Visit( UNUSED_ATTRIBUTE parser::TransactionStatement *op) {} void QueryToOperatorTransformer::Visit(parser::UpdateStatement *op) { - auto target_table = catalog::Catalog::GetInstance()->GetTableObject( - op->table->GetDatabaseName(), op->table->GetSchemaName(), - op->table->GetTableName(), txn_); + auto target_table = + catalog::Catalog::GetInstance()->GetTableCatalogEntry(txn_, + op->table->GetDatabaseName(), + op->table->GetSchemaName(), + op->table->GetTableName()); std::shared_ptr table_scan; auto update_expr = std::make_shared( @@ -382,9 +390,9 @@ void QueryToOperatorTransformer::Visit(parser::CopyStatement *op) { auto target_table = catalog::Catalog::GetInstance() - ->GetDatabaseObject(op->table->GetDatabaseName(), txn_) - ->GetTableObject(op->table->GetTableName(), - op->table->GetSchemaName()); + ->GetDatabaseCatalogEntry(txn_, op->table->GetDatabaseName()) + ->GetTableCatalogEntry(op->table->GetTableName(), + op->table->GetSchemaName()); auto insert_expr = std::make_shared( LogicalInsertSelect::make(target_table)); diff --git a/src/optimizer/rule_impls.cpp b/src/optimizer/rule_impls.cpp index 6d5a1a1ebce..8574e00f337 100644 --- a/src/optimizer/rule_impls.cpp +++ b/src/optimizer/rule_impls.cpp @@ -246,7 +246,7 @@ bool GetToIndexScan::Check(std::shared_ptr plan, const LogicalGet *get = plan->Op().As(); bool index_exist = false; if (get != nullptr && get->table != nullptr && - !get->table->GetIndexObjects().empty()) { + !get->table->GetIndexCatalogEntries().empty()) { index_exist = true; } return index_exist; @@ -281,7 +281,7 @@ void GetToIndexScan::Transform( } // Check whether any index can fulfill sort property if (sort_by_asc_base_column) { - for (auto &index_id_object_pair : get->table->GetIndexObjects()) { + for (auto &index_id_object_pair : get->table->GetIndexCatalogEntries()) { auto &index_id = index_id_object_pair.first; auto &index = index_id_object_pair.second; auto &index_col_ids = index->GetKeyAttrs(); @@ -350,7 +350,7 @@ void GetToIndexScan::Transform( auto column_ref = (expression::TupleValueExpression *)tv_expr; std::string col_name(column_ref->GetColumnName()); LOG_TRACE("Column name: %s", col_name.c_str()); - auto column_id = get->table->GetColumnObject(col_name)->GetColumnId(); + auto column_id = get->table->GetColumnCatalogEntry(col_name)->GetColumnId(); key_column_id_list.push_back(column_id); expr_type_list.push_back(expr_type); @@ -374,7 +374,7 @@ void GetToIndexScan::Transform( } // Loop predicates end // Find match index for the predicates - auto index_objects = get->table->GetIndexObjects(); + auto index_objects = get->table->GetIndexCatalogEntries(); for (auto &index_id_object_pair : index_objects) { auto &index_id = index_id_object_pair.first; auto &index_object = index_id_object_pair.second; diff --git a/src/optimizer/stats/stats_storage.cpp b/src/optimizer/stats/stats_storage.cpp index d1b2fed6b12..4231e95ee9e 100644 --- a/src/optimizer/stats/stats_storage.cpp +++ b/src/optimizer/stats/stats_storage.cpp @@ -117,12 +117,20 @@ void StatsStorage::InsertOrUpdateColumnStats( single_statement_txn = true; txn = txn_manager.BeginTransaction(); } - column_stats_catalog->DeleteColumnStats(database_id, table_id, column_id, - txn); - column_stats_catalog->InsertColumnStats( - database_id, table_id, column_id, num_rows, cardinality, frac_null, - most_common_vals, most_common_freqs, histogram_bounds, column_name, - has_index, pool_.get(), txn); + column_stats_catalog->DeleteColumnStats(txn, database_id, table_id, column_id); + column_stats_catalog->InsertColumnStats(txn, + database_id, + table_id, + column_id, + column_name, + num_rows, + frac_null, + most_common_vals, + most_common_freqs, + histogram_bounds, + cardinality, + has_index, + pool_.get()); if (single_statement_txn) { txn_manager.CommitTransaction(txn); @@ -140,8 +148,10 @@ std::shared_ptr StatsStorage::GetColumnStatsByID(oid_t database_id, auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); // std::unique_ptr> column_stats_vector - auto column_stats_vector = column_stats_catalog->GetColumnStats( - database_id, table_id, column_id, txn); + auto column_stats_vector = column_stats_catalog->GetColumnStats(txn, + database_id, + table_id, + column_id); txn_manager.CommitTransaction(txn); return ConvertVectorToColumnStats(database_id, table_id, column_id, @@ -220,7 +230,9 @@ std::shared_ptr StatsStorage::GetTableStats( oid_t database_id, oid_t table_id, concurrency::TransactionContext *txn) { auto column_stats_catalog = catalog::ColumnStatsCatalog::GetInstance(nullptr); std::map>> column_stats_map; - column_stats_catalog->GetTableStats(database_id, table_id, txn, + column_stats_catalog->GetTableStats(txn, + database_id, + table_id, column_stats_map); std::vector> column_stats_ptrs; @@ -244,7 +256,9 @@ std::shared_ptr StatsStorage::GetTableStats( concurrency::TransactionContext *txn) { auto column_stats_catalog = catalog::ColumnStatsCatalog::GetInstance(nullptr); std::map>> column_stats_map; - column_stats_catalog->GetTableStats(database_id, table_id, txn, + column_stats_catalog->GetTableStats(txn, + database_id, + table_id, column_stats_map); std::vector> column_stats_ptrs; diff --git a/src/optimizer/stats/tuple_samples_storage.cpp b/src/optimizer/stats/tuple_samples_storage.cpp index edb41c2f820..64bd55ddb21 100644 --- a/src/optimizer/stats/tuple_samples_storage.cpp +++ b/src/optimizer/stats/tuple_samples_storage.cpp @@ -44,7 +44,7 @@ TupleSamplesStorage::TupleSamplesStorage() { void TupleSamplesStorage::CreateSamplesDatabase() { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(SAMPLES_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, SAMPLES_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -65,13 +65,17 @@ void TupleSamplesStorage::AddSamplesTable( auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog->CreateTable(std::string(SAMPLES_DB_NAME), - std::string(DEFAULT_SCHEMA_NAME), samples_table_name, - std::move(schema_ptr), txn, is_catalog); - - auto samples_table = catalog->GetTableWithName( - std::string(SAMPLES_DB_NAME), std::string(DEFAULT_SCHEMA_NAME), - samples_table_name, txn); + catalog->CreateTable(txn, + std::string(SAMPLES_DB_NAME), + std::string(DEFAULT_SCHEMA_NAME), + std::move(schema_ptr), + samples_table_name, + is_catalog); + + auto samples_table = catalog->GetTableWithName(txn, + std::string(SAMPLES_DB_NAME), + std::string(DEFAULT_SCHEMA_NAME), + samples_table_name); for (auto &tuple : sampled_tuples) { InsertSampleTuple(samples_table, std::move(tuple), txn); @@ -93,9 +97,10 @@ ResultType TupleSamplesStorage::DeleteSamplesTable( GenerateSamplesTableName(database_id, table_id); ResultType result = ResultType::FAILURE; try { - result = catalog->DropTable(std::string(SAMPLES_DB_NAME), + result = catalog->DropTable(txn, + std::string(SAMPLES_DB_NAME), std::string(DEFAULT_SCHEMA_NAME), - samples_table_name, txn); + samples_table_name); } catch (CatalogException &e) { // Samples table does not exist, no need to drop } @@ -186,9 +191,10 @@ TupleSamplesStorage::GetTupleSamples(oid_t database_id, oid_t table_id) { GenerateSamplesTableName(database_id, table_id); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - auto data_table = catalog->GetTableWithName(std::string(SAMPLES_DB_NAME), + auto data_table = catalog->GetTableWithName(txn, + std::string(SAMPLES_DB_NAME), std::string(DEFAULT_SCHEMA_NAME), - samples_table_name, txn); + samples_table_name); auto col_count = data_table->GetSchema()->GetColumnCount(); std::vector column_ids; @@ -213,9 +219,10 @@ void TupleSamplesStorage::GetColumnSamples( GenerateSamplesTableName(database_id, table_id); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - auto data_table = catalog->GetTableWithName(std::string(SAMPLES_DB_NAME), + auto data_table = catalog->GetTableWithName(txn, + std::string(SAMPLES_DB_NAME), std::string(DEFAULT_SCHEMA_NAME), - samples_table_name, txn); + samples_table_name); std::vector column_ids({column_id}); auto result_tiles = GetTuplesWithSeqScan(data_table, column_ids, txn); diff --git a/src/planner/analyze_plan.cpp b/src/planner/analyze_plan.cpp index 5de53476b14..faec07c5853 100644 --- a/src/planner/analyze_plan.cpp +++ b/src/planner/analyze_plan.cpp @@ -25,8 +25,10 @@ AnalyzePlan::AnalyzePlan(std::string table_name, std::string schema_name, std::string database_name, concurrency::TransactionContext *txn) : table_name_(table_name) { - target_table_ = catalog::Catalog::GetInstance()->GetTableWithName( - database_name, schema_name, table_name, txn); + target_table_ = catalog::Catalog::GetInstance()->GetTableWithName(txn, + database_name, + schema_name, + table_name); } AnalyzePlan::AnalyzePlan(std::string table_name, std::string schema_name, @@ -34,8 +36,10 @@ AnalyzePlan::AnalyzePlan(std::string table_name, std::string schema_name, std::vector column_names, concurrency::TransactionContext *txn) : table_name_(table_name), column_names_(column_names) { - target_table_ = catalog::Catalog::GetInstance()->GetTableWithName( - database_name, schema_name, table_name, txn); + target_table_ = catalog::Catalog::GetInstance()->GetTableWithName(txn, + database_name, + schema_name, + table_name); } AnalyzePlan::AnalyzePlan(parser::AnalyzeStatement *analyze_stmt, @@ -45,9 +49,10 @@ AnalyzePlan::AnalyzePlan(parser::AnalyzeStatement *analyze_stmt, for (auto &name : analyze_stmt->GetColumnNames()) column_names_.push_back((char *)name.c_str()); if (!table_name_.empty()) { - target_table_ = catalog::Catalog::GetInstance()->GetTableWithName( - analyze_stmt->GetDatabaseName(), analyze_stmt->GetSchemaName(), - table_name_, txn); + target_table_ = catalog::Catalog::GetInstance()->GetTableWithName(txn, + analyze_stmt->GetDatabaseName(), + analyze_stmt->GetSchemaName(), + table_name_); } } diff --git a/src/planner/plan_util.cpp b/src/planner/plan_util.cpp index cb013fb1531..aecedb80486 100644 --- a/src/planner/plan_util.cpp +++ b/src/planner/plan_util.cpp @@ -57,8 +57,8 @@ const std::set PlanUtil::GetAffectedIndexes( schema_name = delete_stmt.GetSchemaName(); } auto indexes_map = catalog_cache.GetDatabaseObject(db_name) - ->GetTableObject(table_name, schema_name) - ->GetIndexObjects(); + ->GetTableCatalogEntry(table_name, schema_name) + ->GetIndexCatalogEntries(); for (auto &index : indexes_map) { index_oids.insert(index.first); } @@ -70,18 +70,19 @@ const std::set PlanUtil::GetAffectedIndexes( table_name = update_stmt.table->GetTableName(); schema_name = update_stmt.table->GetSchemaName(); auto table_object = catalog_cache.GetDatabaseObject(db_name) - ->GetTableObject(table_name, schema_name); + ->GetTableCatalogEntry(table_name, schema_name); auto &update_clauses = update_stmt.updates; std::set update_oids; for (const auto &update_clause : update_clauses) { LOG_TRACE("Affected column name for table(%s) in UPDATE query: %s", table_name.c_str(), update_clause->column.c_str()); - auto col_object = table_object->GetColumnObject(update_clause->column); + auto col_object = + table_object->GetColumnCatalogEntry(update_clause->column); update_oids.insert(col_object->GetColumnId()); } - auto indexes_map = table_object->GetIndexObjects(); + auto indexes_map = table_object->GetIndexCatalogEntries(); for (auto &index : indexes_map) { LOG_TRACE("Checking if UPDATE query affects index: %s", index.second->GetIndexName().c_str()); diff --git a/src/settings/settings_manager.cpp b/src/settings/settings_manager.cpp index 10841886303..18de619defd 100644 --- a/src/settings/settings_manager.cpp +++ b/src/settings/settings_manager.cpp @@ -73,12 +73,18 @@ void SettingsManager::InitializeCatalog() { for (auto s : settings_) { // TODO: Use Update instead Delete & Insert - settings_catalog.DeleteSetting(s.second.name, txn); - if (!settings_catalog.InsertSetting( - s.second.name, s.second.value.ToString(), - s.second.value.GetTypeId(), s.second.desc, "", "", - s.second.default_value.ToString(), s.second.is_mutable, - s.second.is_persistent, pool, txn)) { + settings_catalog.DeleteSetting(txn, s.second.name); + if (!settings_catalog.InsertSetting(txn, + s.second.name, + s.second.value.ToString(), + s.second.value.GetTypeId(), + s.second.desc, + "", + "", + s.second.default_value.ToString(), + s.second.is_mutable, + s.second.is_persistent, + pool)) { txn_manager.AbortTransaction(txn); throw SettingsException("failed to initialize catalog pg_settings on " + s.second.name); @@ -174,11 +180,18 @@ bool SettingsManager::InsertIntoCatalog(const Param ¶m) { auto txn = txn_manager.BeginTransaction(); type::AbstractPool *pool = pool_.get(); // TODO: Use Update instead Delete & Insert - settings_catalog.DeleteSetting(param.name, txn); - if (!settings_catalog.InsertSetting( - param.name, param.value.ToString(), param.value.GetTypeId(), - param.desc, "", "", param.default_value.ToString(), param.is_mutable, - param.is_persistent, pool, txn)) { + settings_catalog.DeleteSetting(txn, param.name); + if (!settings_catalog.InsertSetting(txn, + param.name, + param.value.ToString(), + param.value.GetTypeId(), + param.desc, + "", + "", + param.default_value.ToString(), + param.is_mutable, + param.is_persistent, + pool)) { txn_manager.AbortTransaction(txn); return false; } diff --git a/src/statistics/stats_aggregator.cpp b/src/statistics/stats_aggregator.cpp index 6d4a39847a3..29a710bdd86 100644 --- a/src/statistics/stats_aggregator.cpp +++ b/src/statistics/stats_aggregator.cpp @@ -164,11 +164,21 @@ void StatsAggregator::UpdateQueryMetrics(int64_t time_stamp, catalog::Catalog::GetInstance() ->GetSystemCatalogs(query_metric->GetDatabaseId()) ->GetQueryMetricsCatalog() - ->InsertQueryMetrics( - query_metric->GetName(), query_metric->GetDatabaseId(), num_params, - type_buf, format_buf, value_buf, reads, updates, deletes, inserts, - (int64_t)latency, (int64_t)(cpu_system + cpu_user), time_stamp, - pool_.get(), txn); + ->InsertQueryMetrics(txn, + query_metric->GetName(), + query_metric->GetDatabaseId(), + num_params, + type_buf, + format_buf, + value_buf, + reads, + updates, + deletes, + inserts, + (int64_t) latency, + (int64_t) (cpu_system + cpu_user), + time_stamp, + pool_.get()); LOG_TRACE("Query Metric Tuple inserted"); } @@ -199,7 +209,8 @@ void StatsAggregator::UpdateMetrics() { std::string database_name; try { auto database_object = - catalog::Catalog::GetInstance()->GetDatabaseObject(database_oid, txn); + catalog::Catalog::GetInstance()->GetDatabaseCatalogEntry(txn, + database_oid); database_name = database_object->GetDatabaseName(); } catch (CatalogException &e) { continue; @@ -211,8 +222,12 @@ void StatsAggregator::UpdateMetrics() { auto txn_committed = database_metric->GetTxnCommitted().GetCounter(); auto txn_aborted = database_metric->GetTxnAborted().GetCounter(); - catalog::DatabaseMetricsCatalog::GetInstance()->InsertDatabaseMetrics( - database_oid, txn_committed, txn_aborted, time_stamp, pool_.get(), txn); + catalog::DatabaseMetricsCatalog::GetInstance()->InsertDatabaseMetrics(txn, + database_oid, + txn_committed, + txn_aborted, + time_stamp, + pool_.get()); LOG_TRACE("DB Metric Tuple inserted"); // Update all the indices of this database @@ -245,9 +260,14 @@ void StatsAggregator::UpdateTableMetrics(storage::Database *database, auto table_metrics_catalog = catalog::Catalog::GetInstance() ->GetSystemCatalogs(database_oid) ->GetTableMetricsCatalog(); - table_metrics_catalog->InsertTableMetrics(table_oid, reads, updates, - deletes, inserts, time_stamp, - pool_.get(), txn); + table_metrics_catalog->InsertTableMetrics(txn, + table_oid, + reads, + updates, + deletes, + inserts, + time_stamp, + pool_.get()); LOG_TRACE("Table Metric Tuple inserted"); UpdateIndexMetrics(database, table, time_stamp, txn); @@ -277,9 +297,14 @@ void StatsAggregator::UpdateIndexMetrics(storage::Database *database, auto index_metrics_catalog = catalog::Catalog::GetInstance() ->GetSystemCatalogs(database_oid) ->GetIndexMetricsCatalog(); - index_metrics_catalog->InsertIndexMetrics(table_oid, index_oid, reads, - deletes, inserts, time_stamp, - pool_.get(), txn); + index_metrics_catalog->InsertIndexMetrics(txn, + table_oid, + index_oid, + reads, + deletes, + inserts, + time_stamp, + pool_.get()); } } diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 9615691d54f..c292e8d3718 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -1391,9 +1391,9 @@ trigger::TriggerList *DataTable::GetTriggerList() { void DataTable::UpdateTriggerListFromCatalog( concurrency::TransactionContext *txn) { trigger_list_ = catalog::Catalog::GetInstance() - ->GetSystemCatalogs(database_oid) - ->GetTriggerCatalog() - ->GetTriggers(table_oid, txn); + ->GetSystemCatalogs(database_oid) + ->GetTriggerCatalog() + ->GetTriggers(txn, table_oid); } hash_t DataTable::Hash() const { diff --git a/src/storage/zone_map_manager.cpp b/src/storage/zone_map_manager.cpp index 2e6269bf170..01f75070aea 100644 --- a/src/storage/zone_map_manager.cpp +++ b/src/storage/zone_map_manager.cpp @@ -144,12 +144,21 @@ void ZoneMapManager::CreateOrUpdateZoneMapInCatalog( single_statement_txn = true; txn = txn_manager.BeginTransaction(); } - stats_catalog->DeleteColumnStatistics(database_id, table_id, tile_group_idx, - column_id, txn); + stats_catalog->DeleteColumnStatistics(txn, + database_id, + table_id, + tile_group_idx, + column_id); - stats_catalog->InsertColumnStatistics(database_id, table_id, tile_group_idx, - column_id, min, max, type, pool_.get(), - txn); + stats_catalog->InsertColumnStatistics(txn, + database_id, + table_id, + tile_group_idx, + column_id, + min, + max, + type, + pool_.get()); if (single_statement_txn) { txn_manager.CommitTransaction(txn); @@ -172,8 +181,11 @@ ZoneMapManager::GetZoneMapFromCatalog(oid_t database_id, oid_t table_id, auto stats_catalog = catalog::ZoneMapCatalog::GetInstance(nullptr); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - auto result_vector = stats_catalog->GetColumnStatistics( - database_id, table_id, tile_group_idx, column_id, txn); + auto result_vector = stats_catalog->GetColumnStatistics(txn, + database_id, + table_id, + tile_group_idx, + column_id); txn_manager.CommitTransaction(txn); if (result_vector == nullptr) { diff --git a/src/traffic_cop/traffic_cop.cpp b/src/traffic_cop/traffic_cop.cpp index 7bfffebb4c0..bbf0846ac9a 100644 --- a/src/traffic_cop/traffic_cop.cpp +++ b/src/traffic_cop/traffic_cop.cpp @@ -420,8 +420,10 @@ void TrafficCop::GetTableColumns(parser::TableRef *from_table, auto columns = static_cast( catalog::Catalog::GetInstance()->GetTableWithName( - from_table->GetDatabaseName(), from_table->GetSchemaName(), - from_table->GetTableName(), GetCurrentTxnState().first)) + GetCurrentTxnState().first, + from_table->GetDatabaseName(), + from_table->GetSchemaName(), + from_table->GetTableName())) ->GetSchema() ->GetColumns(); target_columns.insert(target_columns.end(), columns.begin(), diff --git a/src/trigger/trigger.cpp b/src/trigger/trigger.cpp index cb992d0e06d..e84cf9fdab4 100644 --- a/src/trigger/trigger.cpp +++ b/src/trigger/trigger.cpp @@ -90,10 +90,13 @@ void Trigger::SerializeWhen(SerializeOutput &output, oid_t database_oid, case ExpressionType::VALUE_TUPLE: { auto e = static_cast(expr); - auto table_object = catalog::Catalog::GetInstance()->GetTableObject( - database_oid, table_oid, txn); + auto table_object = + catalog::Catalog::GetInstance()->GetTableCatalogEntry( + txn, + database_oid, + table_oid); auto column_object = - table_object->GetColumnObject(e->GetColumnName()); + table_object->GetColumnCatalogEntry(e->GetColumnName()); output.WriteInt(static_cast(column_object->GetColumnType())); output.WriteInt(static_cast(column_object->GetColumnId())); break; diff --git a/src/tuning/index_tuner.cpp b/src/tuning/index_tuner.cpp index cbd1bd57926..7a20082e252 100644 --- a/src/tuning/index_tuner.cpp +++ b/src/tuning/index_tuner.cpp @@ -597,8 +597,10 @@ void IndexTuner::BootstrapTPCC(const std::string &path) { // Locate table in catalog auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - auto table = catalog->GetTableWithName( - database_name, std::string(DEFAULT_SCHEMA_NAME), table_name, txn); + auto table = catalog->GetTableWithName(txn, + database_name, + std::string(DEFAULT_SCHEMA_NAME), + table_name); txn_manager.CommitTransaction(txn); PELOTON_ASSERT(table != nullptr); for (auto &sample : samples) { diff --git a/src/tuning/layout_tuner.cpp b/src/tuning/layout_tuner.cpp index 5953a4e8dfc..567c8fec5f8 100644 --- a/src/tuning/layout_tuner.cpp +++ b/src/tuning/layout_tuner.cpp @@ -118,7 +118,7 @@ bool LayoutTuner::UpdateDefaultPartition(storage::DataTable *table) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto *txn = txn_manager.BeginTransaction(); auto catalog = catalog::Catalog::GetInstance(); - if (catalog->CreateDefaultLayout(database_oid, table_oid, column_map, txn) == + if (catalog->CreateDefaultLayout(txn, database_oid, table_oid, column_map) == nullptr) { txn_manager.AbortTransaction(txn); LOG_DEBUG("Layout Update to failed."); diff --git a/test/binder/binder_test.cpp b/test/binder/binder_test.cpp index b5b266c21cf..e581ad5152f 100644 --- a/test/binder/binder_test.cpp +++ b/test/binder/binder_test.cpp @@ -55,7 +55,7 @@ void SetupTables(std::string database_name) { LOG_INFO("Creating database %s", database_name.c_str()); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(database_name, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, database_name); txn_manager.CommitTransaction(txn); LOG_INFO("database %s created!", database_name.c_str()); @@ -126,14 +126,12 @@ TEST_F(BinderCorrectnessTest, SelectStatementTest) { binder->BindNameToNode(selectStmt); oid_t db_oid = - catalog_ptr->GetDatabaseWithName(default_database_name, txn)->GetOid(); + catalog_ptr->GetDatabaseWithName(txn, default_database_name)->GetOid(); oid_t tableA_oid = catalog_ptr - ->GetTableWithName(default_database_name, - DEFAULT_SCHEMA_NAME, "a", txn) + ->GetTableWithName(txn, default_database_name, DEFAULT_SCHEMA_NAME, "a") ->GetOid(); oid_t tableB_oid = catalog_ptr - ->GetTableWithName(default_database_name, - DEFAULT_SCHEMA_NAME, "b", txn) + ->GetTableWithName(txn, default_database_name, DEFAULT_SCHEMA_NAME, "b") ->GetOid(); txn_manager.CommitTransaction(txn); @@ -242,7 +240,7 @@ TEST_F(BinderCorrectnessTest, SelectStatementTest) { txn_manager.CommitTransaction(txn); // Delete the test database txn = txn_manager.BeginTransaction(); - catalog_ptr->DropDatabaseWithName(default_database_name, txn); + catalog_ptr->DropDatabaseWithName(txn, default_database_name); txn_manager.CommitTransaction(txn); } @@ -259,10 +257,9 @@ TEST_F(BinderCorrectnessTest, DeleteStatementTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); oid_t db_oid = - catalog_ptr->GetDatabaseWithName(default_database_name, txn)->GetOid(); + catalog_ptr->GetDatabaseWithName(txn, default_database_name)->GetOid(); oid_t tableB_oid = catalog_ptr - ->GetTableWithName(default_database_name, - DEFAULT_SCHEMA_NAME, "b", txn) + ->GetTableWithName(txn, default_database_name, DEFAULT_SCHEMA_NAME, "b") ->GetOid(); string deleteSQL = "DELETE FROM b WHERE 1 = b1 AND b2 = 'str'"; @@ -288,7 +285,7 @@ TEST_F(BinderCorrectnessTest, DeleteStatementTest) { // Delete the test database txn = txn_manager.BeginTransaction(); - catalog_ptr->DropDatabaseWithName(default_database_name, txn); + catalog_ptr->DropDatabaseWithName(txn, default_database_name); txn_manager.CommitTransaction(txn); } @@ -378,7 +375,7 @@ TEST_F(BinderCorrectnessTest, BindDepthTest) { // Delete the test database catalog::Catalog *catalog_ptr = catalog::Catalog::GetInstance(); txn = txn_manager.BeginTransaction(); - catalog_ptr->DropDatabaseWithName(default_database_name, txn); + catalog_ptr->DropDatabaseWithName(txn, default_database_name); txn_manager.CommitTransaction(txn); } diff --git a/test/catalog/catalog_test.cpp b/test/catalog/catalog_test.cpp index d99ba49bf25..bda84540f89 100644 --- a/test/catalog/catalog_test.cpp +++ b/test/catalog/catalog_test.cpp @@ -42,11 +42,13 @@ TEST_F(CatalogTests, BootstrappingCatalog) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); storage::Database *database = - catalog->GetDatabaseWithName(CATALOG_DATABASE_NAME, txn); + catalog->GetDatabaseWithName(txn, CATALOG_DATABASE_NAME); // Check database metric table storage::DataTable *db_metric_table = - catalog->GetTableWithName(CATALOG_DATABASE_NAME, CATALOG_SCHEMA_NAME, - DATABASE_METRICS_CATALOG_NAME, txn); + catalog->GetTableWithName(txn, + CATALOG_DATABASE_NAME, + CATALOG_SCHEMA_NAME, + DATABASE_METRICS_CATALOG_NAME); txn_manager.CommitTransaction(txn); EXPECT_NE(nullptr, database); EXPECT_NE(nullptr, db_metric_table); @@ -55,10 +57,10 @@ TEST_F(CatalogTests, BootstrappingCatalog) { TEST_F(CatalogTests, CreatingDatabase) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase("emp_db", txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, "emp_db"); EXPECT_EQ("emp_db", catalog::Catalog::GetInstance() - ->GetDatabaseWithName("emp_db", txn) - ->GetDBName()); + ->GetDatabaseWithName(txn, "emp_db") + ->GetDBName()); txn_manager.CommitTransaction(txn); } @@ -79,44 +81,72 @@ TEST_F(CatalogTests, CreatingTable) { std::unique_ptr table_schema_3( new catalog::Schema({id_column, name_column})); - catalog::Catalog::GetInstance()->CreateTable( - "emp_db", DEFAULT_SCHEMA_NAME, "emp_table", std::move(table_schema), txn); - catalog::Catalog::GetInstance()->CreateTable("emp_db", DEFAULT_SCHEMA_NAME, + catalog::Catalog::GetInstance()->CreateTable(txn, + "emp_db", + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + "emp_table", + false); + catalog::Catalog::GetInstance()->CreateTable(txn, + "emp_db", + DEFAULT_SCHEMA_NAME, + std::move(table_schema_2), "department_table", - std::move(table_schema_2), txn); - catalog::Catalog::GetInstance()->CreateTable("emp_db", DEFAULT_SCHEMA_NAME, + false); + catalog::Catalog::GetInstance()->CreateTable(txn, + "emp_db", + DEFAULT_SCHEMA_NAME, + std::move(table_schema_3), "salary_table", - std::move(table_schema_3), txn); + false); // insert random tuple into DATABASE_METRICS_CATALOG and check std::unique_ptr pool(new type::EphemeralPool()); - catalog::DatabaseMetricsCatalog::GetInstance()->InsertDatabaseMetrics( - 2, 3, 4, 5, pool.get(), txn); + catalog::DatabaseMetricsCatalog::GetInstance()->InsertDatabaseMetrics(txn, + 2, + 3, + 4, + 5, + pool.get()); // inset meaningless tuple into QUERY_METRICS_CATALOG and check stats::QueryMetric::QueryParamBuf param; param.len = 1; - param.buf = (unsigned char *)pool->Allocate(1); + param.buf = (unsigned char *) pool->Allocate(1); *param.buf = 'a'; auto database_object = - catalog::Catalog::GetInstance()->GetDatabaseObject("emp_db", txn); + catalog::Catalog::GetInstance()->GetDatabaseCatalogEntry(txn, "emp_db"); catalog::Catalog::GetInstance() ->GetSystemCatalogs(database_object->GetDatabaseOid()) ->GetQueryMetricsCatalog() - ->InsertQueryMetrics("a query", database_object->GetDatabaseOid(), 1, - param, param, param, 1, 1, 1, 1, 1, 1, 1, pool.get(), - txn); + ->InsertQueryMetrics(txn, + "a query", + database_object->GetDatabaseOid(), + 1, + param, + param, + param, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + pool.get()); auto param1 = catalog::Catalog::GetInstance() - ->GetSystemCatalogs(database_object->GetDatabaseOid()) - ->GetQueryMetricsCatalog() - ->GetParamTypes("a query", txn); + ->GetSystemCatalogs(database_object->GetDatabaseOid()) + ->GetQueryMetricsCatalog() + ->GetParamTypes(txn, "a query"); EXPECT_EQ(1, param1.len); EXPECT_EQ('a', *param1.buf); // check colum object EXPECT_EQ("name", catalog::Catalog::GetInstance() - ->GetTableObject("emp_db", DEFAULT_SCHEMA_NAME, - "department_table", txn) - ->GetColumnObject(1) - ->GetColumnName()); + ->GetTableCatalogEntry(txn, + "emp_db", + DEFAULT_SCHEMA_NAME, + "department_table") + ->GetColumnCatalogEntry(1) + ->GetColumnName()); txn_manager.CommitTransaction(txn); } @@ -125,19 +155,23 @@ TEST_F(CatalogTests, TestingCatalogCache) { auto txn = txn_manager.BeginTransaction(); auto catalog = catalog::Catalog::GetInstance(); - auto catalog_db_object = catalog->GetDatabaseObject(CATALOG_DATABASE_OID, txn); - auto catalog_table_objects = catalog_db_object->GetTableObjects(); + auto + catalog_db_object = + catalog->GetDatabaseCatalogEntry(txn, CATALOG_DATABASE_OID); + auto catalog_table_objects = catalog_db_object->GetTableCatalogEntries(); EXPECT_NE(0, catalog_table_objects.size()); - auto user_db_object = catalog->GetDatabaseObject("emp_db", txn); + auto user_db_object = catalog->GetDatabaseCatalogEntry(txn, "emp_db"); auto user_database = storage::StorageManager::GetInstance() - ->GetDatabaseWithOid(user_db_object->GetDatabaseOid()); + ->GetDatabaseWithOid(user_db_object->GetDatabaseOid()); // check expected table object is acquired - for (oid_t table_idx = 0; table_idx < user_database->GetTableCount(); table_idx++) { + for (oid_t table_idx = 0; table_idx < user_database->GetTableCount(); + table_idx++) { auto table = user_database->GetTable(table_idx); - auto user_table_object = user_db_object->GetTableObject(table->GetOid()); - EXPECT_EQ(user_db_object->GetDatabaseOid(), + auto user_table_object = + user_db_object->GetTableCatalogEntry(table->GetOid()); + EXPECT_EQ(user_db_object->GetDatabaseOid(), user_table_object->GetDatabaseOid()); } @@ -148,11 +182,13 @@ TEST_F(CatalogTests, TableObject) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - auto table_object = catalog::Catalog::GetInstance()->GetTableObject( - "emp_db", DEFAULT_SCHEMA_NAME, "department_table", txn); + auto table_object = catalog::Catalog::GetInstance()->GetTableCatalogEntry(txn, + "emp_db", + DEFAULT_SCHEMA_NAME, + "department_table"); - auto index_objects = table_object->GetIndexObjects(); - auto column_objects = table_object->GetColumnObjects(); + auto index_objects = table_object->GetIndexCatalogEntries(); + auto column_objects = table_object->GetColumnCatalogEntries(); EXPECT_EQ(1, index_objects.size()); EXPECT_EQ(2, column_objects.size()); @@ -163,7 +199,7 @@ TEST_F(CatalogTests, TableObject) { EXPECT_EQ(0, column_objects[0]->GetColumnOffset()); EXPECT_EQ(type::TypeId::INTEGER, column_objects[0]->GetColumnType()); EXPECT_EQ(type::Type::GetTypeSize(type::TypeId::INTEGER), - column_objects[0]->GetColumnLength()); + column_objects[0]->GetColumnLength()); EXPECT_TRUE(column_objects[0]->IsInlined()); EXPECT_TRUE(column_objects[0]->IsPrimary()); EXPECT_FALSE(column_objects[0]->IsNotNull()); @@ -173,7 +209,7 @@ TEST_F(CatalogTests, TableObject) { EXPECT_EQ(1, column_objects[1]->GetColumnId()); EXPECT_EQ(4, column_objects[1]->GetColumnOffset()); EXPECT_EQ(type::TypeId::VARCHAR, column_objects[1]->GetColumnType()); - EXPECT_EQ(32, column_objects[1]->GetColumnLength()); + EXPECT_EQ(32, column_objects[1]->GetColumnLength()); EXPECT_TRUE(column_objects[1]->IsInlined()); EXPECT_FALSE(column_objects[1]->IsPrimary()); EXPECT_FALSE(column_objects[1]->IsNotNull()); @@ -181,12 +217,14 @@ TEST_F(CatalogTests, TableObject) { // update pg_table SET version_oid = 1 where table_name = department_table oid_t department_table_oid = table_object->GetTableOid(); auto pg_table = catalog::Catalog::GetInstance() - ->GetSystemCatalogs(table_object->GetDatabaseOid()) - ->GetTableCatalog(); - bool update_result = pg_table->UpdateVersionId(1, department_table_oid, txn); + ->GetSystemCatalogs(table_object->GetDatabaseOid()) + ->GetTableCatalog(); + bool update_result = pg_table->UpdateVersionId(txn, department_table_oid, 1); // get version id after update, invalidate old cache - table_object = catalog::Catalog::GetInstance()->GetTableObject( - "emp_db", DEFAULT_SCHEMA_NAME, "department_table", txn); + table_object = catalog::Catalog::GetInstance()->GetTableCatalogEntry(txn, + "emp_db", + DEFAULT_SCHEMA_NAME, + "department_table"); uint32_t version_oid = table_object->GetVersionId(); EXPECT_NE(department_table_oid, INVALID_OID); EXPECT_EQ(update_result, true); @@ -199,7 +237,7 @@ TEST_F(CatalogTests, TestingNamespace) { EXPECT_EQ(ResultType::SUCCESS, TestingSQLUtil::ExecuteSQLQuery("begin;")); // create namespaces emp_ns0 and emp_ns1 EXPECT_EQ(ResultType::SUCCESS, TestingSQLUtil::ExecuteSQLQuery( - "create database default_database;")); + "create database default_database;")); EXPECT_EQ(ResultType::SUCCESS, TestingSQLUtil::ExecuteSQLQuery("create schema emp_ns0;")); EXPECT_EQ(ResultType::SUCCESS, @@ -240,7 +278,7 @@ TEST_F(CatalogTests, TestingNamespace) { EXPECT_EQ(ResultType::SUCCESS, TestingSQLUtil::ExecuteSQLQuery("commit;")); EXPECT_EQ(ResultType::SUCCESS, TestingSQLUtil::ExecuteSQLQuery("begin;")); EXPECT_EQ(ResultType::FAILURE, TestingSQLUtil::ExecuteSQLQuery( - "select * from emp_ns1.emp_table1;")); + "select * from emp_ns1.emp_table1;")); EXPECT_EQ(ResultType::ABORTED, TestingSQLUtil::ExecuteSQLQuery("commit;")); // drop namespace emp_ns0 and emp_ns1 @@ -254,7 +292,7 @@ TEST_F(CatalogTests, TestingNamespace) { EXPECT_EQ(ResultType::FAILURE, TestingSQLUtil::ExecuteSQLQuery("drop schema emp_ns0;")); EXPECT_EQ(ResultType::FAILURE, TestingSQLUtil::ExecuteSQLQuery( - "select * from emp_ns0.emp_table1;")); + "select * from emp_ns0.emp_table1;")); EXPECT_EQ(ResultType::ABORTED, TestingSQLUtil::ExecuteSQLQuery("commit;")); } @@ -267,66 +305,80 @@ TEST_F(CatalogTests, DroppingTable) { oid_t expected_table_count = CATALOG_TABLES_COUNT + 3; EXPECT_EQ( expected_table_count, - (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); + (int) catalog->GetDatabaseCatalogEntry(txn, + "emp_db")->GetTableCatalogEntries().size()); auto database_object = - catalog::Catalog::GetInstance()->GetDatabaseObject("emp_db", txn); + catalog::Catalog::GetInstance()->GetDatabaseCatalogEntry(txn, "emp_db"); EXPECT_NE(nullptr, database_object); - catalog::Catalog::GetInstance()->DropTable("emp_db", DEFAULT_SCHEMA_NAME, - "department_table", txn); + catalog::Catalog::GetInstance()->DropTable(txn, + "emp_db", + DEFAULT_SCHEMA_NAME, + "department_table"); database_object = - catalog::Catalog::GetInstance()->GetDatabaseObject("emp_db", txn); + catalog::Catalog::GetInstance()->GetDatabaseCatalogEntry(txn, "emp_db"); EXPECT_NE(nullptr, database_object); auto department_table_object = - database_object->GetTableObject("department_table", DEFAULT_SCHEMA_NAME); + database_object->GetTableCatalogEntry("department_table", + DEFAULT_SCHEMA_NAME); // Decrement expected_table_count to account for the dropped table. expected_table_count--; EXPECT_EQ( expected_table_count, - (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); + (int) catalog->GetDatabaseCatalogEntry(txn, + "emp_db")->GetTableCatalogEntries().size()); txn_manager.CommitTransaction(txn); EXPECT_EQ(nullptr, department_table_object); // Try to drop again txn = txn_manager.BeginTransaction(); - EXPECT_THROW(catalog::Catalog::GetInstance()->DropTable( - "emp_db", DEFAULT_SCHEMA_NAME, "department_table", txn), + EXPECT_THROW(catalog::Catalog::GetInstance()->DropTable(txn, + "emp_db", + DEFAULT_SCHEMA_NAME, + "department_table"), CatalogException); EXPECT_EQ( expected_table_count, - (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); + (int) catalog->GetDatabaseCatalogEntry(txn, + "emp_db")->GetTableCatalogEntries().size()); txn_manager.CommitTransaction(txn); // Drop a table that does not exist txn = txn_manager.BeginTransaction(); - EXPECT_THROW(catalog::Catalog::GetInstance()->DropTable( - "emp_db", DEFAULT_SCHEMA_NAME, "void_table", txn), + EXPECT_THROW(catalog::Catalog::GetInstance()->DropTable(txn, + "emp_db", + DEFAULT_SCHEMA_NAME, + "void_table"), CatalogException); EXPECT_EQ( expected_table_count, - (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); + (int) catalog->GetDatabaseCatalogEntry(txn, + "emp_db")->GetTableCatalogEntries().size()); txn_manager.CommitTransaction(txn); // Drop the other table txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropTable("emp_db", DEFAULT_SCHEMA_NAME, - "emp_table", txn); + catalog::Catalog::GetInstance()->DropTable(txn, + "emp_db", + DEFAULT_SCHEMA_NAME, + "emp_table"); // Account for the dropped table. expected_table_count--; EXPECT_EQ( expected_table_count, - (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); + (int) catalog->GetDatabaseCatalogEntry(txn, + "emp_db")->GetTableCatalogEntries().size()); txn_manager.CommitTransaction(txn); } TEST_F(CatalogTests, DroppingDatabase) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName("emp_db", txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, "emp_db"); EXPECT_THROW( - catalog::Catalog::GetInstance()->GetDatabaseWithName("emp_db", txn), + catalog::Catalog::GetInstance()->GetDatabaseWithName(txn, "emp_db"), CatalogException); txn_manager.CommitTransaction(txn); } @@ -350,7 +402,7 @@ TEST_F(CatalogTests, LayoutCatalogTest) { // Create database. auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - EXPECT_EQ(ResultType::SUCCESS, catalog->CreateDatabase(db_name, txn)); + EXPECT_EQ(ResultType::SUCCESS, catalog->CreateDatabase(txn, db_name)); // Create table. auto val0 = catalog::Column(type::TypeId::INTEGER, @@ -368,18 +420,25 @@ TEST_F(CatalogTests, LayoutCatalogTest) { std::unique_ptr table_schema( new catalog::Schema({val0, val1, val2, val3})); EXPECT_EQ(ResultType::SUCCESS, - catalog->CreateTable(db_name, DEFAULT_SCHEMA_NAME, table_name, - std::move(table_schema), txn)); + catalog->CreateTable(txn, + db_name, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + table_name, + false)); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); auto database_oid = - catalog->GetDatabaseObject(db_name, txn)->GetDatabaseOid(); + catalog->GetDatabaseCatalogEntry(txn, db_name)->GetDatabaseOid(); auto table_object = - catalog->GetTableObject(db_name, DEFAULT_SCHEMA_NAME, table_name, txn); + catalog->GetTableCatalogEntry(txn, + db_name, + DEFAULT_SCHEMA_NAME, + table_name); auto table_oid = table_object->GetTableOid(); auto table = - catalog->GetTableWithName(db_name, DEFAULT_SCHEMA_NAME, table_name, txn); + catalog->GetTableWithName(txn, db_name, DEFAULT_SCHEMA_NAME, table_name); auto pg_layout = catalog->GetSystemCatalogs(database_oid)->GetLayoutCatalog(); txn_manager.CommitTransaction(txn); @@ -395,9 +454,11 @@ TEST_F(CatalogTests, LayoutCatalogTest) { auto first_layout_oid = first_default_layout->GetOid(); EXPECT_EQ( *(first_default_layout.get()), - *(pg_layout->GetLayoutWithOid(table_oid, first_layout_oid, txn).get())); + *(pg_layout->GetLayoutWithOid(txn, table_oid, first_layout_oid).get())); EXPECT_EQ(first_layout_oid, - catalog->GetTableObject(database_oid, table_oid, txn)->GetDefaultLayoutOid()); + catalog->GetTableCatalogEntry(txn, + database_oid, + table_oid)->GetDefaultLayoutOid()); txn_manager.CommitTransaction(txn); // Change default layout. @@ -409,7 +470,7 @@ TEST_F(CatalogTests, LayoutCatalogTest) { txn = txn_manager.BeginTransaction(); auto default_layout = - catalog->CreateDefaultLayout(database_oid, table_oid, default_map, txn); + catalog->CreateDefaultLayout(txn, database_oid, table_oid, default_map); EXPECT_NE(nullptr, default_layout); txn_manager.CommitTransaction(txn); @@ -424,9 +485,11 @@ TEST_F(CatalogTests, LayoutCatalogTest) { txn = txn_manager.BeginTransaction(); EXPECT_EQ( *(default_layout.get()), - *(pg_layout->GetLayoutWithOid(table_oid, default_layout_oid, txn).get())); + *(pg_layout->GetLayoutWithOid(txn, table_oid, default_layout_oid).get())); EXPECT_EQ(default_layout_oid, - catalog->GetTableObject(database_oid, table_oid, txn)->GetDefaultLayoutOid()); + catalog->GetTableCatalogEntry(txn, + database_oid, + table_oid)->GetDefaultLayoutOid()); txn_manager.CommitTransaction(txn); // Create additional layout. @@ -438,7 +501,7 @@ TEST_F(CatalogTests, LayoutCatalogTest) { txn = txn_manager.BeginTransaction(); auto other_layout = - catalog->CreateLayout(database_oid, table_oid, non_default_map, txn); + catalog->CreateLayout(txn, database_oid, table_oid, non_default_map); EXPECT_NE(nullptr, other_layout); txn_manager.CommitTransaction(txn); @@ -452,18 +515,23 @@ TEST_F(CatalogTests, LayoutCatalogTest) { auto other_layout_oid = other_layout->GetOid(); EXPECT_EQ( *(other_layout.get()), - *(pg_layout->GetLayoutWithOid(table_oid, other_layout_oid, txn).get())); + *(pg_layout->GetLayoutWithOid(txn, table_oid, other_layout_oid).get())); // Check that the default layout is still the same. EXPECT_NE(other_layout, table->GetDefaultLayout()); EXPECT_NE(other_layout_oid, - catalog->GetTableObject(database_oid, table_oid, txn)->GetDefaultLayoutOid()); + catalog->GetTableCatalogEntry(txn, + database_oid, + table_oid)->GetDefaultLayoutOid()); txn_manager.CommitTransaction(txn); // Drop the default layout. txn = txn_manager.BeginTransaction(); - EXPECT_EQ(ResultType::SUCCESS, catalog->DropLayout(database_oid, table_oid, - default_layout_oid, txn)); + EXPECT_EQ(ResultType::SUCCESS, + catalog->DropLayout(txn, + database_oid, + table_oid, + default_layout_oid)); txn_manager.CommitTransaction(txn); // Check that default layout is reset and set to row_store. @@ -476,19 +544,21 @@ TEST_F(CatalogTests, LayoutCatalogTest) { // Query pg_layout and pg_table to ensure that the entry is dropped txn = txn_manager.BeginTransaction(); EXPECT_EQ(nullptr, - pg_layout->GetLayoutWithOid(table_oid, default_layout_oid, txn)); + pg_layout->GetLayoutWithOid(txn, table_oid, default_layout_oid)); EXPECT_EQ(ROW_STORE_LAYOUT_OID, - catalog->GetTableObject(database_oid, table_oid, txn)->GetDefaultLayoutOid()); + catalog->GetTableCatalogEntry(txn, + database_oid, + table_oid)->GetDefaultLayoutOid()); // The additional layout must be present in pg_layout EXPECT_EQ( *(other_layout.get()), - *(pg_layout->GetLayoutWithOid(table_oid, other_layout_oid, txn).get())); + *(pg_layout->GetLayoutWithOid(txn, table_oid, other_layout_oid).get())); txn_manager.CommitTransaction(txn); // Drop database txn = txn_manager.BeginTransaction(); - catalog->DropDatabaseWithName(db_name, txn); + catalog->DropDatabaseWithName(txn, db_name); txn_manager.CommitTransaction(txn); } diff --git a/test/catalog/constraints_test.cpp b/test/catalog/constraints_test.cpp index dfd8b015a81..8235e85e9a9 100644 --- a/test/catalog/constraints_test.cpp +++ b/test/catalog/constraints_test.cpp @@ -106,7 +106,7 @@ TEST_F(ConstraintsTests, NOTNULLTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } #endif @@ -236,9 +236,11 @@ TEST_F(ConstraintsTests, UNIQUETest) { std::unique_ptr table_schema( new catalog::Schema({column1, column2})); std::string table_name("TEST_TABLE"); - catalog::Catalog::GetInstance()->CreateTable(DEFAULT_DB_NAME, - DEFAULT_SCHEMA_NAME, table_name, - std::move(table_schema), txn); + catalog::Catalog::GetInstance()->CreateTable(txn, DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + table_name, + false); storage::DataTable *table = catalog::Catalog::GetInstance()->GetTableWithName( DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table_name, txn); txn_manager.CommitTransaction(txn); diff --git a/test/codegen/bloom_filter_test.cpp b/test/codegen/bloom_filter_test.cpp index ef6bf168c05..7dd0d32a7c1 100644 --- a/test/codegen/bloom_filter_test.cpp +++ b/test/codegen/bloom_filter_test.cpp @@ -42,7 +42,7 @@ class BloomFilterCodegenTest : public PelotonCodeGenTest { // Create test db auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -50,7 +50,7 @@ class BloomFilterCodegenTest : public PelotonCodeGenTest { // Drop test db auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -213,8 +213,10 @@ TEST_F(BloomFilterCodegenTest, PerformanceTest) { int curr_size = 0; std::vector numbers; std::unordered_set number_set; - auto *table1 = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, - table1_name, txn); + auto *table1 = catalog->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + table1_name); while (curr_size < table1_target_size) { // Find a unique random number int random; @@ -234,8 +236,10 @@ TEST_F(BloomFilterCodegenTest, PerformanceTest) { LOG_INFO("Finish populating test1"); // Load the inner table which contains twice tuples as the outer table - auto *table2 = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, - table2_name, txn); + auto *table2 = catalog->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + table2_name); unsigned outer_table_cardinality = numbers.size() * outer_to_inner_ratio; for (unsigned i = 0; i < outer_table_cardinality; i++) { int number; @@ -335,9 +339,12 @@ void BloomFilterCodegenTest::CreateTable(std::string table_name, int tuple_size, curr_size += bigint_size; } auto *catalog = catalog::Catalog::GetInstance(); - catalog->CreateTable( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table_name, - std::unique_ptr(new catalog::Schema(cols)), txn); + catalog->CreateTable(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + std::unique_ptr(new catalog::Schema(cols)), + table_name, + false); } // Insert a tuple to specific table diff --git a/test/codegen/table_scan_translator_test.cpp b/test/codegen/table_scan_translator_test.cpp index f1080afda6d..6780aadf852 100644 --- a/test/codegen/table_scan_translator_test.cpp +++ b/test/codegen/table_scan_translator_test.cpp @@ -113,11 +113,17 @@ class TableScanTranslatorTest : public PelotonCodeGenTest { std::unique_ptr schema{new catalog::Schema(cols)}; // Insert table in catalog - catalog->CreateTable(test_db_name, DEFAULT_SCHEMA_NAME, all_cols_table_name, - std::move(schema), txn); - - all_cols_table = catalog->GetTableWithName( - test_db_name, DEFAULT_SCHEMA_NAME, all_cols_table_name, txn); + catalog->CreateTable(txn, + test_db_name, + DEFAULT_SCHEMA_NAME, + std::move(schema), + all_cols_table_name, + false); + + all_cols_table = catalog->GetTableWithName(txn, + test_db_name, + DEFAULT_SCHEMA_NAME, + all_cols_table_name); auto *table_schema = all_cols_table->GetSchema(); // Insert one row where all columns are NULL @@ -702,12 +708,19 @@ TEST_F(TableScanTranslatorTest, MultiLayoutScan) { auto txn = txn_manager.BeginTransaction(); // Insert table in catalog - catalog->CreateTable(test_db_name, DEFAULT_SCHEMA_NAME, table_name, - std::move(table_schema), txn, is_catalog, - tuples_per_tilegroup, LayoutType::ROW); + catalog->CreateTable(txn, + test_db_name, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + table_name, + is_catalog, + tuples_per_tilegroup, + LayoutType::ROW); // Get table reference - auto table = catalog->GetTableWithName(test_db_name, DEFAULT_SCHEMA_NAME, - table_name, txn); + auto table = catalog->GetTableWithName(txn, + test_db_name, + DEFAULT_SCHEMA_NAME, + table_name); txn_manager.CommitTransaction(txn); ///////////////////////////////////////////////////////// @@ -716,8 +729,9 @@ TEST_F(TableScanTranslatorTest, MultiLayoutScan) { txn = txn_manager.BeginTransaction(); table->ResetDefaultLayout(LayoutType::COLUMN); catalog->GetSystemCatalogs(table->GetDatabaseOid())->GetTableCatalog() - ->UpdateDefaultLayoutOid(table->GetDefaultLayout()->GetOid(), - table->GetOid(), txn); + ->UpdateDefaultLayoutOid(txn, + table->GetOid(), + table->GetDefaultLayout()->GetOid()); txn_manager.CommitTransaction(txn); ///////////////////////////////////////////////////////// @@ -766,7 +780,7 @@ TEST_F(TableScanTranslatorTest, MultiLayoutScan) { txn = txn_manager.BeginTransaction(); auto layout = - catalog->CreateDefaultLayout(database_oid, table_oid, column_map, txn); + catalog->CreateDefaultLayout(txn, database_oid, table_oid, column_map); EXPECT_NE(nullptr, layout); txn_manager.CommitTransaction(txn); diff --git a/test/codegen/testing_codegen_util.cpp b/test/codegen/testing_codegen_util.cpp index b587aeac199..98879d6de59 100644 --- a/test/codegen/testing_codegen_util.cpp +++ b/test/codegen/testing_codegen_util.cpp @@ -44,8 +44,8 @@ PelotonCodeGenTest::PelotonCodeGenTest(oid_t tuples_per_tilegroup, auto txn = txn_manager.BeginTransaction(); // create test db - catalog->CreateDatabase(test_db_name, txn); - test_db = catalog->GetDatabaseWithName(test_db_name, txn); + catalog->CreateDatabase(txn, test_db_name); + test_db = catalog->GetDatabaseWithName(txn, test_db_name); // Create test table CreateTestTables(txn, tuples_per_tilegroup, layout_type); @@ -57,7 +57,7 @@ PelotonCodeGenTest::~PelotonCodeGenTest() { auto *catalog = catalog::Catalog::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - auto result = catalog->DropDatabaseWithName(test_db_name, txn); + auto result = catalog->DropDatabaseWithName(txn, test_db_name); txn_manager.CommitTransaction(txn); EXPECT_EQ(ResultType::SUCCESS, result); codegen::QueryCache::Instance().Clear(); @@ -115,24 +115,36 @@ void PelotonCodeGenTest::CreateTestTables(concurrency::TransactionContext *txn, auto *catalog = catalog::Catalog::GetInstance(); for (int i = 0; i < 4; i++) { auto table_schema = CreateTestSchema(); - catalog->CreateTable(test_db_name, DEFAULT_SCHEMA_NAME, test_table_names[i], - std::move(table_schema), txn, false, - tuples_per_tilegroup, layout_type); + catalog->CreateTable(txn, + test_db_name, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + test_table_names[i], + false, + tuples_per_tilegroup, + layout_type); test_table_oids.push_back(catalog - ->GetTableObject(test_db_name, - DEFAULT_SCHEMA_NAME, - test_table_names[i], txn) + ->GetTableCatalogEntry(txn, + test_db_name, + DEFAULT_SCHEMA_NAME, + test_table_names[i]) ->GetTableOid()); } for (int i = 4; i < 5; i++) { auto table_schema = CreateTestSchema(true); - catalog->CreateTable(test_db_name, DEFAULT_SCHEMA_NAME, test_table_names[i], - std::move(table_schema), txn, false, - tuples_per_tilegroup, layout_type); + catalog->CreateTable(txn, + test_db_name, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + test_table_names[i], + false, + tuples_per_tilegroup, + layout_type); test_table_oids.push_back(catalog - ->GetTableObject(test_db_name, - DEFAULT_SCHEMA_NAME, - test_table_names[i], txn) + ->GetTableCatalogEntry(txn, + test_db_name, + DEFAULT_SCHEMA_NAME, + test_table_names[i]) ->GetTableOid()); } } @@ -216,12 +228,19 @@ void PelotonCodeGenTest::CreateAndLoadTableWithLayout( auto txn = txn_manager.BeginTransaction(); // Insert table in catalog - catalog->CreateTable(test_db_name, DEFAULT_SCHEMA_NAME, table_name, - std::move(table_schema), txn, is_catalog, - tuples_per_tilegroup, layout_type); + catalog->CreateTable(txn, + test_db_name, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + table_name, + is_catalog, + tuples_per_tilegroup, + layout_type); // Get table reference - layout_table = catalog->GetTableWithName(test_db_name, DEFAULT_SCHEMA_NAME, - table_name, txn); + layout_table = catalog->GetTableWithName(txn, + test_db_name, + DEFAULT_SCHEMA_NAME, + table_name); txn_manager.EndTransaction(txn); ///////////////////////////////////////////////////////// diff --git a/test/executor/copy_test.cpp b/test/executor/copy_test.cpp index 9b9291f4111..c49e1d3848f 100644 --- a/test/executor/copy_test.cpp +++ b/test/executor/copy_test.cpp @@ -44,7 +44,7 @@ TEST_F(CopyTests, Copying) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); const auto &db_name = "emp_db"; - catalog->CreateDatabase(db_name, txn); + catalog->CreateDatabase(txn, db_name); txn_manager.CommitTransaction(txn); std::unique_ptr optimizer( @@ -158,7 +158,7 @@ TEST_F(CopyTests, Copying) { txn_manager.CommitTransaction(txn); // free the database just created txn = txn_manager.BeginTransaction(); - catalog->DropDatabaseWithName("emp_db", txn); + catalog->DropDatabaseWithName(txn, "emp_db"); txn_manager.CommitTransaction(txn); } diff --git a/test/executor/create_index_test.cpp b/test/executor/create_index_test.cpp index 9d53642935e..4068113c1a1 100644 --- a/test/executor/create_index_test.cpp +++ b/test/executor/create_index_test.cpp @@ -49,7 +49,7 @@ TEST_F(CreateIndexTests, CreatingIndex) { LOG_INFO("Bootstrapping..."); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); LOG_INFO("Bootstrapping completed!"); @@ -204,13 +204,15 @@ TEST_F(CreateIndexTests, CreatingIndex) { traffic_cop.CommitQueryHelper(); txn = txn_manager.BeginTransaction(); - auto target_table_ = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); + auto target_table_ = catalog::Catalog::GetInstance()->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + "department_table"); // Expected 2 , Primary key index + created index EXPECT_EQ(target_table_->GetIndexCount(), 2); // free the database just created - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/executor/create_test.cpp b/test/executor/create_test.cpp index 214f32fa44d..55d90a727cb 100644 --- a/test/executor/create_test.cpp +++ b/test/executor/create_test.cpp @@ -58,7 +58,7 @@ TEST_F(CreateTests, CreatingDB) { executor.Execute(); // Check if the database exists in the same txn EXPECT_EQ(0, catalog::Catalog::GetInstance() - ->GetDatabaseObject("pelotondb", txn) + ->GetDatabaseCatalogEntry(txn, "pelotondb") ->GetDatabaseName() .compare("pelotondb")); @@ -68,12 +68,12 @@ TEST_F(CreateTests, CreatingDB) { txn = txn_manager.BeginTransaction(); // Check if the database exists in a new txn EXPECT_EQ(0, catalog::Catalog::GetInstance() - ->GetDatabaseObject("pelotondb", txn) + ->GetDatabaseCatalogEntry(txn, "pelotondb") ->GetDatabaseName() .compare("pelotondb")); // free the database just created - catalog::Catalog::GetInstance()->DropDatabaseWithName("pelotondb", txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, "pelotondb"); txn_manager.CommitTransaction(txn); } @@ -82,7 +82,7 @@ TEST_F(CreateTests, CreatingTable) { // Bootstrap auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); // Insert a table first auto id_column = catalog::Column( @@ -113,7 +113,7 @@ TEST_F(CreateTests, CreatingTable) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -124,7 +124,7 @@ TEST_F(CreateTests, CreatingUDFs) { // catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); auto catalog = catalog::Catalog::GetInstance(); catalog->Bootstrap(); - catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); // Insert a table first auto id_column = catalog::Column( @@ -205,7 +205,7 @@ TEST_F(CreateTests, CreatingUDFs) { std::string func_name = "increment"; std::vector arg_types{type::TypeId::DECIMAL}; - auto inserted_proc = pg_proc.GetProcByName(func_name, arg_types, txn); + auto inserted_proc = pg_proc.GetProcByName(txn, func_name, arg_types); EXPECT_NE(nullptr, inserted_proc); type::TypeId ret_type = inserted_proc->GetRetType(); EXPECT_EQ(type::TypeId::DECIMAL, ret_type); @@ -218,7 +218,7 @@ TEST_F(CreateTests, CreatingUDFs) { EXPECT_NE(nullptr, func_data.func_context_); txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -226,7 +226,7 @@ TEST_F(CreateTests, CreatingTrigger) { // Bootstrap auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); // Insert a table first auto id_column = catalog::Column( @@ -332,8 +332,10 @@ TEST_F(CreateTests, CreatingTrigger) { // Check the effect of creation storage::DataTable *target_table = - catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "accounts", txn); + catalog::Catalog::GetInstance()->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + "accounts"); txn_manager.CommitTransaction(txn); EXPECT_EQ(1, target_table->GetTriggerNumber()); trigger::Trigger *new_trigger = target_table->GetTriggerByIndex(0); @@ -345,7 +347,7 @@ TEST_F(CreateTests, CreatingTrigger) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -356,7 +358,7 @@ TEST_F(CreateTests, CreatingTriggerWithoutWhen) { // Bootstrap auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); // Insert a table first auto id_column = catalog::Column( @@ -420,8 +422,10 @@ TEST_F(CreateTests, CreatingTriggerWithoutWhen) { // Check the effect of creation storage::DataTable *target_table = - catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "accounts", txn); + catalog::Catalog::GetInstance()->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + "accounts"); txn_manager.CommitTransaction(txn); EXPECT_EQ(1, target_table->GetTriggerNumber()); trigger::Trigger *new_trigger = target_table->GetTriggerByIndex(0); @@ -433,7 +437,7 @@ TEST_F(CreateTests, CreatingTriggerWithoutWhen) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -445,7 +449,7 @@ TEST_F(CreateTests, CreatingTriggerInCatalog) { // NOTE: Catalog::GetInstance()->Bootstrap() has been called in previous tests // you can only call it once! // catalog->Bootstrap(); - catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); // Insert a table first auto id_column = catalog::Column( @@ -501,16 +505,18 @@ TEST_F(CreateTests, CreatingTriggerInCatalog) { createTriggerExecutor.Execute(); // check whether the trigger catalog table contains this new trigger - auto table_object = catalog::Catalog::GetInstance()->GetTableObject( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "accounts", txn); + auto table_object = catalog::Catalog::GetInstance()->GetTableCatalogEntry(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + "accounts"); auto trigger_list = catalog::Catalog::GetInstance() ->GetSystemCatalogs(table_object->GetDatabaseOid()) ->GetTriggerCatalog() - ->GetTriggersByType( - table_object->GetTableOid(), - (TRIGGER_TYPE_ROW | TRIGGER_TYPE_BEFORE | TRIGGER_TYPE_UPDATE), - txn); + ->GetTriggersByType(txn, + table_object->GetTableOid(), + (TRIGGER_TYPE_ROW | TRIGGER_TYPE_BEFORE + | TRIGGER_TYPE_UPDATE)); txn_manager.CommitTransaction(txn); @@ -519,7 +525,7 @@ TEST_F(CreateTests, CreatingTriggerInCatalog) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/executor/drop_test.cpp b/test/executor/drop_test.cpp index f7451d3dd8d..2c1b9e5d1fb 100644 --- a/test/executor/drop_test.cpp +++ b/test/executor/drop_test.cpp @@ -45,11 +45,11 @@ TEST_F(DropTests, DroppingDatabase) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog->CreateDatabase(TEST_DB_NAME, txn); + catalog->CreateDatabase(txn, TEST_DB_NAME); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - EXPECT_TRUE(catalog->GetDatabaseObject(TEST_DB_NAME, txn).get() != NULL); + EXPECT_TRUE(catalog->GetDatabaseCatalogEntry(txn, TEST_DB_NAME).get() != NULL); txn_manager.CommitTransaction(txn); parser::DropStatement drop_statement( @@ -70,7 +70,7 @@ TEST_F(DropTests, DroppingDatabase) { // The database should be deleted now txn = txn_manager.BeginTransaction(); - EXPECT_ANY_THROW(catalog->GetDatabaseObject(TEST_DB_NAME, txn);); + EXPECT_ANY_THROW(catalog->GetDatabaseCatalogEntry(txn, TEST_DB_NAME);); txn_manager.CommitTransaction(txn); } @@ -93,40 +93,50 @@ TEST_F(DropTests, DroppingTable) { std::unique_ptr table_schema2( new catalog::Schema({id_column, name_column})); - catalog->CreateDatabase(TEST_DB_NAME, txn); + catalog->CreateDatabase(txn, TEST_DB_NAME); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - catalog->CreateTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", - std::move(table_schema), txn); + catalog->CreateTable(txn, + TEST_DB_NAME, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + "department_table", + false); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - catalog->CreateTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table_2", - std::move(table_schema2), txn); + catalog->CreateTable(txn, + TEST_DB_NAME, + DEFAULT_SCHEMA_NAME, + std::move(table_schema2), + "department_table_2", + false); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); // NOTE: everytime we create a database, there will be 9 catalog tables // inside. In this test case, we have created two additional tables. oid_t expeected_table_count = CATALOG_TABLES_COUNT + 2; - EXPECT_EQ((int)catalog->GetDatabaseObject(TEST_DB_NAME, txn) - ->GetTableObjects() - .size(), + EXPECT_EQ((int) catalog->GetDatabaseCatalogEntry(txn, TEST_DB_NAME) + ->GetTableCatalogEntries() + .size(), expeected_table_count); // Now dropping the table using the executor - catalog->DropTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", - txn); + catalog->DropTable(txn, + TEST_DB_NAME, + DEFAULT_SCHEMA_NAME, + "department_table"); // Account for the dropped table. expeected_table_count--; - EXPECT_EQ((int)catalog->GetDatabaseObject(TEST_DB_NAME, txn) - ->GetTableObjects() - .size(), + EXPECT_EQ((int) catalog->GetDatabaseCatalogEntry(txn, TEST_DB_NAME) + ->GetTableCatalogEntries() + .size(), expeected_table_count); // free the database just created - catalog->DropDatabaseWithName(TEST_DB_NAME, txn); + catalog->DropDatabaseWithName(txn, TEST_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -149,12 +159,16 @@ TEST_F(DropTests, DroppingTrigger) { std::unique_ptr table_schema( new catalog::Schema({id_column, name_column})); - catalog->CreateDatabase(TEST_DB_NAME, txn); + catalog->CreateDatabase(txn, TEST_DB_NAME); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - catalog->CreateTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", - std::move(table_schema), txn); + catalog->CreateTable(txn, + TEST_DB_NAME, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + "department_table", + false); txn_manager.CommitTransaction(txn); // Create a trigger @@ -184,8 +198,10 @@ TEST_F(DropTests, DroppingTrigger) { // Check the effect of creation storage::DataTable *target_table = - catalog::Catalog::GetInstance()->GetTableWithName( - TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); + catalog::Catalog::GetInstance()->GetTableWithName(txn, + TEST_DB_NAME, + DEFAULT_SCHEMA_NAME, + "department_table"); txn_manager.CommitTransaction(txn); EXPECT_EQ(1, target_table->GetTriggerNumber()); trigger::Trigger *new_trigger = target_table->GetTriggerByIndex(0); @@ -217,17 +233,19 @@ TEST_F(DropTests, DroppingTrigger) { // Now dropping the table using the executer txn = txn_manager.BeginTransaction(); - catalog->DropTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", - txn); - EXPECT_EQ(CATALOG_TABLES_COUNT, (int)catalog::Catalog::GetInstance() - ->GetDatabaseObject(TEST_DB_NAME, txn) - ->GetTableObjects() - .size()); + catalog->DropTable(txn, + TEST_DB_NAME, + DEFAULT_SCHEMA_NAME, + "department_table"); + EXPECT_EQ(CATALOG_TABLES_COUNT, (int) catalog::Catalog::GetInstance() + ->GetDatabaseCatalogEntry(txn, TEST_DB_NAME) + ->GetTableCatalogEntries() + .size()); txn_manager.CommitTransaction(txn); // free the database just created txn = txn_manager.BeginTransaction(); - catalog->DropDatabaseWithName(TEST_DB_NAME, txn); + catalog->DropDatabaseWithName(txn, TEST_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -240,7 +258,7 @@ TEST_F(DropTests, DroppingIndexByName) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); // create database - catalog->CreateDatabase(TEST_DB_NAME, txn); + catalog->CreateDatabase(txn, TEST_DB_NAME); // Insert a table first auto id_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), @@ -255,60 +273,81 @@ TEST_F(DropTests, DroppingIndexByName) { txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - catalog->CreateTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table_01", - std::move(table_schema), txn); + catalog->CreateTable(txn, + TEST_DB_NAME, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + "department_table_01", + false); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - auto source_table = catalog->GetTableWithName( - TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table_01", txn); + auto source_table = catalog->GetTableWithName(txn, + TEST_DB_NAME, + DEFAULT_SCHEMA_NAME, + "department_table_01"); oid_t col_id = source_table->GetSchema()->GetColumnID(id_column.column_name); std::vector source_col_ids; source_col_ids.push_back(col_id); std::string index_name1 = "Testing_Drop_Index_By_Name"; - catalog->CreateIndex(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table_01", - source_col_ids, index_name1, false, IndexType::BWTREE, - txn); + catalog->CreateIndex(txn, + TEST_DB_NAME, + DEFAULT_SCHEMA_NAME, + "department_table_01", + index_name1, + source_col_ids, + false, + IndexType::BWTREE); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); // retrieve pg_index catalog table auto database_object = - catalog::Catalog::GetInstance()->GetDatabaseObject(TEST_DB_NAME, txn); + catalog::Catalog::GetInstance()->GetDatabaseCatalogEntry(txn, + TEST_DB_NAME); EXPECT_NE(nullptr, database_object); auto pg_index = catalog::Catalog::GetInstance() - ->GetSystemCatalogs(database_object->GetDatabaseOid()) - ->GetIndexCatalog(); + ->GetSystemCatalogs(database_object->GetDatabaseOid()) + ->GetIndexCatalog(); auto index_object = - pg_index->GetIndexObject(database_object->GetDatabaseName(), - index_name1, DEFAULT_SCHEMA_NAME, txn); + pg_index->GetIndexCatalogEntry(txn, + database_object->GetDatabaseName(), + DEFAULT_SCHEMA_NAME, + index_name1); EXPECT_NE(nullptr, index_object); // Check the effect of drop // Most major check in this test case // Now dropping the index using the DropIndex functionality - catalog->DropIndex(database_object->GetDatabaseOid(), - index_object->GetIndexOid(), txn); - EXPECT_EQ(pg_index->GetIndexObject(database_object->GetDatabaseName(), - index_name1, DEFAULT_SCHEMA_NAME, txn), + catalog->DropIndex(txn, + database_object->GetDatabaseOid(), + index_object->GetIndexOid()); + EXPECT_EQ(pg_index->GetIndexCatalogEntry(txn, + database_object->GetDatabaseName(), + DEFAULT_SCHEMA_NAME, + index_name1), nullptr); txn_manager.CommitTransaction(txn); // Drop the table just created txn = txn_manager.BeginTransaction(); // Check the effect of drop index - EXPECT_EQ(pg_index->GetIndexObject(database_object->GetDatabaseName(), - index_name1, DEFAULT_SCHEMA_NAME, txn), + EXPECT_EQ(pg_index->GetIndexCatalogEntry(txn, + database_object->GetDatabaseName(), + DEFAULT_SCHEMA_NAME, + index_name1), nullptr); // Now dropping the table - catalog->DropTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table_01", - txn); + catalog->DropTable(txn, + TEST_DB_NAME, + DEFAULT_SCHEMA_NAME, + "department_table_01"); txn_manager.CommitTransaction(txn); // free the database just created txn = txn_manager.BeginTransaction(); - catalog->DropDatabaseWithName(TEST_DB_NAME, txn); + catalog->DropDatabaseWithName(txn, TEST_DB_NAME); txn_manager.CommitTransaction(txn); } } // namespace test diff --git a/test/executor/insert_test.cpp b/test/executor/insert_test.cpp index 489b871121d..a76050b3e53 100644 --- a/test/executor/insert_test.cpp +++ b/test/executor/insert_test.cpp @@ -48,16 +48,21 @@ TEST_F(InsertTests, InsertRecord) { std::unique_ptr table_schema( new catalog::Schema({id_column, name_column})); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateTable( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "TEST_TABLE", - std::move(table_schema), txn); - - auto table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "TEST_TABLE", txn); + catalog::Catalog::GetInstance()->CreateTable(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + "TEST_TABLE", + false); + + auto table = catalog::Catalog::GetInstance()->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + "TEST_TABLE"); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); @@ -141,7 +146,7 @@ TEST_F(InsertTests, InsertRecord) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/executor/testing_executor_util.cpp b/test/executor/testing_executor_util.cpp index 455cf33e5e7..eeb9c88095d 100644 --- a/test/executor/testing_executor_util.cpp +++ b/test/executor/testing_executor_util.cpp @@ -52,9 +52,9 @@ storage::Database *TestingExecutorUtil::InitializeDatabase( auto catalog = catalog::Catalog::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - auto result = catalog->CreateDatabase(db_name, txn); + auto result = catalog->CreateDatabase(txn, db_name); EXPECT_EQ(ResultType::SUCCESS, result); - auto database = catalog->GetDatabaseWithName(db_name, txn); + auto database = catalog->GetDatabaseWithName(txn, db_name); txn_manager.CommitTransaction(txn); return (database); } @@ -63,7 +63,7 @@ void TestingExecutorUtil::DeleteDatabase(const std::string &db_name) { auto catalog = catalog::Catalog::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - auto result = catalog->DropDatabaseWithName(db_name, txn); + auto result = catalog->DropDatabaseWithName(txn, db_name); txn_manager.CommitTransaction(txn); EXPECT_EQ(ResultType::SUCCESS, result); } @@ -433,14 +433,18 @@ storage::DataTable *TestingExecutorUtil::CreateTableUpdateCatalog( auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); // Insert table in catalog - catalog->CreateTable(db_name, DEFAULT_SCHEMA_NAME, table_name, - std::move(table_schema), txn, is_catalog, + catalog->CreateTable(txn, + db_name, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + table_name, + is_catalog, tuples_per_tilegroup_count); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); auto table = - catalog->GetTableWithName(db_name, DEFAULT_SCHEMA_NAME, table_name, txn); + catalog->GetTableWithName(txn, db_name, DEFAULT_SCHEMA_NAME, table_name); txn_manager.CommitTransaction(txn); return table; diff --git a/test/executor/update_test.cpp b/test/executor/update_test.cpp index faa5526efea..80cbc4bce7c 100644 --- a/test/executor/update_test.cpp +++ b/test/executor/update_test.cpp @@ -159,7 +159,7 @@ TEST_F(UpdateTests, UpdatingOld) { auto catalog = catalog::Catalog::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); LOG_INFO("Bootstrapping completed!"); std::unique_ptr optimizer( @@ -193,8 +193,10 @@ TEST_F(UpdateTests, UpdatingOld) { LOG_INFO("Table created!"); - storage::DataTable *table = catalog->GetTableWithName( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); + storage::DataTable *table = catalog->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + "department_table"); txn_manager.CommitTransaction(txn); // Inserting a tuple end-to-end @@ -420,7 +422,7 @@ TEST_F(UpdateTests, UpdatingOld) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } } // namespace diff --git a/test/function/functions_test.cpp b/test/function/functions_test.cpp index 25b56ab5823..a20d24c40fa 100644 --- a/test/function/functions_test.cpp +++ b/test/function/functions_test.cpp @@ -44,23 +44,23 @@ TEST_F(FunctionsTests, CatalogTest) { auto &pg_language = catalog::LanguageCatalog::GetInstance(); // Test "internal" language auto txn = txn_manager.BeginTransaction(); - auto internal_lang = pg_language.GetLanguageByName("internal", txn); + auto internal_lang = pg_language.GetLanguageByName(txn, "internal"); EXPECT_NE(nullptr, internal_lang); - internal_lang = pg_language.GetLanguageByOid(internal_lang->GetOid(), txn); + internal_lang = pg_language.GetLanguageByOid(txn, internal_lang->GetOid()); EXPECT_NE(nullptr, internal_lang); EXPECT_EQ("internal", internal_lang->GetName()); // test add/del language type::EphemeralPool pool; std::string lanname = "foo_lang"; - pg_language.InsertLanguage(lanname, &pool, txn); - auto inserted_lang = pg_language.GetLanguageByName(lanname, txn); + pg_language.InsertLanguage(txn, lanname, &pool); + auto inserted_lang = pg_language.GetLanguageByName(txn, lanname); EXPECT_NE(nullptr, inserted_lang); - inserted_lang = pg_language.GetLanguageByOid(inserted_lang->GetOid(), txn); + inserted_lang = pg_language.GetLanguageByOid(txn, inserted_lang->GetOid()); EXPECT_NE(nullptr, inserted_lang); EXPECT_EQ(lanname, inserted_lang->GetName()); - pg_language.DeleteLanguage(lanname, txn); - inserted_lang = pg_language.GetLanguageByName(lanname, txn); + pg_language.DeleteLanguage(txn, lanname); + inserted_lang = pg_language.GetLanguageByName(txn, lanname); EXPECT_EQ(nullptr, inserted_lang); txn_manager.CommitTransaction(txn); @@ -72,11 +72,16 @@ TEST_F(FunctionsTests, CatalogTest) { std::vector arg_types{type::TypeId::VARCHAR, type::TypeId::INTEGER}; - catalog->AddBuiltinFunction( - func_name, arg_types, type::TypeId::INTEGER, internal_lang->GetOid(), - "TestFunc", function::BuiltInFuncType{OperatorId::Add, TestFunc}, txn); + catalog->AddBuiltinFunction(txn, + func_name, + function::BuiltInFuncType{OperatorId::Add, + TestFunc}, + "TestFunc", + type::TypeId::INTEGER, + arg_types, + internal_lang->GetOid()); - auto inserted_proc = pg_proc.GetProcByName(func_name, arg_types, txn); + auto inserted_proc = pg_proc.GetProcByName(txn, func_name, arg_types); EXPECT_NE(nullptr, inserted_proc); EXPECT_EQ(internal_lang->GetOid(), inserted_proc->GetLangOid()); type::TypeId ret_type = inserted_proc->GetRetType(); @@ -94,7 +99,7 @@ TEST_F(FunctionsTests, CatalogTest) { TEST_F(FunctionsTests, FuncCallTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); TestingSQLUtil::ExecuteSQLQuery( @@ -123,14 +128,14 @@ TEST_F(FunctionsTests, FuncCallTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(FunctionsTests, SubstrFuncCallTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); TestingSQLUtil::ExecuteSQLQuery("CREATE TABLE test(a DECIMAL, s VARCHAR);"); @@ -148,7 +153,7 @@ TEST_F(FunctionsTests, SubstrFuncCallTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/gc/garbage_collection_test.cpp b/test/gc/garbage_collection_test.cpp index d3b24b878fc..886131e5fb0 100644 --- a/test/gc/garbage_collection_test.cpp +++ b/test/gc/garbage_collection_test.cpp @@ -200,7 +200,8 @@ TEST_F(GarbageCollectionTests, UpdateTest) { TestingExecutorUtil::DeleteDatabase("update_db"); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - EXPECT_THROW(catalog::Catalog::GetInstance()->GetDatabaseObject(db_id, txn), + EXPECT_THROW(catalog::Catalog::GetInstance()->GetDatabaseCatalogEntry(txn, + db_id), CatalogException); txn_manager.CommitTransaction(txn); // EXPECT_FALSE(storage_manager->HasDatabase(db_id)); @@ -299,7 +300,7 @@ TEST_F(GarbageCollectionTests, DeleteTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); EXPECT_THROW( - catalog::Catalog::GetInstance()->GetDatabaseObject("DATABASE0", txn), + catalog::Catalog::GetInstance()->GetDatabaseCatalogEntry(txn, "DATABASE0"), CatalogException); txn_manager.CommitTransaction(txn); // EXPECT_FALSE(storage_manager->HasDatabase(db_id)); diff --git a/test/gc/transaction_level_gc_manager_test.cpp b/test/gc/transaction_level_gc_manager_test.cpp index cef62e0cf73..e574f5bd4d8 100644 --- a/test/gc/transaction_level_gc_manager_test.cpp +++ b/test/gc/transaction_level_gc_manager_test.cpp @@ -205,7 +205,7 @@ TEST_F(TransactionLevelGCManagerTests, UpdateDeleteTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); EXPECT_THROW( - catalog::Catalog::GetInstance()->GetDatabaseObject("database0", txn), + catalog::Catalog::GetInstance()->GetDatabaseCatalogEntry(txn, "database0"), CatalogException); txn_manager.CommitTransaction(txn); // EXPECT_FALSE(storage_manager->HasDatabase(db_id)); @@ -368,7 +368,7 @@ TEST_F(TransactionLevelGCManagerTests, ReInsertTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); EXPECT_THROW( - catalog::Catalog::GetInstance()->GetDatabaseObject("database0", txn), + catalog::Catalog::GetInstance()->GetDatabaseCatalogEntry(txn, "database0"), CatalogException); txn_manager.CommitTransaction(txn); // EXPECT_FALSE(storage_manager->HasDatabase(db_id)); @@ -481,7 +481,8 @@ TEST_F(TransactionLevelGCManagerTests, ImmutabilityTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); EXPECT_THROW( - catalog::Catalog::GetInstance()->GetDatabaseObject("immutabilitydb", txn), + catalog::Catalog::GetInstance()->GetDatabaseCatalogEntry(txn, + "immutabilitydb"), CatalogException); txn_manager.CommitTransaction(txn); } diff --git a/test/include/catalog/testing_constraints_util.h b/test/include/catalog/testing_constraints_util.h index 1c96b995fa9..ab39e311bd1 100644 --- a/test/include/catalog/testing_constraints_util.h +++ b/test/include/catalog/testing_constraints_util.h @@ -116,7 +116,7 @@ class TestingConstraintsUtil { auto catalog = catalog::Catalog::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); // First populate the list of catalog::Columns that we @@ -132,14 +132,14 @@ class TestingConstraintsUtil { // Create table. txn = txn_manager.BeginTransaction(); auto result = - catalog->CreateTable(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table_name, - std::move(table_schema), txn, false); + catalog->CreateTable(txn, DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, + std::move(table_schema), table_name, false); txn_manager.CommitTransaction(txn); EXPECT_EQ(ResultType::SUCCESS, result); txn = txn_manager.BeginTransaction(); - auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, - table_name, txn); + auto table = catalog->GetTableWithName(txn, DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, + table_name); txn_manager.CommitTransaction(txn); EXPECT_NE(nullptr, table); diff --git a/test/optimizer/old_optimizer_test.cpp b/test/optimizer/old_optimizer_test.cpp index 92949cc8521..348c5bbdf52 100644 --- a/test/optimizer/old_optimizer_test.cpp +++ b/test/optimizer/old_optimizer_test.cpp @@ -44,7 +44,7 @@ TEST_F(OldOptimizerTests, UpdateDelWithIndexScanTest) { LOG_TRACE("Bootstrapping..."); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); LOG_TRACE("Bootstrapping completed!"); @@ -170,8 +170,10 @@ TEST_F(OldOptimizerTests, UpdateDelWithIndexScanTest) { traffic_cop.CommitQueryHelper(); txn = txn_manager.BeginTransaction(); - auto target_table_ = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); + auto target_table_ = catalog::Catalog::GetInstance()->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + "department_table"); // Expected 1 , Primary key index + created index EXPECT_EQ(target_table_->GetIndexCount(), 2); txn_manager.CommitTransaction(txn); @@ -247,7 +249,7 @@ TEST_F(OldOptimizerTests, UpdateDelWithIndexScanTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/optimizer/operator_transformer_test.cpp b/test/optimizer/operator_transformer_test.cpp index f1d5229798c..b1b466991f0 100644 --- a/test/optimizer/operator_transformer_test.cpp +++ b/test/optimizer/operator_transformer_test.cpp @@ -39,7 +39,7 @@ class OperatorTransformerTests : public PelotonTest { // Create test database auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); // Create table @@ -94,7 +94,7 @@ class OperatorTransformerTests : public PelotonTest { // Destroy test database auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); // Call parent virtual function diff --git a/test/optimizer/optimizer_test.cpp b/test/optimizer/optimizer_test.cpp index 50696017bb5..f1ffd6add66 100644 --- a/test/optimizer/optimizer_test.cpp +++ b/test/optimizer/optimizer_test.cpp @@ -61,7 +61,7 @@ class OptimizerTests : public PelotonTest { // Destroy test database auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); // Call parent virtual function @@ -75,7 +75,7 @@ TEST_F(OptimizerTests, HashJoinTest) { LOG_INFO("Bootstrapping..."); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); LOG_INFO("Bootstrapping completed!"); @@ -125,7 +125,7 @@ TEST_F(OptimizerTests, HashJoinTest) { oid_t expected_table_count = CATALOG_TABLES_COUNT + 1; txn = txn_manager.BeginTransaction(); EXPECT_EQ(catalog::Catalog::GetInstance() - ->GetDatabaseWithName(DEFAULT_DB_NAME, txn) + ->GetDatabaseWithName(txn, DEFAULT_DB_NAME) ->GetTableCount(), expected_table_count); @@ -162,7 +162,7 @@ TEST_F(OptimizerTests, HashJoinTest) { expected_table_count++; txn = txn_manager.BeginTransaction(); EXPECT_EQ(catalog::Catalog::GetInstance() - ->GetDatabaseWithName(DEFAULT_DB_NAME, txn) + ->GetDatabaseWithName(txn, DEFAULT_DB_NAME) ->GetTableCount(), expected_table_count); @@ -261,7 +261,7 @@ TEST_F(OptimizerTests, HashJoinTest) { TEST_F(OptimizerTests, PredicatePushDownTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); TestingSQLUtil::ExecuteSQLQuery( @@ -315,7 +315,7 @@ TEST_F(OptimizerTests, PredicatePushDownTest) { TEST_F(OptimizerTests, PushFilterThroughJoinTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); TestingSQLUtil::ExecuteSQLQuery( @@ -397,7 +397,7 @@ TEST_F(OptimizerTests, PushFilterThroughJoinTest) { TEST_F(OptimizerTests, PredicatePushDownRewriteTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); TestingSQLUtil::ExecuteSQLQuery( @@ -479,7 +479,7 @@ TEST_F(OptimizerTests, ExecuteTaskStackTest) { // Currently need database for test teardown auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); const auto num_tasks = 2; diff --git a/test/optimizer/selectivity_test.cpp b/test/optimizer/selectivity_test.cpp index cbb8df08d2c..6794fa29b9f 100644 --- a/test/optimizer/selectivity_test.cpp +++ b/test/optimizer/selectivity_test.cpp @@ -56,7 +56,7 @@ void ExpectSelectivityEqual(double actual, double expected, TEST_F(SelectivityTests, RangeSelectivityTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -71,9 +71,11 @@ TEST_F(SelectivityTests, RangeSelectivityTest) { txn = txn_manager.BeginTransaction(); auto catalog = catalog::Catalog::GetInstance(); - auto database = catalog->GetDatabaseWithName(DEFAULT_DB_NAME, txn); - auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, - TEST_TABLE_NAME, txn); + auto database = catalog->GetDatabaseWithName(txn, DEFAULT_DB_NAME); + auto table = catalog->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + TEST_TABLE_NAME); txn_manager.CommitTransaction(txn); oid_t db_id = database->GetOid(); oid_t table_id = table->GetOid(); @@ -109,7 +111,7 @@ TEST_F(SelectivityTests, RangeSelectivityTest) { // Free the database txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -165,7 +167,7 @@ TEST_F(SelectivityTests, LikeSelectivityTest) { TEST_F(SelectivityTests, EqualSelectivityTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -180,9 +182,11 @@ TEST_F(SelectivityTests, EqualSelectivityTest) { txn = txn_manager.BeginTransaction(); auto catalog = catalog::Catalog::GetInstance(); - auto database = catalog->GetDatabaseWithName(DEFAULT_DB_NAME, txn); - auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, - TEST_TABLE_NAME, txn); + auto database = catalog->GetDatabaseWithName(txn, DEFAULT_DB_NAME); + auto table = catalog->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + TEST_TABLE_NAME); txn_manager.CommitTransaction(txn); oid_t db_id = database->GetOid(); oid_t table_id = table->GetOid(); @@ -258,7 +262,7 @@ TEST_F(SelectivityTests, EqualSelectivityTest) { // Free the database txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/optimizer/table_stats_collector_test.cpp b/test/optimizer/table_stats_collector_test.cpp index e98f1a54bf9..fcce0079777 100644 --- a/test/optimizer/table_stats_collector_test.cpp +++ b/test/optimizer/table_stats_collector_test.cpp @@ -48,7 +48,7 @@ TEST_F(TableStatsCollectorTests, SingleColumnTableTest) { auto catalog = catalog::Catalog::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); TestingSQLUtil::ExecuteSQLQuery("CREATE TABLE test(id integer);"); @@ -60,8 +60,10 @@ TEST_F(TableStatsCollectorTests, SingleColumnTableTest) { } txn = txn_manager.BeginTransaction(); - auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, - "test", txn); + auto table = catalog->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + "test"); txn_manager.CommitTransaction(txn); TableStatsCollector stats{table}; stats.CollectColumnStats(); @@ -80,7 +82,7 @@ TEST_F(TableStatsCollectorTests, SingleColumnTableTest) { // Free the database txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -91,7 +93,7 @@ TEST_F(TableStatsCollectorTests, MultiColumnTableTest) { auto catalog = catalog::Catalog::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); TestingSQLUtil::ExecuteSQLQuery( @@ -111,8 +113,10 @@ TEST_F(TableStatsCollectorTests, MultiColumnTableTest) { } txn = txn_manager.BeginTransaction(); - auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, - "test", txn); + auto table = catalog->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + "test"); txn_manager.CommitTransaction(txn); TableStatsCollector stats{table}; stats.CollectColumnStats(); @@ -141,7 +145,7 @@ TEST_F(TableStatsCollectorTests, MultiColumnTableTest) { // Free the database txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/optimizer/tuple_samples_storage_test.cpp b/test/optimizer/tuple_samples_storage_test.cpp index 72ed769f270..62e2b595d95 100644 --- a/test/optimizer/tuple_samples_storage_test.cpp +++ b/test/optimizer/tuple_samples_storage_test.cpp @@ -46,7 +46,7 @@ TEST_F(TupleSamplesStorageTests, SamplesDBTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); storage::Database *samples_db = - catalog->GetDatabaseWithName(SAMPLES_DB_NAME, txn); + catalog->GetDatabaseWithName(txn, SAMPLES_DB_NAME); txn_manager.CommitTransaction(txn); EXPECT_TRUE(samples_db != nullptr); EXPECT_EQ(samples_db->GetDBName(), SAMPLES_DB_NAME); @@ -84,8 +84,10 @@ TEST_F(TupleSamplesStorageTests, AddSamplesTableTest) { tuple_samples_storage->GenerateSamplesTableName( data_table->GetDatabaseOid(), data_table->GetOid()); txn = txn_manager.BeginTransaction(); - storage::DataTable *samples_table = catalog->GetTableWithName( - SAMPLES_DB_NAME, DEFAULT_SCHEMA_NAME, samples_table_name, txn); + storage::DataTable *samples_table = catalog->GetTableWithName(txn, + SAMPLES_DB_NAME, + DEFAULT_SCHEMA_NAME, + samples_table_name); txn_manager.CommitTransaction(txn); EXPECT_TRUE(samples_table != nullptr); diff --git a/test/planner/plan_util_test.cpp b/test/planner/plan_util_test.cpp index 77df6f54e88..6130d29d260 100644 --- a/test/planner/plan_util_test.cpp +++ b/test/planner/plan_util_test.cpp @@ -39,7 +39,7 @@ TEST_F(PlanUtilTests, GetAffectedIndexesTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog->CreateDatabase(TEST_DB_NAME, txn); + catalog->CreateDatabase(txn, TEST_DB_NAME); // Insert a table first auto id_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), @@ -54,10 +54,16 @@ TEST_F(PlanUtilTests, GetAffectedIndexesTest) { txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - catalog->CreateTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "test_table", - std::move(table_schema), txn); - auto source_table = catalog->GetTableWithName( - TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "test_table", txn); + catalog->CreateTable(txn, + TEST_DB_NAME, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + "test_table", + false); + auto source_table = catalog->GetTableWithName(txn, + TEST_DB_NAME, + DEFAULT_SCHEMA_NAME, + "test_table"); EXPECT_NE(source_table, nullptr); txn_manager.CommitTransaction(txn); @@ -67,35 +73,45 @@ TEST_F(PlanUtilTests, GetAffectedIndexesTest) { source_col_ids.push_back(col_id); // create index on 'id' - catalog->CreateIndex(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "test_table", - source_col_ids, "test_id_idx", false, IndexType::BWTREE, - txn); + catalog->CreateIndex(txn, + TEST_DB_NAME, + DEFAULT_SCHEMA_NAME, + "test_table", + "test_id_idx", + source_col_ids, + false, + IndexType::BWTREE); // create index on 'id' and 'first_name' col_id = source_table->GetSchema()->GetColumnID(fname_column.column_name); source_col_ids.push_back(col_id); - catalog->CreateIndex(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "test_table", - source_col_ids, "test_fname_idx", false, - IndexType::BWTREE, txn); + catalog->CreateIndex(txn, + TEST_DB_NAME, + DEFAULT_SCHEMA_NAME, + "test_table", + "test_fname_idx", + source_col_ids, + false, + IndexType::BWTREE); txn_manager.CommitTransaction(txn); // dummy txn to get the catalog_cache object txn = txn_manager.BeginTransaction(); // This is also required so that database objects are cached - auto db_object = catalog->GetDatabaseObject(TEST_DB_NAME, txn); + auto db_object = catalog->GetDatabaseCatalogEntry(txn, TEST_DB_NAME); // Till now, we have a table : id, first_name, last_name // And two indexes on following columns: // 1) id // 2) id and first_name auto table_object = - db_object->GetTableObject("test_table", DEFAULT_SCHEMA_NAME); + db_object->GetTableCatalogEntry("test_table", DEFAULT_SCHEMA_NAME); EXPECT_NE(table_object, nullptr); - oid_t id_idx_oid = table_object->GetIndexObject("test_id_idx")->GetIndexOid(); + oid_t id_idx_oid = table_object->GetIndexCatalogEntry("test_id_idx")->GetIndexOid(); oid_t fname_idx_oid = - table_object->GetIndexObject("test_fname_idx")->GetIndexOid(); + table_object->GetIndexCatalogEntry("test_fname_idx")->GetIndexOid(); // An update query affecting both indexes std::string query_string = "UPDATE test_table SET id = 0;"; @@ -178,11 +194,11 @@ TEST_F(PlanUtilTests, GetIndexableColumnsTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog->CreateDatabase(TEST_DB_COLUMNS, txn); - auto db = catalog->GetDatabaseWithName(TEST_DB_COLUMNS, txn); + catalog->CreateDatabase(txn, TEST_DB_COLUMNS); + auto db = catalog->GetDatabaseWithName(txn, TEST_DB_COLUMNS); oid_t database_id = db->GetOid(); - auto db_object = catalog->GetDatabaseObject(TEST_DB_COLUMNS, txn); - int table_count = db_object->GetTableObjects().size(); + auto db_object = catalog->GetDatabaseCatalogEntry(txn, TEST_DB_COLUMNS); + int table_count = db_object->GetTableCatalogEntries().size(); txn_manager.CommitTransaction(txn); // Insert a 'test_table' with 'id', 'first_name' and 'last_name' @@ -198,14 +214,20 @@ TEST_F(PlanUtilTests, GetIndexableColumnsTest) { new catalog::Schema({id_column, fname_column, lname_column})); txn = txn_manager.BeginTransaction(); - catalog->CreateTable(TEST_DB_COLUMNS, DEFAULT_SCHEMA_NAME, "test_table", - std::move(table_schema), txn); + catalog->CreateTable(txn, + TEST_DB_COLUMNS, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + "test_table", + false); txn_manager.CommitTransaction(txn); // Obtain ids for the table and columns txn = txn_manager.BeginTransaction(); - auto source_table = catalog->GetTableWithName( - TEST_DB_COLUMNS, DEFAULT_SCHEMA_NAME, "test_table", txn); + auto source_table = catalog->GetTableWithName(txn, + TEST_DB_COLUMNS, + DEFAULT_SCHEMA_NAME, + "test_table"); txn_manager.CommitTransaction(txn); oid_t table_id = source_table->GetOid(); @@ -228,14 +250,20 @@ TEST_F(PlanUtilTests, GetIndexableColumnsTest) { std::unique_ptr job_table_schema( new catalog::Schema({age_column, job_column, pid_column})); - catalog->CreateTable(TEST_DB_COLUMNS, DEFAULT_SCHEMA_NAME, "test_table_job", - std::move(job_table_schema), txn); + catalog->CreateTable(txn, + TEST_DB_COLUMNS, + DEFAULT_SCHEMA_NAME, + std::move(job_table_schema), + "test_table_job", + false); txn_manager.CommitTransaction(txn); // Obtain ids for the table and columns txn = txn_manager.BeginTransaction(); - auto source_table_job = catalog->GetTableWithName( - TEST_DB_COLUMNS, DEFAULT_SCHEMA_NAME, "test_table_job", txn); + auto source_table_job = catalog->GetTableWithName(txn, + TEST_DB_COLUMNS, + DEFAULT_SCHEMA_NAME, + "test_table_job"); oid_t table_job_id = source_table_job->GetOid(); oid_t age_col_oid = source_table_job->GetSchema()->GetColumnID(age_column.column_name); @@ -247,9 +275,9 @@ TEST_F(PlanUtilTests, GetIndexableColumnsTest) { txn = txn_manager.BeginTransaction(); // This is required so that database objects are cached - db_object = catalog->GetDatabaseObject(TEST_DB_COLUMNS, txn); + db_object = catalog->GetDatabaseCatalogEntry(txn, TEST_DB_COLUMNS); EXPECT_EQ( - 2, static_cast(db_object->GetTableObjects().size()) - table_count); + 2, static_cast(db_object->GetTableCatalogEntries().size()) - table_count); // ====== UPDATE statements check === // id and first_name in test_table are affected diff --git a/test/planner/planner_equality_test.cpp b/test/planner/planner_equality_test.cpp index fe109bc9dff..dbb9feb6c58 100644 --- a/test/planner/planner_equality_test.cpp +++ b/test/planner/planner_equality_test.cpp @@ -35,7 +35,7 @@ class PlannerEqualityTest : public PelotonTest { // Destroy test database auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); // Call parent virtual function @@ -46,7 +46,7 @@ class PlannerEqualityTest : public PelotonTest { // Create database auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); // Create a table 'test' diff --git a/test/planner/planner_test.cpp b/test/planner/planner_test.cpp index 5fd1709eaca..19304048dd9 100644 --- a/test/planner/planner_test.cpp +++ b/test/planner/planner_test.cpp @@ -72,7 +72,7 @@ TEST_F(PlannerTest, DeletePlanTestParameter) { // Bootstrapping peloton auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); // Create table @@ -84,9 +84,12 @@ TEST_F(PlannerTest, DeletePlanTestParameter) { std::unique_ptr table_schema( new catalog::Schema({id_column, name_column})); - catalog::Catalog::GetInstance()->CreateTable( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", - std::move(table_schema), txn); + catalog::Catalog::GetInstance()->CreateTable(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + "department_table", + false); txn_manager.CommitTransaction(txn); // DELETE FROM department_table WHERE id = $0 @@ -99,8 +102,10 @@ TEST_F(PlannerTest, DeletePlanTestParameter) { auto *scan_expr = new expression::ComparisonExpression( ExpressionType::COMPARE_EQUAL, tuple_expr, parameter_expr); - auto target_table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); + auto target_table = catalog::Catalog::GetInstance()->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + "department_table"); // Create delete plan std::unique_ptr delete_plan( @@ -128,7 +133,7 @@ TEST_F(PlannerTest, DeletePlanTestParameter) { delete_plan->SetParameterValues(&values); // free the database just created - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -136,7 +141,7 @@ TEST_F(PlannerTest, UpdatePlanTestParameter) { // Bootstrapping peloton auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); // Create table @@ -148,9 +153,12 @@ TEST_F(PlannerTest, UpdatePlanTestParameter) { std::unique_ptr table_schema( new catalog::Schema({id_column, name_column})); - catalog::Catalog::GetInstance()->CreateTable( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", - std::move(table_schema), txn); + catalog::Catalog::GetInstance()->CreateTable(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + "department_table", + false); txn_manager.CommitTransaction(txn); // UPDATE department_table SET name = $0 WHERE id = $1 @@ -158,8 +166,10 @@ TEST_F(PlannerTest, UpdatePlanTestParameter) { auto table_name = std::string("department_table"); auto database_name = DEFAULT_DB_NAME; - auto target_table = catalog::Catalog::GetInstance()->GetTableWithName( - database_name, DEFAULT_SCHEMA_NAME, table_name, txn); + auto target_table = catalog::Catalog::GetInstance()->GetTableWithName(txn, + database_name, + DEFAULT_SCHEMA_NAME, + table_name); auto schema = target_table->GetSchema(); TargetList tlist; @@ -225,7 +235,7 @@ TEST_F(PlannerTest, UpdatePlanTestParameter) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -233,7 +243,7 @@ TEST_F(PlannerTest, InsertPlanTestParameter) { // Bootstrapping peloton auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); // Create table @@ -245,9 +255,12 @@ TEST_F(PlannerTest, InsertPlanTestParameter) { std::unique_ptr table_schema( new catalog::Schema({id_column, name_column})); - auto ret = catalog::Catalog::GetInstance()->CreateTable( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", - std::move(table_schema), txn); + auto ret = catalog::Catalog::GetInstance()->CreateTable(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + "department_table", + false); if (ret != ResultType::SUCCESS) LOG_TRACE("create table failed"); txn_manager.CommitTransaction(txn); @@ -275,8 +288,10 @@ TEST_F(PlannerTest, InsertPlanTestParameter) { parameter_exprs.push_back( std::unique_ptr(parameter_expr_2)); - auto target_table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); + auto target_table = catalog::Catalog::GetInstance()->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + "department_table"); std::unique_ptr insert_plan( new planner::InsertPlan(target_table, &insert_statement->columns, @@ -299,7 +314,7 @@ TEST_F(PlannerTest, InsertPlanTestParameter) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -307,7 +322,7 @@ TEST_F(PlannerTest, InsertPlanTestParameterColumns) { // Bootstrapping peloton auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); // Create table @@ -319,9 +334,12 @@ TEST_F(PlannerTest, InsertPlanTestParameterColumns) { std::unique_ptr table_schema( new catalog::Schema({id_column, name_column})); - catalog::Catalog::GetInstance()->CreateTable( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", - std::move(table_schema), txn); + catalog::Catalog::GetInstance()->CreateTable(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + "department_table", + false); txn_manager.CommitTransaction(txn); // INSERT INTO department_table VALUES (1, $1) @@ -353,8 +371,10 @@ TEST_F(PlannerTest, InsertPlanTestParameterColumns) { exprs.push_back( std::unique_ptr(parameter_expr_2)); - auto target_table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); + auto target_table = catalog::Catalog::GetInstance()->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + "department_table"); std::unique_ptr insert_plan( new planner::InsertPlan(target_table, &insert_statement->columns, @@ -375,7 +395,7 @@ TEST_F(PlannerTest, InsertPlanTestParameterColumns) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/settings/settings_manager_test.cpp b/test/settings/settings_manager_test.cpp index e9e00162259..251b057648a 100644 --- a/test/settings/settings_manager_test.cpp +++ b/test/settings/settings_manager_test.cpp @@ -35,7 +35,7 @@ TEST_F(SettingsManagerTests, InitializationTest) { auto txn = txn_manager.BeginTransaction(); int32_t port = settings::SettingsManager::GetInt(settings::SettingId::port); int32_t port_default = - atoi(settings_catalog.GetDefaultValue("port", txn).c_str()); + atoi(settings_catalog.GetDefaultValue(txn, "port").c_str()); txn_manager.CommitTransaction(txn); EXPECT_EQ(port, port_default); @@ -44,7 +44,7 @@ TEST_F(SettingsManagerTests, InitializationTest) { std::string socket_family = settings::SettingsManager::GetString(settings::SettingId::socket_family); std::string socket_family_default = - settings_catalog.GetDefaultValue("socket_family", txn); + settings_catalog.GetDefaultValue(txn, "socket_family"); txn_manager.CommitTransaction(txn); EXPECT_EQ(socket_family, socket_family_default); @@ -53,7 +53,7 @@ TEST_F(SettingsManagerTests, InitializationTest) { bool index_tuner = settings::SettingsManager::GetBool(settings::SettingId::index_tuner); bool index_tuner_default = - ("true" == settings_catalog.GetDefaultValue("index_tuner", txn)); + ("true" == settings_catalog.GetDefaultValue(txn, "index_tuner")); txn_manager.CommitTransaction(txn); EXPECT_EQ(index_tuner, index_tuner_default); } @@ -71,7 +71,7 @@ TEST_F(SettingsManagerTests, ModificationTest) { // modify int auto txn = txn_manager.BeginTransaction(); int32_t value1 = settings::SettingsManager::GetInt(settings::SettingId::port); - int32_t value2 = atoi(settings_catalog.GetSettingValue("port", txn).c_str()); + int32_t value2 = atoi(settings_catalog.GetSettingValue(txn, "port").c_str()); EXPECT_EQ(value1, value2); txn_manager.CommitTransaction(txn); @@ -79,7 +79,7 @@ TEST_F(SettingsManagerTests, ModificationTest) { txn = txn_manager.BeginTransaction(); int32_t value3 = settings::SettingsManager::GetInt(settings::SettingId::port); - int32_t value4 = atoi(settings_catalog.GetSettingValue("port", txn).c_str()); + int32_t value4 = atoi(settings_catalog.GetSettingValue(txn, "port").c_str()); EXPECT_EQ(value3, 12345); EXPECT_EQ(value3, value4); txn_manager.CommitTransaction(txn); @@ -89,7 +89,7 @@ TEST_F(SettingsManagerTests, ModificationTest) { bool value5 = settings::SettingsManager::GetBool(settings::SettingId::index_tuner); bool value6 = - ("true" == settings_catalog.GetSettingValue("index_tuner", txn)); + ("true" == settings_catalog.GetSettingValue(txn, "index_tuner")); EXPECT_EQ(value5, value6); txn_manager.CommitTransaction(txn); @@ -99,7 +99,7 @@ TEST_F(SettingsManagerTests, ModificationTest) { bool value7 = settings::SettingsManager::GetBool(settings::SettingId::index_tuner); bool value8 = - ("true" == settings_catalog.GetSettingValue("index_tuner", txn)); + ("true" == settings_catalog.GetSettingValue(txn, "index_tuner")); EXPECT_TRUE(value7); EXPECT_EQ(value7, value8); txn_manager.CommitTransaction(txn); @@ -108,7 +108,7 @@ TEST_F(SettingsManagerTests, ModificationTest) { txn = txn_manager.BeginTransaction(); std::string value9 = settings::SettingsManager::GetString(settings::SettingId::socket_family); - std::string value10 = settings_catalog.GetSettingValue("socket_family", txn); + std::string value10 = settings_catalog.GetSettingValue(txn, "socket_family"); EXPECT_EQ(value9, value10); txn_manager.CommitTransaction(txn); @@ -118,7 +118,7 @@ TEST_F(SettingsManagerTests, ModificationTest) { txn = txn_manager.BeginTransaction(); std::string value11 = settings::SettingsManager::GetString(settings::SettingId::socket_family); - std::string value12 = settings_catalog.GetSettingValue("socket_family", txn); + std::string value12 = settings_catalog.GetSettingValue(txn, "socket_family"); EXPECT_EQ(value11, "test"); EXPECT_EQ(value11, value12); txn_manager.CommitTransaction(txn); diff --git a/test/sql/aggregate_groupby_sql_test.cpp b/test/sql/aggregate_groupby_sql_test.cpp index 527ede393ca..198156cc6b7 100644 --- a/test/sql/aggregate_groupby_sql_test.cpp +++ b/test/sql/aggregate_groupby_sql_test.cpp @@ -42,7 +42,7 @@ class AggregateGroupBySQLTests : public PelotonTest { TEST_F(AggregateGroupBySQLTests, AggregateGroupByManyAVGsSQLTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -53,14 +53,14 @@ TEST_F(AggregateGroupBySQLTests, AggregateGroupByManyAVGsSQLTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(AggregateGroupBySQLTests, AggregateGroupByMixedAVGsSQLTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -73,14 +73,14 @@ TEST_F(AggregateGroupBySQLTests, AggregateGroupByMixedAVGsSQLTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(AggregateGroupBySQLTests, AggregateGroupByAllAggregationsSQLTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -91,14 +91,14 @@ TEST_F(AggregateGroupBySQLTests, AggregateGroupByAllAggregationsSQLTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(AggregateGroupBySQLTests, AggregateGroupBySingleRowPerGroupSQLTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -110,7 +110,7 @@ TEST_F(AggregateGroupBySQLTests, AggregateGroupBySingleRowPerGroupSQLTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/sql/aggregate_sql_test.cpp b/test/sql/aggregate_sql_test.cpp index 62fcaddea8a..d240b72da3e 100644 --- a/test/sql/aggregate_sql_test.cpp +++ b/test/sql/aggregate_sql_test.cpp @@ -29,7 +29,7 @@ TEST_F(AggregateSQLTests, EmptyTableTest) { PELOTON_ASSERT(&TestingSQLUtil::traffic_cop_); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); // Create a table first @@ -200,7 +200,7 @@ TEST_F(AggregateSQLTests, MinMaxTest) { // free the database just created auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); LOG_DEBUG("Finish test"); } diff --git a/test/sql/analyze_sql_test.cpp b/test/sql/analyze_sql_test.cpp index 16191ec000d..2504b839342 100644 --- a/test/sql/analyze_sql_test.cpp +++ b/test/sql/analyze_sql_test.cpp @@ -44,7 +44,7 @@ void CreateAndLoadTable() { TEST_F(AnalyzeSQLTests, AnalyzeAllTablesTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -54,14 +54,14 @@ TEST_F(AnalyzeSQLTests, AnalyzeAllTablesTest) { // Free the database txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(AnalyzeSQLTests, AnalyzeSingleTableTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -74,20 +74,22 @@ TEST_F(AnalyzeSQLTests, AnalyzeSingleTableTest) { txn = txn_manager.BeginTransaction(); auto catalog = catalog::Catalog::GetInstance(); storage::DataTable *db_column_stats_collector_table = - catalog->GetTableWithName(CATALOG_DATABASE_NAME, CATALOG_SCHEMA_NAME, - COLUMN_STATS_CATALOG_NAME, txn); + catalog->GetTableWithName(txn, + CATALOG_DATABASE_NAME, + CATALOG_SCHEMA_NAME, + COLUMN_STATS_CATALOG_NAME); EXPECT_NE(db_column_stats_collector_table, nullptr); EXPECT_EQ(db_column_stats_collector_table->GetTupleCount(), 4); // Free the database - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(AnalyzeSQLTests, AnalyzeTableWithColumnsTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -97,7 +99,7 @@ TEST_F(AnalyzeSQLTests, AnalyzeTableWithColumnsTest) { // Free the database txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/sql/case_sql_test.cpp b/test/sql/case_sql_test.cpp index 7fe464b2974..7d0078d1383 100644 --- a/test/sql/case_sql_test.cpp +++ b/test/sql/case_sql_test.cpp @@ -43,7 +43,7 @@ TEST_F(CaseSQLTests, Simple) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -72,7 +72,7 @@ TEST_F(CaseSQLTests, Simple) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -81,7 +81,7 @@ TEST_F(CaseSQLTests, SimpleWithArg) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -109,7 +109,7 @@ TEST_F(CaseSQLTests, SimpleWithArg) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -118,7 +118,7 @@ TEST_F(CaseSQLTests, SimpleWithArgStringResult) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -146,7 +146,7 @@ TEST_F(CaseSQLTests, SimpleWithArgStringResult) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -156,7 +156,7 @@ TEST_F(CaseSQLTests, SimpleMultipleWhen) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -184,7 +184,7 @@ TEST_F(CaseSQLTests, SimpleMultipleWhen) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -194,7 +194,7 @@ TEST_F(CaseSQLTests, SimpleMultipleWhenWithoutElse) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -222,7 +222,7 @@ TEST_F(CaseSQLTests, SimpleMultipleWhenWithoutElse) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/sql/decimal_functions_sql_test.cpp b/test/sql/decimal_functions_sql_test.cpp index 850e075a62c..366d2b2f781 100644 --- a/test/sql/decimal_functions_sql_test.cpp +++ b/test/sql/decimal_functions_sql_test.cpp @@ -32,7 +32,7 @@ class DecimalSQLTestsBase : public PelotonTest { // Create database auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -50,7 +50,7 @@ class DecimalSQLTestsBase : public PelotonTest { // Destroy test database auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); // Call parent virtual function @@ -63,7 +63,7 @@ class DecimalFunctionsSQLTest : public PelotonTest {}; TEST_F(DecimalFunctionsSQLTest, FloorTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); catalog::Catalog::GetInstance()->Bootstrap(); txn_manager.CommitTransaction(txn); @@ -107,7 +107,7 @@ TEST_F(DecimalFunctionsSQLTest, FloorTest) { } // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -297,7 +297,7 @@ TEST_F(DecimalSQLTestsBase, DecimalAbsTest) { TEST_F(DecimalFunctionsSQLTest, CeilTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); // NOTE: Catalog::GetInstance()->Bootstrap() has been called in previous tests // you can only call it once! // catalog::Catalog::GetInstance()->Bootstrap(); @@ -360,7 +360,7 @@ TEST_F(DecimalFunctionsSQLTest, CeilTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/sql/delete_sql_test.cpp b/test/sql/delete_sql_test.cpp index 1648e0a3b1b..eefbc22ba1c 100644 --- a/test/sql/delete_sql_test.cpp +++ b/test/sql/delete_sql_test.cpp @@ -28,7 +28,7 @@ TEST_F(DeleteSQLTests, SimpleDeleteSQLTest) { auto catalog = catalog::Catalog::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); LOG_DEBUG("Bootstrapping completed!"); @@ -118,7 +118,7 @@ TEST_F(DeleteSQLTests, SimpleDeleteSQLTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/sql/distinct_aggregate_sql_test.cpp b/test/sql/distinct_aggregate_sql_test.cpp index a48412e0b68..ae4ba2fc2e2 100644 --- a/test/sql/distinct_aggregate_sql_test.cpp +++ b/test/sql/distinct_aggregate_sql_test.cpp @@ -87,7 +87,7 @@ class DistinctAggregateSQLTests : public PelotonTest { TEST_F(DistinctAggregateSQLTests, DistinctAggregateCountTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -100,14 +100,14 @@ TEST_F(DistinctAggregateSQLTests, DistinctAggregateCountTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(DistinctAggregateSQLTests, DistinctAggregateMaxTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -120,14 +120,14 @@ TEST_F(DistinctAggregateSQLTests, DistinctAggregateMaxTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(DistinctAggregateSQLTests, DistinctAggregateMinTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -140,14 +140,14 @@ TEST_F(DistinctAggregateSQLTests, DistinctAggregateMinTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(DistinctAggregateSQLTests, DistinctAggregateAvgTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -159,14 +159,14 @@ TEST_F(DistinctAggregateSQLTests, DistinctAggregateAvgTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(DistinctAggregateSQLTests, DistinctAggregateSumTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -178,14 +178,14 @@ TEST_F(DistinctAggregateSQLTests, DistinctAggregateSumTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(DistinctAggregateSQLTests, DistinctAggregateGroupByCountTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -198,14 +198,14 @@ TEST_F(DistinctAggregateSQLTests, DistinctAggregateGroupByCountTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(DistinctAggregateSQLTests, DistinctAggregateGroupByMaxTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -220,14 +220,14 @@ TEST_F(DistinctAggregateSQLTests, DistinctAggregateGroupByMaxTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(DistinctAggregateSQLTests, DistinctAggregateGroupByMinTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -242,14 +242,14 @@ TEST_F(DistinctAggregateSQLTests, DistinctAggregateGroupByMinTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(DistinctAggregateSQLTests, DistinctAggregateGroupByAvgTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -261,14 +261,14 @@ TEST_F(DistinctAggregateSQLTests, DistinctAggregateGroupByAvgTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(DistinctAggregateSQLTests, DistinctAggregateGroupBySumTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -280,7 +280,7 @@ TEST_F(DistinctAggregateSQLTests, DistinctAggregateGroupBySumTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/sql/distinct_sql_test.cpp b/test/sql/distinct_sql_test.cpp index d2bfa637ddf..86520d28474 100644 --- a/test/sql/distinct_sql_test.cpp +++ b/test/sql/distinct_sql_test.cpp @@ -81,7 +81,7 @@ class DistinctSQLTests : public PelotonTest { TEST_F(DistinctSQLTests, DistinctIntTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -91,14 +91,14 @@ TEST_F(DistinctSQLTests, DistinctIntTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(DistinctSQLTests, DistinctVarcharTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -108,14 +108,14 @@ TEST_F(DistinctSQLTests, DistinctVarcharTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(DistinctSQLTests, DistinctTupleTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -126,14 +126,14 @@ TEST_F(DistinctSQLTests, DistinctTupleTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(DistinctSQLTests, DistinctStarTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); TestingSQLUtil::ExecuteSQLQuery( @@ -153,14 +153,14 @@ TEST_F(DistinctSQLTests, DistinctStarTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(DistinctSQLTests, DistinctDateTimeTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); TestingSQLUtil::ExecuteSQLQuery( @@ -180,7 +180,7 @@ TEST_F(DistinctSQLTests, DistinctDateTimeTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/sql/drop_sql_test.cpp b/test/sql/drop_sql_test.cpp index 2e0dccc63f7..b541437ba71 100644 --- a/test/sql/drop_sql_test.cpp +++ b/test/sql/drop_sql_test.cpp @@ -29,7 +29,7 @@ class DropSQLTests : public PelotonTest {}; TEST_F(DropSQLTests, DropTableTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); // Create a table first @@ -40,8 +40,10 @@ TEST_F(DropSQLTests, DropTableTest) { storage::DataTable *table; txn = txn_manager.BeginTransaction(); try { - table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "test", txn); + table = catalog::Catalog::GetInstance()->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + "test"); } catch (CatalogException &e) { table = nullptr; } @@ -76,8 +78,10 @@ TEST_F(DropSQLTests, DropTableTest) { // Check the table does not exist txn = txn_manager.BeginTransaction(); try { - table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "test", txn); + table = catalog::Catalog::GetInstance()->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + "test"); } catch (CatalogException &e) { txn_manager.CommitTransaction(txn); table = nullptr; @@ -86,16 +90,17 @@ TEST_F(DropSQLTests, DropTableTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(DropSQLTests, DropIndexTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); auto database_object = - catalog::Catalog::GetInstance()->GetDatabaseObject(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->GetDatabaseCatalogEntry(txn, + DEFAULT_DB_NAME); EXPECT_NE(nullptr, database_object); txn_manager.CommitTransaction(txn); @@ -111,11 +116,13 @@ TEST_F(DropSQLTests, DropIndexTest) { ->GetIndexCatalog(); EXPECT_NE(nullptr, pg_index); // Check if the index is in catalog - std::shared_ptr index; + std::shared_ptr index; txn = txn_manager.BeginTransaction(); try { - index = pg_index->GetIndexObject(database_object->GetDatabaseName(), - "idx", DEFAULT_SCHEMA_NAME, txn); + index = pg_index->GetIndexCatalogEntry(txn, + database_object->GetDatabaseName(), + DEFAULT_SCHEMA_NAME, + "idx"); } catch (CatalogException &e) { index = nullptr; @@ -129,12 +136,14 @@ TEST_F(DropSQLTests, DropIndexTest) { // Check if index is not in catalog txn = txn_manager.BeginTransaction(); - index = pg_index->GetIndexObject(database_object->GetDatabaseName(), - "idx", DEFAULT_SCHEMA_NAME, txn); + index = pg_index->GetIndexCatalogEntry(txn, + database_object->GetDatabaseName(), + DEFAULT_SCHEMA_NAME, + "idx"); EXPECT_EQ(index, nullptr); // Free the database just created - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/sql/foreign_key_sql_test.cpp b/test/sql/foreign_key_sql_test.cpp index da29b9883b7..ade6ef7bc05 100644 --- a/test/sql/foreign_key_sql_test.cpp +++ b/test/sql/foreign_key_sql_test.cpp @@ -17,7 +17,7 @@ TEST_F(ForeignKeySQLTests, NoActionTest) { auto catalog = catalog::Catalog::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); TestingSQLUtil::ExecuteSQLQuery( @@ -43,7 +43,7 @@ TEST_F(ForeignKeySQLTests, NoActionTest) { "DELETE FROM tb1 WHERE id = 1;"), ResultType::SUCCESS); txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -51,7 +51,7 @@ TEST_F(ForeignKeySQLTests, CascadeTest) { auto catalog = catalog::Catalog::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); TestingSQLUtil::ExecuteSQLQuery( @@ -78,7 +78,7 @@ TEST_F(ForeignKeySQLTests, CascadeTest) { true); txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/sql/index_scan_sql_test.cpp b/test/sql/index_scan_sql_test.cpp index a0d6df7e3ba..e6929118080 100644 --- a/test/sql/index_scan_sql_test.cpp +++ b/test/sql/index_scan_sql_test.cpp @@ -41,7 +41,7 @@ void CreateAndLoadTable() { TEST_F(IndexScanSQLTests, CreateIndexAfterInsertTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -65,14 +65,14 @@ TEST_F(IndexScanSQLTests, CreateIndexAfterInsertTest) { EXPECT_EQ("33", TestingSQLUtil::GetResultValueAsString(result, 1)); // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(IndexScanSQLTests, CreateIndexAfterInsertOnMultipleColumnsTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -96,14 +96,14 @@ TEST_F(IndexScanSQLTests, CreateIndexAfterInsertOnMultipleColumnsTest) { EXPECT_EQ("3", TestingSQLUtil::GetResultValueAsString(result, 1)); // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(IndexScanSQLTests, SQLTest) { LOG_INFO("Bootstrapping..."); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); LOG_INFO("Bootstrapping completed!"); @@ -227,7 +227,7 @@ TEST_F(IndexScanSQLTests, SQLTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/sql/insert_sql_test.cpp b/test/sql/insert_sql_test.cpp index 74b767bfa5c..9d57ced992c 100644 --- a/test/sql/insert_sql_test.cpp +++ b/test/sql/insert_sql_test.cpp @@ -113,7 +113,7 @@ void CreateAndLoadTable8() { TEST_F(InsertSQLTests, InsertOneValue) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -149,14 +149,14 @@ TEST_F(InsertSQLTests, InsertOneValue) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(InsertSQLTests, InsertMultipleValues) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -201,14 +201,14 @@ TEST_F(InsertSQLTests, InsertMultipleValues) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(InsertSQLTests, InsertSpecifyColumns) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable8(); @@ -245,14 +245,14 @@ TEST_F(InsertSQLTests, InsertSpecifyColumns) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(InsertSQLTests, InsertTooLargeVarchar) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable3(); @@ -288,14 +288,14 @@ TEST_F(InsertSQLTests, InsertTooLargeVarchar) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(InsertSQLTests, InsertIntoSelectSimple) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -363,14 +363,14 @@ TEST_F(InsertSQLTests, InsertIntoSelectSimple) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(InsertSQLTests, InsertIntoSelectSimpleAllType) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable4(); @@ -451,14 +451,14 @@ TEST_F(InsertSQLTests, InsertIntoSelectSimpleAllType) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(InsertSQLTests, InsertIntoSelectColumn) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable6(); @@ -526,14 +526,14 @@ TEST_F(InsertSQLTests, InsertIntoSelectColumn) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(InsertSQLTests, UniqueColumn) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); std::string create_table( @@ -571,14 +571,14 @@ TEST_F(InsertSQLTests, UniqueColumn) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(InsertSQLTests, BadTypes) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); std::string error_message; std::unique_ptr optimizer( @@ -610,14 +610,14 @@ TEST_F(InsertSQLTests, BadTypes) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(InsertSQLTests, NonExistentTable) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); std::string error_message; std::unique_ptr optimizer( @@ -631,14 +631,14 @@ TEST_F(InsertSQLTests, NonExistentTable) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(InsertSQLTests, BadInserts) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable8(); @@ -707,7 +707,7 @@ TEST_F(InsertSQLTests, BadInserts) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/sql/is_null_sql_test.cpp b/test/sql/is_null_sql_test.cpp index 6e7d1eaab20..c4121a2efd1 100644 --- a/test/sql/is_null_sql_test.cpp +++ b/test/sql/is_null_sql_test.cpp @@ -54,7 +54,7 @@ TEST_F(IsNullSqlTests, InsertNullTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); auto catalog = catalog::Catalog::GetInstance(); - catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); LOG_TRACE("Bootstrapping completed!"); @@ -77,7 +77,7 @@ TEST_F(IsNullSqlTests, InsertNullTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -87,7 +87,7 @@ TEST_F(IsNullSqlTests, IsNullWhereTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); auto catalog = catalog::Catalog::GetInstance(); - catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); LOG_TRACE("Bootstrapping completed!"); @@ -117,7 +117,7 @@ TEST_F(IsNullSqlTests, IsNullWhereTest) { TestingSQLUtil::GetResultValueAsString(result, 1).c_str()); // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -127,7 +127,7 @@ TEST_F(IsNullSqlTests, IsNotNullWhereTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); auto catalog = catalog::Catalog::GetInstance(); - catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); LOG_TRACE("Bootstrapping completed!"); @@ -156,7 +156,7 @@ TEST_F(IsNullSqlTests, IsNotNullWhereTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/sql/optimizer_sql_test.cpp b/test/sql/optimizer_sql_test.cpp index 3855c015e20..9ebd230346c 100644 --- a/test/sql/optimizer_sql_test.cpp +++ b/test/sql/optimizer_sql_test.cpp @@ -45,7 +45,7 @@ class OptimizerSQLTests : public PelotonTest { // Destroy test database auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); // Call parent virtual function @@ -57,7 +57,7 @@ class OptimizerSQLTests : public PelotonTest { // Create database auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); // Create a table first @@ -331,8 +331,10 @@ TEST_F(OptimizerSQLTests, DDLSqlTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); // using transaction to get table from catalog - auto table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "test2", txn); + auto table = catalog::Catalog::GetInstance()->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + "test2"); EXPECT_NE(nullptr, table); auto cols = table->GetSchema()->GetColumns(); EXPECT_EQ(3, cols.size()); @@ -353,8 +355,10 @@ TEST_F(OptimizerSQLTests, DDLSqlTest) { LOG_DEBUG("here"); txn = txn_manager.BeginTransaction(); - EXPECT_THROW(catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "test2", txn), + EXPECT_THROW(catalog::Catalog::GetInstance()->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + "test2"), peloton::Exception); txn_manager.CommitTransaction(txn); } diff --git a/test/sql/order_by_sql_test.cpp b/test/sql/order_by_sql_test.cpp index f3643389bfd..b0abed429dd 100644 --- a/test/sql/order_by_sql_test.cpp +++ b/test/sql/order_by_sql_test.cpp @@ -41,7 +41,7 @@ void CreateAndLoadTable() { TEST_F(OrderBySQLTests, PerformanceTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); // Create a table first @@ -108,14 +108,14 @@ TEST_F(OrderBySQLTests, PerformanceTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(OrderBySQLTests, OrderByWithColumnsTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -138,14 +138,14 @@ TEST_F(OrderBySQLTests, OrderByWithColumnsTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(OrderBySQLTests, OrderByWithColumnsDescTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -168,14 +168,14 @@ TEST_F(OrderBySQLTests, OrderByWithColumnsDescTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(OrderBySQLTests, OrderByWithoutColumnsTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -198,14 +198,14 @@ TEST_F(OrderBySQLTests, OrderByWithoutColumnsTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(OrderBySQLTests, OrderByWithoutColumnsDescTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -228,14 +228,14 @@ TEST_F(OrderBySQLTests, OrderByWithoutColumnsDescTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(OrderBySQLTests, OrderByWithColumnsAndLimitTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -259,14 +259,14 @@ TEST_F(OrderBySQLTests, OrderByWithColumnsAndLimitTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(OrderBySQLTests, OrderByWithColumnsAndLimitDescTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -290,14 +290,14 @@ TEST_F(OrderBySQLTests, OrderByWithColumnsAndLimitDescTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(OrderBySQLTests, OrderByWithoutColumnsAndLimitTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -321,14 +321,14 @@ TEST_F(OrderBySQLTests, OrderByWithoutColumnsAndLimitTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(OrderBySQLTests, OrderByWithoutColumnsAndLimitDescTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -352,14 +352,14 @@ TEST_F(OrderBySQLTests, OrderByWithoutColumnsAndLimitDescTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(OrderBySQLTests, OrderByStar) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -385,14 +385,14 @@ TEST_F(OrderBySQLTests, OrderByStar) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(OrderBySQLTests, OrderByStarDesc) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -418,14 +418,14 @@ TEST_F(OrderBySQLTests, OrderByStarDesc) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(OrderBySQLTests, OrderByStarWithLimit) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -450,14 +450,14 @@ TEST_F(OrderBySQLTests, OrderByStarWithLimit) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(OrderBySQLTests, OrderByStarWithLimitDesc) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -482,14 +482,14 @@ TEST_F(OrderBySQLTests, OrderByStarWithLimitDesc) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(OrderBySQLTests, OrderByWithProjectionTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -517,13 +517,13 @@ TEST_F(OrderBySQLTests, OrderByWithProjectionTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(OrderBySQLTests, OrderByWithProjectionDescTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -551,13 +551,13 @@ TEST_F(OrderBySQLTests, OrderByWithProjectionDescTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(OrderBySQLTests, OrderByWithProjectionLimitTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -586,14 +586,14 @@ TEST_F(OrderBySQLTests, OrderByWithProjectionLimitTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(OrderBySQLTests, OrderByWithProjectionLimitDescTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -622,14 +622,14 @@ TEST_F(OrderBySQLTests, OrderByWithProjectionLimitDescTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(OrderBySQLTests, OrderByWithNullCheck) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -664,7 +664,7 @@ TEST_F(OrderBySQLTests, OrderByWithNullCheck) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/sql/projection_sql_test.cpp b/test/sql/projection_sql_test.cpp index 98ed0e4d45c..f57e0c0bd84 100644 --- a/test/sql/projection_sql_test.cpp +++ b/test/sql/projection_sql_test.cpp @@ -26,14 +26,14 @@ class ProjectionSQLTests : public PelotonTest { ProjectionSQLTests() { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto *txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } ~ProjectionSQLTests() { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto *txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/sql/string_functions_sql_test.cpp b/test/sql/string_functions_sql_test.cpp index 4a57c8a3b45..4cc33d454d3 100644 --- a/test/sql/string_functions_sql_test.cpp +++ b/test/sql/string_functions_sql_test.cpp @@ -32,7 +32,7 @@ of the second column. TEST_F(StringFunctionTest, LengthTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); catalog::Catalog::GetInstance()->Bootstrap(); txn_manager.CommitTransaction(txn); // Create a t @@ -69,7 +69,7 @@ TEST_F(StringFunctionTest, LengthTest) { } txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/sql/timestamp_functions_sql_test.cpp b/test/sql/timestamp_functions_sql_test.cpp index c368f84ad1c..a004c206bd0 100644 --- a/test/sql/timestamp_functions_sql_test.cpp +++ b/test/sql/timestamp_functions_sql_test.cpp @@ -25,7 +25,7 @@ class TimestampFunctionsSQLTest : public PelotonTest {}; TEST_F(TimestampFunctionsSQLTest, DateTruncTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); catalog::Catalog::GetInstance()->Bootstrap(); txn_manager.CommitTransaction(txn); // Create a t @@ -72,14 +72,14 @@ TEST_F(TimestampFunctionsSQLTest, DateTruncTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(TimestampFunctionsSQLTest, DatePartTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); // NOTE: Catalog::GetInstance()->Bootstrap() has been called in previous tests // you can only call it once! txn_manager.CommitTransaction(txn); @@ -142,7 +142,7 @@ TEST_F(TimestampFunctionsSQLTest, DatePartTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/sql/type_sql_test.cpp b/test/sql/type_sql_test.cpp index 1da4f602646..6c46359f13b 100644 --- a/test/sql/type_sql_test.cpp +++ b/test/sql/type_sql_test.cpp @@ -27,14 +27,14 @@ class TypeSQLTests : public PelotonTest { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } virtual void TearDown() override { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); PelotonTest::TearDown(); diff --git a/test/sql/update_primary_index_sql_test.cpp b/test/sql/update_primary_index_sql_test.cpp index 0b6f91b39f1..0ae64bf36cb 100644 --- a/test/sql/update_primary_index_sql_test.cpp +++ b/test/sql/update_primary_index_sql_test.cpp @@ -27,7 +27,7 @@ class UpdatePrimaryIndexSQLTests : public PelotonTest {}; TEST_F(UpdatePrimaryIndexSQLTests, UpdatePrimaryIndexTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); // Create a table first @@ -91,7 +91,7 @@ TEST_F(UpdatePrimaryIndexSQLTests, UpdatePrimaryIndexTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/sql/update_secondary_index_sql_test.cpp b/test/sql/update_secondary_index_sql_test.cpp index 842bc1ca9ee..fb810eb3963 100644 --- a/test/sql/update_secondary_index_sql_test.cpp +++ b/test/sql/update_secondary_index_sql_test.cpp @@ -27,7 +27,7 @@ class UpdateSecondaryIndexSQLTests : public PelotonTest {}; TEST_F(UpdateSecondaryIndexSQLTests, UpdateSecondaryIndexTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); // Create a table first @@ -69,7 +69,7 @@ TEST_F(UpdateSecondaryIndexSQLTests, UpdateSecondaryIndexTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/sql/update_sql_test.cpp b/test/sql/update_sql_test.cpp index f3584843918..6df7faa5903 100644 --- a/test/sql/update_sql_test.cpp +++ b/test/sql/update_sql_test.cpp @@ -30,7 +30,7 @@ TEST_F(UpdateSQLTests, SimpleUpdateSQLTest) { auto catalog = catalog::Catalog::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); LOG_DEBUG("Bootstrapping completed!"); @@ -99,7 +99,7 @@ TEST_F(UpdateSQLTests, SimpleUpdateSQLTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -109,7 +109,7 @@ TEST_F(UpdateSQLTests, ComplexUpdateSQLTest) { auto catalog = catalog::Catalog::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); LOG_DEBUG("Bootstrapping completed!"); @@ -191,7 +191,7 @@ TEST_F(UpdateSQLTests, ComplexUpdateSQLTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -201,7 +201,7 @@ TEST_F(UpdateSQLTests, UpdateSQLCastTest) { auto catalog = catalog::Catalog::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); LOG_DEBUG("Bootstrapping completed!"); @@ -274,7 +274,7 @@ TEST_F(UpdateSQLTests, UpdateSQLCastTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -287,7 +287,7 @@ TEST_F(UpdateSQLTests, HalloweenProblemTest) { auto catalog = catalog::Catalog::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); LOG_DEBUG("Bootstrapping completed!"); @@ -351,7 +351,7 @@ TEST_F(UpdateSQLTests, HalloweenProblemTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -365,7 +365,7 @@ TEST_F(UpdateSQLTests, HalloweenProblemTestWithPK) { auto catalog = catalog::Catalog::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); LOG_DEBUG("Bootstrapping completed!"); @@ -452,7 +452,7 @@ TEST_F(UpdateSQLTests, HalloweenProblemTestWithPK) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -462,7 +462,7 @@ TEST_F(UpdateSQLTests, MultiTileGroupUpdateSQLTest) { auto catalog = catalog::Catalog::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); LOG_DEBUG("Bootstrapping completed!"); @@ -523,7 +523,7 @@ TEST_F(UpdateSQLTests, MultiTileGroupUpdateSQLTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -537,7 +537,7 @@ TEST_F(UpdateSQLTests, AttributeOrderUpdateSQLTest) { auto catalog = catalog::Catalog::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); LOG_DEBUG("Bootstrapping completed!"); @@ -629,7 +629,7 @@ TEST_F(UpdateSQLTests, AttributeOrderUpdateSQLTest) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } diff --git a/test/statistics/stats_test.cpp b/test/statistics/stats_test.cpp index ef3c7da6cba..b40efd823e6 100644 --- a/test/statistics/stats_test.cpp +++ b/test/statistics/stats_test.cpp @@ -135,17 +135,22 @@ TEST_F(StatsTests, MultiThreadStatsTest) { std::unique_ptr table_schema( new catalog::Schema({id_column, name_column})); - catalog->CreateDatabase("emp_db", txn); - catalog::Catalog::GetInstance()->CreateTable("emp_db", DEFAULT_SCHEMA_NAME, + catalog->CreateDatabase(txn, "emp_db"); + catalog::Catalog::GetInstance()->CreateTable(txn, + "emp_db", + DEFAULT_SCHEMA_NAME, + std::move(table_schema), "department_table", - std::move(table_schema), txn); + false); // Create multiple stat worker threads int num_threads = 8; storage::Database *database = - catalog->GetDatabaseWithName("emp_db", txn); - storage::DataTable *table = catalog->GetTableWithName( - "emp_db", DEFAULT_SCHEMA_NAME, "department_table", txn); + catalog->GetDatabaseWithName(txn, "emp_db"); + storage::DataTable *table = catalog->GetTableWithName(txn, + "emp_db", + DEFAULT_SCHEMA_NAME, + "department_table"); txn_manager.CommitTransaction(txn); LaunchParallelTest(num_threads, TransactionTest, database, table); // Wait for aggregation to finish @@ -194,7 +199,7 @@ TEST_F(StatsTests, MultiThreadStatsTest) { num_threads * NUM_ITERATION * NUM_INDEX_INSERT); txn = txn_manager.BeginTransaction(); - catalog->DropDatabaseWithName("emp_db", txn); + catalog->DropDatabaseWithName(txn, "emp_db"); txn_manager.CommitTransaction(txn); } // diff --git a/test/storage/database_test.cpp b/test/storage/database_test.cpp index 427d4c220e3..fee76bd33d5 100644 --- a/test/storage/database_test.cpp +++ b/test/storage/database_test.cpp @@ -41,7 +41,8 @@ TEST_F(DatabaseTests, AddDropTest) { TestingExecutorUtil::DeleteDatabase(DEFAULT_DB_NAME); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - EXPECT_THROW(catalog::Catalog::GetInstance()->GetDatabaseObject(db_id, txn), + EXPECT_THROW(catalog::Catalog::GetInstance()->GetDatabaseCatalogEntry(txn, + db_id), CatalogException); txn_manager.CommitTransaction(txn); // Only GC will remove the actual database object @@ -80,7 +81,8 @@ TEST_F(DatabaseTests, AddDropTableTest) { TestingExecutorUtil::DeleteDatabase(DEFAULT_DB_NAME); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - EXPECT_THROW(catalog::Catalog::GetInstance()->GetDatabaseObject(db_id, txn), + EXPECT_THROW(catalog::Catalog::GetInstance()->GetDatabaseCatalogEntry(txn, + db_id), CatalogException); txn_manager.CommitTransaction(txn); // Only GC will remove the actual database object diff --git a/test/trigger/trigger_test.cpp b/test/trigger/trigger_test.cpp index 7238d7f9e6d..e199d1f69e0 100644 --- a/test/trigger/trigger_test.cpp +++ b/test/trigger/trigger_test.cpp @@ -42,7 +42,7 @@ class TriggerTests : public PelotonTest { void CreateTableHelper() { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); // Insert a table first auto id_column = catalog::Column( @@ -74,8 +74,11 @@ class TriggerTests : public PelotonTest { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - auto table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, std::string(table_name), txn); + auto table = catalog::Catalog::GetInstance()->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + std::string( + table_name)); std::unique_ptr context( new executor::ExecutorContext(txn)); @@ -149,8 +152,10 @@ class TriggerTests : public PelotonTest { // Check the effect of creation storage::DataTable *target_table = - catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table_name, txn); + catalog::Catalog::GetInstance()->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + table_name); txn_manager.CommitTransaction(txn); EXPECT_EQ(trigger_number, target_table->GetTriggerNumber()); trigger::Trigger *new_trigger = target_table->GetTriggerByIndex(0); @@ -277,8 +282,10 @@ TEST_F(TriggerTests, BeforeAndAfterRowInsertTriggers) { // Check the effect of creation storage::DataTable *target_table = - catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "accounts", txn); + catalog::Catalog::GetInstance()->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + "accounts"); txn_manager.CommitTransaction(txn); EXPECT_EQ(1, target_table->GetTriggerNumber()); trigger::Trigger *new_trigger = target_table->GetTriggerByIndex(0); @@ -304,7 +311,7 @@ TEST_F(TriggerTests, BeforeAndAfterRowInsertTriggers) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -364,8 +371,10 @@ TEST_F(TriggerTests, AfterStatmentInsertTriggers) { // Check the effect of creation storage::DataTable *target_table = - catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "accounts", txn); + catalog::Catalog::GetInstance()->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + "accounts"); txn_manager.CommitTransaction(txn); EXPECT_EQ(1, target_table->GetTriggerNumber()); trigger::Trigger *new_trigger = target_table->GetTriggerByIndex(0); @@ -384,7 +393,7 @@ TEST_F(TriggerTests, AfterStatmentInsertTriggers) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -468,8 +477,10 @@ TEST_F(TriggerTests, OtherTypesTriggers) { auto txn = txn_manager.BeginTransaction(); storage::DataTable *target_table = - catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table_name, txn); + catalog::Catalog::GetInstance()->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + table_name); txn_manager.CommitTransaction(txn); trigger::TriggerList *new_trigger_list = target_table->GetTriggerList(); @@ -503,7 +514,7 @@ TEST_F(TriggerTests, OtherTypesTriggers) { // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } } // namespace test diff --git a/test/udf/udf_test.cpp b/test/udf/udf_test.cpp index b45b29510b3..725bdd9f5ff 100644 --- a/test/udf/udf_test.cpp +++ b/test/udf/udf_test.cpp @@ -25,7 +25,7 @@ class UDFTest : public PelotonTest {}; TEST_F(UDFTest, SimpleExpressionTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); catalog::Catalog::GetInstance()->Bootstrap(); txn_manager.CommitTransaction(txn); // Create a t @@ -72,14 +72,14 @@ TEST_F(UDFTest, SimpleExpressionTest) { } // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(UDFTest, ComplexExpressionTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); // Create a txn txn = txn_manager.BeginTransaction(); @@ -128,14 +128,14 @@ TEST_F(UDFTest, ComplexExpressionTest) { } // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(UDFTest, IfElseExpressionTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); // Create a txn txn = txn_manager.BeginTransaction(); @@ -186,14 +186,14 @@ TEST_F(UDFTest, IfElseExpressionTest) { } // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } TEST_F(UDFTest, RecursiveFunctionTest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); // Create a txn txn = txn_manager.BeginTransaction(); @@ -233,7 +233,7 @@ TEST_F(UDFTest, RecursiveFunctionTest) { } // free the database just created txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } From 40acdf664651e93e65476884af99e36d96656536 Mon Sep 17 00:00:00 2001 From: Tianyu Li Date: Wed, 27 Jun 2018 21:29:23 -0400 Subject: [PATCH 12/19] Fix ssl copy bug (#1436) * Fix ssl copy bug * Update network_io_wrapper_factory.cpp --- src/network/network_io_wrapper_factory.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/network/network_io_wrapper_factory.cpp b/src/network/network_io_wrapper_factory.cpp index 2c675ea0d63..ce7eca4244d 100644 --- a/src/network/network_io_wrapper_factory.cpp +++ b/src/network/network_io_wrapper_factory.cpp @@ -24,8 +24,7 @@ std::shared_ptr NetworkIoWrapperFactory::NewNetworkIoWrapper( conn_fd, std::make_shared(), std::make_shared()); reusable_wrappers_[conn_fd] = - std::static_pointer_cast( - wrapper); + std::static_pointer_cast(wrapper); return wrapper; } @@ -52,6 +51,7 @@ Transition NetworkIoWrapperFactory::PerformSslHandshake( throw NetworkProcessException("Failed to set ssl fd"); io_wrapper = std::make_shared(std::move(*io_wrapper), context); + reusable_wrappers_[io_wrapper->sock_fd_] = io_wrapper; } else { auto ptr = std::dynamic_pointer_cast( io_wrapper); @@ -75,4 +75,4 @@ Transition NetworkIoWrapperFactory::PerformSslHandshake( } } } // namespace network -} // namespace peloton \ No newline at end of file +} // namespace peloton From 2b67714fcbbf26b75c962e92afee374b4a5208de Mon Sep 17 00:00:00 2001 From: pervazea <34660288+pervazea@users.noreply.github.com> Date: Thu, 28 Jun 2018 13:30:44 -0400 Subject: [PATCH 13/19] Compilation fix (#1439) * Fix for python-six upgrade problem Uninstall python-six on Ubuntu 14.04 so new version can be installed during tensorflow and dependencies install * - Fix for compilation error (from Marcel) --- src/codegen/interpreter/bytecode_builder.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/codegen/interpreter/bytecode_builder.cpp b/src/codegen/interpreter/bytecode_builder.cpp index 57295da2567..795f116c51c 100644 --- a/src/codegen/interpreter/bytecode_builder.cpp +++ b/src/codegen/interpreter/bytecode_builder.cpp @@ -488,7 +488,7 @@ void BytecodeBuilder::AnalyseFunction() { // which are labeled and explained below. for (llvm::BasicBlock::const_iterator instr_iterator = bb->begin(); instr_iterator != bb->end(); ++instr_iterator, ++instruction_index) { - const llvm::Instruction *instruction = instr_iterator; + const llvm::Instruction *instruction = &(*instr_iterator); bool is_non_zero_gep = false; if (instruction->getOpcode() == llvm::Instruction::GetElementPtr && @@ -828,7 +828,7 @@ void BytecodeBuilder::TranslateFunction() { // Interate all instruction in the basic block for (llvm::BasicBlock::const_iterator instr_iterator = bb->begin(); instr_iterator != bb->end(); ++instr_iterator) { - const llvm::Instruction *instruction = instr_iterator; + const llvm::Instruction *instruction = &(*instr_iterator); // Dispatch to the respective translator function switch (instruction->getOpcode()) { From 6cc5559477a3362a7bd7d56a0117ed05678e8e3d Mon Sep 17 00:00:00 2001 From: Wan Shen Lim Date: Fri, 29 Jun 2018 15:27:24 -0400 Subject: [PATCH 14/19] Remove microbenchmarks (#1442) * This is one of those commits that you just dream about for years. And yet here we are! Removing the microbenchmarks! No longer will we get TPC-C numbers by running embedded logic. We will keep everything real over JDBC. I would like to give a shout out to KB for sticking with me all this time. I am sitting in a German hotel waiting for her to fly in from Pittsburgh tomorrow. --- src/CMakeLists.txt | 29 +- src/include/benchmark/benchmark_common.h | 124 -- .../benchmark/logger/logger_configuration.h | 95 - .../benchmark/logger/logger_workload.h | 61 - src/include/benchmark/peloton/main.h | 32 - .../benchmark/sdbench/sdbench_configuration.h | 151 -- .../benchmark/sdbench/sdbench_loader.h | 36 - .../benchmark/sdbench/sdbench_workload.h | 29 - src/include/benchmark/tpcc/tpcc.sql | 140 -- .../benchmark/tpcc/tpcc_configuration.h | 238 --- src/include/benchmark/tpcc/tpcc_loader.h | 215 -- src/include/benchmark/tpcc/tpcc_workload.h | 58 - .../benchmark/tpch/tpch_configuration.h | 120 -- src/include/benchmark/tpch/tpch_database.h | 107 - src/include/benchmark/tpch/tpch_workload.h | 90 - .../benchmark/ycsb/ycsb_configuration.h | 129 -- src/include/benchmark/ycsb/ycsb_loader.h | 31 - src/include/benchmark/ycsb/ycsb_workload.h | 48 - src/main/logger/logger.cpp | 102 - src/main/logger/logger_configuration.cpp | 441 ----- src/main/logger/logger_workload.cpp | 421 ---- src/main/sdbench/sdbench.cpp | 72 - src/main/sdbench/sdbench_configuration.cpp | 566 ------ src/main/sdbench/sdbench_loader.cpp | 136 -- src/main/sdbench/sdbench_workload.cpp | 1545 --------------- src/main/tpcc/tpcc.cpp | 103 - src/main/tpcc/tpcc_configuration.cpp | 271 --- src/main/tpcc/tpcc_delivery.cpp | 572 ------ src/main/tpcc/tpcc_loader.cpp | 1764 ----------------- src/main/tpcc/tpcc_new_order.cpp | 732 ------- src/main/tpcc/tpcc_order_status.cpp | 354 ---- src/main/tpcc/tpcc_payment.cpp | 743 ------- src/main/tpcc/tpcc_stock_level.cpp | 273 --- src/main/tpcc/tpcc_workload.cpp | 421 ---- src/main/tpch/tpch.cpp | 125 -- src/main/tpch/tpch_configuration.cpp | 117 -- src/main/tpch/tpch_database.cpp | 690 ------- src/main/tpch/tpch_workload.cpp | 214 -- src/main/tpch/tpch_workload_q1.cpp | 170 -- src/main/tpch/tpch_workload_q3.cpp | 253 --- src/main/tpch/tpch_workload_q6.cpp | 138 -- src/main/ycsb/ycsb.cpp | 102 - src/main/ycsb/ycsb_configuration.cpp | 318 --- src/main/ycsb/ycsb_loader.cpp | 209 -- src/main/ycsb/ycsb_mixed.cpp | 231 --- src/main/ycsb/ycsb_workload.cpp | 321 --- 46 files changed, 1 insertion(+), 13136 deletions(-) delete mode 100644 src/include/benchmark/benchmark_common.h delete mode 100644 src/include/benchmark/logger/logger_configuration.h delete mode 100644 src/include/benchmark/logger/logger_workload.h delete mode 100644 src/include/benchmark/peloton/main.h delete mode 100644 src/include/benchmark/sdbench/sdbench_configuration.h delete mode 100644 src/include/benchmark/sdbench/sdbench_loader.h delete mode 100644 src/include/benchmark/sdbench/sdbench_workload.h delete mode 100644 src/include/benchmark/tpcc/tpcc.sql delete mode 100644 src/include/benchmark/tpcc/tpcc_configuration.h delete mode 100644 src/include/benchmark/tpcc/tpcc_loader.h delete mode 100644 src/include/benchmark/tpcc/tpcc_workload.h delete mode 100644 src/include/benchmark/tpch/tpch_configuration.h delete mode 100644 src/include/benchmark/tpch/tpch_database.h delete mode 100644 src/include/benchmark/tpch/tpch_workload.h delete mode 100644 src/include/benchmark/ycsb/ycsb_configuration.h delete mode 100644 src/include/benchmark/ycsb/ycsb_loader.h delete mode 100644 src/include/benchmark/ycsb/ycsb_workload.h delete mode 100644 src/main/logger/logger.cpp delete mode 100644 src/main/logger/logger_configuration.cpp delete mode 100644 src/main/logger/logger_workload.cpp delete mode 100644 src/main/sdbench/sdbench.cpp delete mode 100644 src/main/sdbench/sdbench_configuration.cpp delete mode 100644 src/main/sdbench/sdbench_loader.cpp delete mode 100644 src/main/sdbench/sdbench_workload.cpp delete mode 100644 src/main/tpcc/tpcc.cpp delete mode 100644 src/main/tpcc/tpcc_configuration.cpp delete mode 100644 src/main/tpcc/tpcc_delivery.cpp delete mode 100644 src/main/tpcc/tpcc_loader.cpp delete mode 100644 src/main/tpcc/tpcc_new_order.cpp delete mode 100644 src/main/tpcc/tpcc_order_status.cpp delete mode 100644 src/main/tpcc/tpcc_payment.cpp delete mode 100644 src/main/tpcc/tpcc_stock_level.cpp delete mode 100644 src/main/tpcc/tpcc_workload.cpp delete mode 100644 src/main/tpch/tpch.cpp delete mode 100644 src/main/tpch/tpch_configuration.cpp delete mode 100644 src/main/tpch/tpch_database.cpp delete mode 100644 src/main/tpch/tpch_workload.cpp delete mode 100644 src/main/tpch/tpch_workload_q1.cpp delete mode 100644 src/main/tpch/tpch_workload_q3.cpp delete mode 100644 src/main/tpch/tpch_workload_q6.cpp delete mode 100644 src/main/ycsb/ycsb.cpp delete mode 100644 src/main/ycsb/ycsb_configuration.cpp delete mode 100644 src/main/ycsb/ycsb_loader.cpp delete mode 100644 src/main/ycsb/ycsb_mixed.cpp delete mode 100644 src/main/ycsb/ycsb_workload.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 559ac4e0ed5..a35ee7df695 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -62,28 +62,6 @@ set_target_properties(peloton-bin PROPERTIES OUTPUT_NAME peloton) ################################################################################## -# --[ Benchmarks - -# --[ tpcc -file(GLOB_RECURSE tpcc_srcs ${PROJECT_SOURCE_DIR}/src/main/tpcc/*.cpp) -add_executable(tpcc EXCLUDE_FROM_ALL ${tpcc_srcs}) -target_link_libraries(tpcc peloton) - -# --[ ycsb -file(GLOB_RECURSE ycsb_srcs ${PROJECT_SOURCE_DIR}/src/main/ycsb/*.cpp) -add_executable(ycsb EXCLUDE_FROM_ALL ${ycsb_srcs}) -target_link_libraries(ycsb peloton) - -# --[ sdbench -file(GLOB_RECURSE sdbench_srcs ${PROJECT_SOURCE_DIR}/src/main/sdbench/*.cpp) -add_executable(sdbench EXCLUDE_FROM_ALL ${sdbench_srcs}) -target_link_libraries(sdbench peloton) - -# --[ tpch -file(GLOB_RECURSE tpch_srcs ${PROJECT_SOURCE_DIR}/src/main/tpch/*.cpp) -add_executable(tpch EXCLUDE_FROM_ALL ${tpch_srcs}) -target_link_libraries(tpch peloton) - # --[ logger #file(GLOB_RECURSE logger_srcs ${PROJECT_SOURCE_DIR}/src/main/logger/*.cpp) #list(APPEND logger_srcs ${ycsb_srcs}) @@ -96,7 +74,7 @@ target_link_libraries(tpch peloton) # --[ link to jemalloc set(EXE_LINK_LIBRARIES ${JEMALLOC_LIBRARIES}) set(EXE_LINK_FLAGS "-Wl,--no-as-needed") -set(EXE_LIST peloton-bin ycsb tpcc sdbench tpch) +set(EXE_LIST peloton-bin) foreach(exe_name ${EXE_LIST}) target_link_libraries(${exe_name} ${EXE_LINK_LIBRARIES}) if (LINUX) @@ -104,8 +82,3 @@ foreach(exe_name ${EXE_LIST}) endif () endforeach() -# --[ benchmark - -add_custom_target(benchmark) -add_dependencies(benchmark tpcc ycsb sdbench) - diff --git a/src/include/benchmark/benchmark_common.h b/src/include/benchmark/benchmark_common.h deleted file mode 100644 index 698d4a7b793..00000000000 --- a/src/include/benchmark/benchmark_common.h +++ /dev/null @@ -1,124 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// ycsb_workload.h -// -// Identification: src/include/benchmark/benchmark_common.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include -#include -#include - -#include "common/platform.h" - -namespace peloton { -namespace benchmark { - -// Fast random number generator -class FastRandom { - public: - FastRandom(unsigned long seed) : seed(0) { set_seed0(seed); } - - inline unsigned long next() { - return ((unsigned long)next(32) << 32) + next(32); - } - - inline uint32_t next_u32() { return next(32); } - - inline uint16_t next_u16() { return (uint16_t)next(16); } - - /** [0.0, 1.0) */ - inline double NextUniform() { - return (((unsigned long)next(26) << 27) + next(27)) / (double)(1L << 53); - } - - inline char next_char() { return next(8) % 256; } - - inline char next_readable_char() { - static const char readables[] = - "0123456789@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz"; - return readables[next(6)]; - } - - inline std::string next_string(size_t len) { - std::string s(len, 0); - for (size_t i = 0; i < len; i++) s[i] = next_char(); - return s; - } - - inline std::string next_readable_string(size_t len) { - std::string s(len, 0); - for (size_t i = 0; i < len; i++) s[i] = next_readable_char(); - return s; - } - - inline unsigned long get_seed() { return seed; } - - inline void set_seed(unsigned long seed) { this->seed = seed; } - - private: - inline void set_seed0(unsigned long seed) { - this->seed = (seed ^ 0x5DEECE66DL) & ((1L << 48) - 1); - } - - inline unsigned long next(unsigned int bits) { - seed = (seed * 0x5DEECE66DL + 0xBL) & ((1L << 48) - 1); - return (unsigned long)(seed >> (48 - bits)); - } - - unsigned long seed; -}; - - -class ZipfDistribution { - public: - ZipfDistribution(const uint64_t &n, const double &theta) - : rand_generator(rand()) { - // range: 1-n - the_n = n; - zipf_theta = theta; - zeta_2_theta = zeta(2, zipf_theta); - denom = zeta(the_n, zipf_theta); - } - double zeta(uint64_t n, double theta) { - double sum = 0; - for (uint64_t i = 1; i <= n; i++) sum += pow(1.0 / i, theta); - return sum; - } - int GenerateInteger(const int &min, const int &max) { - return rand_generator.next() % (max - min + 1) + min; - } - uint64_t GetNextNumber() { - double alpha = 1 / (1 - zipf_theta); - double zetan = denom; - double eta = - (1 - pow(2.0 / the_n, 1 - zipf_theta)) / (1 - zeta_2_theta / zetan); - double u = (double)(GenerateInteger(1, 10000000) % 10000000) / 10000000; - double uz = u * zetan; - if (uz < 1) return 1; - if (uz < 1 + pow(0.5, zipf_theta)) return 2; - return 1 + (uint64_t)(the_n * pow(eta * u - eta + 1, alpha)); - } - - uint64_t the_n; - double zipf_theta; - double denom; - double zeta_2_theta; - FastRandom rand_generator; -}; - -struct PadInt { - PadInt() { data = 0; } - uint64_t data; - uint32_t padding[CACHELINE_SIZE - sizeof(uint64_t)]; -}; - -} // namespace benchmark -} // namespace peloton diff --git a/src/include/benchmark/logger/logger_configuration.h b/src/include/benchmark/logger/logger_configuration.h deleted file mode 100644 index 31cc1018919..00000000000 --- a/src/include/benchmark/logger/logger_configuration.h +++ /dev/null @@ -1,95 +0,0 @@ -// //===----------------------------------------------------------------------===// -// // -// // Peloton -// // -// // logger_configuration.h -// // -// // Identification: src/include/benchmark/logger/logger_configuration.h -// // -// // Copyright (c) 2015-16, Carnegie Mellon University Database Group -// // -// //===----------------------------------------------------------------------===// - - -// #pragma once - -// #include -// #include -// #include -// #include -// #include - -// #include "type/types.h" - -// namespace peloton { -// namespace benchmark { -// namespace logger { - -// enum ExperimentType { -// EXPERIMENT_TYPE_INVALID = 0, - -// EXPERIMENT_TYPE_THROUGHPUT = 1, -// EXPERIMENT_TYPE_RECOVERY = 2, -// EXPERIMENT_TYPE_STORAGE = 3, -// EXPERIMENT_TYPE_LATENCY = 4 -// }; - -// enum BenchmarkType { -// BENCHMARK_TYPE_INVALID = 0, - -// BENCHMARK_TYPE_YCSB = 1, -// BENCHMARK_TYPE_TPCC = 2 -// }; - -// enum AsynchronousType { -// ASYNCHRONOUS_TYPE_INVALID = 0, - -// ASYNCHRONOUS_TYPE_SYNC = 1, // logging enabled + sync commits -// ASYNCHRONOUS_TYPE_ASYNC = 2, // logging enabled + async commits -// ASYNCHRONOUS_TYPE_DISABLED = 3, // logging disabled -// ASYNCHRONOUS_TYPE_NO_WRITE = 4 -// }; - -// class configuration { -// public: -// // experiment type -// ExperimentType experiment_type; - -// // logging type -// LoggingType logging_type; - -// // checkpoint type -// CheckpointType checkpoint_type; - -// // log file dir -// std::string log_file_dir; - -// // size of the pmem file (in MB) -// size_t data_file_size; - -// // frequency with which the logger flushes -// int wait_timeout; - -// // Benchmark type -// BenchmarkType benchmark_type; - -// // clflush or clwb -// int flush_mode; - -// // nvm latency -// int nvm_latency; - -// // pcommit latency -// int pcommit_latency; - -// // asynchronous_mode -// AsynchronousType asynchronous_mode; -// }; - -// void Usage(FILE *out); - -// void ParseArguments(int argc, char *argv[], configuration &state); - -// } // namespace logger -// } // namespace benchmark -// } // namespace peloton diff --git a/src/include/benchmark/logger/logger_workload.h b/src/include/benchmark/logger/logger_workload.h deleted file mode 100644 index 3f148d37d25..00000000000 --- a/src/include/benchmark/logger/logger_workload.h +++ /dev/null @@ -1,61 +0,0 @@ -// //===----------------------------------------------------------------------===// -// // -// // Peloton -// // -// // logger_workload.h -// // -// // Identification: src/include/benchmark/logger/logger_workload.h -// // -// // Copyright (c) 2015-16, Carnegie Mellon University Database Group -// // -// //===----------------------------------------------------------------------===// - - -// #pragma once - -// #include - -// #include "benchmark/logger/logger_configuration.h" - -// namespace peloton { - -// namespace catalog { -// class Column; -// class Schema; -// } - -// namespace storage { -// class Tuple; -// class DataTable; -// } - -// namespace benchmark { -// namespace logger { - -// extern configuration state; - -// //===--------------------------------------------------------------------===// -// // PREPARE LOG FILE -// //===--------------------------------------------------------------------===// - -// bool SetupLoggingOnFollower(); - -// bool PrepareLogFile(); - -// //===--------------------------------------------------------------------===// -// // CHECK RECOVERY -// //===--------------------------------------------------------------------===// - -// void ResetSystem(void); - -// void DoRecovery(); - -// //===--------------------------------------------------------------------===// -// // WRITING LOG RECORD -// //===--------------------------------------------------------------------===// - -// void BuildLog(); - -// } // namespace logger -// } // namespace benchmark -// } // namespace peloton diff --git a/src/include/benchmark/peloton/main.h b/src/include/benchmark/peloton/main.h deleted file mode 100644 index f3df65d550c..00000000000 --- a/src/include/benchmark/peloton/main.h +++ /dev/null @@ -1,32 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// main.h -// -// Identification: src/include/benchmark/peloton/main.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#pragma once - -#include -#include -#include - -/*! - \brief Peloton DBMS namespace. - */ -namespace peloton { - -class configuration { - public: - std::string filesystem_path; -}; - -// sample test helper -int SampleFunc(int a, int b) { return a + b; } -} diff --git a/src/include/benchmark/sdbench/sdbench_configuration.h b/src/include/benchmark/sdbench/sdbench_configuration.h deleted file mode 100644 index 52edf622749..00000000000 --- a/src/include/benchmark/sdbench/sdbench_configuration.h +++ /dev/null @@ -1,151 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// sdbench_configuration.h -// -// Identification: src/include/benchmark/sdbench/sdbench_configuration.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include -#include -#include -#include -#include - -#include "storage/data_table.h" - -namespace peloton { -namespace benchmark { -namespace sdbench { - -enum IndexUsageType { - INDEX_USAGE_TYPE_INVALID = 0, - - INDEX_USAGE_TYPE_PARTIAL_FAST = 1, // use partially materialized indexes (fast) - INDEX_USAGE_TYPE_PARTIAL_MEDIUM = 2, // use partially materialized indexes (medium) - INDEX_USAGE_TYPE_PARTIAL_SLOW = 3, // use partially materialized indexes (slow) - INDEX_USAGE_TYPE_FULL = 4, // use only fully materialized indexes - INDEX_USAGE_TYPE_NEVER = 5, // never use ad-hoc indexes -}; - -enum QueryComplexityType { - QUERY_COMPLEXITY_TYPE_INVALID = 0, - - QUERY_COMPLEXITY_TYPE_SIMPLE = 1, - QUERY_COMPLEXITY_TYPE_MODERATE = 2, - QUERY_COMPLEXITY_TYPE_COMPLEX = 3 - -}; - -enum WriteComplexityType { - WRITE_COMPLEXITY_TYPE_INVALID = 0, - - WRITE_COMPLEXITY_TYPE_SIMPLE = 1, - WRITE_COMPLEXITY_TYPE_COMPLEX = 2, - // This is a special complexity type, where we do insert instead of update. - WRITE_COMPLEXITY_TYPE_INSERT = 3 -}; - -extern int orig_scale_factor; - -static const int generator_seed = 50; - -class configuration { - public: - // What kind of indexes can be used ? - IndexUsageType index_usage_type; - - // Complexity of the query. - QueryComplexityType query_complexity_type; - - // Complexity of update. - WriteComplexityType write_complexity_type; - - // size of the table - int scale_factor; - - int tuples_per_tilegroup; - - // tile group layout - LayoutType layout_mode; - - double selectivity; - - double projectivity; - - // column count - oid_t attribute_count; - - // write ratio - double write_ratio; - - // # of times to run operator - std::size_t phase_length; - - // total number of ops - size_t total_ops; - - // Verbose output - bool verbose; - - // Convergence test? - bool convergence; - - // INDEX TUNER PARAMETERS - - // duration between pauses - oid_t duration_between_pauses; - - // duration of pause - oid_t duration_of_pause; - - // sample count threshold after which - // tuner analyze iteration takes place - oid_t analyze_sample_count_threshold; - - // max tile groups indexed per tuning iteration per table - oid_t tile_groups_indexed_per_iteration; - - // CONVERGENCE PARAMETER - - // number of queries for which index configuration must remain stable - oid_t convergence_op_threshold; - - // VARIABILITY PARAMETER - oid_t variability_threshold; - - // DROP PARAMETER - - // index utility threshold - double index_utility_threshold; - - // maximum # of indexes per table - oid_t index_count_threshold; - - // write intensive workload ratio threshold - double write_ratio_threshold; - - // wether run multi stage experiment or not. - bool multi_stage; - - // whether in holistic indexing mode or not. - bool holistic_indexing; - - oid_t multi_stage_idx = 0; -}; - -void Usage(FILE *out); - -void ParseArguments(int argc, char *argv[], configuration &state); - -void GenerateSequence(oid_t column_count); - -} // namespace sdbench -} // namespace benchmark -} // namespace peloton diff --git a/src/include/benchmark/sdbench/sdbench_loader.h b/src/include/benchmark/sdbench/sdbench_loader.h deleted file mode 100644 index 26b4f2ed8d1..00000000000 --- a/src/include/benchmark/sdbench/sdbench_loader.h +++ /dev/null @@ -1,36 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// sdbench_loader.h -// -// Identification: src/include/benchmark/sdbench/sdbench_loader.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#pragma once - -#include "benchmark/sdbench/sdbench_configuration.h" - -namespace peloton { -namespace benchmark { -namespace sdbench { - -extern configuration state; - -extern std::unique_ptr sdbench_table; - -void CreateTable(peloton::LayoutType layout_type); - -void LoadTable(); - -void CreateAndLoadTable(LayoutType layout_type); - -void DropIndexes(); - -} // namespace sdbench -} // namespace benchmark -} // namespace peloton diff --git a/src/include/benchmark/sdbench/sdbench_workload.h b/src/include/benchmark/sdbench/sdbench_workload.h deleted file mode 100644 index 3d63dc75581..00000000000 --- a/src/include/benchmark/sdbench/sdbench_workload.h +++ /dev/null @@ -1,29 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// sdbench_workload.h -// -// Identification: src/include/benchmark/sdbench/sdbench_workload.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#pragma once - -#include "benchmark/sdbench/sdbench_configuration.h" - -namespace peloton { -namespace benchmark { -namespace sdbench { - -extern configuration state; - -void RunSDBenchTest(); -void RunMultiStageBenchmark(); - -} // namespace sdbench -} // namespace benchmark -} // namespace peloton diff --git a/src/include/benchmark/tpcc/tpcc.sql b/src/include/benchmark/tpcc/tpcc.sql deleted file mode 100644 index 42f6dc2887d..00000000000 --- a/src/include/benchmark/tpcc/tpcc.sql +++ /dev/null @@ -1,140 +0,0 @@ -CREATE TABLE WAREHOUSE ( - W_ID SMALLINT DEFAULT '0' NOT NULL, - W_NAME VARCHAR(16) DEFAULT NULL, - W_STREET_1 VARCHAR(32) DEFAULT NULL, - W_STREET_2 VARCHAR(32) DEFAULT NULL, - W_CITY VARCHAR(32) DEFAULT NULL, - W_STATE VARCHAR(2) DEFAULT NULL, - W_ZIP VARCHAR(9) DEFAULT NULL, - W_TAX FLOAT DEFAULT NULL, - W_YTD FLOAT DEFAULT NULL, - CONSTRAINT W_PK_ARRAY PRIMARY KEY (W_ID) -); - -CREATE TABLE DISTRICT ( - D_ID TINYINT DEFAULT '0' NOT NULL, - D_W_ID SMALLINT DEFAULT '0' NOT NULL REFERENCES WAREHOUSE (W_ID), - D_NAME VARCHAR(16) DEFAULT NULL, - D_STREET_1 VARCHAR(32) DEFAULT NULL, - D_STREET_2 VARCHAR(32) DEFAULT NULL, - D_CITY VARCHAR(32) DEFAULT NULL, - D_STATE VARCHAR(2) DEFAULT NULL, - D_ZIP VARCHAR(9) DEFAULT NULL, - D_TAX FLOAT DEFAULT NULL, - D_YTD FLOAT DEFAULT NULL, - D_NEXT_O_ID INT DEFAULT NULL, - PRIMARY KEY (D_W_ID,D_ID) -); - -CREATE TABLE ITEM ( - I_ID INTEGER DEFAULT '0' NOT NULL, - I_IM_ID INTEGER DEFAULT NULL, - I_NAME VARCHAR(32) DEFAULT NULL, - I_PRICE FLOAT DEFAULT NULL, - I_DATA VARCHAR(64) DEFAULT NULL, - CONSTRAINT I_PK_ARRAY PRIMARY KEY (I_ID) -); - -CREATE TABLE CUSTOMER ( - C_ID INTEGER DEFAULT '0' NOT NULL, - C_D_ID TINYINT DEFAULT '0' NOT NULL, - C_W_ID SMALLINT DEFAULT '0' NOT NULL, - C_FIRST VARCHAR(32) DEFAULT NULL, - C_MIDDLE VARCHAR(2) DEFAULT NULL, - C_LAST VARCHAR(32) DEFAULT NULL, - C_STREET_1 VARCHAR(32) DEFAULT NULL, - C_STREET_2 VARCHAR(32) DEFAULT NULL, - C_CITY VARCHAR(32) DEFAULT NULL, - C_STATE VARCHAR(2) DEFAULT NULL, - C_ZIP VARCHAR(9) DEFAULT NULL, - C_PHONE VARCHAR(32) DEFAULT NULL, - C_SINCE TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, - C_CREDIT VARCHAR(2) DEFAULT NULL, - C_CREDIT_LIM FLOAT DEFAULT NULL, - C_DISCOUNT FLOAT DEFAULT NULL, - C_BALANCE FLOAT DEFAULT NULL, - C_YTD_PAYMENT FLOAT DEFAULT NULL, - C_PAYMENT_CNT INTEGER DEFAULT NULL, - C_DELIVERY_CNT INTEGER DEFAULT NULL, - C_DATA VARCHAR(500), - PRIMARY KEY (C_W_ID,C_D_ID,C_ID), - UNIQUE (C_W_ID,C_D_ID,C_LAST,C_FIRST), - CONSTRAINT C_FKEY_D FOREIGN KEY (C_D_ID, C_W_ID) REFERENCES DISTRICT (D_ID, D_W_ID) -); -CREATE INDEX IDX_CUSTOMER ON CUSTOMER (C_W_ID,C_D_ID,C_LAST); - -CREATE TABLE HISTORY ( - H_C_ID INTEGER DEFAULT NULL, - H_C_D_ID TINYINT DEFAULT NULL, - H_C_W_ID SMALLINT DEFAULT NULL, - H_D_ID TINYINT DEFAULT NULL, - H_W_ID SMALLINT DEFAULT '0' NOT NULL, - H_DATE TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, - H_AMOUNT FLOAT DEFAULT NULL, - H_DATA VARCHAR(32) DEFAULT NULL, - CONSTRAINT H_FKEY_C FOREIGN KEY (H_C_ID, H_C_D_ID, H_C_W_ID) REFERENCES CUSTOMER (C_ID, C_D_ID, C_W_ID), - CONSTRAINT H_FKEY_D FOREIGN KEY (H_D_ID, H_W_ID) REFERENCES DISTRICT (D_ID, D_W_ID) -); - -CREATE TABLE STOCK ( - S_I_ID INTEGER DEFAULT '0' NOT NULL REFERENCES ITEM (I_ID), - S_W_ID SMALLINT DEFAULT '0 ' NOT NULL REFERENCES WAREHOUSE (W_ID), - S_QUANTITY INTEGER DEFAULT '0' NOT NULL, - S_DIST_01 VARCHAR(32) DEFAULT NULL, - S_DIST_02 VARCHAR(32) DEFAULT NULL, - S_DIST_03 VARCHAR(32) DEFAULT NULL, - S_DIST_04 VARCHAR(32) DEFAULT NULL, - S_DIST_05 VARCHAR(32) DEFAULT NULL, - S_DIST_06 VARCHAR(32) DEFAULT NULL, - S_DIST_07 VARCHAR(32) DEFAULT NULL, - S_DIST_08 VARCHAR(32) DEFAULT NULL, - S_DIST_09 VARCHAR(32) DEFAULT NULL, - S_DIST_10 VARCHAR(32) DEFAULT NULL, - S_YTD INTEGER DEFAULT NULL, - S_ORDER_CNT INTEGER DEFAULT NULL, - S_REMOTE_CNT INTEGER DEFAULT NULL, - S_DATA VARCHAR(64) DEFAULT NULL, - PRIMARY KEY (S_W_ID,S_I_ID) -); - -CREATE TABLE ORDERS ( - O_ID INTEGER DEFAULT '0' NOT NULL, - O_C_ID INTEGER DEFAULT NULL, - O_D_ID TINYINT DEFAULT '0' NOT NULL, - O_W_ID SMALLINT DEFAULT '0' NOT NULL, - O_ENTRY_D TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, - O_CARRIER_ID INTEGER DEFAULT NULL, - O_OL_CNT INTEGER DEFAULT NULL, - O_ALL_LOCAL INTEGER DEFAULT NULL, - PRIMARY KEY (O_W_ID,O_D_ID,O_ID), - UNIQUE (O_W_ID,O_D_ID,O_C_ID,O_ID), - CONSTRAINT O_FKEY_C FOREIGN KEY (O_C_ID, O_D_ID, O_W_ID) REFERENCES CUSTOMER (C_ID, C_D_ID, C_W_ID) -); -CREATE INDEX IDX_ORDERS ON ORDERS (O_W_ID,O_D_ID,O_C_ID); - -CREATE TABLE NEW_ORDER ( - NO_O_ID INTEGER DEFAULT '0' NOT NULL, - NO_D_ID TINYINT DEFAULT '0' NOT NULL, - NO_W_ID SMALLINT DEFAULT '0' NOT NULL, - CONSTRAINT NO_PK_TREE PRIMARY KEY (NO_D_ID,NO_W_ID,NO_O_ID), - CONSTRAINT NO_FKEY_O FOREIGN KEY (NO_O_ID, NO_D_ID, NO_W_ID) REFERENCES ORDERS (O_ID, O_D_ID, O_W_ID) -); - -CREATE TABLE ORDER_LINE ( - OL_O_ID INTEGER DEFAULT '0' NOT NULL, - OL_D_ID TINYINT DEFAULT '0' NOT NULL, - OL_W_ID SMALLINT DEFAULT '0' NOT NULL, - OL_NUMBER INTEGER DEFAULT '0' NOT NULL, - OL_I_ID INTEGER DEFAULT NULL, - OL_SUPPLY_W_ID SMALLINT DEFAULT NULL, - OL_DELIVERY_D TIMESTAMP DEFAULT NULL, - OL_QUANTITY INTEGER DEFAULT NULL, - OL_AMOUNT FLOAT DEFAULT NULL, - OL_DIST_INFO VARCHAR(32) DEFAULT NULL, - PRIMARY KEY (OL_W_ID,OL_D_ID,OL_O_ID,OL_NUMBER), - CONSTRAINT OL_FKEY_O FOREIGN KEY (OL_O_ID, OL_D_ID, OL_W_ID) REFERENCES ORDERS (O_ID, O_D_ID, O_W_ID), - CONSTRAINT OL_FKEY_S FOREIGN KEY (OL_I_ID, OL_SUPPLY_W_ID) REFERENCES STOCK (S_I_ID, S_W_ID) -); ---CREATE INDEX IDX_ORDER_LINE_3COL ON ORDER_LINE (OL_W_ID,OL_D_ID,OL_O_ID); ---CREATE INDEX IDX_ORDER_LINE_2COL ON ORDER_LINE (OL_W_ID,OL_D_ID); -CREATE INDEX IDX_ORDER_LINE_TREE ON ORDER_LINE (OL_W_ID,OL_D_ID,OL_O_ID); diff --git a/src/include/benchmark/tpcc/tpcc_configuration.h b/src/include/benchmark/tpcc/tpcc_configuration.h deleted file mode 100644 index c46a8333b41..00000000000 --- a/src/include/benchmark/tpcc/tpcc_configuration.h +++ /dev/null @@ -1,238 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpcc_configuration.h -// -// Identification: src/include/benchmark/tpcc/tpcc_configuration.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "common/internal_types.h" - -namespace peloton { -namespace benchmark { -namespace tpcc { - -static const oid_t tpcc_database_oid = 100; - -static const oid_t warehouse_table_oid = 1001; -static const oid_t warehouse_table_pkey_index_oid = 20010; // W_ID - -static const oid_t district_table_oid = 1002; -static const oid_t district_table_pkey_index_oid = 20021; // D_ID, D_W_ID - -static const oid_t item_table_oid = 1003; -static const oid_t item_table_pkey_index_oid = 20030; // I_ID - -static const oid_t customer_table_oid = 1004; -static const oid_t customer_table_pkey_index_oid = - 20040; // C_W_ID, C_D_ID, C_ID -static const oid_t customer_table_skey_index_oid = - 20041; // C_W_ID, C_D_ID, C_LAST - -static const oid_t history_table_oid = 1005; - -static const oid_t stock_table_oid = 1006; -static const oid_t stock_table_pkey_index_oid = 20060; // S_W_ID, S_I_ID - -static const oid_t orders_table_oid = 1007; -static const oid_t orders_table_pkey_index_oid = 20070; // O_W_ID, O_D_ID, O_ID -static const oid_t orders_table_skey_index_oid = - 20071; // O_W_ID, O_D_ID, O_C_ID - -static const oid_t new_order_table_oid = 1008; -static const oid_t new_order_table_pkey_index_oid = - 20080; // NO_D_ID, NO_W_ID, NO_O_ID - -static const oid_t order_line_table_oid = 1008; -static const oid_t order_line_table_pkey_index_oid = - 20080; // OL_W_ID, OL_D_ID, OL_O_ID, OL_NUMBER -static const oid_t order_line_table_skey_index_oid = - 20081; // OL_W_ID, OL_D_ID, OL_O_ID - -//=========== -// Column ids -//=========== - -// NEW_ORDER -#define COL_IDX_NO_O_ID 0 -#define COL_IDX_NO_D_ID 1 -#define COL_IDX_NO_W_ID 2 - -// ORDERS -#define COL_IDX_O_ID 0 -#define COL_IDX_O_C_ID 1 -#define COL_IDX_O_D_ID 2 -#define COL_IDX_O_W_ID 3 -#define COL_IDX_O_ENTRY_D 4 -#define COL_IDX_O_CARRIER_ID 5 -#define COL_IDX_O_OL_CNT 6 -#define COL_IDX_O_ALL_LOCAL 7 - -// ORDER_LINE -#define COL_IDX_OL_O_ID 0 -#define COL_IDX_OL_D_ID 1 -#define COL_IDX_OL_W_ID 2 -#define COL_IDX_OL_NUMBER 3 -#define COL_IDX_OL_I_ID 4 -#define COL_IDX_OL_SUPPLY_W_ID 5 -#define COL_IDX_OL_DELIVERY_D 6 -#define COL_IDX_OL_QUANTITY 7 -#define COL_IDX_OL_AMOUNT 8 -#define COL_IDX_OL_DIST_INFO 9 - -// Customer -#define COL_IDX_C_ID 0 -#define COL_IDX_C_D_ID 1 -#define COL_IDX_C_W_ID 2 -#define COL_IDX_C_FIRST 3 -#define COL_IDX_C_MIDDLE 4 -#define COL_IDX_C_LAST 5 -#define COL_IDX_C_STREET_1 6 -#define COL_IDX_C_STREET_2 7 -#define COL_IDX_C_CITY 8 -#define COL_IDX_C_STATE 9 -#define COL_IDX_C_ZIP 10 -#define COL_IDX_C_PHONE 11 -#define COL_IDX_C_SINCE 12 -#define COL_IDX_C_CREDIT 13 -#define COL_IDX_C_CREDIT_LIM 14 -#define COL_IDX_C_DISCOUNT 15 -#define COL_IDX_C_BALANCE 16 -#define COL_IDX_C_YTD_PAYMENT 17 -#define COL_IDX_C_PAYMENT_CNT 18 -#define COL_IDX_C_DELIVERY_CNT 19 -#define COL_IDX_C_DATA 20 - -// District -#define COL_IDX_D_ID 0 -#define COL_IDX_D_W_ID 1 -#define COL_IDX_D_NAME 2 -#define COL_IDX_D_STREET_1 3 -#define COL_IDX_D_STREET_2 4 -#define COL_IDX_D_CITY 5 -#define COL_IDX_D_STATE 6 -#define COL_IDX_D_ZIP 7 -#define COL_IDX_D_TAX 8 -#define COL_IDX_D_YTD 9 -#define COL_IDX_D_NEXT_O_ID 10 - -// Stock -#define COL_IDX_S_I_ID 0 -#define COL_IDX_S_W_ID 1 -#define COL_IDX_S_QUANTITY 2 -#define COL_IDX_S_DIST_01 3 -#define COL_IDX_S_DIST_02 4 -#define COL_IDX_S_DIST_03 5 -#define COL_IDX_S_DIST_04 6 -#define COL_IDX_S_DIST_05 7 -#define COL_IDX_S_DIST_06 8 -#define COL_IDX_S_DIST_07 9 -#define COL_IDX_S_DIST_08 10 -#define COL_IDX_S_DIST_09 11 -#define COL_IDX_S_DIST_10 12 -#define COL_IDX_S_YTD 13 -#define COL_IDX_S_ORDER_CNT 14 -#define COL_IDX_S_REMOTE_CNT 15 -#define COL_IDX_S_DATA 16 - - -class configuration { - public: - - // index type - IndexType index; - - // epoch type - EpochType epoch; - - // scale factor - double scale_factor; - - // execution duration (in s) - double duration; - - // profile duration (in s) - double profile_duration; - - // number of backends - int backend_count; - - // num of warehouses - int warehouse_count; - - // item count - int item_count; - - int districts_per_warehouse; - - int customers_per_district; - - int new_orders_per_district; - - // exponential backoff - bool exp_backoff; - - // client affinity - bool affinity; - - // garbage collection - bool gc_mode; - - // number of gc threads - bool gc_backend_count; - - // number of loaders - int loader_count; - - // throughput - double throughput = 0; - - // abort rate - double abort_rate = 0; - - std::vector profile_throughput; - - std::vector profile_abort_rate; - - std::vector profile_memory; - -}; - -extern configuration state; - -void Usage(FILE *out); - -void ParseArguments(int argc, char *argv[], configuration &state); - -void ValidateIndex(const configuration &state); - -void ValidateScaleFactor(const configuration &state); - -void ValidateDuration(const configuration &state); - -void ValidateProfileDuration(const configuration &state); - -void ValidateBackendCount(const configuration &state); - -void ValidateWarehouseCount(const configuration &state); - -void ValidateGCBackendCount(const configuration &state); - -void WriteOutput(); - -} // namespace tpcc -} // namespace benchmark -} // namespace peloton diff --git a/src/include/benchmark/tpcc/tpcc_loader.h b/src/include/benchmark/tpcc/tpcc_loader.h deleted file mode 100644 index 91f6e14e56c..00000000000 --- a/src/include/benchmark/tpcc/tpcc_loader.h +++ /dev/null @@ -1,215 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpcc_loader.h -// -// Identification: src/include/benchmark/tpcc/tpcc_loader.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#pragma once - -#include - -#include "benchmark/tpcc/tpcc_configuration.h" - -namespace peloton { -namespace storage { -class Database; -class DataTable; -class Tuple; -} - -class VarlenPool; - -namespace benchmark { -namespace tpcc { - -extern configuration state; - -void CreateTPCCDatabase(); - -void LoadTPCCDatabase(); - -///////////////////////////////////////////////////////// -// Tables -///////////////////////////////////////////////////////// - -extern storage::Database* tpcc_database; - -extern storage::DataTable* warehouse_table; -extern storage::DataTable* district_table; -extern storage::DataTable* item_table; -extern storage::DataTable* customer_table; -extern storage::DataTable* history_table; -extern storage::DataTable* stock_table; -extern storage::DataTable* orders_table; -extern storage::DataTable* new_order_table; -extern storage::DataTable* order_line_table; - -///////////////////////////////////////////////////////// -// Constants -///////////////////////////////////////////////////////// - -extern const size_t name_length; -extern const size_t middle_name_length; -extern const size_t data_length; -extern const size_t state_length; -extern const size_t zip_length; -extern const size_t street_length; -extern const size_t city_length; -extern const size_t credit_length; -extern const size_t phone_length; -extern const size_t dist_length; - -extern double item_min_price; -extern double item_max_price; - -extern double warehouse_name_length; -extern double warehouse_min_tax; -extern double warehouse_max_tax; -extern double warehouse_initial_ytd; - -extern double district_name_length; -extern double district_min_tax; -extern double district_max_tax; -extern double district_initial_ytd; - -extern std::string customers_good_credit; -extern std::string customers_bad_credit; -extern double customers_bad_credit_ratio; -extern double customers_init_credit_lim; -extern double customers_min_discount; -extern double customers_max_discount; -extern double customers_init_balance; -extern double customers_init_ytd; -extern int customers_init_payment_cnt; -extern int customers_init_delivery_cnt; - -extern double history_init_amount; -extern size_t history_data_length; - -extern int orders_min_ol_cnt; -extern int orders_max_ol_cnt; -extern int orders_init_all_local; -extern int orders_null_carrier_id; -extern int orders_min_carrier_id; -extern int orders_max_carrier_id; - -extern int new_orders_per_district; - -extern int order_line_init_quantity; -extern int order_line_max_ol_quantity; -extern double order_line_min_amount; -extern size_t order_line_dist_info_length; - -extern double stock_original_ratio; -extern int stock_min_quantity; -extern int stock_max_quantity; -extern int stock_dist_count; - -extern double payment_min_amount; -extern double payment_max_amount; - -extern int stock_min_threshold; -extern int stock_max_threshold; - -extern double new_order_remote_txns; - -extern const int syllable_count; -extern const char* syllables[]; - -extern const std::string data_constant; - -struct NURandConstant { - int c_last; - int c_id; - int order_line_itme_id; - - NURandConstant(); -}; - -extern NURandConstant nu_rand_const; - -///////////////////////////////////////////////////////// -// Tuple Constructors -///////////////////////////////////////////////////////// - -std::unique_ptr BuildItemTuple( - const int item_id, const std::unique_ptr& pool); - -std::unique_ptr BuildWarehouseTuple( - const int warehouse_id, const std::unique_ptr& pool); - -std::unique_ptr BuildDistrictTuple( - const int district_id, const int warehouse_id, - const std::unique_ptr& pool); - -std::unique_ptr BuildCustomerTuple( - const int customer_id, const int district_id, const int warehouse_id, - const std::unique_ptr& pool); - -std::unique_ptr BuildHistoryTuple( - const int customer_id, const int district_id, const int warehouse_id, - const int history_district_id, const int history_warehouse_id, - const std::unique_ptr& pool); - -std::unique_ptr BuildOrdersTuple(const int orders_id, - const int district_id, - const int warehouse_id, - const bool new_order, - const int o_ol_cnt); - -std::unique_ptr BuildNewOrderTuple(const int orders_id, - const int district_id, - const int warehouse_id); - -std::unique_ptr BuildOrderLineTuple( - const int orders_id, const int district_id, const int warehouse_id, - const int order_line_id, const int ol_supply_w_id, const bool new_order, - const std::unique_ptr& pool); - -std::unique_ptr BuildStockTuple( - const int stock_id, const int s_w_id, - const std::unique_ptr& pool); - -///////////////////////////////////////////////////////// -// Utils -///////////////////////////////////////////////////////// - -std::string GetRandomAlphaNumericString(const size_t string_length); - -int GetNURand(int a, int x, int y); - -std::string GetLastName(int number); - -std::string GetRandomLastName(int max_cid); - -bool GetRandomBoolean(double ratio); - -int GetRandomInteger(const int lower_bound, const int upper_bound); - -int GetRandomIntegerExcluding(const int lower_bound, const int upper_bound, - const int exclude_sample); - -double GetRandomDouble(const double lower_bound, const double upper_bound); - -double GetRandomFixedPoint(int decimal_places, double minimum, double maximum); - -std::string GetStreetName(); - -std::string GetZipCode(); - -std::string GetCityName(); - -std::string GetStateName(); - -int GetTimeStamp(); - -} // namespace tpcc -} // namespace benchmark -} // namespace peloton diff --git a/src/include/benchmark/tpcc/tpcc_workload.h b/src/include/benchmark/tpcc/tpcc_workload.h deleted file mode 100644 index 7978f00f395..00000000000 --- a/src/include/benchmark/tpcc/tpcc_workload.h +++ /dev/null @@ -1,58 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpcc_workload.h -// -// Identification: src/include/benchmark/tpcc/tpcc_workload.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#pragma once - -#include "common/macros.h" -#include "benchmark/tpcc/tpcc_configuration.h" -#include "benchmark/tpcc/tpcc_loader.h" -#include "executor/abstract_executor.h" - -namespace peloton { - -namespace storage { -class DataTable; -} - -namespace benchmark { -namespace tpcc { - -extern configuration state; - -void RunWorkload(); - -bool RunNewOrder(const size_t &thread_id); - -bool RunPayment(const size_t &thread_id); - -bool RunDelivery(const size_t &thread_id); - -bool RunOrderStatus(const size_t &thread_id); - -bool RunStockLevel(const size_t &thread_id); - -size_t GenerateWarehouseId(const size_t &thread_id); - -///////////////////////////////////////////////////////// - -std::vector> ExecuteRead(executor::AbstractExecutor* executor); - -void ExecuteUpdate(executor::AbstractExecutor* executor); - -void ExecuteDelete(executor::AbstractExecutor* executor); - -void PinToCore(size_t core); - -} // namespace tpcc -} // namespace benchmark -} // namespace peloton diff --git a/src/include/benchmark/tpch/tpch_configuration.h b/src/include/benchmark/tpch/tpch_configuration.h deleted file mode 100644 index 037f65b473a..00000000000 --- a/src/include/benchmark/tpch/tpch_configuration.h +++ /dev/null @@ -1,120 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpch_configuration.h -// -// Identification: src/include/benchmark/tpch/tpch_configuration.h -// -// Copyright (c) 2015-17, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include -#include -#include - -#include "common/internal_types.h" - -namespace peloton { -namespace benchmark { -namespace tpch { - -//===----------------------------------------------------------------------===// -// Type size constants -//===----------------------------------------------------------------------===// - -extern oid_t kIntSize; -extern oid_t kDateSize; -extern oid_t kBigIntSize; -extern oid_t kDecimalSize; - -//===----------------------------------------------------------------------===// -// Query and Table types -//===----------------------------------------------------------------------===// - -enum class QueryId : uint32_t { - Q1 = 0, - Q2, - Q3, - Q4, - Q5, - Q6, - Q7, - Q8, - Q9, - Q10, - Q11, - Q12, - Q13, - Q14, - Q15, - Q16, - Q17, - Q18, - Q19, - Q20, - Q21, - Q22, -}; - -enum class TableId : uint32_t { - Part = 44, - Supplier = 45, - PartSupp = 46, - Customer = 47, - Nation = 48, - Lineitem = 49, - Region = 50, - Orders = 51, -}; - -//===----------------------------------------------------------------------===// -// The benchmark configuration -//===----------------------------------------------------------------------===// - -struct Configuration { - // Default 64K tuples per tile group - uint32_t tuples_per_tile_group = 1 << 16; - - // The scale factor of the benchmark - double scale_factor = 1; - - // The number of runs to average over - uint32_t num_runs = 10; - - // The directory where all the data files are - std::string data_dir; - - // The suffix of all the files - std::string suffix; - - // Do we dictionary encode strings? - bool dictionary_encode = true; - - // Which queries will the benchmark run? - bool queries_to_run[22] = {false}; - - bool IsValid() const; - - std::string GetInputPath(std::string file_name) const; - - std::string GetCustomerPath() const; - std::string GetLineitemPath() const; - std::string GetNationPath() const; - std::string GetOrdersPath() const; - std::string GetPartSuppPath() const; - std::string GetPartPath() const; - std::string GetSupplierPath() const; - std::string GetRegionPath() const; - - void SetRunnableQueries(char *query_list); - - bool ShouldRunQuery(QueryId qid) const; -}; - -} // namespace tpch -} // namespace benchmark -} // namespace pelton \ No newline at end of file diff --git a/src/include/benchmark/tpch/tpch_database.h b/src/include/benchmark/tpch/tpch_database.h deleted file mode 100644 index 34b198bff7a..00000000000 --- a/src/include/benchmark/tpch/tpch_database.h +++ /dev/null @@ -1,107 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpch_database.h -// -// Identification: src/include/benchmark/tpch/tpch_database.h -// -// Copyright (c) 2015-17, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include -#include - -#include "benchmark/tpch/tpch_configuration.h" - -namespace peloton { - -namespace storage { -class Database; -class DataTable; -} // namespace storage - -namespace benchmark { -namespace tpch { - -//===----------------------------------------------------------------------===// -// The TPCH Database. This class is responsible for access to all table in the -// DB. Tables are created on instantiation. Individual tables can be loaded -// from files based on the benchmark configuration parameters. Loading supports -// dictionary encoding string values. -//===----------------------------------------------------------------------===// -class TPCHDatabase { - public: - typedef std::unordered_map Dictionary; - - TPCHDatabase(const Configuration &c); - - ~TPCHDatabase(); - - storage::Database &GetDatabase() const; - - // Table accessors - storage::DataTable &GetTable(TableId table_id) const; - - // Create all tables - void CreateTables() const; - - void LoadTable(TableId table_id); - - // Load individual tables - void LoadCustomerTable(); - void LoadLineitemTable(); - void LoadNationTable(); - void LoadOrdersTable(); - void LoadPartTable(); - void LoadPartSupplierTable(); - void LoadRegionTable(); - void LoadSupplierTable(); - - uint32_t CodeForMktSegment(const std::string mktsegment) const; - - private: - uint32_t DictionaryEncode(Dictionary &dict, const std::string &val); - - // Table creators - void CreateCustomerTable() const; - void CreateLineitemTable() const; - void CreateNationTable() const; - void CreateOrdersTable() const; - void CreatePartTable() const; - void CreatePartSupplierTable() const; - void CreateRegionTable() const; - void CreateSupplierTable() const; - - // Has the given table been loaded already? - bool TableIsLoaded(TableId table_id) const { - return loaded_tables_[static_cast(table_id) - - static_cast(TableId::Part)]; - } - - void SetTableIsLoaded(TableId table_id) { - loaded_tables_[static_cast(table_id) - - static_cast(TableId::Part)] = true; - } - - private: - // The configuration - const Configuration &config_; - - // Track which tables have been loaded - bool loaded_tables_[8]; - - // Dictionary codes - Dictionary l_shipinstruct_dict_; - Dictionary l_shipmode_dict_; - Dictionary p_brand_dict_; - Dictionary p_container_dict_; - Dictionary c_mktsegment_dict_; -}; - -} // namespace tpch -} // namespace benchmark -} // namespace peloton \ No newline at end of file diff --git a/src/include/benchmark/tpch/tpch_workload.h b/src/include/benchmark/tpch/tpch_workload.h deleted file mode 100644 index 65e1bc148b2..00000000000 --- a/src/include/benchmark/tpch/tpch_workload.h +++ /dev/null @@ -1,90 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpch_workload.h -// -// Identification: src/include/benchmark/tpch/tpch_workload.h -// -// Copyright (c) 2015-17, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "benchmark/tpch/tpch_configuration.h" -#include "benchmark/tpch/tpch_database.h" - -#include "codegen/compilation_context.h" -#include "codegen/execution_consumer.h" - -namespace peloton { -namespace benchmark { -namespace tpch { - -// The benchmark -class TPCHBenchmark { - public: - TPCHBenchmark(const Configuration &config, TPCHDatabase &db); - - // Run the benchmark - void RunBenchmark(); - - private: - - struct QueryConfig { - public: - // The name of the query - std::string query_name; - - // The ID of the query - QueryId query_id; - - // The list of tables this query uses - std::vector required_tables; - - // A function that constructs a plan for this query - std::function()> PlanConstructor; - }; - - // Run the given query - void RunQuery(const QueryConfig &query_config); - - // Plan constructors - std::unique_ptr ConstructQ1Plan() const; - std::unique_ptr ConstructQ2Plan() const; - std::unique_ptr ConstructQ3Plan() const; - std::unique_ptr ConstructQ4Plan() const; - std::unique_ptr ConstructQ5Plan() const; - std::unique_ptr ConstructQ6Plan() const; - std::unique_ptr ConstructQ7Plan() const; - std::unique_ptr ConstructQ8Plan() const; - std::unique_ptr ConstructQ9Plan() const; - std::unique_ptr ConstructQ10Plan() const; - std::unique_ptr ConstructQ11Plan() const; - std::unique_ptr ConstructQ12Plan() const; - std::unique_ptr ConstructQ13Plan() const; - std::unique_ptr ConstructQ14Plan() const; - std::unique_ptr ConstructQ15Plan() const; - std::unique_ptr ConstructQ16Plan() const; - std::unique_ptr ConstructQ17Plan() const; - std::unique_ptr ConstructQ18Plan() const; - std::unique_ptr ConstructQ19Plan() const; - std::unique_ptr ConstructQ20Plan() const; - std::unique_ptr ConstructQ21Plan() const; - std::unique_ptr ConstructQ22Plan() const; - - private: - // The benchmark configuration - const Configuration &config_; - - // The TPCH database - TPCHDatabase &db_; - - // All query configurations - std::vector query_configs_; -}; - -} // namespace tpch -} // namespace benchmark -} // namespace peloton \ No newline at end of file diff --git a/src/include/benchmark/ycsb/ycsb_configuration.h b/src/include/benchmark/ycsb/ycsb_configuration.h deleted file mode 100644 index 633dba6ddca..00000000000 --- a/src/include/benchmark/ycsb/ycsb_configuration.h +++ /dev/null @@ -1,129 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// ycsb_configuration.h -// -// Identification: src/include/benchmark/ycsb/ycsb_configuration.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "common/internal_types.h" - -namespace peloton { -namespace benchmark { -namespace ycsb { - -static const oid_t ycsb_database_oid = 100; - -static const oid_t user_table_oid = 1001; - -static const oid_t user_table_pkey_index_oid = 2001; - -static const oid_t ycsb_field_length = 100; - -class configuration { - public: - - // index type - IndexType index; - - // epoch type - EpochType epoch; - - // size of the table - int scale_factor; - - // execution duration (in s) - double duration; - - // profile duration (in s) - double profile_duration; - - // number of backends - int backend_count; - - // column count - int column_count; - - // operation count in a transaction - int operation_count; - - // update ratio - double update_ratio; - - // contention level - double zipf_theta; - - // exponential backoff - bool exp_backoff; - - // store strings - bool string_mode; - - // garbage collection - bool gc_mode; - - // number of gc threads - int gc_backend_count; - - // number of loaders - int loader_count; - - // throughput - double throughput = 0; - - // abort rate - double abort_rate = 0; - - std::vector profile_throughput; - - std::vector profile_abort_rate; - - std::vector profile_memory; - -}; - -extern configuration state; - -void Usage(FILE *out); - -void ParseArguments(int argc, char *argv[], configuration &state); - -void ValidateIndex(const configuration &state); - -void ValidateScaleFactor(const configuration &state); - -void ValidateDuration(const configuration &state); - -void ValidateProfileDuration(const configuration &state); - -void ValidateBackendCount(const configuration &state); - -void ValidateColumnCount(const configuration &state); - -void ValidateOperationCount(const configuration &state); - -void ValidateUpdateRatio(const configuration &state); - -void ValidateZipfTheta(const configuration &state); - -void ValidateGCBackendCount(const configuration &state); - -void WriteOutput(); - -} // namespace ycsb -} // namespace benchmark -} // namespace peloton diff --git a/src/include/benchmark/ycsb/ycsb_loader.h b/src/include/benchmark/ycsb/ycsb_loader.h deleted file mode 100644 index de39ed49bd8..00000000000 --- a/src/include/benchmark/ycsb/ycsb_loader.h +++ /dev/null @@ -1,31 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// ycsb_loader.h -// -// Identification: src/include/benchmark/ycsb/ycsb_loader.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#pragma once - -#include "benchmark/ycsb/ycsb_configuration.h" - -namespace peloton { -namespace benchmark { -namespace ycsb { - -extern configuration state; - -void CreateYCSBDatabase(); - -void LoadYCSBDatabase(); -void LoadYCSBRows(const int begin_rowid, const int end_rowid); - -} // namespace ycsb -} // namespace benchmark -} // namespace peloton diff --git a/src/include/benchmark/ycsb/ycsb_workload.h b/src/include/benchmark/ycsb/ycsb_workload.h deleted file mode 100644 index 678a5d1b81c..00000000000 --- a/src/include/benchmark/ycsb/ycsb_workload.h +++ /dev/null @@ -1,48 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// ycsb_workload.h -// -// Identification: src/include/benchmark/ycsb/ycsb_workload.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#pragma once - -#include "benchmark/benchmark_common.h" -#include "benchmark/ycsb/ycsb_configuration.h" -#include "storage/data_table.h" -#include "executor/abstract_executor.h" - -namespace peloton { - -namespace storage { -class DataTable; -} - -namespace benchmark { -namespace ycsb { - -extern configuration state; - -extern storage::DataTable* user_table; - -void RunWorkload(); - -bool RunMixed(const size_t thread_id, ZipfDistribution &zipf, FastRandom &rng); - -///////////////////////////////////////////////////////// - -std::vector> ExecuteRead(executor::AbstractExecutor* executor); - -void ExecuteUpdate(executor::AbstractExecutor* executor); - -void PinToCore(size_t core); - -} // namespace ycsb -} // namespace benchmark -} // namespace peloton diff --git a/src/main/logger/logger.cpp b/src/main/logger/logger.cpp deleted file mode 100644 index aaca3ff2bbe..00000000000 --- a/src/main/logger/logger.cpp +++ /dev/null @@ -1,102 +0,0 @@ -// //===----------------------------------------------------------------------===// -// // -// // Peloton -// // -// // logger.cpp -// // -// // Identification: src/main/logger/logger.cpp -// // -// // Copyright (c) 2015-16, Carnegie Mellon University Database Group -// // -// //===----------------------------------------------------------------------===// - -// #include -// #include -// #include - -// #include "benchmark/logger/logger_configuration.h" -// #include "benchmark/logger/logger_workload.h" -// #include "benchmark/tpcc/tpcc_configuration.h" -// #include "benchmark/ycsb/ycsb_configuration.h" - -// #include "common/logger.h" -// #include "logging/logging_util.h" - -// // Logging mode -// extern peloton::LoggingType peloton_logging_mode; - -// extern peloton::CheckpointType peloton_checkpoint_mode; - -// extern size_t peloton_data_file_size; - -// extern int64_t peloton_wait_timeout; - -// // Flush mode (for NVM WBL) -// extern int peloton_flush_mode; - -// // PCOMMIT latency (for NVM WBL) -// extern int peloton_pcommit_latency; - -// namespace peloton { -// namespace benchmark { - -// namespace ycsb { -// configuration state; -// void CreateYCSBDatabase(); -// } -// namespace tpcc { -// configuration state; -// } - -// namespace logger { - -// void StartLogging(std::thread &thread); - -// // Configuration -// configuration state; - -// // Main Entry Point -// void RunBenchmark() { -// // First, set the global peloton logging mode and pmem file size -// peloton_logging_mode = state.logging_type; -// peloton_data_file_size = state.data_file_size; -// peloton_wait_timeout = state.wait_timeout; -// peloton_flush_mode = state.flush_mode; -// peloton_pcommit_latency = state.pcommit_latency; - -// //===--------------------------------------------------------------------===// -// // WAL -// //===--------------------------------------------------------------------===// -// if (logging::LoggingUtil::IsBasedOnWriteAheadLogging(peloton_logging_mode)) { -// // Prepare a simple log file -// PrepareLogFile(); - -// // Do recovery -// DoRecovery(); -// } -// //===--------------------------------------------------------------------===// -// // WBL -// //===--------------------------------------------------------------------===// -// else if (logging::LoggingUtil::IsBasedOnWriteBehindLogging(peloton_logging_mode)) { -// LOG_ERROR("currently, we do not support write behind logging."); -// PELOTON_ASSERT(false); -// // Test a simple log process -// PrepareLogFile(); - -// // Do recovery -// DoRecovery(); -// } -// } - -// } // namespace logger -// } // namespace benchmark -// } // namespace peloton - -// int main(int argc, char **argv) { -// peloton::benchmark::logger::ParseArguments(argc, argv, -// peloton::benchmark::logger::state); - -// peloton::benchmark::logger::RunBenchmark(); - -// return 0; -// } diff --git a/src/main/logger/logger_configuration.cpp b/src/main/logger/logger_configuration.cpp deleted file mode 100644 index 747cb0cc698..00000000000 --- a/src/main/logger/logger_configuration.cpp +++ /dev/null @@ -1,441 +0,0 @@ -// //===----------------------------------------------------------------------===// -// // -// // Peloton -// // -// // logger_configuration.cpp -// // -// // Identification: src/main/logger/logger_configuration.cpp -// // -// // Copyright (c) 2015-16, Carnegie Mellon University Database Group -// // -// //===----------------------------------------------------------------------===// - - -// #include -// #include -// #include - -// #include "common/exception.h" -// #include "common/logger.h" -// #include "storage/storage_manager.h" - -// #include "benchmark/logger/logger_configuration.h" -// #include "benchmark/ycsb/ycsb_configuration.h" -// #include "benchmark/tpcc/tpcc_configuration.h" - -// extern peloton::CheckpointType peloton_checkpoint_mode; - -// namespace peloton { -// namespace benchmark { -// namespace logger { - -// void Usage(FILE* out) { -// fprintf(out, -// "Command line options : logger \n" -// " -h --help : Print help message \n" -// " -s --asynchronous-mode : Asynchronous mode \n" -// " -x --experiment-type : Experiment Type \n" -// " -f --data-file-size : Data file size (MB) \n" -// " -l --logging-type : Logging type \n" -// " -t --nvm-latency : NVM latency \n" -// " -q --pcommit-latency : pcommit latency \n" -// " -v --flush-mode : Flush mode \n" -// " -r --commit-interval : Group commit interval \n" -// " -j --log-dir : Log directory\n" -// " -y --benchmark-type : Benchmark type \n"); -// } - -// static struct option opts[] = { -// {"asynchronous_mode", optional_argument, NULL, 's'}, -// {"experiment-type", optional_argument, NULL, 'x'}, -// {"data-file-size", optional_argument, NULL, 'f'}, -// {"logging-type", optional_argument, NULL, 'l'}, -// {"nvm-latency", optional_argument, NULL, 't'}, -// {"pcommit-latency", optional_argument, NULL, 'q'}, -// {"flush-mode", optional_argument, NULL, 'v'}, -// {"commit-interval", optional_argument, NULL, 'r'}, -// {"benchmark-type", optional_argument, NULL, 'y'}, -// {"log-dir", optional_argument, NULL, 'j'}, -// {NULL, 0, NULL, 0}}; - -// static void ValidateLoggingType(const configuration& state) { -// if (state.logging_type <= LoggingTypeId::INVALID) { -// LOG_ERROR("Invalid logging_type :: %d", static_cast(state.logging_type)); -// exit(EXIT_FAILURE); -// } - -// LOG_INFO("Logging_type :: %s", -// LoggingTypeToString(state.logging_type).c_str()); -// } - -// std::string BenchmarkTypeToString(BenchmarkType type) { -// switch (type) { -// case BENCHMARK_TYPE_INVALID: -// return "INVALID"; - -// case BENCHMARK_TYPE_YCSB: -// return "YCSB"; -// case BENCHMARK_TYPE_TPCC: -// return "TPCC"; - -// default: -// LOG_ERROR("Invalid benchmark_type :: %d", type); -// exit(EXIT_FAILURE); -// } -// return "INVALID"; -// } - -// std::string ExperimentTypeToString(ExperimentType type) { -// switch (type) { -// case EXPERIMENT_TYPE_INVALID: -// return "INVALID"; - -// case EXPERIMENT_TYPE_THROUGHPUT: -// return "THROUGHPUT"; -// case EXPERIMENT_TYPE_RECOVERY: -// return "RECOVERY"; -// case EXPERIMENT_TYPE_STORAGE: -// return "STORAGE"; -// case EXPERIMENT_TYPE_LATENCY: -// return "LATENCY"; - -// default: -// LOG_ERROR("Invalid experiment_type :: %d", type); -// exit(EXIT_FAILURE); -// } - -// return "INVALID"; -// } - -// std::string AsynchronousTypeToString(AsynchronousType type) { -// switch (type) { -// case ASYNCHRONOUS_TYPE_INVALID: -// return "INVALID"; - -// case ASYNCHRONOUS_TYPE_SYNC: -// return "SYNC"; -// case ASYNCHRONOUS_TYPE_ASYNC: -// return "ASYNC"; -// case ASYNCHRONOUS_TYPE_DISABLED: -// return "DISABLED"; -// case ASYNCHRONOUS_TYPE_NO_WRITE: -// return "NO_WRITE"; - -// default: -// LOG_ERROR("Invalid asynchronous_mode :: %d", type); -// exit(EXIT_FAILURE); -// } - -// return "INVALID"; -// } - -// static void ValidateBenchmarkType(const configuration& state) { -// if (state.benchmark_type <= 0 || state.benchmark_type > 3) { -// LOG_ERROR("Invalid benchmark_type :: %d", state.benchmark_type); -// exit(EXIT_FAILURE); -// } - -// LOG_INFO("%s : %s", "benchmark_type", -// BenchmarkTypeToString(state.benchmark_type).c_str()); -// } - -// static void ValidateDataFileSize(const configuration& state) { -// if (state.data_file_size <= 0) { -// LOG_ERROR("Invalid data_file_size :: %lu", state.data_file_size); -// exit(EXIT_FAILURE); -// } - -// LOG_INFO("data_file_size :: %lu", state.data_file_size); -// } - -// static void ValidateExperimentType(const configuration& state) { -// if (state.experiment_type < 0 || state.experiment_type > 4) { -// LOG_ERROR("Invalid experiment_type :: %d", state.experiment_type); -// exit(EXIT_FAILURE); -// } - -// LOG_INFO("%s : %s", "experiment_type", -// ExperimentTypeToString(state.experiment_type).c_str()); -// } - -// static void ValidateWaitTimeout(const configuration& state) { -// if (state.wait_timeout < 0) { -// LOG_ERROR("Invalid wait_timeout :: %d", state.wait_timeout); -// exit(EXIT_FAILURE); -// } - -// LOG_INFO("wait_timeout :: %d", state.wait_timeout); -// } - -// static void ValidateFlushMode(const configuration& state) { -// if (state.flush_mode <= 0 || state.flush_mode >= 3) { -// LOG_ERROR("Invalid flush_mode :: %d", state.flush_mode); -// exit(EXIT_FAILURE); -// } - -// LOG_INFO("flush_mode :: %d", state.flush_mode); -// } - -// static void ValidateAsynchronousMode(const configuration& state) { -// if (state.asynchronous_mode <= ASYNCHRONOUS_TYPE_INVALID || -// state.asynchronous_mode > ASYNCHRONOUS_TYPE_NO_WRITE) { -// LOG_ERROR("Invalid asynchronous_mode :: %d", state.asynchronous_mode); -// exit(EXIT_FAILURE); -// } - -// LOG_INFO("%s : %s", "asynchronous_mode", -// AsynchronousTypeToString(state.asynchronous_mode).c_str()); -// } - -// static void ValidateNVMLatency(const configuration& state) { -// if (state.nvm_latency < 0) { -// LOG_ERROR("Invalid nvm_latency :: %d", state.nvm_latency); -// exit(EXIT_FAILURE); -// } - -// LOG_INFO("nvm_latency :: %d", state.nvm_latency); -// } - -// static void ValidatePCOMMITLatency(const configuration& state) { -// if (state.pcommit_latency < 0) { -// LOG_ERROR("Invalid pcommit_latency :: %d", state.pcommit_latency); -// exit(EXIT_FAILURE); -// } - -// LOG_INFO("pcommit_latency :: %d", state.pcommit_latency); -// } - -// static void ValidateLogFileDir(configuration& state) { -// struct stat data_stat; -// // Check the existence of the log directory -// if (stat(state.log_file_dir.c_str(), &data_stat) != 0) { -// LOG_ERROR("log_file_dir :: %s does not exist", state.log_file_dir.c_str()); -// exit(EXIT_FAILURE); -// } else if (!(data_stat.st_mode & S_IFDIR)) { -// LOG_ERROR("log_file_dir :: %s is not a directory", state.log_file_dir.c_str()); -// exit(EXIT_FAILURE); -// } - -// LOG_INFO("log_file_dir :: %s", state.log_file_dir.c_str()); -// } - -// void ParseArguments(int argc, char* argv[], configuration& state) { -// // Default Logger Values -// state.logging_type = LoggingType::SSD_WAL; -// state.log_file_dir = TMP_DIR; -// state.data_file_size = 512; - -// state.experiment_type = EXPERIMENT_TYPE_THROUGHPUT; -// state.wait_timeout = 200; -// state.benchmark_type = BENCHMARK_TYPE_YCSB; -// state.flush_mode = 2; -// state.nvm_latency = 0; -// state.pcommit_latency = 0; -// state.asynchronous_mode = ASYNCHRONOUS_TYPE_SYNC; -// state.checkpoint_type = CheckpointTypeId::INVALID; - -// // YCSB Default Values -// ycsb::state.index = IndexType::BWTREE; -// ycsb::state.scale_factor = 1; -// ycsb::state.duration = 10; -// ycsb::state.profile_duration = 1; -// ycsb::state.backend_count = 2; -// ycsb::state.column_count = 10; -// ycsb::state.operation_count = 10; -// ycsb::state.update_ratio = 0.5; -// ycsb::state.zipf_theta = 0.0; -// ycsb::state.exp_backoff = false; -// ycsb::state.string_mode = false; -// ycsb::state.gc_mode = false; -// ycsb::state.gc_backend_count = 1; - -// // TPC-C Default Values -// tpcc::state.index = IndexType::BWTREE; -// tpcc::state.scale_factor = 1; -// tpcc::state.duration = 10; -// tpcc::state.profile_duration = 1; -// tpcc::state.backend_count = 2; -// tpcc::state.warehouse_count = 2; -// tpcc::state.exp_backoff = false; -// tpcc::state.affinity = false; -// tpcc::state.gc_mode = false; -// tpcc::state.gc_backend_count = 1; - - -// // Parse args -// while (1) { -// int idx = 0; -// // logger - hs:x:f:l:t:q:v:r:y: -// // ycsb - hemgi:k:d:p:b:c:o:u:z:n: -// // tpcc - heagi:k:d:p:b:w:n: -// int c = getopt_long(argc, argv, "hs:x:f:l:t:q:v:r:y:emgi:k:d:p:b:c:o:u:z:n:aw:j:", -// opts, &idx); - -// if (c == -1) break; - -// switch (c) { -// case 's': -// state.asynchronous_mode = (AsynchronousType)atoi(optarg); -// break; -// case 'x': -// state.experiment_type = (ExperimentType)atoi(optarg); -// break; -// case 'f': -// state.data_file_size = atoi(optarg); -// break; -// case 'j': -// state.log_file_dir = optarg; -// break; -// case 'l': -// state.logging_type = (LoggingType)atoi(optarg); -// break; -// case 't': -// state.nvm_latency = atoi(optarg); -// break; -// case 'q': -// state.pcommit_latency = atoi(optarg); -// break; -// case 'v': -// state.flush_mode = atoi(optarg); -// break; -// case 'r': -// state.wait_timeout = atoi(optarg); -// break; -// case 'y': -// state.benchmark_type = (BenchmarkType)atoi(optarg); -// break; - -// case 'i': { -// char *index = optarg; -// if (strcmp(index, "btree") == 0) { -// ycsb::state.index = IndexType::BWTREE; -// tpcc::state.index = IndexType::BWTREE; -// } else if (strcmp(index, "bwtree") == 0) { -// ycsb::state.index = IndexType::BWTREE; -// tpcc::state.index = IndexType::BWTREE; -// } else { -// LOG_ERROR("Unknown index: %s", index); -// exit(EXIT_FAILURE); -// } -// break; -// } -// case 'k': -// ycsb::state.scale_factor = atoi(optarg); -// tpcc::state.scale_factor = atof(optarg); -// break; -// case 'd': -// ycsb::state.duration = atof(optarg); -// tpcc::state.duration = atof(optarg); -// break; -// case 'p': -// ycsb::state.profile_duration = atof(optarg); -// tpcc::state.profile_duration = atof(optarg); -// break; -// case 'b': -// ycsb::state.backend_count = atoi(optarg); -// tpcc::state.backend_count = atoi(optarg); -// break; -// case 'c': -// ycsb::state.column_count = atoi(optarg); -// break; -// case 'o': -// ycsb::state.operation_count = atoi(optarg); -// break; -// case 'u': -// ycsb::state.update_ratio = atof(optarg); -// break; -// case 'z': -// ycsb::state.zipf_theta = atof(optarg); -// break; -// case 'e': -// ycsb::state.exp_backoff = true; -// tpcc::state.exp_backoff = true; -// break; -// case 'm': -// ycsb::state.string_mode = true; -// break; -// case 'g': -// ycsb::state.gc_mode = true; -// tpcc::state.gc_mode = true; -// break; -// case 'n': -// ycsb::state.gc_backend_count = atof(optarg); -// tpcc::state.gc_backend_count = atof(optarg); -// break; -// case 'w': -// tpcc::state.warehouse_count = atoi(optarg); -// break; -// case 'a': -// tpcc::state.affinity = true; -// break; - -// case 'h': -// Usage(stderr); -// ycsb::Usage(stderr); -// tpcc::Usage(stderr); -// exit(EXIT_FAILURE); -// break; - -// default: -// fprintf(stderr, "\nUnknown option: -%c-\n", c); -// Usage(stderr); -// ycsb::Usage(stderr); -// tpcc::Usage(stderr); -// exit(EXIT_FAILURE); -// break; -// } -// } - -// if (state.checkpoint_type == CheckpointType::NORMAL && -// (state.logging_type == LoggingType::NVM_WAL || -// state.logging_type == LoggingType::SSD_WAL || -// state.logging_type == LoggingType::HDD_WAL)) { -// peloton_checkpoint_mode = CheckpointType::NORMAL; -// } - -// // Print Logger configuration -// ValidateLoggingType(state); -// ValidateExperimentType(state); -// ValidateAsynchronousMode(state); -// ValidateBenchmarkType(state); -// ValidateDataFileSize(state); -// ValidateLogFileDir(state); -// ValidateWaitTimeout(state); -// ValidateFlushMode(state); -// ValidateNVMLatency(state); -// ValidatePCOMMITLatency(state); - -// // Print YCSB configuration -// if (state.benchmark_type == BENCHMARK_TYPE_YCSB) { -// ycsb::ValidateIndex(ycsb::state); -// ycsb::ValidateScaleFactor(ycsb::state); -// ycsb::ValidateDuration(ycsb::state); -// ycsb::ValidateProfileDuration(ycsb::state); -// ycsb::ValidateBackendCount(ycsb::state); -// ycsb::ValidateColumnCount(ycsb::state); -// ycsb::ValidateOperationCount(ycsb::state); -// ycsb::ValidateUpdateRatio(ycsb::state); -// ycsb::ValidateZipfTheta(ycsb::state); -// ycsb::ValidateGCBackendCount(ycsb::state); -// } -// // Print TPCC configuration -// else if (state.benchmark_type == BENCHMARK_TYPE_TPCC) { -// tpcc::ValidateIndex(tpcc::state); -// tpcc::ValidateScaleFactor(tpcc::state); -// tpcc::ValidateDuration(tpcc::state); -// tpcc::ValidateProfileDuration(tpcc::state); -// tpcc::ValidateBackendCount(tpcc::state); -// tpcc::ValidateWarehouseCount(tpcc::state); -// tpcc::ValidateGCBackendCount(tpcc::state); - -// // Static TPCC parameters -// tpcc::state.item_count = 100000 * tpcc::state.scale_factor; -// tpcc::state.districts_per_warehouse = 10; -// tpcc::state.customers_per_district = 3000 * tpcc::state.scale_factor; -// tpcc::state.new_orders_per_district = 900 * tpcc::state.scale_factor; -// } -// } - -// } // namespace logger -// } // namespace benchmark -// } // namespace peloton diff --git a/src/main/logger/logger_workload.cpp b/src/main/logger/logger_workload.cpp deleted file mode 100644 index dbed1d6dca8..00000000000 --- a/src/main/logger/logger_workload.cpp +++ /dev/null @@ -1,421 +0,0 @@ -// //===----------------------------------------------------------------------===// -// // -// // Peloton -// // -// // logger_workload.cpp -// // -// // Identification: src/main/logger/logger_workload.cpp -// // -// // Copyright (c) 2015-16, Carnegie Mellon University Database Group -// // -// //===----------------------------------------------------------------------===// - -// #include -// #include -// #include -// #include -// #include -// #include - -// #include "type/value_factory.h" -// #include "concurrency/transaction_manager_factory.h" - -// #include "common/exception.h" -// #include "common/logger.h" -// #include "common/timer.h" -// #include "logging/log_manager.h" -// #include "storage/storage_manager.h" - -// #include "benchmark/logger/logger_workload.h" - -// #include "benchmark/ycsb/ycsb_configuration.h" -// #include "benchmark/ycsb/ycsb_loader.h" -// #include "benchmark/ycsb/ycsb_workload.h" - -// #include "benchmark/tpcc/tpcc_configuration.h" -// #include "benchmark/tpcc/tpcc_loader.h" -// #include "benchmark/tpcc/tpcc_workload.h" - -// #include "logging/checkpoint_manager.h" -// #include "logging/loggers/wbl_frontend_logger.h" -// #include "logging/logging_util.h" - -// //===--------------------------------------------------------------------===// -// // GUC Variables -// //===--------------------------------------------------------------------===// - -// extern peloton::CheckpointType peloton_checkpoint_mode; - -// namespace peloton { -// namespace benchmark { -// namespace logger { - -// //===--------------------------------------------------------------------===// -// // PREPARE LOG FILE -// //===--------------------------------------------------------------------===// - -// //===--------------------------------------------------------------------===// -// // 1. Standby -- Bootstrap -// // 2. Recovery -- Optional -// // 3. Logging -- Collect data and flush when commit -// // 4. Terminate -- Collect any remaining data and flush -// // 5. Sleep -- Disconnect backend loggers and frontend logger from manager -// //===--------------------------------------------------------------------===// - -// #define LOGGING_TESTS_DATABASE_OID 20000 -// #define LOGGING_TESTS_TABLE_OID 10000 - -// void WriteOutput() { -// std::ofstream out("outputfile-log.summary"); -// LOG_INFO("----------------------------------------------------------"); -// LOG_INFO("%d %d %d %d %d %d", state.benchmark_type, static_cast(state.logging_type), -// state.nvm_latency, state.pcommit_latency, state.flush_mode, -// state.asynchronous_mode); - -// out << state.benchmark_type << " "; -// out << state.logging_type << " "; -// out << state.nvm_latency << " "; -// out << state.pcommit_latency << " "; -// out << state.flush_mode << " "; -// out << state.asynchronous_mode << "\n"; -// out.flush(); -// } - -// std::string GetFilePath(std::string directory_path, std::string file_name) { -// std::string file_path = directory_path; - -// // Add a trailing slash to a file path if needed -// if (!file_path.empty() && file_path.back() != '/') file_path += '/'; - -// file_path += file_name; - -// return file_path; -// } - -// void StartLogging(std::thread& log_thread, std::thread& checkpoint_thread) { -// auto& log_manager = logging::LogManager::GetInstance(); - -// if (peloton_checkpoint_mode != CheckpointTypeId::INVALID) { -// auto& checkpoint_manager = -// peloton::logging::CheckpointManager::GetInstance(); - -// // launch checkpoint thread -// if (!checkpoint_manager.IsInCheckpointingMode()) { -// // Wait for standby mode -// auto local_thread = -// std::thread(&peloton::logging::CheckpointManager::StartStandbyMode, -// &checkpoint_manager); -// checkpoint_thread.swap(local_thread); -// checkpoint_manager.WaitForModeTransition( -// peloton::CheckpointStatus::STANDBY, true); - -// // Clean up table tile state before recovery from checkpoint -// log_manager.PrepareRecovery(); - -// // Do any recovery -// checkpoint_manager.StartRecoveryMode(); - -// // Wait for standby mode -// checkpoint_manager.WaitForModeTransition( -// peloton::CheckpointStatus::DONE_RECOVERY, true); -// } - -// // start checkpointing mode after recovery -// if (peloton_checkpoint_mode != CheckpointTypeId::INVALID) { -// if (!checkpoint_manager.IsInCheckpointingMode()) { -// // Now, enter CHECKPOINTING mode -// checkpoint_manager.SetCheckpointStatus( -// peloton::CheckpointStatus::CHECKPOINTING); -// } -// } -// } - -// if (peloton_logging_mode != LoggingTypeId::INVALID) { -// // Launching a thread for logging -// if (!log_manager.IsInLoggingMode()) { -// // Wait for standby mode -// auto local_thread = std::thread( -// &peloton::logging::LogManager::StartStandbyMode, &log_manager); -// log_thread.swap(local_thread); -// log_manager.WaitForModeTransition(peloton::LoggingStatusType::STANDBY, -// true); - -// // Clean up database tile state before recovery from checkpoint -// log_manager.PrepareRecovery(); - -// // Do any recovery -// log_manager.StartRecoveryMode(); - -// // Wait for logging mode -// log_manager.WaitForModeTransition(peloton::LoggingStatusType::LOGGING, -// true); - -// // Done recovery -// log_manager.DoneRecovery(); -// } -// } -// } - -// int RemoveDirectory(const char* dir) { -// int ret = 0; -// FTS* ftsp = NULL; -// FTSENT* curr; - -// // Cast needed (in C) because fts_open() takes a "char * const *", instead -// // of a "const char * const *", which is only allowed in C++. fts_open() -// // does not modify the argument. -// char* files[] = {(char*)dir, NULL}; - -// // FTS_NOCHDIR - Avoid changing cwd, which could cause unexpected behavior -// // in multithreaded programs -// // FTS_PHYSICAL - Don't follow symlinks. Prevents deletion of files outside -// // of the specified directory -// // FTS_XDEV - Don't cross filesystem boundaries -// ftsp = fts_open(files, FTS_NOCHDIR | FTS_PHYSICAL | FTS_XDEV, NULL); -// if (!ftsp) { -// fprintf(stderr, "%s: fts_open failed: %s\n", dir, strerror(errno)); -// ret = -1; -// goto finish; -// } - -// while ((curr = fts_read(ftsp))) { -// switch (curr->fts_info) { -// case FTS_NS: -// case FTS_DNR: -// case FTS_ERR: -// break; - -// case FTS_DC: -// case FTS_DOT: -// case FTS_NSOK: -// // Not reached unless FTS_LOGICAL, FTS_SEEDOT, or FTS_NOSTAT were -// // passed to fts_open() -// break; - -// case FTS_D: -// // Do nothing. Need depth-first search, so directories are deleted -// // in FTS_DP -// break; - -// case FTS_DP: -// case FTS_F: -// case FTS_SL: -// case FTS_SLNONE: -// case FTS_DEFAULT: -// if (remove(curr->fts_accpath) < 0) { -// fprintf(stderr, "%s: Failed to remove: %s\n", curr->fts_path, -// strerror(errno)); -// ret = -1; -// } -// break; -// } -// } - -// finish: -// if (ftsp) { -// fts_close(ftsp); -// } - -// return ret; -// } - -// void CleanUpLogDirectory() { -// if (chdir(state.log_file_dir.c_str())) { -// LOG_ERROR("Could not change directory"); -// } -// // remove wbl log file if it exists -// std::string wbl_directory_path = -// state.log_file_dir + logging::WriteBehindFrontendLogger::wbl_log_path; - -// // remove wal log directory (for wal if it exists) -// // for now hardcode for 1 logger -// std::string wal_directory_path = -// state.log_file_dir + -// logging::WriteAheadFrontendLogger::wal_directory_path; - -// std::string checkpoint_dir_path = state.log_file_dir + "pl_checkpoint"; - -// RemoveDirectory(wbl_directory_path.c_str()); - -// RemoveDirectory(wal_directory_path.c_str()); -// } - -// /** -// * @brief writing a simple log file -// */ -// bool PrepareLogFile() { -// // Clean up log directory -// CleanUpLogDirectory(); - -// // start a thread for logging -// auto& log_manager = logging::LogManager::GetInstance(); -// log_manager.SetLogDirectoryName(state.log_file_dir); - -// if (logging::LoggingUtil::IsBasedOnWriteAheadLogging(peloton_logging_mode)) { -// log_manager.SetLogFileName( -// state.log_file_dir + "/" + -// logging::WriteAheadFrontendLogger::wal_directory_path); -// } else { -// LOG_ERROR("currently, we do not support write behind logging."); -// PELOTON_ASSERT(false); -// } - -// UNUSED_ATTRIBUTE auto& checkpoint_manager = -// logging::CheckpointManager::GetInstance(); - -// if (log_manager.ContainsFrontendLogger() == true) { -// LOG_ERROR("another logging thread is running now"); -// return false; -// } - -// // Get an instance of the storage manager to force posix_fallocate -// // to be invoked before we begin benchmarking -// auto& storage_manager = storage::StorageManager::GetInstance(); -// auto tmp = storage_manager.Allocate(BackendType::MM, 1024); -// storage_manager.Release(BackendType::MM, tmp); - -// // Pick sync commit mode -// switch (state.asynchronous_mode) { -// case ASYNCHRONOUS_TYPE_SYNC: -// log_manager.SetSyncCommit(true); -// break; - -// case ASYNCHRONOUS_TYPE_ASYNC: -// log_manager.SetSyncCommit(false); -// break; - -// case ASYNCHRONOUS_TYPE_DISABLED: -// // No logging -// peloton_logging_mode = LoggingTypeId::INVALID; -// break; -// case ASYNCHRONOUS_TYPE_NO_WRITE: -// log_manager.SetNoWrite(true); -// break; - -// case ASYNCHRONOUS_TYPE_INVALID: -// throw Exception("Invalid asynchronous mode : " + -// std::to_string(state.asynchronous_mode)); -// } - -// std::thread logging_thread; -// std::thread checkpoint_thread; - -// // Initializing logging module -// StartLogging(logging_thread, checkpoint_thread); - -// // Build the log -// BuildLog(); - -// // Stop frontend logger if in a valid logging mode -// if (peloton_checkpoint_mode != CheckpointTypeId::INVALID) { -// // Wait for the mode transition :: LOGGING -> TERMINATE -> SLEEP -// checkpoint_manager.SetCheckpointStatus(CheckpointStatus::INVALID); -// checkpoint_manager.WaitForModeTransition(CheckpointStatus::INVALID, true); -// checkpoint_thread.join(); -// } -// // Stop frontend logger if in a valid logging mode -// if (peloton_logging_mode != LoggingTypeId::INVALID) { -// // Wait for the mode transition :: LOGGING -> TERMINATE -> SLEEP -// if (log_manager.EndLogging()) { -// logging_thread.join(); -// } -// } - -// if (state.benchmark_type == BENCHMARK_TYPE_YCSB) { -// ycsb::WriteOutput(); -// } else if (state.benchmark_type == BENCHMARK_TYPE_TPCC) { -// tpcc::WriteOutput(); -// } - -// return true; -// } - -// //===--------------------------------------------------------------------===// -// // CHECK RECOVERY -// //===--------------------------------------------------------------------===// - -// void ResetSystem() { -// auto& txn_manager = concurrency::TransactionManagerFactory::GetInstance(); -// txn_manager.ResetStates(); - -// // Reset database (only needed for WAL not WBL) -// if (state.benchmark_type == BENCHMARK_TYPE_YCSB) { -// ycsb::CreateYCSBDatabase(); -// } else if (state.benchmark_type == BENCHMARK_TYPE_TPCC) { -// tpcc::CreateTPCCDatabase(); -// } -// } - -// /** -// * @brief recover the database and check the tuples -// */ -// void DoRecovery() { -// //===--------------------------------------------------------------------===// -// // RECOVERY -// //===--------------------------------------------------------------------===// - -// // Reset log manager state -// auto& log_manager = peloton::logging::LogManager::GetInstance(); -// log_manager.ResetLogStatus(); -// log_manager.ResetFrontendLoggers(); - -// Timer timer; -// std::thread thread; -// std::thread cp_thread; - -// timer.Start(); - -// // Do recovery -// StartLogging(thread, cp_thread); - -// timer.Stop(); - -// // Synchronize and finish recovery -// if (peloton_logging_mode != LoggingTypeId::INVALID) { -// if (log_manager.EndLogging()) { -// thread.join(); -// } else { -// LOG_ERROR("Failed to terminate logging thread"); -// } -// } -// auto& checkpoint_manager = logging::CheckpointManager::GetInstance(); -// // Synchronize and finish recovery -// if (peloton_checkpoint_mode != CheckpointTypeId::INVALID) { -// checkpoint_manager.SetCheckpointStatus(CheckpointStatus::INVALID); -// checkpoint_manager.WaitForModeTransition(CheckpointStatus::INVALID, true); -// cp_thread.join(); -// } - -// if (state.benchmark_type == BENCHMARK_TYPE_YCSB) { -// ycsb::WriteOutput(); -// } else if (state.benchmark_type == BENCHMARK_TYPE_TPCC) { -// tpcc::WriteOutput(); -// } - -// // Recovery time (in ms) -// LOG_INFO("recovery time: %lf", timer.GetDuration()); -// } - -// //===--------------------------------------------------------------------===// -// // WRITING LOG RECORD -// //===--------------------------------------------------------------------===// - -// void BuildLog() { -// if (state.benchmark_type == BENCHMARK_TYPE_YCSB) { -// ycsb::CreateYCSBDatabase(); - -// ycsb::LoadYCSBDatabase(); - -// ycsb::RunWorkload(); -// } else if (state.benchmark_type == BENCHMARK_TYPE_TPCC) { -// tpcc::CreateTPCCDatabase(); - -// tpcc::LoadTPCCDatabase(); - -// tpcc::RunWorkload(); -// } -// } - -// } // namespace logger -// } // namespace benchmark -// } // namespace peloton diff --git a/src/main/sdbench/sdbench.cpp b/src/main/sdbench/sdbench.cpp deleted file mode 100644 index aaaa834f3d5..00000000000 --- a/src/main/sdbench/sdbench.cpp +++ /dev/null @@ -1,72 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// sdbench.cpp -// -// Identification: src/main/sdbench/sdbench.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include -#include - -#include "common/logger.h" -#include "benchmark/sdbench/sdbench_configuration.h" -#include "benchmark/sdbench/sdbench_workload.h" -#include "benchmark/sdbench/sdbench_loader.h" -#include "concurrency/epoch_manager_factory.h" - -#include - -namespace peloton { -namespace benchmark { -namespace sdbench { - -configuration state; - -// Main Entry Point -void RunBenchmark() { - - concurrency::EpochManagerFactory::Configure(EpochType::DECENTRALIZED_EPOCH); - - std::unique_ptr epoch_thread; - - concurrency::EpochManager &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); - - epoch_manager.RegisterThread(0); - - epoch_manager.StartEpoch(epoch_thread); - - if (state.multi_stage) { - // Run holistic indexing comparison benchmark - RunMultiStageBenchmark(); - } else { - // Run a single sdbench test - RunSDBenchTest(); - } - - epoch_manager.StopEpoch(); - - epoch_thread->join(); -} - -} // namespace sdbench -} // namespace benchmark -} // namespace peloton - -int main(int argc, char **argv) { - peloton::benchmark::sdbench::ParseArguments( - argc, argv, peloton::benchmark::sdbench::state); - - peloton::benchmark::sdbench::RunBenchmark(); - - peloton::benchmark::sdbench::sdbench_table.release(); - - // shutdown protocol buf library - google::protobuf::ShutdownProtobufLibrary(); - - return 0; -} diff --git a/src/main/sdbench/sdbench_configuration.cpp b/src/main/sdbench/sdbench_configuration.cpp deleted file mode 100644 index 09047892eef..00000000000 --- a/src/main/sdbench/sdbench_configuration.cpp +++ /dev/null @@ -1,566 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// sdbench_configuration.cpp -// -// Identification: src/main/sdbench/sdbench_configuration.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include -#include - -#include "benchmark/sdbench/sdbench_configuration.h" -#include "common/logger.h" - -namespace peloton { -namespace benchmark { -namespace sdbench { - -void Usage() { - LOG_INFO( - "\n" - "Command line options : sdbench \n" - " -a --attribute_count : # of attributes\n" - " -b --convergence_query_threshold : # of queries for convergence\n" - " -c --query_complexity_type : Complexity of query\n" - " -d --variability_threshold : Variability threshold\n" - " -e --index_usage_type : Index usage type\n" - " -f --analyze_sample_count_threshold : Analyze speed \n" - " -g --tuples_per_tg : # of tuples per tilegroup\n" - " -h --help : Print help message\n" - " -i --duration_between_pauses : Duration between pauses\n" - " -j --duration_of_pause : Duration of pause\n" - " -k --scale-factor : # of tile groups\n" - " -l --layout : Layout\n" - " -m --max_tile_groups_indexed : Max tile groups indexed\n" - " -n --multi_stage : Run multi stage experiment\n" - " -o --convergence : Convergence\n" - " -p --projectivity : Projectivity\n" - " -q --total_ops : # of operations\n" - " -r --holistic_indexing : Run with holistic indexing\n" - " -s --selectivity : Selectivity\n" - " -t --phase_length : Length of a phase\n" - " -u --write_complexity_type : Complexity of write\n" - " -v --verbose : Output verbosity\n" - " -w --write_ratio : Fraction of writes\n" - " -x --index_count_threshold : Index count threshold\n" - " -y --index_utility_threshold : Index utility threshold\n" - " -z --write_ratio_threshold : Write ratio threshold\n"); - - exit(EXIT_FAILURE); -} - -static struct option opts[] = { - {"attribute_count", optional_argument, NULL, 'a'}, - {"convergence_query_threshold", optional_argument, NULL, 'b'}, - {"query_complexity_type", optional_argument, NULL, 'c'}, - {"variability_threshold", optional_argument, NULL, 'd'}, - {"tuner_mode_type", optional_argument, NULL, 'e'}, - {"tuples_per_tg", optional_argument, NULL, 'g'}, - {"duration_between_pauses", optional_argument, NULL, 'i'}, - {"duration_of_pause", optional_argument, NULL, 'j'}, - {"scale-factor", optional_argument, NULL, 'k'}, - {"layout", optional_argument, NULL, 'l'}, - {"max_tile_groups_indexed", optional_argument, NULL, 'm'}, - {"convergence", optional_argument, NULL, 'o'}, - {"projectivity", optional_argument, NULL, 'p'}, - {"total_ops", optional_argument, NULL, 'q'}, - {"selectivity", optional_argument, NULL, 's'}, - {"phase_length", optional_argument, NULL, 't'}, - {"write_complexity_type", optional_argument, NULL, 'u'}, - {"verbose", optional_argument, NULL, 'v'}, - {"write_ratio", optional_argument, NULL, 'w'}, - {"index_count_threshold", optional_argument, NULL, 'x'}, - {"index_utility_threshold", optional_argument, NULL, 'y'}, - {"write_ratio_threshold", optional_argument, NULL, 'z'}, - {"multi_stage", optional_argument, NULL, 'n'}, - {"holistic_indexing", optional_argument, NULL, 'r'}, - {NULL, 0, NULL, 0}}; - -void GenerateSequence(oid_t column_count) { - // Reset sequence - sdbench_column_ids.clear(); - - // Generate sequence - for (oid_t column_id = 1; column_id <= column_count; column_id++) - sdbench_column_ids.push_back(column_id); - - std::random_shuffle(sdbench_column_ids.begin(), sdbench_column_ids.end()); -} - -static void ValidateIndexUsageType(const configuration &state) { - if (state.index_usage_type < 1 || state.index_usage_type > 5) { - LOG_ERROR("Invalid index_usage_type :: %d", state.index_usage_type); - exit(EXIT_FAILURE); - } else { - switch (state.index_usage_type) { - case INDEX_USAGE_TYPE_PARTIAL_FAST: - LOG_INFO("%s : PARTIAL_FAST", "index_usage_type "); - break; - case INDEX_USAGE_TYPE_PARTIAL_MEDIUM: - LOG_INFO("%s : PARTIAL_MEDIUM", "index_usage_type "); - break; - case INDEX_USAGE_TYPE_PARTIAL_SLOW: - LOG_INFO("%s : PARTIAL_SLOW", "index_usage_type "); - break; - case INDEX_USAGE_TYPE_NEVER: - LOG_INFO("%s : NEVER", "index_usage_type "); - break; - case INDEX_USAGE_TYPE_FULL: - LOG_INFO("%s : FULL", "index_usage_type "); - break; - default: - break; - } - } -} - -static void ValidateQueryComplexityType(const configuration &state) { - if (state.query_complexity_type < 1 || state.query_complexity_type > 3) { - LOG_ERROR("Invalid query_complexity_type :: %d", - state.query_complexity_type); - exit(EXIT_FAILURE); - } else { - switch (state.query_complexity_type) { - case QUERY_COMPLEXITY_TYPE_SIMPLE: - LOG_INFO("%s : SIMPLE", "query_complexity_type "); - break; - case QUERY_COMPLEXITY_TYPE_MODERATE: - LOG_INFO("%s : MODERATE", "query_complexity_type "); - break; - case QUERY_COMPLEXITY_TYPE_COMPLEX: - LOG_INFO("%s : COMPLEX", "query_complexity_type "); - break; - default: - break; - } - } -} - -static void ValidateWriteComplexityType(const configuration &state) { - if (state.write_complexity_type < 1 || state.write_complexity_type > 3) { - LOG_ERROR("Invalid write_complexity_type :: %d", - state.write_complexity_type); - exit(EXIT_FAILURE); - } else { - switch (state.write_complexity_type) { - case WRITE_COMPLEXITY_TYPE_SIMPLE: - LOG_INFO("write_complexity_type : SIMPLE"); - break; - case WRITE_COMPLEXITY_TYPE_COMPLEX: - LOG_INFO("write_complexity_type : COMPLEX"); - break; - case WRITE_COMPLEXITY_TYPE_INSERT: - LOG_INFO("write_complexity_type: INSERT"); - break; - default: - break; - } - } -} - -static void ValidateScaleFactor(const configuration &state) { - if (state.scale_factor <= 0) { - LOG_ERROR("Invalid scale_factor :: %d", state.scale_factor); - exit(EXIT_FAILURE); - } - - LOG_INFO("%s : %d", "scale_factor", state.scale_factor); -} - -static void ValidateLayout(const configuration &state) { - if (state.layout_mode == LayoutType::INVALID) { - LOG_ERROR("Invalid layout :: %s", - LayoutTypeToString(state.layout_mode).c_str()); - exit(EXIT_FAILURE); - } else { - switch (state.layout_mode) { - case LayoutType::ROW: - LOG_INFO("%s : ROW", "layout "); - break; - case LayoutType::COLUMN: - LOG_INFO("%s : COLUMN", "layout "); - break; - case LayoutType::HYBRID: - LOG_INFO("%s : HYBRID", "layout "); - break; - default: - break; - } - } -} - -static void ValidateProjectivity(const configuration &state) { - if (state.projectivity < 0 || state.projectivity > 1) { - LOG_ERROR("Invalid projectivity :: %.1lf", state.projectivity); - exit(EXIT_FAILURE); - } - - LOG_INFO("%s : %.3lf", "projectivity", state.projectivity); -} - -static void ValidateSelectivity(const configuration &state) { - if (state.selectivity < 0 || state.selectivity > 1) { - LOG_ERROR("Invalid selectivity :: %.1lf", state.selectivity); - exit(EXIT_FAILURE); - } - - LOG_INFO("%s : %.3lf", "selectivity", state.selectivity); -} - -static void ValidateAttributeCount(const configuration &state) { - if (state.attribute_count <= 0) { - LOG_ERROR("Invalid column_count :: %d", state.attribute_count); - exit(EXIT_FAILURE); - } - - LOG_INFO("%s : %d", "column_count", state.attribute_count); -} - -static void ValidateWriteRatio(const configuration &state) { - if (state.write_ratio < 0 || state.write_ratio > 1) { - LOG_ERROR("Invalid write_ratio :: %.1lf", state.write_ratio); - exit(EXIT_FAILURE); - } - - if (state.write_ratio == 0) { - LOG_INFO("%s : READ_ONLY", "write_ratio"); - } else if (state.write_ratio == 0.1) { - LOG_INFO("%s : READ_HEAVY", "write_ratio"); - } else if (state.write_ratio == 0.5) { - LOG_INFO("%s : BALANCED", "write_ratio"); - } else if (state.write_ratio == 0.9) { - LOG_INFO("%s : WRITE_HEAVY", "write_ratio"); - } else { - LOG_INFO("%s : %.1lf", "write_ratio", state.write_ratio); - } -} - -static void ValidateTotalOps(const configuration &state) { - if (state.total_ops <= 0) { - LOG_ERROR("Invalid total_ops :: %lu", state.total_ops); - exit(EXIT_FAILURE); - } - - LOG_INFO("%s : %ld", "total_ops", state.total_ops); -} - -static void ValidatePhaseLength(const configuration &state) { - if (state.phase_length <= 0) { - LOG_ERROR("Invalid phase_length :: %lu", state.phase_length); - exit(EXIT_FAILURE); - } - - LOG_INFO("%s : %lu", "phase_length", state.phase_length); -} - -static void ValidateTuplesPerTileGroup(const configuration &state) { - if (state.tuples_per_tilegroup <= 0) { - LOG_ERROR("Invalid tuples_per_tilegroup :: %d", state.tuples_per_tilegroup); - exit(EXIT_FAILURE); - } - - LOG_INFO("%s : %d", "tuples_per_tilegroup", state.tuples_per_tilegroup); -} - -static void ValidateDurationBetweenPauses(const configuration &state) { - if (state.duration_between_pauses <= 0) { - LOG_ERROR("Invalid duration_between_pauses :: %u", - state.duration_between_pauses); - exit(EXIT_FAILURE); - } - - LOG_INFO("%s : %u", "duration_between_pauses", state.duration_between_pauses); -} - -static void ValidateDurationOfPause(const configuration &state) { - if (state.duration_of_pause <= 0) { - LOG_ERROR("Invalid duration_of_pause :: %u", state.duration_of_pause); - exit(EXIT_FAILURE); - } - - LOG_INFO("%s : %u", "duration_of_pause", state.duration_of_pause); -} - -static void ValidateAnalyzeSampleCountThreshold(const configuration &state) { - if (state.analyze_sample_count_threshold <= 0) { - LOG_ERROR("Invalid analyze_sample_count_threshold :: %u", - state.analyze_sample_count_threshold); - exit(EXIT_FAILURE); - } - - LOG_INFO("%s : %u", "analyze_sample_count_threshold", - state.analyze_sample_count_threshold); -} - -static void ValidateMaxTileGroupsIndexed(const configuration &state) { - if (state.tile_groups_indexed_per_iteration <= 0) { - LOG_ERROR("Invalid max_tile_groups_indexed :: %u", - state.tile_groups_indexed_per_iteration); - exit(EXIT_FAILURE); - } - - LOG_INFO("%s : %u", "max_tile_groups_indexed", - state.tile_groups_indexed_per_iteration); -} - -static void ValidateConvergence(const configuration &state) { - if (state.convergence == true) { - LOG_INFO("%s : %s", "convergence", "true"); - } -} - -static void ValidateQueryConvergenceThreshold(const configuration &state) { - if (state.convergence_op_threshold <= 0) { - LOG_ERROR("Invalid convergence_query_threshold :: %u", - state.convergence_op_threshold); - exit(EXIT_FAILURE); - } - - LOG_INFO("%s : %u", "convergence_query_threshold", - state.convergence_op_threshold); -} - -static void ValidateVariabilityThreshold(const configuration &state) { - if (state.variability_threshold <= 0 || state.variability_threshold > 1000) { - LOG_ERROR("Invalid variability_threshold :: %u", - state.variability_threshold); - exit(EXIT_FAILURE); - } - - LOG_INFO("%s : %u", "variability_threshold", state.variability_threshold); -} - -static void ValidateIndexCountThreshold(const configuration &state) { - if (state.index_count_threshold == 0) { - LOG_ERROR("Invalid index_count_threshold :: %u", - state.index_count_threshold); - exit(EXIT_FAILURE); - } - - LOG_INFO("%s : %u", "index_count_threshold", state.index_count_threshold); -} - -static void ValidateIndexUtilityThreshold(const configuration &state) { - if (state.index_utility_threshold < 0 || state.index_utility_threshold > 1) { - LOG_ERROR("Invalid index_utility_threshold :: %.2lf", - state.index_utility_threshold); - exit(EXIT_FAILURE); - } - - LOG_INFO("%s : %.2lf", "index_utility_threshold", - state.index_utility_threshold); -} - -static void ValidateWriteRatioThreshold(const configuration &state) { - if (state.write_ratio_threshold < 0 || state.write_ratio_threshold > 1) { - LOG_ERROR("Invalid write_ratio_threshold :: %.2lf", - state.write_ratio_threshold); - exit(EXIT_FAILURE); - } - - LOG_INFO("%s : %.2lf", "write_ratio_threshold", state.write_ratio_threshold); -} - -static void ValidateMultiStage(const configuration &state) { - LOG_INFO("multi_stage: %d", state.multi_stage); -} - -static void ValidateHolisticIndexing(const configuration &state) { - LOG_INFO("holistic_indexing : %d", state.holistic_indexing); -} - -void ParseArguments(int argc, char *argv[], configuration &state) { - state.verbose = false; - - // Default Values - state.index_usage_type = INDEX_USAGE_TYPE_PARTIAL_FAST; - state.query_complexity_type = QUERY_COMPLEXITY_TYPE_SIMPLE; - state.write_complexity_type = WRITE_COMPLEXITY_TYPE_SIMPLE; - - // Scale and attribute count - state.scale_factor = 100.0; - state.attribute_count = 200; - - state.write_ratio = 0.0; - state.tuples_per_tilegroup = DEFAULT_TUPLES_PER_TILEGROUP; - - // Phase parameters - state.total_ops = 10; - state.phase_length = 10; - - // Query parameters - state.selectivity = 0.001; - state.projectivity = 0.01; - - // Layout parameter - state.layout_mode = LayoutType::ROW; - - // Learning rate - state.analyze_sample_count_threshold = 100; - state.duration_between_pauses = 10; - state.duration_of_pause = 100; - state.tile_groups_indexed_per_iteration = 10; - - // Convergence parameters - state.convergence = false; - state.convergence_op_threshold = 200; - - // Variability parameters - state.variability_threshold = 100; - - // Drop parameters - state.index_utility_threshold = 0.25; - state.index_count_threshold = 10; - state.write_ratio_threshold = 0.75; - state.multi_stage = false; - state.holistic_indexing = false; - state.multi_stage_idx = 0; - - // Parse args - while (1) { - int idx = 0; - int c = getopt_long(argc, argv, - "a:b:c:d:e:f:g:hi:j:k:l:m:n:o:p:q:r:s:t:u:v:w:x:y:z:", - opts, &idx); - - if (c == -1) break; - - switch (c) { - // AVAILABLE FLAGS: rABCDEFGHIJKLMNOPQRSTUVWXYZ - case 'a': - state.attribute_count = atoi(optarg); - break; - case 'b': - state.convergence_op_threshold = atoi(optarg); - break; - case 'c': - state.query_complexity_type = (QueryComplexityType)atoi(optarg); - break; - case 'd': - state.variability_threshold = atoi(optarg); - break; - case 'e': - state.index_usage_type = (IndexUsageType)atoi(optarg); - break; - case 'f': - state.analyze_sample_count_threshold = atoi(optarg); - break; - case 'g': - state.tuples_per_tilegroup = atoi(optarg); - break; - case 'h': - Usage(); - break; - case 'i': - state.duration_between_pauses = atoi(optarg); - break; - case 'j': - state.duration_of_pause = atoi(optarg); - break; - case 'k': - state.scale_factor = atoi(optarg); - break; - case 'l': - state.layout_mode = (LayoutType)atoi(optarg); - break; - case 'm': - state.tile_groups_indexed_per_iteration = atoi(optarg); - break; - case 'n': - state.multi_stage = atoi(optarg); - break; - case 'o': - state.convergence = atoi(optarg); - break; - case 'p': - state.projectivity = atof(optarg); - break; - case 'q': - state.total_ops = atol(optarg); - break; - case 'r': - state.holistic_indexing = atoi(optarg); - break; - case 's': - state.selectivity = atof(optarg); - break; - case 't': - state.phase_length = atol(optarg); - break; - case 'u': - state.write_complexity_type = (WriteComplexityType)atoi(optarg); - break; - case 'v': - state.verbose = atoi(optarg); - break; - case 'w': - state.write_ratio = atof(optarg); - break; - case 'x': - state.index_count_threshold = atoi(optarg); - break; - case 'y': - state.index_utility_threshold = atof(optarg); - break; - case 'z': - state.write_ratio_threshold = atof(optarg); - break; - - default: - LOG_ERROR("Unknown option: -%c-", c); - Usage(); - } - } - - ValidateIndexUsageType(state); - - /// Set duration between pauses based on index usage type - if (state.index_usage_type == INDEX_USAGE_TYPE_PARTIAL_FAST) { - state.duration_between_pauses = 10000; - } else if (state.index_usage_type == INDEX_USAGE_TYPE_PARTIAL_MEDIUM) { - state.duration_between_pauses = 1000; - } else if (state.index_usage_type == INDEX_USAGE_TYPE_PARTIAL_SLOW) { - state.duration_between_pauses = 100; - } else if (state.index_usage_type == INDEX_USAGE_TYPE_FULL) { - state.duration_between_pauses = 10000; - } - - /// Check variability threshold - if(state.variability_threshold >= state.attribute_count){ - LOG_ERROR("Variability threshold higher than attribute count"); - exit(EXIT_FAILURE); - } - - ValidateWriteRatio(state); - ValidateQueryComplexityType(state); - ValidateWriteComplexityType(state); - ValidateScaleFactor(state); - ValidateAttributeCount(state); - ValidateTuplesPerTileGroup(state); - ValidateTotalOps(state); - ValidatePhaseLength(state); - ValidateSelectivity(state); - ValidateProjectivity(state); - ValidateLayout(state); - ValidateIndexCountThreshold(state); - ValidateIndexUtilityThreshold(state); - ValidateWriteRatioThreshold(state); - ValidateDurationOfPause(state); - ValidateDurationBetweenPauses(state); - ValidateAnalyzeSampleCountThreshold(state); - ValidateMaxTileGroupsIndexed(state); - ValidateConvergence(state); - ValidateQueryConvergenceThreshold(state); - ValidateVariabilityThreshold(state); - ValidateMultiStage(state); - ValidateHolisticIndexing(state); -} - -} // namespace sdbench -} // namespace benchmark -} // namespace peloton diff --git a/src/main/sdbench/sdbench_loader.cpp b/src/main/sdbench/sdbench_loader.cpp deleted file mode 100644 index 41ce49e4efa..00000000000 --- a/src/main/sdbench/sdbench_loader.cpp +++ /dev/null @@ -1,136 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// sdbench_loader.cpp -// -// Identification: src/main/sdbench/sdbench_loader.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "benchmark/sdbench/sdbench_loader.h" -#include "benchmark/sdbench/sdbench_configuration.h" - -#include -#include -#include -#include -#include -#include -#include - -#include "catalog/manager.h" -#include "catalog/schema.h" -#include "common/item_pointer.h" -#include "common/logger.h" -#include "common/macros.h" -#include "concurrency/transaction_context.h" -#include "concurrency/transaction_manager_factory.h" -#include "executor/abstract_executor.h" -#include "executor/insert_executor.h" -#include "executor/executor_context.h" -#include "expression/constant_value_expression.h" -#include "index/index_factory.h" -#include "planner/insert_plan.h" -#include "storage/data_table.h" -#include "storage/table_factory.h" -#include "storage/tile.h" -#include "storage/tile_group.h" -#include "type/ephemeral_pool.h" -#include "type/value_factory.h" - -namespace peloton { -namespace benchmark { -namespace sdbench { - -std::unique_ptr sdbench_table; - -void CreateTable(UNUSED_ATTRIBUTE peloton::LayoutType layout_type) { - const oid_t col_count = state.attribute_count + 1; - const bool is_inlined = true; - - // Create schema first - std::vector columns; - - for (oid_t col_itr = 0; col_itr < col_count; col_itr++) { - auto column = catalog::Column(type::TypeId::INTEGER, - type::Type::GetTypeSize(type::TypeId::INTEGER), - "" + std::to_string(col_itr), is_inlined); - - columns.push_back(column); - } - - catalog::Schema *table_schema = new catalog::Schema(columns); - std::string table_name("SDBENCHTABLE"); - - ///////////////////////////////////////////////////////// - // Create table. - ///////////////////////////////////////////////////////// - - bool own_schema = true; - bool adapt_table = true; - sdbench_table.reset(storage::TableFactory::GetDataTable( - INVALID_OID, INVALID_OID, table_schema, table_name, - state.tuples_per_tilegroup, own_schema, adapt_table)); -} - -void LoadTable() { - const oid_t col_count = state.attribute_count + 1; - const int tuple_count = state.scale_factor * state.tuples_per_tilegroup; - - auto table_schema = sdbench_table->GetSchema(); - - ///////////////////////////////////////////////////////// - // Load in the data - ///////////////////////////////////////////////////////// - - // Insert tuples into tile_group. - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - const bool allocate = true; - auto txn = txn_manager.BeginTransaction(); - std::unique_ptr pool(new type::EphemeralPool()); - - std::unique_ptr context( - new executor::ExecutorContext(txn)); - - for (int rowid = 0; rowid < tuple_count; rowid++) { - int populate_value = rowid; - - std::unique_ptr tuple(new storage::Tuple(table_schema, allocate)); - - for (oid_t col_itr = 0; col_itr < col_count; col_itr++) { - auto value = type::ValueFactory::GetIntegerValue(populate_value); - tuple->SetValue(col_itr, value, pool.get()); - } - - planner::InsertPlan node(sdbench_table.get(), std::move(tuple)); - executor::InsertExecutor executor(&node, context.get()); - executor.Execute(); - } - - auto result = txn_manager.CommitTransaction(txn); - - if (result == ResultType::SUCCESS) { - LOG_TRACE("commit successfully"); - } else { - LOG_TRACE("commit failed"); - } -} - -void CreateAndLoadTable(LayoutType layout_type) { - - CreateTable(layout_type); - - LoadTable(); -} - -void DropIndexes() { - // Drop index - sdbench_table->DropIndexWithOid(0); -} - -} // namespace sdbench -} // namespace benchmark -} // namespace peloton diff --git a/src/main/sdbench/sdbench_workload.cpp b/src/main/sdbench/sdbench_workload.cpp deleted file mode 100644 index 3d50b037e8c..00000000000 --- a/src/main/sdbench/sdbench_workload.cpp +++ /dev/null @@ -1,1545 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// sdbench_workload.cpp -// -// Identification: src/main/sdbench/sdbench_workload.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "tuning/index_tuner.h" -#include "tuning/layout_tuner.h" -#include "tuning/sample.h" - -#include "benchmark/sdbench/sdbench_loader.h" -#include "benchmark/sdbench/sdbench_workload.h" - -#include "catalog/manager.h" -#include "catalog/schema.h" -#include "common/internal_types.h" -#include "common/logger.h" -#include "common/macros.h" -#include "common/timer.h" -#include "concurrency/transaction_context.h" -#include "concurrency/transaction_manager_factory.h" -#include "type/value.h" -#include "type/value_factory.h" - -#include "executor/abstract_executor.h" -#include "executor/aggregate_executor.h" -#include "executor/executor_context.h" -#include "executor/hybrid_scan_executor.h" -#include "executor/insert_executor.h" -#include "executor/logical_tile.h" -#include "executor/logical_tile_factory.h" -#include "executor/materialization_executor.h" -#include "executor/nested_loop_join_executor.h" -#include "executor/projection_executor.h" -#include "executor/seq_scan_executor.h" -#include "executor/update_executor.h" - -#include "expression/abstract_expression.h" -#include "expression/comparison_expression.h" -#include "expression/conjunction_expression.h" -#include "expression/constant_value_expression.h" -#include "expression/expression_util.h" -#include "expression/operator_expression.h" -#include "expression/tuple_value_expression.h" - -#include "planner/abstract_plan.h" -#include "planner/aggregate_plan.h" -#include "planner/hybrid_scan_plan.h" -#include "planner/insert_plan.h" -#include "planner/materialization_plan.h" -#include "planner/nested_loop_join_plan.h" -#include "planner/projection_plan.h" -#include "planner/seq_scan_plan.h" -#include "planner/update_plan.h" - -#include "storage/data_table.h" -#include "storage/table_factory.h" -#include "storage/tile.h" -#include "storage/tile_group.h" -#include "storage/tile_group_header.h" - -namespace peloton { -namespace benchmark { -namespace sdbench { - -// Function definitions -static std::shared_ptr PickIndex(storage::DataTable *table, - std::vector query_attrs); - -static void AggregateQueryHelper(const std::vector &tuple_key_attrs, - const std::vector &index_key_attrs); - -static void JoinQueryHelper( - const std::vector &left_table_tuple_key_attrs, - const std::vector &left_table_index_key_attrs, - const std::vector &right_table_tuple_key_attrs, - const std::vector &right_table_index_key_attrs, - const oid_t left_table_join_column, const oid_t right_table_join_column); - -// Tuple id counter -oid_t sdbench_tuple_counter = -1000000; - -std::vector column_counts = {50, 500}; - -// Index tuner -tuning::IndexTuner &index_tuner = tuning::IndexTuner::GetInstance(); - -// Layout tuner -tuning::LayoutTuner &layout_tuner = tuning::LayoutTuner::GetInstance(); - -// Predicate generator -std::vector> predicate_distribution; - -// Bitmap for already used predicate -#define MAX_PREDICATE_ATTR 10 -bool predicate_used[MAX_PREDICATE_ATTR][MAX_PREDICATE_ATTR] - [MAX_PREDICATE_ATTR] = {}; - -std::size_t predicate_distribution_size = 0; - -static void CopyColumn(oid_t col_itr); - -static void GeneratePredicateDistribution() { - for (oid_t i = 1; i <= 9; i++) { - for (oid_t j = 1; j <= 9; j++) { - for (oid_t k = 1; k <= 9; k++) { - if (i != j && j != k && i != k) { - predicate_distribution.push_back(std::vector{i, j, k}); - } - } - } - } - - predicate_distribution_size = predicate_distribution.size(); -} - -static std::vector GetPredicate() { - if (state.variability_threshold >= predicate_distribution_size) { - LOG_ERROR("Don't have enough samples"); - exit(EXIT_FAILURE); - } - - auto sample = rand() % state.variability_threshold; - LOG_INFO("Predicate : %u", sample); - auto predicate = predicate_distribution[sample]; - return predicate; -} - -static int GetLowerBound() { - int tuple_count = state.scale_factor * state.tuples_per_tilegroup; - int predicate_offset = 0.1 * tuple_count; - - LOG_TRACE("Tuple count : %d", tuple_count); - - int lower_bound = predicate_offset; - return lower_bound; -} - -static int GetUpperBound() { - int tuple_count = state.scale_factor * state.tuples_per_tilegroup; - int selected_tuple_count = state.selectivity * tuple_count; - int predicate_offset = 0.1 * tuple_count; - - int upper_bound = predicate_offset + selected_tuple_count; - return upper_bound; -} - -static expression::AbstractExpression *CreateSimpleScanPredicate( - oid_t key_attr, ExpressionType expression_type, oid_t constant) { - // First, create tuple value expression. - oid_t left_tuple_idx = 0; - expression::AbstractExpression *tuple_value_expr_left = - expression::ExpressionUtil::TupleValueFactory(type::TypeId::INTEGER, - left_tuple_idx, key_attr); - - // Second, create constant value expression. - type::Value constant_value_left = - type::ValueFactory::GetIntegerValue(constant); - - expression::AbstractExpression *constant_value_expr_left = - expression::ExpressionUtil::ConstantValueFactory(constant_value_left); - - // Finally, link them together using an greater than expression. - expression::AbstractExpression *predicate = - expression::ExpressionUtil::ComparisonFactory( - expression_type, tuple_value_expr_left, constant_value_expr_left); - - return predicate; -} - -/** - * @brief Create the scan predicate given a set of attributes. The predicate - * will be attr >= LOWER_BOUND AND attr < UPPER_BOUND. - * LOWER_BOUND and UPPER_BOUND are determined by the selectivity config. - */ -static expression::AbstractExpression *CreateScanPredicate( - std::vector key_attrs) { - const int tuple_start_offset = GetLowerBound(); - const int tuple_end_offset = GetUpperBound(); - - LOG_TRACE("Lower bound : %d", tuple_start_offset); - LOG_TRACE("Upper bound : %d", tuple_end_offset); - - expression::AbstractExpression *predicate = nullptr; - - // Go over all key_attrs - for (auto key_attr : key_attrs) { - // ATTR >= LOWER_BOUND && < UPPER_BOUND - - auto left_predicate = CreateSimpleScanPredicate( - key_attr, ExpressionType::COMPARE_GREATERTHANOREQUALTO, - tuple_start_offset); - - auto right_predicate = CreateSimpleScanPredicate( - key_attr, ExpressionType::COMPARE_LESSTHAN, tuple_end_offset); - - expression::AbstractExpression *attr_predicate = - expression::ExpressionUtil::ConjunctionFactory( - ExpressionType::CONJUNCTION_AND, left_predicate, right_predicate); - - // Build complex predicate - if (predicate == nullptr) { - predicate = attr_predicate; - } else { - // Join predicate with given attribute predicate - predicate = expression::ExpressionUtil::ConjunctionFactory( - ExpressionType::CONJUNCTION_AND, predicate, attr_predicate); - } - } - - return predicate; -} - -static void CreateIndexScanPredicate(std::vector key_attrs, - std::vector &key_column_ids, - std::vector &expr_types, - std::vector &values) { - const int tuple_start_offset = GetLowerBound(); - const int tuple_end_offset = GetUpperBound(); - - // Go over all key_attrs - for (auto key_attr : key_attrs) { - key_column_ids.push_back(key_attr); - expr_types.push_back(ExpressionType::COMPARE_GREATERTHANOREQUALTO); - values.push_back(type::ValueFactory::GetIntegerValue(tuple_start_offset)); - - key_column_ids.push_back(key_attr); - expr_types.push_back(ExpressionType::COMPARE_LESSTHAN); - values.push_back(type::ValueFactory::GetIntegerValue(tuple_end_offset)); - } -} - -/** - * @brief Get the string for a list of oids. - */ -static inline std::string GetOidVectorString(const std::vector &oids) { - std::string oid_str = ""; - for (oid_t o : oids) { - oid_str += " " + std::to_string(o); - } - return oid_str; -} - -/** - * @brief Create a hybrid scan executor based on selected key columns. - * @param tuple_key_attrs The columns which the seq scan predicate is on. - * @param index_key_attrs The columns in the *index key tuple* which the index - * scan predicate is on. It should match the corresponding columns in - * \b tuple_key_columns. - * @param column_ids Column ids to be added to the result tile after scan. - * @return A hybrid scan executor based on the key columns. - */ -static std::shared_ptr CreateHybridScanPlan( - const std::vector &tuple_key_attrs, - const std::vector &index_key_attrs, - const std::vector &column_ids) { - // Create and set up seq scan executor - auto predicate = CreateScanPredicate(tuple_key_attrs); - - planner::IndexScanPlan::IndexScanDesc index_scan_desc; - - std::vector key_column_ids; - std::vector expr_types; - std::vector values; - std::vector runtime_keys; - - // Create index scan predicate - CreateIndexScanPredicate(index_key_attrs, key_column_ids, expr_types, values); - - // Determine hybrid scan type - auto hybrid_scan_type = HybridScanType::SEQUENTIAL; - - // Pick index - auto index = PickIndex(sdbench_table.get(), tuple_key_attrs); - - if (index != nullptr) { - index_scan_desc = planner::IndexScanPlan::IndexScanDesc( - index->GetOid(), key_column_ids, expr_types, values, runtime_keys); - - hybrid_scan_type = HybridScanType::HYBRID; - } - - LOG_TRACE("Hybrid scan type : %d", hybrid_scan_type); - - std::shared_ptr hybrid_scan_node( - new planner::HybridScanPlan(sdbench_table.get(), predicate, column_ids, - index_scan_desc, hybrid_scan_type)); - - return hybrid_scan_node; -} - -const static std::string OUTPUT_FILE = "outputfile.summary"; -std::ofstream out(OUTPUT_FILE); - -oid_t query_itr; - -double total_duration = 0; - -UNUSED_ATTRIBUTE static void WriteOutput(double duration) { - // Convert to ms - duration *= 1000; - - auto index_count = index_tuner.GetIndexCount(); - - // Write out output in verbose mode - if (state.verbose == true) { - LOG_INFO("----------------------------------------------------------"); - LOG_INFO("%d %d %.3lf %.3lf %u %.1lf %d %d %d %u :: %.1lf ms", - state.index_usage_type, state.query_complexity_type, - state.selectivity, state.projectivity, query_itr, - state.write_ratio, state.scale_factor, state.attribute_count, - state.tuples_per_tilegroup, index_count, duration); - } - - out << state.index_usage_type << " "; - out << state.query_complexity_type << " "; - out << state.selectivity << " "; - out << state.projectivity << " "; - out << query_itr << " "; - out << state.write_ratio << " "; - out << state.scale_factor << " "; - out << state.attribute_count << " "; - out << state.tuples_per_tilegroup << " "; - out << index_count << " "; - out << std::fixed << std::setprecision(2) << duration << "\n"; - - out.flush(); -} - -/** - * @brief Map the accsessed columns to a access bitmap. - * @details We should use the output of this method to construct a Sample for - * layout tuning instead of passing in the accessed columns directly!! - */ -static std::vector GetColumnsAccessed( - const std::vector &column_ids) { - std::vector columns_accessed; - std::map columns_accessed_map; - - // Init map - for (auto col : column_ids) columns_accessed_map[(int)col] = 1; - - for (oid_t column_itr = 0; column_itr < state.attribute_count + 1; - column_itr++) { - auto location = columns_accessed_map.find(column_itr); - auto end = columns_accessed_map.end(); - if (location != end) - columns_accessed.push_back(1); - else - columns_accessed.push_back(0); - } - - return columns_accessed; -} - -/** - * @brief Execute a set of executors and update access information. - * - * @param executors Executors to be executed. - * @param index_columns_accessed Columns that are accessed by index scan, used - * fpr index tuning. - * @param tuple_columns_accessed Columns of the tuples that are accessed, used - * for layout tuning. - * @param selectivity The selectivity of the operation. - */ -static void ExecuteTest(std::vector &executors, - tuning::SampleType sample_type, - std::vector> index_columns_accessed, - std::vector> tuple_columns_accessed, - UNUSED_ATTRIBUTE double selectivity) { - Timer<> timer; - - bool status = false; - - // Increment query counter - query_itr++; - - // Reset timer - timer.Reset(); - timer.Start(); - - // Run all the executors - for (auto executor : executors) { - status = executor->Init(); - if (status == false) { - throw Exception("Init failed"); - } - - std::vector> result_tiles; - - while (executor->Execute() == true) { - std::unique_ptr result_tile(executor->GetOutput()); - result_tiles.emplace_back(result_tile.release()); - } - - size_t sum = 0; - for (auto &result_tile : result_tiles) { - if (result_tile != nullptr) sum += result_tile->GetTupleCount(); - } - - LOG_TRACE("result tiles have %d tuples", (int)sum); - - // Execute stuff - executor->Execute(); - } - - // For holistic index - if (state.holistic_indexing) { - for (auto index_columns : index_columns_accessed) { - if (index_columns.size() == 3) { // It should be so for moderate query - oid_t i = oid_t(index_columns[0]); - oid_t j = oid_t(index_columns[1]); - oid_t k = oid_t(index_columns[2]); - if (!predicate_used[i][j][k]) { - // Copy the predicate column - CopyColumn(i); - CopyColumn(j); - CopyColumn(k); - predicate_used[i][j][k] = true; - } - } - } - } - - // Emit time - timer.Stop(); - auto duration = timer.GetDuration(); - total_duration += duration; - - WriteOutput(duration); - - // Record index sample - for (auto &index_columns : index_columns_accessed) { - tuning::Sample index_access_sample( - index_columns, duration / index_columns_accessed.size(), sample_type); - // ???, selectivity); - sdbench_table->RecordIndexSample(index_access_sample); - } - - // Record layout sample - for (auto &tuple_columns : tuple_columns_accessed) { - // Record layout sample - tuning::Sample tuple_access_bitmap( - GetColumnsAccessed(tuple_columns), - duration / tuple_columns_accessed.size()); - sdbench_table->RecordLayoutSample(tuple_access_bitmap); - } -} - -static std::shared_ptr PickIndex(storage::DataTable *table, - std::vector query_attrs) { - // Construct set - std::set query_attrs_set(query_attrs.begin(), query_attrs.end()); - - oid_t index_count = table->GetIndexCount(); - - // Empty index pointer - std::shared_ptr index; - - // Can't use indexes => return empty index - if (state.index_usage_type == INDEX_USAGE_TYPE_NEVER) { - return index; - } - - // Go over all indices - bool query_index_found = false; - oid_t index_itr = 0; - for (index_itr = 0; index_itr < index_count; index_itr++) { - auto index_attrs = table->GetIndexAttrs(index_itr); - - auto index = table->GetIndex(index_itr); - // Check if index exists - if (index == nullptr) { - continue; - } - - // Some attribute did not match - if (index_attrs != query_attrs_set) { - continue; - } - - // Can only use full indexes ? - if (state.index_usage_type == INDEX_USAGE_TYPE_FULL) { - auto indexed_tg_count = index->GetIndexedTileGroupOff(); - auto table_tg_count = table->GetTileGroupCount(); - - LOG_TRACE("Indexed TG Count : %lu", indexed_tg_count); - LOG_TRACE("Table TG Count : %lu", table_tg_count); - - if (indexed_tg_count < table_tg_count) { - continue; - } - } - - // Exact match - query_index_found = true; - break; - - // update index count - index_count = table->GetIndexCount(); - } - - // Found index - if (query_index_found == true) { - LOG_TRACE("Found available Index"); - index = table->GetIndex(index_itr); - } else { - LOG_TRACE("Did not find available index"); - } - - return index; -} - -/** - * @brief Copy a column from the table. - */ -static void CopyColumn(oid_t col_itr) { - auto tile_group_count = sdbench_table->GetTileGroupCount(); - for (oid_t tile_group_itr = 0; tile_group_itr < tile_group_count; - tile_group_itr++) { - // Prepare a tile for copying - std::vector columns; - - catalog::Column column1(type::TypeId::INTEGER, - type::Type::GetTypeSize(type::TypeId::INTEGER), "A", - true); - columns.push_back(column1); - - // Schema - catalog::Schema *schema = new catalog::Schema(columns); - - // Column Names - std::vector column_names; - - column_names.push_back("COL 1"); - - // TG Header - storage::TileGroupHeader *header = new storage::TileGroupHeader( - BackendType::MM, state.tuples_per_tilegroup); - - storage::Tile *new_tile = storage::TileFactory::GetTile( - BackendType::MM, INVALID_OID, INVALID_OID, INVALID_OID, INVALID_OID, - header, *schema, nullptr, state.tuples_per_tilegroup); - - // Begin copy - oid_t orig_tile_offset, orig_tile_column_offset; - auto orig_tile_group = sdbench_table->GetTileGroup(tile_group_itr); - auto orig_layout = orig_tile_group->GetLayout(); - orig_layout.LocateTileAndColumn(col_itr, orig_tile_offset, - orig_tile_column_offset); - auto orig_tile = orig_tile_group->GetTile(orig_tile_offset); - oid_t tuple_count = state.tuples_per_tilegroup; - for (oid_t tuple_itr = 0; tuple_itr < tuple_count; tuple_itr++) { - auto val = orig_tile->GetValue(tuple_itr, orig_tile_column_offset); - new_tile->SetValue(val, tuple_itr, 0); - } - - delete new_tile; - delete header; - delete schema; - } -} - -static void RunSimpleQuery() { - std::vector tuple_key_attrs; - std::vector index_key_attrs; - - oid_t predicate = rand() % state.variability_threshold; - oid_t first_attribute = predicate; - tuple_key_attrs = {first_attribute}; - index_key_attrs = {0}; - - // PHASE LENGTH - for (oid_t txn_itr = 0; txn_itr < state.phase_length; txn_itr++) { - AggregateQueryHelper(tuple_key_attrs, index_key_attrs); - } -} - -static void RunModerateQuery() { - LOG_TRACE("Moderate Query"); - - std::vector tuple_key_attrs; - std::vector index_key_attrs; - - auto predicate = GetPredicate(); - tuple_key_attrs = predicate; - index_key_attrs = {0, 1, 2}; - - LOG_TRACE("Moderate :: %s", GetOidVectorString(tuple_key_attrs).c_str()); - - // PHASE LENGTH - for (oid_t txn_itr = 0; txn_itr < state.phase_length; txn_itr++) { - AggregateQueryHelper(tuple_key_attrs, index_key_attrs); - } -} - -/** - * @brief Run complex query - * @details 60% join test, 30% moderate query, 10% simple query - */ -static void RunComplexQuery() { - LOG_TRACE("Complex Query"); - - std::vector left_table_tuple_key_attrs; - std::vector left_table_index_key_attrs; - std::vector right_table_tuple_key_attrs; - std::vector right_table_index_key_attrs; - oid_t left_table_join_column; - oid_t right_table_join_column; - - bool is_join_query = false; - bool is_aggregate_query = false; - - // Assume there are 20 columns, - // 10 for the left table, 10 for the right table - auto predicate = GetPredicate(); - left_table_tuple_key_attrs = predicate; - left_table_index_key_attrs = {0, 1, 2}; - std::vector tuple_key_attrs = predicate; - std::vector index_key_attrs = {0, 1, 2}; - right_table_tuple_key_attrs = {predicate[0] + 10, predicate[1] + 10, - predicate[2] + 10}; - right_table_index_key_attrs = {0, 1, 2}; - - predicate = GetPredicate(); - left_table_join_column = predicate[0]; - right_table_join_column = predicate[1]; - - // Pick join or aggregate - // is_join_query = true; - // is_aggregate_query = false; - auto sample = rand() % 10; - if (sample > 5) { - is_join_query = true; - is_aggregate_query = false; - } else { - is_aggregate_query = true; - is_join_query = false; - } - - if (is_join_query == true) { - LOG_INFO("Complex :: %s, %s, c1: %d, c2: %d", - GetOidVectorString(left_table_tuple_key_attrs).c_str(), - GetOidVectorString(right_table_tuple_key_attrs).c_str(), - (int)left_table_join_column, (int)right_table_join_column); - } else if (is_aggregate_query == true) { - LOG_TRACE("Complex :: %s", GetOidVectorString(tuple_key_attrs).c_str()); - } else { - LOG_ERROR("Invalid query \n"); - return; - } - - // PHASE LENGTH - for (oid_t txn_itr = 0; txn_itr < state.phase_length; txn_itr++) { - // Invoke appropriate query - if (is_join_query == true) { - JoinQueryHelper(left_table_tuple_key_attrs, left_table_index_key_attrs, - right_table_tuple_key_attrs, right_table_index_key_attrs, - left_table_join_column, right_table_join_column); - } else if (is_aggregate_query == true) { - AggregateQueryHelper(tuple_key_attrs, index_key_attrs); - } else { - LOG_ERROR("Invalid query \n"); - return; - } - } -} - -static void JoinQueryHelper( - const std::vector &left_table_tuple_key_attrs, - const std::vector &left_table_index_key_attrs, - const std::vector &right_table_tuple_key_attrs, - const std::vector &right_table_index_key_attrs, - const oid_t left_table_join_column, const oid_t right_table_join_column) { - LOG_TRACE("Run join query on left table: %s and right table: %s", - GetOidVectorString(left_table_tuple_key_attrs).c_str(), - GetOidVectorString(right_table_tuple_key_attrs).c_str()); - const bool is_inlined = true; - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - - auto txn = txn_manager.BeginTransaction(); - - ///////////////////////////////////////////////////////// - // SEQ SCAN + PREDICATE - ///////////////////////////////////////////////////////// - - std::unique_ptr context( - new executor::ExecutorContext(txn)); - - // Column ids to be added to logical tile after scan. - // Left half of the columns are considered left table, - // right half of the columns are considered right table. - std::vector column_ids; - oid_t column_count = state.attribute_count; - - for (oid_t col_itr = 0; col_itr < column_count; col_itr++) { - column_ids.push_back(sdbench_column_ids[col_itr]); - } - - // Create and set up seq scan executor - auto left_table_scan_node = CreateHybridScanPlan( - left_table_tuple_key_attrs, left_table_index_key_attrs, column_ids); - auto right_table_scan_node = CreateHybridScanPlan( - right_table_tuple_key_attrs, right_table_index_key_attrs, column_ids); - - executor::HybridScanExecutor left_table_hybrid_scan_executor( - left_table_scan_node.get(), context.get()); - executor::HybridScanExecutor right_table_hybrid_scan_executor( - right_table_scan_node.get(), context.get()); - - ///////////////////////////////////////////////////////// - // JOIN EXECUTOR - ///////////////////////////////////////////////////////// - - auto join_type = JoinType::INNER; - - // Create join predicate - expression::TupleValueExpression *left_table_attr = - new expression::TupleValueExpression(type::TypeId::INTEGER, 0, - left_table_join_column); - expression::TupleValueExpression *right_table_attr = - new expression::TupleValueExpression(type::TypeId::INTEGER, 1, - right_table_join_column); - - std::unique_ptr join_predicate( - new expression::ComparisonExpression(ExpressionType::COMPARE_LESSTHAN, - left_table_attr, right_table_attr)); - - std::unique_ptr project_info(nullptr); - std::shared_ptr schema(nullptr); - - planner::NestedLoopJoinPlan nested_loop_join_node( - join_type, std::move(join_predicate), std::move(project_info), schema, - {left_table_join_column}, {right_table_join_column}); - - // Run the nested loop join executor - executor::NestedLoopJoinExecutor nested_loop_join_executor( - &nested_loop_join_node, nullptr); - - // Construct the executor tree - nested_loop_join_executor.AddChild(&left_table_hybrid_scan_executor); - nested_loop_join_executor.AddChild(&right_table_hybrid_scan_executor); - - ///////////////////////////////////////////////////////// - // MATERIALIZE - ///////////////////////////////////////////////////////// - - // Create and set up materialization executor - // FIXME: this will always retreive all columns, projectivity is ignored - std::vector output_columns; - std::unordered_map old_to_new_cols; - oid_t join_column_count = column_count * 2; - for (oid_t col_itr = 0; col_itr < join_column_count; col_itr++) { - auto column = catalog::Column( - type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "" + std::to_string(col_itr), is_inlined); - output_columns.push_back(column); - - old_to_new_cols[col_itr] = col_itr; - } - - std::shared_ptr output_schema( - new catalog::Schema(output_columns)); - bool physify_flag = true; // is going to create a physical tile - planner::MaterializationPlan mat_node(old_to_new_cols, output_schema, - physify_flag); - - executor::MaterializationExecutor mat_executor(&mat_node, nullptr); - mat_executor.AddChild(&nested_loop_join_executor); - - ///////////////////////////////////////////////////////// - // EXECUTE - ///////////////////////////////////////////////////////// - - std::vector executors; - executors.push_back(&mat_executor); - - ///////////////////////////////////////////////////////// - // COLLECT STATS - ///////////////////////////////////////////////////////// - - std::vector left_table_index_columns_accessed( - left_table_tuple_key_attrs.begin(), left_table_tuple_key_attrs.end()); - std::vector right_table_index_columns_accessed( - right_table_tuple_key_attrs.begin(), right_table_tuple_key_attrs.end()); - - // Prepare tuple columns accessed - auto left_table_tuple_columns_accessed = left_table_tuple_key_attrs; - auto right_table_tuple_columns_accessed = right_table_tuple_key_attrs; - left_table_tuple_columns_accessed.push_back(left_table_join_column); - right_table_tuple_columns_accessed.push_back(right_table_join_column); - - auto selectivity = state.selectivity; - - ExecuteTest( - executors, tuning::SampleType::ACCESS, - {left_table_index_columns_accessed, right_table_index_columns_accessed}, - {left_table_tuple_columns_accessed, right_table_tuple_columns_accessed}, - selectivity); - - auto result = txn_manager.CommitTransaction(txn); - - if (result == ResultType::SUCCESS) { - LOG_TRACE("commit successfully"); - } else { - LOG_TRACE("commit failed"); - } -} - -static void AggregateQueryHelper(const std::vector &tuple_key_attrs, - const std::vector &index_key_attrs) { - if (state.verbose) { - LOG_INFO("Run aggregate query on %s ", - GetOidVectorString(tuple_key_attrs).c_str()); - } - - const bool is_inlined = true; - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - - auto txn = txn_manager.BeginTransaction(); - - ///////////////////////////////////////////////////////// - // SEQ SCAN + PREDICATE - ///////////////////////////////////////////////////////// - - // Column ids to be added to logical tile after scan. - // We need all columns because projection can require any column - std::vector column_ids; - oid_t column_count = state.attribute_count; - - column_ids.push_back(0); - for (oid_t col_itr = 0; col_itr < column_count; col_itr++) { - column_ids.push_back(sdbench_column_ids[col_itr]); - } - - column_count = state.projectivity * state.attribute_count; - column_ids.resize(column_count); - - std::unique_ptr context( - new executor::ExecutorContext(txn)); - - auto hybrid_scan_node = - CreateHybridScanPlan(tuple_key_attrs, index_key_attrs, column_ids); - executor::HybridScanExecutor hybrid_scan_executor(hybrid_scan_node.get(), - context.get()); - - ///////////////////////////////////////////////////////// - // AGGREGATION - ///////////////////////////////////////////////////////// - - // Resize column ids to contain only columns - // over which we compute aggregates - - // (1-5) Setup plan node - - // 1) Set up group-by columns - std::vector group_by_columns; - - // 2) Set up project info - DirectMapList direct_map_list; - oid_t col_itr = 0; - oid_t tuple_idx = 1; // tuple2 - for (col_itr = 0; col_itr < column_count; col_itr++) { - direct_map_list.push_back({col_itr, {tuple_idx, col_itr}}); - } - - std::unique_ptr proj_info( - new planner::ProjectInfo(TargetList(), std::move(direct_map_list))); - - // 3) Set up aggregates - std::vector agg_terms; - for (col_itr = 0; col_itr < column_count; col_itr++) { - planner::AggregatePlan::AggTerm max_column_agg( - ExpressionType::AGGREGATE_MAX, - expression::ExpressionUtil::TupleValueFactory(type::TypeId::INTEGER, 0, - col_itr), - false); - agg_terms.push_back(max_column_agg); - } - - // 4) Set up predicate (empty) - std::unique_ptr aggregate_predicate( - nullptr); - - // 5) Create output table schema - auto data_table_schema = sdbench_table->GetSchema(); - std::vector columns; - for (auto column_id : column_ids) { - columns.push_back(data_table_schema->GetColumn(column_id)); - } - - std::shared_ptr output_table_schema( - new catalog::Schema(columns)); - - // OK) Create the plan node - planner::AggregatePlan aggregation_node( - std::move(proj_info), std::move(aggregate_predicate), - std::move(agg_terms), std::move(group_by_columns), output_table_schema, - AggregateType::PLAIN); - - executor::AggregateExecutor aggregation_executor(&aggregation_node, - context.get()); - - aggregation_executor.AddChild(&hybrid_scan_executor); - - ///////////////////////////////////////////////////////// - // MATERIALIZE - ///////////////////////////////////////////////////////// - - // Create and set up materialization executor - std::vector output_columns; - std::unordered_map old_to_new_cols; - col_itr = 0; - for (auto column_id : column_ids) { - auto column = catalog::Column( - type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - std::to_string(column_id), is_inlined); - output_columns.push_back(column); - - old_to_new_cols[col_itr] = col_itr; - - col_itr++; - } - - std::shared_ptr output_schema( - new catalog::Schema(output_columns)); - bool physify_flag = true; // is going to create a physical tile - planner::MaterializationPlan mat_node(old_to_new_cols, output_schema, - physify_flag); - - executor::MaterializationExecutor mat_executor(&mat_node, nullptr); - - mat_executor.AddChild(&aggregation_executor); - - ///////////////////////////////////////////////////////// - // EXECUTE - ///////////////////////////////////////////////////////// - - std::vector executors; - executors.push_back(&mat_executor); - - ///////////////////////////////////////////////////////// - // COLLECT STATS - ///////////////////////////////////////////////////////// - std::vector index_columns_accessed(tuple_key_attrs.begin(), - tuple_key_attrs.end()); - auto selectivity = state.selectivity; - - auto tuple_columns_accessed = tuple_key_attrs; - for (auto column_id : column_ids) { - tuple_columns_accessed.push_back(column_id); - } - - ExecuteTest(executors, tuning::SampleType::ACCESS, {index_columns_accessed}, - {tuple_columns_accessed}, selectivity); - - auto result = txn_manager.CommitTransaction(txn); - - if (result == ResultType::SUCCESS) { - LOG_TRACE("commit successfully"); - } else { - LOG_TRACE("commit failed"); - } -} - -/** - * @brief Run write transactions - * - * @param tuple_key_attrs Tuple attributes to query on. - * @param index_key_attrs Index attributes to query on. - * @param update_attrs Columns to be updated. The value value of each attribute - * in update_attrs will be updated to -v, where v is the original value, and -v - * is minus original value. - */ -static void UpdateHelper(const std::vector &tuple_key_attrs, - const std::vector &index_key_attrs, - const std::vector &update_attrs) { - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - auto txn = txn_manager.BeginTransaction(); - - ////////////////////////////////////////// - // SCAN + PREDICATE - ////////////////////////////////////////// - - std::vector column_ids; - oid_t column_count = state.attribute_count; - - column_ids.push_back(0); - for (oid_t col_itr = 0; col_itr < column_count; col_itr++) { - column_ids.push_back(col_itr); - } - - std::unique_ptr context( - new executor::ExecutorContext(txn)); - - auto hybrid_scan_node = - CreateHybridScanPlan(tuple_key_attrs, index_key_attrs, column_ids); - executor::HybridScanExecutor hybrid_scan_executor(hybrid_scan_node.get(), - context.get()); - - ////////////////////////////////////////// - // UPDATE - ////////////////////////////////////////// - - // Update the value of each attribute in update_attrs to -v, where v is the - // original value, and -v is minus original value. - - std::vector values; - TargetList target_list; - DirectMapList direct_map_list; - - target_list.clear(); - direct_map_list.clear(); - - // Build target_list: -value for update_attrs - for (oid_t update_attr : update_attrs) { - auto tuple_value_expression = new expression::TupleValueExpression( - type::TypeId::INTEGER, 0, update_attr); - auto minus_value_expression = - new expression::OperatorUnaryMinusExpression(tuple_value_expression); - planner::DerivedAttribute attribute{minus_value_expression}; - target_list.emplace_back(update_attr, attribute); - } - - // Build direct_map_list: value unchanged for other attributes - oid_t update_attr_itr = 0; - for (oid_t col_itr = 0; col_itr < column_count; col_itr++) { - // Skip the updated column - if (update_attr_itr > update_attrs.size() || - col_itr != update_attrs[update_attr_itr]) { - direct_map_list.emplace_back(col_itr, - std::pair(0, col_itr)); - } else { - update_attr_itr++; - } - } - - std::unique_ptr project_info( - new planner::ProjectInfo(std::move(target_list), - std::move(direct_map_list))); - planner::UpdatePlan update_node(sdbench_table.get(), std::move(project_info)); - - executor::UpdateExecutor update_executor(&update_node, context.get()); - update_executor.AddChild(&hybrid_scan_executor); - - ///////////////////////////////////////////////////////// - // EXECUTE - ///////////////////////////////////////////////////////// - - std::vector executors; - executors.push_back(&update_executor); - - ///////////////////////////////////////////////////////// - // COLLECT STATS - ///////////////////////////////////////////////////////// - std::vector index_columns_accessed(tuple_key_attrs.begin(), - tuple_key_attrs.end()); - auto selectivity = state.selectivity; - - auto tuple_columns_accessed = tuple_key_attrs; - for (oid_t update_attr : update_attrs) { - tuple_columns_accessed.push_back(update_attr); - } - - ExecuteTest(executors, tuning::SampleType::ACCESS, {index_columns_accessed}, - {tuple_columns_accessed}, selectivity); - - auto result = txn_manager.CommitTransaction(txn); - - if (result == ResultType::SUCCESS) { - LOG_TRACE("commit successfully"); - } else { - LOG_TRACE("commit failed"); - } -} - -static void InsertHelper() { - const int BULK_INSERT_COUNT = 1000; - - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - - auto txn = txn_manager.BeginTransaction(); - - ///////////////////////////////////////////////////////// - // INSERT - ///////////////////////////////////////////////////////// - - std::unique_ptr context( - new executor::ExecutorContext(txn)); - - std::vector values; - type::Value insert_val = - type::ValueFactory::GetIntegerValue(++sdbench_tuple_counter); - TargetList target_list; - DirectMapList direct_map_list; - std::vector column_ids; - - target_list.clear(); - direct_map_list.clear(); - - for (oid_t col_id = 0; col_id <= state.attribute_count; col_id++) { - auto expression = - expression::ExpressionUtil::ConstantValueFactory(insert_val); - planner::DerivedAttribute attribute{expression}; - target_list.emplace_back(col_id, attribute); - column_ids.push_back(col_id); - } - - std::unique_ptr project_info( - new planner::ProjectInfo(std::move(target_list), - std::move(direct_map_list))); - - LOG_TRACE("Bulk insert count : %d", BULK_INSERT_COUNT); - planner::InsertPlan insert_node(sdbench_table.get(), std::move(project_info), - BULK_INSERT_COUNT); - executor::InsertExecutor insert_executor(&insert_node, context.get()); - - ///////////////////////////////////////////////////////// - // EXECUTE - ///////////////////////////////////////////////////////// - - std::vector executors; - executors.push_back(&insert_executor); - - ///////////////////////////////////////////////////////// - // COLLECT STATS - ///////////////////////////////////////////////////////// - std::vector index_columns_accessed; - double selectivity = 0; - - ExecuteTest(executors, tuning::SampleType::UPDATE, {{}}, {}, selectivity); - - auto result = txn_manager.CommitTransaction(txn); - - if (result == ResultType::SUCCESS) { - LOG_TRACE("commit successfully"); - } else { - LOG_TRACE("commit failed"); - } -} - -/** - * @brief Run bulk insert workload. - */ -static void RunInsert() { - LOG_TRACE("Run insert"); - - for (oid_t txn_itr = 0; txn_itr < state.phase_length; txn_itr++) { - InsertHelper(); - } -} - -static void RunSimpleUpdate() { - std::vector tuple_key_attrs; - std::vector index_key_attrs; - std::vector update_attrs; - - update_attrs = {15, 16, 17}; - - oid_t predicate = rand() % state.variability_threshold; - oid_t first_attribute = predicate; - tuple_key_attrs = {first_attribute}; - index_key_attrs = {0}; - - UNUSED_ATTRIBUTE std::stringstream os; - os << "Simple Update :: "; - for (auto tuple_key_attr : tuple_key_attrs) { - os << tuple_key_attr << " "; - } - if (state.verbose) { - LOG_INFO("%s", os.str().c_str()); - } - - // PHASE LENGTH - for (oid_t txn_itr = 0; txn_itr < state.phase_length; txn_itr++) { - UpdateHelper(tuple_key_attrs, index_key_attrs, update_attrs); - } -} - -static void RunComplexUpdate() { - std::vector tuple_key_attrs; - std::vector index_key_attrs; - std::vector update_attrs; - - update_attrs = {15, 16, 17}; - - auto predicate = GetPredicate(); - tuple_key_attrs = predicate; - index_key_attrs = {0, 1, 2}; - - UNUSED_ATTRIBUTE std::stringstream os; - os << "Complex Update :: "; - for (auto tuple_key_attr : tuple_key_attrs) { - os << tuple_key_attr << " "; - } - LOG_TRACE("%s", os.str().c_str()); - - // PHASE LENGTH - for (oid_t txn_itr = 0; txn_itr < state.phase_length; txn_itr++) { - UpdateHelper(tuple_key_attrs, index_key_attrs, update_attrs); - } -} - -/** - * @brief Run query depending on query type - */ -static void RunQuery() { - LOG_TRACE("Run query"); - switch (state.query_complexity_type) { - case QUERY_COMPLEXITY_TYPE_SIMPLE: - RunSimpleQuery(); - break; - case QUERY_COMPLEXITY_TYPE_MODERATE: - RunModerateQuery(); - break; - case QUERY_COMPLEXITY_TYPE_COMPLEX: - RunComplexQuery(); - break; - default: - break; - } -} - -/** - * @brief Run write txn depending on write type - */ -static void RunWrite() { - LOG_TRACE("Run write"); - switch (state.write_complexity_type) { - case WRITE_COMPLEXITY_TYPE_SIMPLE: - RunSimpleUpdate(); - break; - case WRITE_COMPLEXITY_TYPE_COMPLEX: - RunComplexUpdate(); - break; - case WRITE_COMPLEXITY_TYPE_INSERT: - RunInsert(); - break; - default: - break; - } -} - -/** - * @brief A data structure to hold index information of a table. - */ -struct IndexSummary { - // Index oids - std::vector index_oids; - - // Index has complete built? - bool completed; -}; - -static size_t stable_index_configuration_op_count = 0; - -/** - * @brief Check if index scheme has converged. - * Determine by looking at how many times index has not been changed. - * - * @return true if the index configuration has converged. False otherwise. - */ -static bool HasIndexConfigurationConverged() { - static IndexSummary prev_index_summary; - // If the index configuration stays the same - // for "convergence_query_threshold" continuous queries, - // then it's considered as converged. - - IndexSummary index_summary; - index_summary.completed = true; - - // Get index summary - oid_t index_count = sdbench_table->GetIndexCount(); - auto table_tile_group_count = sdbench_table->GetTileGroupCount(); - for (oid_t index_itr = 0; index_itr < index_count; index_itr++) { - // Get index - auto index = sdbench_table->GetIndex(index_itr); - if (index == nullptr) { - continue; - } - - auto indexed_tile_group_offset = index->GetIndexedTileGroupOff(); - - // Get percentage completion - double fraction = 0.0; - if (table_tile_group_count != 0) { - fraction = - (double)indexed_tile_group_offset / (double)table_tile_group_count; - fraction *= 100; - } - - if (fraction < 0) { - index_summary.completed = false; - } - - // Get index columns - index_summary.index_oids.push_back(index->GetOid()); - } - - if (index_summary.completed == false) { - prev_index_summary = index_summary; - stable_index_configuration_op_count = 0; - return false; - } - - // Check if the index summary is the same - bool identical = true; - if (index_summary.index_oids.size() == prev_index_summary.index_oids.size()) { - for (size_t i = 0; i < index_summary.index_oids.size(); i++) { - if (index_summary.index_oids[i] != prev_index_summary.index_oids[i]) { - identical = false; - break; - } - } - } else { - identical = false; - } - - // Update index unchanged phase count - if (identical) { - stable_index_configuration_op_count += 1; - } else { - stable_index_configuration_op_count = 0; - } - - prev_index_summary = index_summary; - - // Check threshold # of ops - if (stable_index_configuration_op_count >= state.convergence_op_threshold) { - LOG_INFO("Has converged"); - return true; - } - - return false; -} - -/** - * @brief Do any preparation before running a benchmark. - */ -void BenchmarkPrepare() { - // Setup index tuner - index_tuner.SetAnalyzeSampleCountThreshold( - state.analyze_sample_count_threshold); - index_tuner.SetTileGroupsIndexedPerIteration( - state.tile_groups_indexed_per_iteration); - index_tuner.SetDurationBetweenPauses(state.duration_between_pauses); - index_tuner.SetDurationOfPause(state.duration_of_pause); - index_tuner.SetAnalyzeSampleCountThreshold( - state.analyze_sample_count_threshold); - index_tuner.SetTileGroupsIndexedPerIteration( - state.tile_groups_indexed_per_iteration); - index_tuner.SetIndexUtilityThreshold(state.index_utility_threshold); - index_tuner.SetIndexCountThreshold(state.index_count_threshold); - index_tuner.SetWriteRatioThreshold(state.write_ratio_threshold); - index_tuner.SetTileGroupsIndexedPerIteration( - state.tile_groups_indexed_per_iteration); - - // seed generator - srand(generator_seed); - - // Generate sequence - GenerateSequence(state.attribute_count); - - // Generate distribution - GeneratePredicateDistribution(); - - CreateAndLoadTable((LayoutType)state.layout_mode); - - // Start index tuner - if (state.index_usage_type != INDEX_USAGE_TYPE_NEVER) { - index_tuner.AddTable(sdbench_table.get()); - - // Start after adding tables - index_tuner.Start(); - } - - // Start layout tuner - if (state.layout_mode == LayoutType::HYBRID) { - layout_tuner.AddTable(sdbench_table.get()); - - // Start layout tuner - layout_tuner.Start(); - } -} - -/** - * @brief Do any clean up after running a benchmark. - */ -void BenchmarkCleanUp() { - // Stop index tuner - if (state.index_usage_type != INDEX_USAGE_TYPE_NEVER) { - index_tuner.Stop(); - index_tuner.ClearTables(); - } - - if (state.layout_mode == LayoutType::HYBRID) { - layout_tuner.Stop(); - layout_tuner.ClearTables(); - } - - // Drop Indexes - DropIndexes(); - - // Reset - query_itr = 0; - - out.close(); -} - -static void SDBenchHelper() { - double write_ratio = state.write_ratio; - - // Reset total duration - total_duration = 0; - - // Reset query counter - query_itr = 0; - - Timer<> index_unchanged_timer; - - // cache original phase length - size_t original_phase_length = state.phase_length; - if (original_phase_length < 5) { - LOG_ERROR("Phase length must be greater than 5"); - return; - } - - // run desired number of ops - oid_t phase_count = 0; - for (oid_t op_itr = 0; op_itr < state.total_ops;) { - // set phase length (NOTE: uneven across phases) - size_t minimum_op_count = (original_phase_length / 5); - size_t rest_op_count = original_phase_length - minimum_op_count; - size_t current_phase_length = minimum_op_count + rand() % rest_op_count; - if (current_phase_length > state.total_ops - op_itr) { - current_phase_length = state.total_ops - op_itr; - } - - state.phase_length = current_phase_length; - op_itr += current_phase_length; - phase_count++; - - double rand_sample = (double)rand() / RAND_MAX; - - // Do write - if (rand_sample < write_ratio) { - RunWrite(); - } - // Do read - else { - RunQuery(); - } - - // Randomly add some access sample to build indices - if (state.holistic_indexing && state.multi_stage_idx == 1) { - auto predicate = GetPredicate(); - std::vector index_columns_accessed(predicate.begin(), - predicate.end()); - // double selectivity = state.selectivity; - double duration = rand() % 100; - tuning::Sample index_access_sample(index_columns_accessed, duration, - tuning::SampleType::ACCESS); - // ??? , selectivity); - for (oid_t i = 0; i < state.analyze_sample_count_threshold; i++) { - sdbench_table->RecordIndexSample(index_access_sample); - } - } - - // Check index convergence - if (state.convergence == true) { - bool converged = HasIndexConfigurationConverged(); - if (converged == true) { - break; - } - } - } - - LOG_INFO("Average phase length : %.0lf", - (double)state.total_ops / phase_count); - LOG_INFO("Duration : %.2lf", total_duration); -} - -void RunMultiStageBenchmark() { - BenchmarkPrepare(); - - int orig_analyze_sample_count_threshold = - state.analyze_sample_count_threshold; - // The first stage - if (state.holistic_indexing) { - // Make the index build speed faster - index_tuner.SetAnalyzeSampleCountThreshold( - (int)(orig_analyze_sample_count_threshold * 0.6)); - } - state.multi_stage_idx = 0; - SDBenchHelper(); - // The second stage - if (state.holistic_indexing) { - // Make the index build speed slower - index_tuner.SetAnalyzeSampleCountThreshold( - (int)(orig_analyze_sample_count_threshold * 2)); - } - state.multi_stage_idx = 1; - SDBenchHelper(); - // The third stage - state.write_ratio = 1.00; - state.multi_stage_idx = 2; - if (state.holistic_indexing) { - // Holistic doesn't drop index - index_tuner.SetAnalyzeSampleCountThreshold( - (int)(orig_analyze_sample_count_threshold)); - index_tuner.SetWriteRatioThreshold(1.0); - } - SDBenchHelper(); - - BenchmarkCleanUp(); -} - -void RunSDBenchTest() { - BenchmarkPrepare(); - - // Run the benchmark once - SDBenchHelper(); - - BenchmarkCleanUp(); -} - -} // namespace sdbench -} // namespace benchmark -} // namespace peloton diff --git a/src/main/tpcc/tpcc.cpp b/src/main/tpcc/tpcc.cpp deleted file mode 100644 index 5d9af1a28be..00000000000 --- a/src/main/tpcc/tpcc.cpp +++ /dev/null @@ -1,103 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpcc.cpp -// -// Identification: src/main/tpcc/tpcc.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include -#include -#include - -#include "common/logger.h" -#include "benchmark/tpcc/tpcc_configuration.h" -#include "benchmark/tpcc/tpcc_loader.h" -#include "benchmark/tpcc/tpcc_workload.h" - -#include "gc/gc_manager_factory.h" -#include "concurrency/epoch_manager_factory.h" - -namespace peloton { -namespace benchmark { -namespace tpcc { - -configuration state; - -// Main Entry Point -void RunBenchmark() { - - if (state.gc_mode == false) { - gc::GCManagerFactory::Configure(0); - } else { - gc::GCManagerFactory::Configure(state.gc_backend_count); - } - - concurrency::EpochManagerFactory::Configure(state.epoch); - - std::unique_ptr epoch_thread; - std::vector> gc_threads; - - concurrency::EpochManager &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); - - if (concurrency::EpochManagerFactory::GetEpochType() == EpochType::DECENTRALIZED_EPOCH) { - for (size_t i = 0; i < (size_t) state.backend_count; ++i) { - // register thread to epoch manager - epoch_manager.RegisterThread(i); - } - } - - // start epoch. - epoch_manager.StartEpoch(epoch_thread); - - gc::GCManager &gc_manager = gc::GCManagerFactory::GetInstance(); - - // start GC. - gc_manager.StartGC(gc_threads); - - // Create the database - CreateTPCCDatabase(); - - // Load the database - LoadTPCCDatabase(); - - // Run the workload - RunWorkload(); - - // stop GC. - gc_manager.StopGC(); - - // stop epoch. - epoch_manager.StopEpoch(); - - // join all gc threads - for (auto &gc_thread : gc_threads) { - PELOTON_ASSERT(gc_thread != nullptr); - gc_thread->join(); - } - - // join epoch thread - PELOTON_ASSERT(epoch_thread != nullptr); - epoch_thread->join(); - - - // Emit throughput - WriteOutput(); -} - -} // namespace tpcc -} // namespace benchmark -} // namespace peloton - -int main(int argc, char **argv) { - peloton::benchmark::tpcc::ParseArguments(argc, argv, - peloton::benchmark::tpcc::state); - - peloton::benchmark::tpcc::RunBenchmark(); - - return 0; -} diff --git a/src/main/tpcc/tpcc_configuration.cpp b/src/main/tpcc/tpcc_configuration.cpp deleted file mode 100644 index 3cd63cbf7da..00000000000 --- a/src/main/tpcc/tpcc_configuration.cpp +++ /dev/null @@ -1,271 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpcc_configuration.cpp -// -// Identification: src/main/tpcc/tpcc_configuration.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#include -#include -#include - -#include "benchmark/tpcc/tpcc_configuration.h" -#include "common/logger.h" - -namespace peloton { -namespace benchmark { -namespace tpcc { - -void Usage(FILE *out) { - fprintf(out, - "Command line options : tpcc \n" - " -h --help : print help message \n" - " -i --index : index type: bwtree (default) \n" - " -k --scale_factor : scale factor \n" - " -d --duration : execution duration \n" - " -p --profile_duration : profile duration \n" - " -b --backend_count : # of backends \n" - " -w --warehouse_count : # of warehouses \n" - " -e --exp_backoff : enable exponential backoff \n" - " -a --affinity : enable client affinity \n" - " -g --gc_mode : enable garbage collection \n" - " -n --gc_backend_count : # of gc backends \n" - " -l --loader_count : # of loaders \n" - " -y --epoch : epoch type: centralized or decentralized \n" - ); -} - -static struct option opts[] = { - { "index", optional_argument, NULL, 'i' }, - { "scale_factor", optional_argument, NULL, 'k' }, - { "duration", optional_argument, NULL, 'd' }, - { "profile_duration", optional_argument, NULL, 'p' }, - { "backend_count", optional_argument, NULL, 'b' }, - { "warehouse_count", optional_argument, NULL, 'w' }, - { "exp_backoff", no_argument, NULL, 'e' }, - { "affinity", no_argument, NULL, 'a' }, - { "gc_mode", no_argument, NULL, 'g' }, - { "gc_backend_count", optional_argument, NULL, 'n' }, - { "loader_count", optional_argument, NULL, 'n' }, - { "epoch", optional_argument, NULL, 'y' }, - { NULL, 0, NULL, 0 } -}; - -void ValidateIndex(const configuration &state) { - if (state.index != IndexType::BWTREE) { - LOG_ERROR("Invalid index"); - exit(EXIT_FAILURE); - } -} - -void ValidateScaleFactor(const configuration &state) { - if (state.scale_factor <= 0) { - LOG_ERROR("Invalid scale_factor :: %lf", state.scale_factor); - exit(EXIT_FAILURE); - } - - LOG_TRACE("%s : %lf", "scale_factor", state.scale_factor); -} - -void ValidateDuration(const configuration &state) { - if (state.duration <= 0) { - LOG_ERROR("Invalid duration :: %lf", state.duration); - exit(EXIT_FAILURE); - } - - LOG_TRACE("%s : %lf", "duration", state.duration); -} - -void ValidateProfileDuration(const configuration &state) { - if (state.profile_duration <= 0) { - LOG_ERROR("Invalid profile_duration :: %lf", state.profile_duration); - exit(EXIT_FAILURE); - } - - LOG_TRACE("%s : %lf", "profile_duration", state.profile_duration); -} - -void ValidateBackendCount(const configuration &state) { - if (state.backend_count <= 0) { - LOG_ERROR("Invalid backend_count :: %d", state.backend_count); - exit(EXIT_FAILURE); - } - - LOG_TRACE("%s : %d", "backend_count", state.backend_count); -} - -void ValidateWarehouseCount(const configuration &state) { - if (state.warehouse_count <= 0) { - LOG_ERROR("Invalid warehouse_count :: %d", state.warehouse_count); - exit(EXIT_FAILURE); - } - - LOG_TRACE("%s : %d", "warehouse_count", state.warehouse_count); -} - -void ValidateGCBackendCount(const configuration &state) { - if (state.gc_backend_count <= 0) { - LOG_ERROR("Invalid gc_backend_count :: %d", state.gc_backend_count); - exit(EXIT_FAILURE); - } - - LOG_TRACE("%s : %d", "gc_backend_count", state.gc_backend_count); -} - - -void ParseArguments(int argc, char *argv[], configuration &state) { - // Default Values - state.index = IndexType::BWTREE; - state.epoch = EpochType::DECENTRALIZED_EPOCH; - state.scale_factor = 1; - state.duration = 10; - state.profile_duration = 1; - state.backend_count = 2; - state.warehouse_count = 2; - state.exp_backoff = false; - state.affinity = false; - state.gc_mode = false; - state.gc_backend_count = 1; - state.loader_count = 1; - - - // Parse args - while (1) { - int idx = 0; - int c = getopt_long(argc, argv, "heagi:k:d:p:b:w:n:l:y:", opts, &idx); - - if (c == -1) break; - - switch (c) { - case 'i': { - char *index = optarg; - if (strcmp(index, "bwtree") == 0) { - state.index = IndexType::BWTREE; - } else { - LOG_ERROR("Unknown index: %s", index); - exit(EXIT_FAILURE); - } - break; - } - case 'y': { - char *epoch = optarg; - if (strcmp(epoch, "decentralized") == 0) { - state.epoch = EpochType::DECENTRALIZED_EPOCH; - } else { - LOG_ERROR("Unknown epoch: %s", epoch); - exit(EXIT_FAILURE); - } - break; - } - case 'l': - state.loader_count = atoi(optarg); - break; - case 'k': - state.scale_factor = atof(optarg); - break; - case 'd': - state.duration = atof(optarg); - break; - case 'p': - state.profile_duration = atof(optarg); - break; - case 'b': - state.backend_count = atoi(optarg); - break; - case 'w': - state.warehouse_count = atoi(optarg); - break; - case 'e': - state.exp_backoff = true; - break; - case 'a': - state.affinity = true; - break; - case 'g': - state.gc_mode = true; - break; - case 'n': - state.gc_backend_count = atoi(optarg); - break; - - case 'h': - Usage(stderr); - exit(EXIT_FAILURE); - break; - - default: - LOG_ERROR("Unknown option: -%c-", c); - Usage(stderr); - exit(EXIT_FAILURE); - } - } - - // Static TPCC parameters - state.item_count = 100000 * state.scale_factor; - state.districts_per_warehouse = 10; - state.customers_per_district = 3000 * state.scale_factor; - state.new_orders_per_district = 900 * state.scale_factor; - - // Print configuration - ValidateIndex(state); - ValidateScaleFactor(state); - ValidateDuration(state); - ValidateProfileDuration(state); - ValidateBackendCount(state); - ValidateWarehouseCount(state); - ValidateGCBackendCount(state); - - LOG_TRACE("%s : %d", "Run client affinity", state.affinity); - LOG_TRACE("%s : %d", "Run exponential backoff", state.exp_backoff); - LOG_TRACE("%s : %d", "Run garbage collection", state.gc_mode); -} - - - -void WriteOutput() { - std::ofstream out("outputfile.summary"); - - oid_t total_profile_memory = 0; - for (auto &entry : state.profile_memory) { - total_profile_memory += entry; - } - - LOG_INFO("----------------------------------------------------------"); - LOG_INFO("%lf %d %d :: %lf %lf %d", - state.scale_factor, - state.backend_count, - state.warehouse_count, - state.throughput, - state.abort_rate, - total_profile_memory); - - out << state.scale_factor << " "; - out << state.backend_count << " "; - out << state.warehouse_count << " "; - out << state.throughput << " "; - out << state.abort_rate << " "; - out << total_profile_memory << "\n"; - - for (size_t round_id = 0; round_id < state.profile_throughput.size(); - ++round_id) { - out << "[" << std::setw(3) << std::left - << state.profile_duration * round_id << " - " << std::setw(3) - << std::left << state.profile_duration * (round_id + 1) - << " s]: " << state.profile_throughput[round_id] << " " - << state.profile_abort_rate[round_id] << " " - << state.profile_memory[round_id] << "\n"; - } - out.flush(); - out.close(); -} - - -} // namespace tpcc -} // namespace benchmark -} // namespace peloton diff --git a/src/main/tpcc/tpcc_delivery.cpp b/src/main/tpcc/tpcc_delivery.cpp deleted file mode 100644 index 8b688e1a4c2..00000000000 --- a/src/main/tpcc/tpcc_delivery.cpp +++ /dev/null @@ -1,572 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpcc_payment.cpp -// -// Identification: src/main/tpcc/tpcc_delivery.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "benchmark/tpcc/tpcc_configuration.h" -#include "benchmark/tpcc/tpcc_loader.h" -#include "benchmark/tpcc/tpcc_workload.h" - -#include "catalog/column.h" -#include "catalog/manager.h" -#include "catalog/schema.h" - -#include "common/generator.h" -#include "common/internal_types.h" -#include "common/logger.h" -#include "common/timer.h" -#include "type/value.h" -#include "type/value_factory.h" - -#include "concurrency/transaction_context.h" -#include "concurrency/transaction_manager_factory.h" - -#include "executor/abstract_executor.h" -#include "executor/aggregate_executor.h" -#include "executor/delete_executor.h" -#include "executor/executor_context.h" -#include "executor/index_scan_executor.h" -#include "executor/insert_executor.h" -#include "executor/limit_executor.h" -#include "executor/logical_tile.h" -#include "executor/logical_tile_factory.h" -#include "executor/materialization_executor.h" -#include "executor/update_executor.h" - -#include "common/container_tuple.h" -#include "expression/abstract_expression.h" -#include "expression/comparison_expression.h" -#include "expression/constant_value_expression.h" -#include "expression/expression_util.h" -#include "expression/tuple_value_expression.h" - -#include "index/index_factory.h" - -#include "logging/log_manager.h" - -#include "planner/abstract_plan.h" -#include "planner/aggregate_plan.h" -#include "planner/delete_plan.h" -#include "planner/index_scan_plan.h" -#include "planner/insert_plan.h" -#include "planner/limit_plan.h" -#include "planner/materialization_plan.h" -#include "planner/project_info.h" -#include "planner/update_plan.h" - -#include "storage/data_table.h" -#include "storage/table_factory.h" - -namespace peloton { -namespace benchmark { -namespace tpcc { - -bool RunDelivery(const size_t &thread_id) { - /* - "DELIVERY": { - "getNewOrder": "SELECT NO_O_ID FROM NEW_ORDER WHERE NO_D_ID = ? AND NO_W_ID = - ? AND NO_O_ID > -1 LIMIT 1", # - "deleteNewOrder": "DELETE FROM NEW_ORDER WHERE NO_D_ID = ? AND NO_W_ID = ? - AND NO_O_ID = ?", # d_id, w_id, no_o_id - "getCId": "SELECT O_C_ID FROM ORDERS WHERE O_ID = ? AND O_D_ID = ? AND O_W_ID - = ?", # no_o_id, d_id, w_id - "updateOrders": "UPDATE ORDERS SET O_CARRIER_ID = ? WHERE O_ID = ? AND O_D_ID - = ? AND O_W_ID = ?", # o_carrier_id, no_o_id, d_id, w_id - "updateOrderLine": "UPDATE ORDER_LINE SET OL_DELIVERY_D = ? WHERE OL_O_ID = ? - AND OL_D_ID = ? AND OL_W_ID = ?", # o_entry_d, no_o_id, d_id, w_id - "sumOLAmount": "SELECT SUM(OL_AMOUNT) FROM ORDER_LINE WHERE OL_O_ID = ? AND - OL_D_ID = ? AND OL_W_ID = ?", # no_o_id, d_id, w_id - "updateCustomer": "UPDATE CUSTOMER SET C_BALANCE = C_BALANCE + ? WHERE C_ID = - ? AND C_D_ID = ? AND C_W_ID = ?", # ol_total, c_id, d_id, w_id - } - */ - - LOG_TRACE("-------------------------------------"); - - ///////////////////////////////////////////////////////// - // PREPARE ARGUMENTS - ///////////////////////////////////////////////////////// - int warehouse_id = GenerateWarehouseId(thread_id); - int o_carrier_id = - GetRandomInteger(orders_min_carrier_id, orders_max_carrier_id); - - std::vector runtime_keys; - - ///////////////////////////////////////////////////////// - // BEGIN TRANSACTION - ///////////////////////////////////////////////////////// - - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - - auto txn = txn_manager.BeginTransaction(thread_id); - - std::unique_ptr context( - new executor::ExecutorContext(txn)); - - for (int d_id = 0; d_id < state.districts_per_warehouse; ++d_id) { - LOG_TRACE( - "getNewOrder: SELECT NO_O_ID FROM NEW_ORDER WHERE NO_D_ID = ? AND " - "NO_W_ID = ? AND NO_O_ID > -1 LIMIT 1"); - - // Construct index scan executor - std::vector new_order_column_ids = {COL_IDX_NO_O_ID}; - std::vector new_order_key_column_ids = { - COL_IDX_NO_D_ID, COL_IDX_NO_W_ID, COL_IDX_NO_O_ID}; - - std::vector new_order_expr_types; - - new_order_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - new_order_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - new_order_expr_types.push_back(ExpressionType::COMPARE_GREATERTHAN); - - std::vector new_order_key_values; - - new_order_key_values.push_back( - type::ValueFactory::GetIntegerValue(d_id).Copy()); - new_order_key_values.push_back( - type::ValueFactory::GetIntegerValue(warehouse_id).Copy()); - new_order_key_values.push_back( - type::ValueFactory::GetIntegerValue(-1).Copy()); - - planner::IndexScanPlan::IndexScanDesc new_order_idex_scan_desc( - new_order_table_pkey_index_oid, new_order_key_column_ids, - new_order_expr_types, new_order_key_values, runtime_keys); - - planner::IndexScanPlan new_order_idex_scan_node(new_order_table, nullptr, - new_order_column_ids, - new_order_idex_scan_desc); - - executor::IndexScanExecutor new_order_index_scan_executor( - &new_order_idex_scan_node, context.get()); - - // Construct limit executor - size_t limit = 1; - size_t offset = 0; - planner::LimitPlan limit_node(limit, offset); - executor::LimitExecutor limit_executor(&limit_node, context.get()); - limit_executor.AddChild(&new_order_index_scan_executor); - - auto new_order_ids = ExecuteRead(&limit_executor); - - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - if (new_order_ids.size() == 0) { - // TODO: No orders for this district: skip it. Note: This must be - // reported if > 1% - continue; - } - - assert(new_order_ids.size() == 1); - assert(new_order_ids[0].size() == 1); - - // result: NO_O_ID - auto no_o_id = new_order_ids[0][0]; - - LOG_TRACE("no_o_id = %d", type::ValuePeeker::PeekInteger(no_o_id)); - - LOG_TRACE( - "getCId: SELECT O_C_ID FROM ORDERS WHERE O_ID = ? AND O_D_ID = ? AND " - "O_W_ID = ?"); - - std::vector orders_column_ids = {COL_IDX_O_C_ID}; - std::vector orders_key_column_ids = {COL_IDX_O_ID, COL_IDX_O_D_ID, - COL_IDX_O_W_ID}; - - std::vector orders_expr_types; - - orders_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - orders_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - orders_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - - std::vector orders_key_values; - - orders_key_values.push_back(no_o_id); - orders_key_values.push_back( - type::ValueFactory::GetIntegerValue(d_id).Copy()); - orders_key_values.push_back( - type::ValueFactory::GetIntegerValue(warehouse_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc orders_index_scan_desc( - orders_table_pkey_index_oid, orders_key_column_ids, orders_expr_types, - orders_key_values, runtime_keys); - - // Create the index scan plan node - planner::IndexScanPlan orders_index_scan_node( - orders_table, nullptr, orders_column_ids, orders_index_scan_desc); - - // Create the executors - executor::IndexScanExecutor orders_index_scan_executor( - &orders_index_scan_node, context.get()); - - auto orders_ids = ExecuteRead(&orders_index_scan_executor); - - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - assert(orders_ids.size() == 1); - assert(orders_ids[0].size() == 1); - - // Result: O_C_ID - auto c_id = orders_ids[0][0]; - - LOG_TRACE( - "sumOLAmount: SELECT SUM(OL_AMOUNT) FROM ORDER_LINE WHERE OL_O_ID = ? " - "AND OL_D_ID = ? AND OL_W_ID = ?"); - - // Construct index scan executor - std::vector order_line_column_ids = {COL_IDX_OL_AMOUNT}; - std::vector order_line_key_column_ids = { - COL_IDX_OL_O_ID, COL_IDX_OL_D_ID, COL_IDX_OL_W_ID}; - - std::vector order_line_expr_types; - - order_line_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - order_line_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - order_line_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - - std::vector order_line_key_values; - - order_line_key_values.push_back(no_o_id); - order_line_key_values.push_back( - type::ValueFactory::GetIntegerValue(d_id).Copy()); - order_line_key_values.push_back( - type::ValueFactory::GetIntegerValue(warehouse_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc order_line_index_scan_desc( - order_line_table_pkey_index_oid, order_line_key_column_ids, - order_line_expr_types, order_line_key_values, runtime_keys); - - planner::IndexScanPlan order_line_index_scan_node( - order_line_table, nullptr, order_line_column_ids, - order_line_index_scan_desc); - - executor::IndexScanExecutor order_line_index_scan_executor( - &order_line_index_scan_node, context.get()); - - auto order_line_index_scan_res = - ExecuteRead(&order_line_index_scan_executor); - - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - double sum_res = 0.0; - - // Workaround: Externanl sum - for (auto v : order_line_index_scan_res) { - assert(v.size() == 1); - sum_res += type::ValuePeeker::PeekDouble(v[0]); - } - - auto ol_total = type::ValueFactory::GetDecimalValue(sum_res); - - LOG_TRACE( - "deleteNewOrder: DELETE FROM NEW_ORDER WHERE NO_D_ID = ? AND NO_W_ID = " - "? AND NO_O_ID = ?"); - - // Construct index scan executor - std::vector new_order_delete_column_ids = {0}; - - std::vector new_order_delete_expr_types; - - new_order_delete_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - new_order_delete_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - new_order_delete_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - - std::vector new_order_delete_key_values; - - new_order_delete_key_values.push_back( - type::ValueFactory::GetIntegerValue(d_id).Copy()); - new_order_delete_key_values.push_back( - type::ValueFactory::GetIntegerValue(warehouse_id).Copy()); - new_order_delete_key_values.push_back(no_o_id); - - planner::IndexScanPlan::IndexScanDesc new_order_delete_idex_scan_desc( - new_order_table_pkey_index_oid, new_order_key_column_ids, - new_order_delete_expr_types, new_order_delete_key_values, runtime_keys); - - // Create index scan plan node - planner::IndexScanPlan new_order_delete_idex_scan_node( - new_order_table, nullptr, new_order_delete_column_ids, - new_order_delete_idex_scan_desc); - - // Create executors - executor::IndexScanExecutor new_order_delete_index_scan_executor( - &new_order_delete_idex_scan_node, context.get()); - - // Construct delete executor - planner::DeletePlan new_order_delete_node(new_order_table); - - executor::DeleteExecutor new_order_delete_executor(&new_order_delete_node, - context.get()); - - new_order_delete_executor.AddChild(&new_order_delete_index_scan_executor); - - // Execute the query - ExecuteDelete(&new_order_delete_executor); - - // Check if aborted - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - LOG_TRACE( - "updateOrders: UPDATE ORDERS SET O_CARRIER_ID = ? WHERE O_ID = ? AND " - "O_D_ID = ? AND O_W_ID = ?"); - - // Construct index scan executor - std::vector orders_update_column_ids = {COL_IDX_O_CARRIER_ID}; - - std::vector orders_update_key_values; - - orders_update_key_values.push_back(no_o_id); - orders_update_key_values.push_back( - type::ValueFactory::GetIntegerValue(d_id).Copy()); - orders_update_key_values.push_back( - type::ValueFactory::GetIntegerValue(warehouse_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc orders_update_index_scan_desc( - orders_table_pkey_index_oid, orders_key_column_ids, orders_expr_types, - orders_update_key_values, runtime_keys); - - // Reuse the index scan desc created above since nothing different - planner::IndexScanPlan orders_update_index_scan_node( - orders_table, nullptr, orders_update_column_ids, - orders_update_index_scan_desc); - - executor::IndexScanExecutor orders_update_index_scan_executor( - &orders_update_index_scan_node, context.get()); - - // Construct update executor - TargetList orders_target_list; - DirectMapList orders_direct_map_list; - - size_t orders_column_count = 8; - for (oid_t col_itr = 0; col_itr < orders_column_count; col_itr++) { - // Skip O_CARRIER_ID - if (col_itr != COL_IDX_O_CARRIER_ID) { - orders_direct_map_list.emplace_back(col_itr, - std::make_pair(0, col_itr)); - } - } - type::Value orders_update_val = - type::ValueFactory::GetIntegerValue(o_carrier_id).Copy(); - - planner::DerivedAttribute carrier_id{ - expression::ExpressionUtil::ConstantValueFactory(orders_update_val)}; - orders_target_list.emplace_back(COL_IDX_O_CARRIER_ID, carrier_id); - - std::unique_ptr orders_project_info( - new planner::ProjectInfo(std::move(orders_target_list), - std::move(orders_direct_map_list))); - planner::UpdatePlan orders_update_node(orders_table, - std::move(orders_project_info)); - - executor::UpdateExecutor orders_update_executor(&orders_update_node, - context.get()); - - orders_update_executor.AddChild(&orders_update_index_scan_executor); - - // Execute the query - ExecuteUpdate(&orders_update_executor); - - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - LOG_TRACE( - "updateOrderLine: UPDATE ORDER_LINE SET OL_DELIVERY_D = ? WHERE " - "OL_O_ID = ? AND OL_D_ID = ? AND OL_W_ID = ?"); - - // Construct index scan executor - std::vector order_line_update_column_ids = {COL_IDX_OL_DELIVERY_D}; - - std::vector order_line_update_key_values; - - order_line_update_key_values.push_back(no_o_id); - order_line_update_key_values.push_back( - type::ValueFactory::GetIntegerValue(d_id).Copy()); - order_line_update_key_values.push_back( - type::ValueFactory::GetIntegerValue(warehouse_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc order_line_update_index_scan_desc( - order_line_table_pkey_index_oid, order_line_key_column_ids, - order_line_expr_types, order_line_update_key_values, runtime_keys); - - planner::IndexScanPlan order_line_update_index_scan_node( - order_line_table, nullptr, order_line_update_column_ids, - order_line_update_index_scan_desc); - - executor::IndexScanExecutor order_line_update_index_scan_executor( - &order_line_update_index_scan_node, context.get()); - - // Construct update executor - TargetList order_line_target_list; - DirectMapList order_line_direct_map_list; - - size_t order_line_column_count = 10; - for (oid_t col_itr = 0; col_itr < order_line_column_count; col_itr++) { - // Skip OL_DELIVERY_D - if (col_itr != COL_IDX_OL_DELIVERY_D) { - order_line_direct_map_list.emplace_back(col_itr, - std::make_pair(0, col_itr)); - } - } - type::Value order_line_update_val = - type::ValueFactory::GetTimestampValue(0).Copy(); - - planner::DerivedAttribute delivery_id{ - expression::ExpressionUtil::ConstantValueFactory( - order_line_update_val)}; - order_line_target_list.emplace_back(COL_IDX_OL_DELIVERY_D, delivery_id); - - std::unique_ptr order_line_project_info( - new planner::ProjectInfo(std::move(order_line_target_list), - std::move(order_line_direct_map_list))); - planner::UpdatePlan order_line_update_node( - order_line_table, std::move(order_line_project_info)); - - executor::UpdateExecutor order_line_update_executor(&order_line_update_node, - context.get()); - - order_line_update_executor.AddChild(&order_line_update_index_scan_executor); - - ExecuteUpdate(&order_line_update_executor); - - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - LOG_TRACE( - "updateCustomer: UPDATE CUSTOMER SET C_BALANCE = C_BALANCE + ? WHERE " - "C_ID = ? AND C_D_ID = ? AND C_W_ID = ?"); - - // Construct index scan executor - std::vector customer_column_ids = {COL_IDX_C_BALANCE}; - std::vector customer_key_column_ids = {COL_IDX_C_ID, COL_IDX_C_D_ID, - COL_IDX_C_W_ID}; - - std::vector customer_expr_types; - - customer_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - customer_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - customer_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - - std::vector customer_key_values; - - customer_key_values.push_back(c_id); - customer_key_values.push_back( - type::ValueFactory::GetIntegerValue(d_id).Copy()); - customer_key_values.push_back( - type::ValueFactory::GetIntegerValue(warehouse_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc customer_index_scan_desc( - customer_table_pkey_index_oid, customer_key_column_ids, - customer_expr_types, customer_key_values, runtime_keys); - - planner::IndexScanPlan customer_index_scan_node( - customer_table, nullptr, customer_column_ids, customer_index_scan_desc); - - executor::IndexScanExecutor customer_index_scan_executor( - &customer_index_scan_node, context.get()); - - // Construct update executor - TargetList customer_target_list; - DirectMapList customer_direct_map_list; - - size_t customer_column_count = 21; - for (oid_t col_itr = 0; col_itr < customer_column_count; col_itr++) { - // Skip OL_DELIVERY_D - if (col_itr != COL_IDX_C_BALANCE) { - customer_direct_map_list.emplace_back(col_itr, - std::make_pair(0, col_itr)); - } - } - - // Expressions - // Tuple value expression - auto tuple_val_expr = expression::ExpressionUtil::TupleValueFactory( - type::TypeId::INTEGER, 0, COL_IDX_C_BALANCE); - // Constant value expression - auto constant_val_expr = - expression::ExpressionUtil::ConstantValueFactory(ol_total); - // + operator expression - auto plus_operator_expr = expression::ExpressionUtil::OperatorFactory( - ExpressionType::OPERATOR_PLUS, type::TypeId::INTEGER, tuple_val_expr, - constant_val_expr); - - planner::DerivedAttribute c_balance{plus_operator_expr}; - customer_target_list.emplace_back(COL_IDX_C_BALANCE, c_balance); - - std::unique_ptr customer_project_info( - new planner::ProjectInfo(std::move(customer_target_list), - std::move(customer_direct_map_list))); - planner::UpdatePlan customer_update_node(customer_table, - std::move(customer_project_info)); - - executor::UpdateExecutor customer_update_executor(&customer_update_node, - context.get()); - - customer_update_executor.AddChild(&customer_index_scan_executor); - - // Execute the query - ExecuteUpdate(&customer_update_executor); - - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - } - - assert(txn->GetResult() == ResultType::SUCCESS); - - auto result = txn_manager.CommitTransaction(txn); - - if (result == ResultType::SUCCESS) { - LOG_TRACE("commit successfully"); - return true; - } else { - assert(result == ResultType::ABORTED || result == ResultType::FAILURE); - return false; - } -} -} -} -} diff --git a/src/main/tpcc/tpcc_loader.cpp b/src/main/tpcc/tpcc_loader.cpp deleted file mode 100644 index 44d8e07e4b7..00000000000 --- a/src/main/tpcc/tpcc_loader.cpp +++ /dev/null @@ -1,1764 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpcc_loader.cpp -// -// Identification: src/main/tpcc/tpcc_loader.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "benchmark/tpcc/tpcc_loader.h" -#include "benchmark/tpcc/tpcc_configuration.h" -#include "catalog/catalog.h" -#include "catalog/schema.h" -#include "concurrency/transaction_context.h" -#include "concurrency/transaction_manager_factory.h" -#include "executor/abstract_executor.h" -#include "executor/insert_executor.h" -#include "executor/executor_context.h" -#include "expression/constant_value_expression.h" -#include "expression/expression_util.h" -#include "index/index_factory.h" -#include "planner/insert_plan.h" -#include "storage/tile.h" -#include "storage/tile_group.h" -#include "storage/data_table.h" -#include "storage/table_factory.h" -#include "storage/database.h" -#include "common/internal_types.h" - - -// Logging mode -// extern peloton::LoggingType peloton_logging_mode; - -// disable unused const variable warning for clang -#ifdef __APPLE__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wunused-const-variable" -#endif - -namespace peloton { -namespace benchmark { -namespace tpcc { - -///////////////////////////////////////////////////////// -// Constants -///////////////////////////////////////////////////////// - -const size_t name_length = 32; -const size_t middle_name_length = 2; -const size_t data_length = 64; -const size_t state_length = 16; -const size_t zip_length = 9; -const size_t street_length = 32; -const size_t city_length = 32; -const size_t credit_length = 2; -const size_t phone_length = 32; -const size_t dist_length = 32; - -double item_min_price = 1.0; -double item_max_price = 100.0; - -double warehouse_name_length = 16; -double warehouse_min_tax = 0.0; -double warehouse_max_tax = 0.2; -double warehouse_initial_ytd = 300000.00f; - -double district_name_length = 16; -double district_min_tax = 0.0; -double district_max_tax = 0.2; -double district_initial_ytd = 30000.00f; - -std::string customers_good_credit = "GC"; -std::string customers_bad_credit = "BC"; -double customers_bad_credit_ratio = 0.1; -double customers_init_credit_lim = 50000.0; -double customers_min_discount = 0; -double customers_max_discount = 0.5; -double customers_init_balance = -10.0; -double customers_init_ytd = 10.0; -int customers_init_payment_cnt = 1; -int customers_init_delivery_cnt = 0; - -double history_init_amount = 10.0; -size_t history_data_length = 32; - -int orders_min_ol_cnt = 5; -int orders_max_ol_cnt = 15; -int orders_init_all_local = 1; -int orders_null_carrier_id = 0; -int orders_min_carrier_id = 1; -int orders_max_carrier_id = 10; - -int new_orders_per_district = 900; // 900 - -int order_line_init_quantity = 5; -int order_line_max_ol_quantity = 10; -double order_line_min_amount = 0.01; -size_t order_line_dist_info_length = 32; - -double stock_original_ratio = 0.1; -int stock_min_quantity = 10; -int stock_max_quantity = 100; -int stock_dist_count = 10; - -double payment_min_amount = 1.0; -double payment_max_amount = 5000.0; - -int stock_min_threshold = 10; -int stock_max_threshold = 20; - -double new_order_remote_txns = 0.01; - -const int syllable_count = 10; -const char *syllables[syllable_count] = {"BAR", "OUGHT", "ABLE", "PRI", - "PRES", "ESES", "ANTI", "CALLY", - "ATION", "EING"}; - -const std::string data_constant = std::string("FOO"); - -NURandConstant nu_rand_const; - -///////////////////////////////////////////////////////// -// Create the tables -///////////////////////////////////////////////////////// - -storage::Database *tpcc_database; -storage::DataTable *warehouse_table; -storage::DataTable *district_table; -storage::DataTable *item_table; -storage::DataTable *customer_table; -storage::DataTable *history_table; -storage::DataTable *stock_table; -storage::DataTable *orders_table; -storage::DataTable *new_order_table; -storage::DataTable *order_line_table; - -const bool own_schema = true; -const bool adapt_table = false; -const bool is_inlined = false; -const bool unique_index = false; -const bool allocate = true; - -void CreateWarehouseTable() { - /* - CREATE TABLE WAREHOUSE ( - W_ID SMALLINT DEFAULT '0' NOT NULL, - W_NAME VARCHAR(16) DEFAULT NULL, - W_STREET_1 VARCHAR(32) DEFAULT NULL, - W_STREET_2 VARCHAR(32) DEFAULT NULL, - W_CITY VARCHAR(32) DEFAULT NULL, - W_STATE VARCHAR(2) DEFAULT NULL, - W_ZIP VARCHAR(9) DEFAULT NULL, - W_TAX FLOAT DEFAULT NULL, - W_YTD FLOAT DEFAULT NULL, - CONSTRAINT W_PK_ARRAY PRIMARY KEY (W_ID) - ); - */ - - // Create schema first - std::vector warehouse_columns; - - auto w_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "W_ID", is_inlined); - warehouse_columns.push_back(w_id_column); - auto w_name_column = catalog::Column( - type::TypeId::VARCHAR, warehouse_name_length, "W_NAME", is_inlined); - warehouse_columns.push_back(w_name_column); - auto w_street_1_column = catalog::Column(type::TypeId::VARCHAR, street_length, - "W_STREET_1", is_inlined); - warehouse_columns.push_back(w_street_1_column); - auto w_street_2_column = catalog::Column(type::TypeId::VARCHAR, street_length, - "W_STREET_2", is_inlined); - warehouse_columns.push_back(w_street_2_column); - auto w_city_column = - catalog::Column(type::TypeId::VARCHAR, city_length, "W_CITY", is_inlined); - warehouse_columns.push_back(w_city_column); - auto w_state_column = - catalog::Column(type::TypeId::VARCHAR, state_length, "W_STATE", is_inlined); - warehouse_columns.push_back(w_state_column); - auto w_zip_column = - catalog::Column(type::TypeId::VARCHAR, zip_length, "W_ZIP", is_inlined); - warehouse_columns.push_back(w_zip_column); - auto w_tax_column = catalog::Column( - type::TypeId::DECIMAL, type::Type::GetTypeSize(type::TypeId::DECIMAL), "W_TAX", is_inlined); - warehouse_columns.push_back(w_tax_column); - auto w_ytd_column = catalog::Column( - type::TypeId::DECIMAL, type::Type::GetTypeSize(type::TypeId::DECIMAL), "W_YTD", is_inlined); - warehouse_columns.push_back(w_ytd_column); - - catalog::Schema *table_schema = new catalog::Schema(warehouse_columns); - std::string table_name("WAREHOUSE"); - - warehouse_table = storage::TableFactory::GetDataTable( - tpcc_database_oid, warehouse_table_oid, table_schema, table_name, - DEFAULT_TUPLES_PER_TILEGROUP, own_schema, adapt_table); - - tpcc_database->AddTable(warehouse_table); - - // Primary index on W_ID - std::vector key_attrs = {0}; - - auto tuple_schema = warehouse_table->GetSchema(); - catalog::Schema *key_schema = - catalog::Schema::CopySchema(tuple_schema, key_attrs); - key_schema->SetIndexedColumns(key_attrs); - bool unique = true; - - index::IndexMetadata *index_metadata = new index::IndexMetadata( - "warehouse_pkey", warehouse_table_pkey_index_oid, warehouse_table_oid, - tpcc_database_oid, state.index, IndexConstraintType::PRIMARY_KEY, - tuple_schema, key_schema, key_attrs, unique); - - std::shared_ptr pkey_index( - index::IndexFactory::GetIndex(index_metadata)); - - warehouse_table->AddIndex(pkey_index); -} - -void CreateDistrictTable() { - /* - CREATE TABLE DISTRICT ( - D_ID TINYINT DEFAULT '0' NOT NULL, - D_W_ID SMALLINT DEFAULT '0' NOT NULL REFERENCES WAREHOUSE (W_ID), - D_NAME VARCHAR(16) DEFAULT NULL, - D_STREET_1 VARCHAR(32) DEFAULT NULL, - D_STREET_2 VARCHAR(32) DEFAULT NULL, - D_CITY VARCHAR(32) DEFAULT NULL, - D_STATE VARCHAR(2) DEFAULT NULL, - D_ZIP VARCHAR(9) DEFAULT NULL, - D_TAX FLOAT DEFAULT NULL, - D_YTD FLOAT DEFAULT NULL, - D_NEXT_O_ID INT DEFAULT NULL, - PRIMARY KEY (D_W_ID,D_ID) - ); - */ - - // Create schema first - std::vector district_columns; - - auto d_id_column = catalog::Column( - type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "D_ID", is_inlined); - district_columns.push_back(d_id_column); - auto d_w_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "D_W_ID", is_inlined); - district_columns.push_back(d_w_id_column); - auto d_name_column = catalog::Column(type::TypeId::VARCHAR, district_name_length, - "D_NAME", is_inlined); - district_columns.push_back(d_name_column); - auto d_street_1_column = catalog::Column(type::TypeId::VARCHAR, street_length, - "D_STREET_1", is_inlined); - district_columns.push_back(d_street_1_column); - auto d_street_2_column = catalog::Column(type::TypeId::VARCHAR, street_length, - "D_STREET_2", is_inlined); - district_columns.push_back(d_street_2_column); - auto d_city_column = - catalog::Column(type::TypeId::VARCHAR, city_length, "D_CITY", is_inlined); - district_columns.push_back(d_city_column); - auto d_state_column = - catalog::Column(type::TypeId::VARCHAR, state_length, "D_STATE", is_inlined); - district_columns.push_back(d_state_column); - auto d_zip_column = - catalog::Column(type::TypeId::VARCHAR, zip_length, "D_ZIP", is_inlined); - district_columns.push_back(d_zip_column); - auto d_tax_column = catalog::Column( - type::TypeId::DECIMAL, type::Type::GetTypeSize(type::TypeId::DECIMAL), "D_TAX", is_inlined); - district_columns.push_back(d_tax_column); - auto d_ytd_column = catalog::Column( - type::TypeId::DECIMAL, type::Type::GetTypeSize(type::TypeId::DECIMAL), "D_YTD", is_inlined); - district_columns.push_back(d_ytd_column); - auto d_next_o_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "D_NEXT_O_ID", is_inlined); - district_columns.push_back(d_next_o_id_column); - - catalog::Schema *table_schema = new catalog::Schema(district_columns); - std::string table_name("DISTRICT"); - - district_table = storage::TableFactory::GetDataTable( - tpcc_database_oid, district_table_oid, table_schema, table_name, - DEFAULT_TUPLES_PER_TILEGROUP, own_schema, adapt_table); - - tpcc_database->AddTable(district_table); - - // Primary index on D_ID, D_W_ID - std::vector key_attrs = {0, 1}; - - auto tuple_schema = district_table->GetSchema(); - catalog::Schema *key_schema = - catalog::Schema::CopySchema(tuple_schema, key_attrs); - key_schema->SetIndexedColumns(key_attrs); - bool unique = true; - - index::IndexMetadata *index_metadata = new index::IndexMetadata( - "district_pkey", district_table_pkey_index_oid, - district_table_pkey_index_oid, district_table_oid, state.index, - IndexConstraintType::PRIMARY_KEY, tuple_schema, key_schema, key_attrs, - unique); - - std::shared_ptr pkey_index( - index::IndexFactory::GetIndex(index_metadata)); - - district_table->AddIndex(pkey_index); -} - -void CreateItemTable() { - /* - CREATE TABLE ITEM ( - I_ID INTEGER DEFAULT '0' NOT NULL, - I_IM_ID INTEGER DEFAULT NULL, - I_NAME VARCHAR(32) DEFAULT NULL, - I_PRICE FLOAT DEFAULT NULL, - I_DATA VARCHAR(64) DEFAULT NULL, - CONSTRAINT I_PK_ARRAY PRIMARY KEY (I_ID) - ); - */ - - // Create schema first - std::vector item_columns; - - auto i_id_column = catalog::Column( - type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "I_ID", is_inlined); - item_columns.push_back(i_id_column); - auto i_im_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "I_IM_ID", is_inlined); - item_columns.push_back(i_im_id_column); - auto i_name_column = - catalog::Column(type::TypeId::VARCHAR, name_length, "I_NAME", is_inlined); - item_columns.push_back(i_name_column); - auto i_price_column = catalog::Column( - type::TypeId::DECIMAL, type::Type::GetTypeSize(type::TypeId::DECIMAL), "I_PRICE", is_inlined); - item_columns.push_back(i_price_column); - auto i_data_column = - catalog::Column(type::TypeId::VARCHAR, data_length, "I_DATA", is_inlined); - item_columns.push_back(i_data_column); - - catalog::Schema *table_schema = new catalog::Schema(item_columns); - std::string table_name("ITEM"); - - item_table = storage::TableFactory::GetDataTable( - tpcc_database_oid, item_table_oid, table_schema, table_name, - DEFAULT_TUPLES_PER_TILEGROUP, own_schema, adapt_table); - - tpcc_database->AddTable(item_table); - - // Primary index on I_ID - std::vector key_attrs = {0}; - - auto tuple_schema = item_table->GetSchema(); - catalog::Schema *key_schema = - catalog::Schema::CopySchema(tuple_schema, key_attrs); - key_schema->SetIndexedColumns(key_attrs); - bool unique = true; - - index::IndexMetadata *index_metadata = new index::IndexMetadata( - "item_pkey", item_table_pkey_index_oid, item_table_oid, tpcc_database_oid, - state.index, IndexConstraintType::PRIMARY_KEY, tuple_schema, - key_schema, key_attrs, unique); - - std::shared_ptr pkey_index( - index::IndexFactory::GetIndex(index_metadata)); - item_table->AddIndex(pkey_index); -} - -void CreateCustomerTable() { - /* - CREATE TABLE CUSTOMER ( - C_ID INTEGER DEFAULT '0' NOT NULL, - C_D_ID TINYINT DEFAULT '0' NOT NULL, - C_W_ID SMALLINT DEFAULT '0' NOT NULL, - C_FIRST VARCHAR(32) DEFAULT NULL, - C_MIDDLE VARCHAR(2) DEFAULT NULL, - C_LAST VARCHAR(32) DEFAULT NULL, - C_STREET_1 VARCHAR(32) DEFAULT NULL, - C_STREET_2 VARCHAR(32) DEFAULT NULL, - C_CITY VARCHAR(32) DEFAULT NULL, - C_STATE VARCHAR(2) DEFAULT NULL, - C_ZIP VARCHAR(9) DEFAULT NULL, - C_PHONE VARCHAR(32) DEFAULT NULL, - C_SINCE TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, - C_CREDIT VARCHAR(2) DEFAULT NULL, - C_CREDIT_LIM FLOAT DEFAULT NULL, - C_DISCOUNT FLOAT DEFAULT NULL, - C_BALANCE FLOAT DEFAULT NULL, - C_YTD_PAYMENT FLOAT DEFAULT NULL, - C_PAYMENT_CNT INTEGER DEFAULT NULL, - C_DELIVERY_CNT INTEGER DEFAULT NULL, - C_DATA VARCHAR(500), - PRIMARY KEY (C_W_ID,C_D_ID,C_ID), - UNIQUE (C_W_ID,C_D_ID,C_LAST,C_FIRST), - CONSTRAINT C_FKEY_D FOREIGN KEY (C_D_ID, C_W_ID) REFERENCES DISTRICT (D_ID, - D_W_ID) - ); - CREATE INDEX IDX_CUSTOMER ON CUSTOMER (C_W_ID,C_D_ID,C_LAST); - */ - - // Create schema first - std::vector customer_columns; - - auto c_id_column = catalog::Column( - type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "C_ID", is_inlined); - customer_columns.push_back(c_id_column); - auto c_d_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "C_D_ID", is_inlined); - customer_columns.push_back(c_d_id_column); - auto c_w_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "C_W_ID", is_inlined); - customer_columns.push_back(c_w_id_column); - auto c_first_name_column = - catalog::Column(type::TypeId::VARCHAR, name_length, "C_FIRST", is_inlined); - customer_columns.push_back(c_first_name_column); - auto c_middle_name_column = catalog::Column( - type::TypeId::VARCHAR, middle_name_length, "C_MIDDLE", is_inlined); - customer_columns.push_back(c_middle_name_column); - auto c_last_name_column = - catalog::Column(type::TypeId::VARCHAR, name_length, "C_LAST", is_inlined); - customer_columns.push_back(c_last_name_column); - auto c_street_1_column = catalog::Column(type::TypeId::VARCHAR, street_length, - "C_STREET_1", is_inlined); - customer_columns.push_back(c_street_1_column); - auto c_street_2_column = catalog::Column(type::TypeId::VARCHAR, street_length, - "C_STREET_2", is_inlined); - customer_columns.push_back(c_street_2_column); - auto c_city_column = - catalog::Column(type::TypeId::VARCHAR, city_length, "C_CITY", is_inlined); - customer_columns.push_back(c_city_column); - auto c_state_column = - catalog::Column(type::TypeId::VARCHAR, state_length, "C_STATE", is_inlined); - customer_columns.push_back(c_state_column); - auto c_zip_column = - catalog::Column(type::TypeId::VARCHAR, zip_length, "C_ZIP", is_inlined); - customer_columns.push_back(c_zip_column); - auto c_phone_column = - catalog::Column(type::TypeId::VARCHAR, phone_length, "C_PHONE", is_inlined); - customer_columns.push_back(c_phone_column); - auto c_since_column = - catalog::Column(type::TypeId::TIMESTAMP, type::Type::GetTypeSize(type::TypeId::TIMESTAMP), - "C_SINCE", is_inlined); - customer_columns.push_back(c_since_column); - auto c_credit_column = catalog::Column(type::TypeId::VARCHAR, credit_length, - "C_CREDIT", is_inlined); - customer_columns.push_back(c_credit_column); - auto c_credit_lim_column = - catalog::Column(type::TypeId::DECIMAL, type::Type::GetTypeSize(type::TypeId::DECIMAL), - "C_CREDIT_LIM", is_inlined); - customer_columns.push_back(c_credit_lim_column); - auto c_discount_column = - catalog::Column(type::TypeId::DECIMAL, type::Type::GetTypeSize(type::TypeId::DECIMAL), - "C_DISCOUNT", is_inlined); - customer_columns.push_back(c_discount_column); - auto c_balance_column = - catalog::Column(type::TypeId::DECIMAL, type::Type::GetTypeSize(type::TypeId::DECIMAL), - "C_BALANCE", is_inlined); - customer_columns.push_back(c_balance_column); - auto c_ytd_payment_column = - catalog::Column(type::TypeId::DECIMAL, type::Type::GetTypeSize(type::TypeId::DECIMAL), - "C_YTD_PAYMENT", is_inlined); - customer_columns.push_back(c_ytd_payment_column); - auto c_payment_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "C_PAYMENT_CNT", is_inlined); - customer_columns.push_back(c_payment_column); - auto c_delivery_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "C_DELIVERY_CNT", is_inlined); - customer_columns.push_back(c_delivery_column); - auto c_data_column = - catalog::Column(type::TypeId::VARCHAR, data_length, "C_DATA", is_inlined); - customer_columns.push_back(c_data_column); - - catalog::Schema *table_schema = new catalog::Schema(customer_columns); - std::string table_name("CUSTOMER"); - - customer_table = storage::TableFactory::GetDataTable( - tpcc_database_oid, customer_table_oid, table_schema, table_name, - DEFAULT_TUPLES_PER_TILEGROUP, own_schema, adapt_table); - - tpcc_database->AddTable(customer_table); - - auto tuple_schema = customer_table->GetSchema(); - std::vector key_attrs; - catalog::Schema *key_schema = nullptr; - index::IndexMetadata *index_metadata = nullptr; - - // Primary index on C_ID, C_D_ID, C_W_ID - key_attrs = {0, 1, 2}; - key_schema = catalog::Schema::CopySchema(tuple_schema, key_attrs); - key_schema->SetIndexedColumns(key_attrs); - - index_metadata = new index::IndexMetadata( - "customer_pkey", customer_table_pkey_index_oid, customer_table_oid, - tpcc_database_oid, state.index, IndexConstraintType::PRIMARY_KEY, - tuple_schema, key_schema, key_attrs, true); - - std::shared_ptr pkey_index( - index::IndexFactory::GetIndex(index_metadata)); - customer_table->AddIndex(pkey_index); - - // Secondary index on C_W_ID, C_D_ID, C_LAST - key_attrs = {1, 2, 5}; - key_schema = catalog::Schema::CopySchema(tuple_schema, key_attrs); - key_schema->SetIndexedColumns(key_attrs); - - index_metadata = new index::IndexMetadata( - "customer_skey", customer_table_skey_index_oid, customer_table_oid, - tpcc_database_oid, state.index, IndexConstraintType::INVALID, - tuple_schema, key_schema, key_attrs, false); - - std::shared_ptr skey_index( - index::IndexFactory::GetIndex(index_metadata)); - customer_table->AddIndex(skey_index); -} - -void CreateHistoryTable() { - /* - CREATE TABLE HISTORY ( - H_C_ID INTEGER DEFAULT NULL, - H_C_D_ID TINYINT DEFAULT NULL, - H_C_W_ID SMALLINT DEFAULT NULL, - H_D_ID TINYINT DEFAULT NULL, - H_W_ID SMALLINT DEFAULT '0' NOT NULL, - H_DATE TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, - H_AMOUNT FLOAT DEFAULT NULL, - H_DATA VARCHAR(32) DEFAULT NULL, - CONSTRAINT H_FKEY_C FOREIGN KEY (H_C_ID, H_C_D_ID, H_C_W_ID) REFERENCES - CUSTOMER (C_ID, C_D_ID, C_W_ID), - CONSTRAINT H_FKEY_D FOREIGN KEY (H_D_ID, H_W_ID) REFERENCES DISTRICT (D_ID, - D_W_ID) - ); - */ - - // Create schema first - std::vector history_columns; - - auto h_c_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "H_C_ID", is_inlined); - history_columns.push_back(h_c_id_column); - auto h_c_d_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "H_C_D_ID", is_inlined); - history_columns.push_back(h_c_d_id_column); - auto h_c_w_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "H_C_W_ID", is_inlined); - history_columns.push_back(h_c_w_id_column); - auto h_d_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "H_D_ID", is_inlined); - history_columns.push_back(h_d_id_column); - auto h_w_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "H_W_ID", is_inlined); - history_columns.push_back(h_w_id_column); - auto h_date_column = - catalog::Column(type::TypeId::TIMESTAMP, type::Type::GetTypeSize(type::TypeId::TIMESTAMP), - "H_DATE", is_inlined); - history_columns.push_back(h_date_column); - auto h_amount_column = - catalog::Column(type::TypeId::DECIMAL, type::Type::GetTypeSize(type::TypeId::DECIMAL), - "H_AMOUNT", is_inlined); - history_columns.push_back(h_amount_column); - auto h_data_column = catalog::Column(type::TypeId::VARCHAR, history_data_length, - "H_DATA", is_inlined); - history_columns.push_back(h_data_column); - - catalog::Schema *table_schema = new catalog::Schema(history_columns); - std::string table_name("HISTORY"); - - history_table = storage::TableFactory::GetDataTable( - tpcc_database_oid, history_table_oid, table_schema, table_name, - DEFAULT_TUPLES_PER_TILEGROUP, own_schema, adapt_table); - - tpcc_database->AddTable(history_table); -} - -void CreateStockTable() { - /* - CREATE TABLE STOCK ( - S_I_ID INTEGER DEFAULT '0' NOT NULL REFERENCES ITEM (I_ID), - S_W_ID SMALLINT DEFAULT '0 ' NOT NULL REFERENCES WAREHOUSE (W_ID), - S_QUANTITY INTEGER DEFAULT '0' NOT NULL, - S_DIST_01 VARCHAR(32) DEFAULT NULL, - S_DIST_02 VARCHAR(32) DEFAULT NULL, - S_DIST_03 VARCHAR(32) DEFAULT NULL, - S_DIST_04 VARCHAR(32) DEFAULT NULL, - S_DIST_05 VARCHAR(32) DEFAULT NULL, - S_DIST_06 VARCHAR(32) DEFAULT NULL, - S_DIST_07 VARCHAR(32) DEFAULT NULL, - S_DIST_08 VARCHAR(32) DEFAULT NULL, - S_DIST_09 VARCHAR(32) DEFAULT NULL, - S_DIST_10 VARCHAR(32) DEFAULT NULL, - S_YTD INTEGER DEFAULT NULL, - S_ORDER_CNT INTEGER DEFAULT NULL, - S_REMOTE_CNT INTEGER DEFAULT NULL, - S_DATA VARCHAR(64) DEFAULT NULL, - PRIMARY KEY (S_W_ID,S_I_ID) - ); - */ - - // Create schema first - std::vector stock_columns; - - auto s_i_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "S_I_ID", is_inlined); - stock_columns.push_back(s_i_id_column); - auto s_w_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "S_W_ID", is_inlined); - stock_columns.push_back(s_w_id_column); - auto s_quantity_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "S_QUANTITY", is_inlined); - stock_columns.push_back(s_quantity_column); - auto s_dist_01_column = - catalog::Column(type::TypeId::VARCHAR, dist_length, "S_DIST_01", is_inlined); - stock_columns.push_back(s_dist_01_column); - auto s_dist_02_column = - catalog::Column(type::TypeId::VARCHAR, dist_length, "S_DIST_02", is_inlined); - stock_columns.push_back(s_dist_02_column); - auto s_dist_03_column = - catalog::Column(type::TypeId::VARCHAR, dist_length, "S_DIST_03", is_inlined); - stock_columns.push_back(s_dist_03_column); - auto s_dist_04_column = - catalog::Column(type::TypeId::VARCHAR, dist_length, "S_DIST_04", is_inlined); - stock_columns.push_back(s_dist_04_column); - auto s_dist_05_column = - catalog::Column(type::TypeId::VARCHAR, dist_length, "S_DIST_05", is_inlined); - stock_columns.push_back(s_dist_05_column); - auto s_dist_06_column = - catalog::Column(type::TypeId::VARCHAR, dist_length, "S_DIST_06", is_inlined); - stock_columns.push_back(s_dist_06_column); - auto s_dist_07_column = - catalog::Column(type::TypeId::VARCHAR, dist_length, "S_DIST_07", is_inlined); - stock_columns.push_back(s_dist_07_column); - auto s_dist_08_column = - catalog::Column(type::TypeId::VARCHAR, dist_length, "S_DIST_08", is_inlined); - stock_columns.push_back(s_dist_08_column); - auto s_dist_09_column = - catalog::Column(type::TypeId::VARCHAR, dist_length, "S_DIST_09", is_inlined); - stock_columns.push_back(s_dist_09_column); - auto s_dist_10_column = - catalog::Column(type::TypeId::VARCHAR, dist_length, "S_DIST_10", is_inlined); - stock_columns.push_back(s_dist_10_column); - auto s_ytd_column = catalog::Column( - type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "S_YTD", is_inlined); - stock_columns.push_back(s_ytd_column); - auto s_order_cnt_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "S_ORDER_CNT", is_inlined); - stock_columns.push_back(s_order_cnt_column); - auto s_discount_cnt_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "S_REMOTE_CNT", is_inlined); - stock_columns.push_back(s_discount_cnt_column); - auto s_data_column = - catalog::Column(type::TypeId::VARCHAR, data_length, "S_DATA", is_inlined); - stock_columns.push_back(s_data_column); - - catalog::Schema *table_schema = new catalog::Schema(stock_columns); - std::string table_name("STOCK"); - - stock_table = storage::TableFactory::GetDataTable( - tpcc_database_oid, stock_table_oid, table_schema, table_name, - DEFAULT_TUPLES_PER_TILEGROUP, own_schema, adapt_table); - - tpcc_database->AddTable(stock_table); - - // Primary index on S_I_ID, S_W_ID - std::vector key_attrs = {0, 1}; - - auto tuple_schema = stock_table->GetSchema(); - catalog::Schema *key_schema = - catalog::Schema::CopySchema(tuple_schema, key_attrs); - key_schema->SetIndexedColumns(key_attrs); - bool unique = true; - - index::IndexMetadata *index_metadata = new index::IndexMetadata( - "stock_pkey", stock_table_pkey_index_oid, stock_table_oid, - tpcc_database_oid, state.index, IndexConstraintType::PRIMARY_KEY, - tuple_schema, key_schema, key_attrs, unique); - - std::shared_ptr pkey_index( - index::IndexFactory::GetIndex(index_metadata)); - stock_table->AddIndex(pkey_index); -} - -void CreateOrdersTable() { - /* - CREATE TABLE ORDERS ( - O_ID INTEGER DEFAULT '0' NOT NULL, - O_C_ID INTEGER DEFAULT NULL, - O_D_ID TINYINT DEFAULT '0' NOT NULL, - O_W_ID SMALLINT DEFAULT '0' NOT NULL, - O_ENTRY_D TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, - O_CARRIER_ID INTEGER DEFAULT NULL, - O_OL_CNT INTEGER DEFAULT NULL, - O_ALL_LOCAL INTEGER DEFAULT NULL, - PRIMARY KEY (O_W_ID,O_D_ID,O_ID), - UNIQUE (O_W_ID,O_D_ID,O_C_ID,O_ID), - CONSTRAINT O_FKEY_C FOREIGN KEY (O_C_ID, O_D_ID, O_W_ID) REFERENCES CUSTOMER - (C_ID, C_D_ID, C_W_ID) - ); - CREATE INDEX IDX_ORDERS ON ORDERS (O_W_ID,O_D_ID,O_C_ID); - */ - - // Create schema first - std::vector orders_columns; - - auto o_id_column = catalog::Column( - type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "O_ID", is_inlined); - orders_columns.push_back(o_id_column); - auto o_c_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "O_C_ID", is_inlined); - orders_columns.push_back(o_c_id_column); - auto o_d_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "O_D_ID", is_inlined); - orders_columns.push_back(o_d_id_column); - auto o_w_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "O_W_ID", is_inlined); - orders_columns.push_back(o_w_id_column); - auto o_entry_d_column = - catalog::Column(type::TypeId::TIMESTAMP, type::Type::GetTypeSize(type::TypeId::TIMESTAMP), - "O_ENTRY_D", is_inlined); - orders_columns.push_back(o_entry_d_column); - auto o_carrier_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "O_CARRIER_ID", is_inlined); - orders_columns.push_back(o_carrier_id_column); - auto o_ol_cnt_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "O_OL_CNT", is_inlined); - orders_columns.push_back(o_ol_cnt_column); - auto o_all_local_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "O_ALL_LOCAL", is_inlined); - orders_columns.push_back(o_all_local_column); - - catalog::Schema *table_schema = new catalog::Schema(orders_columns); - std::string table_name("ORDERS"); - - orders_table = storage::TableFactory::GetDataTable( - tpcc_database_oid, orders_table_oid, table_schema, table_name, - DEFAULT_TUPLES_PER_TILEGROUP, own_schema, adapt_table); - - tpcc_database->AddTable(orders_table); - - auto tuple_schema = customer_table->GetSchema(); - std::vector key_attrs; - catalog::Schema *key_schema = nullptr; - index::IndexMetadata *index_metadata = nullptr; - - // Primary index on O_ID, O_D_ID, O_W_ID - key_attrs = {0, 2, 3}; - key_schema = catalog::Schema::CopySchema(tuple_schema, key_attrs); - key_schema->SetIndexedColumns(key_attrs); - - index_metadata = new index::IndexMetadata( - "orders_pkey", orders_table_pkey_index_oid, orders_table_oid, - tpcc_database_oid, state.index, IndexConstraintType::PRIMARY_KEY, - tuple_schema, key_schema, key_attrs, true); - - - std::shared_ptr pkey_index( - index::IndexFactory::GetIndex(index_metadata)); - orders_table->AddIndex(pkey_index); - - // Secondary index on O_C_ID, O_D_ID, O_W_ID - key_attrs = {1, 2, 3}; - key_schema = catalog::Schema::CopySchema(tuple_schema, key_attrs); - key_schema->SetIndexedColumns(key_attrs); - - index_metadata = new index::IndexMetadata( - "orders_skey", orders_table_skey_index_oid, orders_table_oid, - tpcc_database_oid, state.index, IndexConstraintType::INVALID, - tuple_schema, key_schema, key_attrs, false); - - std::shared_ptr skey_index( - index::IndexFactory::GetIndex(index_metadata)); - orders_table->AddIndex(skey_index); -} - -void CreateNewOrderTable() { - /* - CREATE TABLE NEW_ORDER ( - NO_O_ID INTEGER DEFAULT '0' NOT NULL, - NO_D_ID TINYINT DEFAULT '0' NOT NULL, - NO_W_ID SMALLINT DEFAULT '0' NOT NULL, - CONSTRAINT NO_PK_TREE PRIMARY KEY (NO_D_ID,NO_W_ID,NO_O_ID), - CONSTRAINT NO_FKEY_O FOREIGN KEY (NO_O_ID, NO_D_ID, NO_W_ID) REFERENCES - ORDERS (O_ID, O_D_ID, O_W_ID) - ); - */ - - // Create schema first - std::vector new_order_columns; - - auto no_o_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "NO_O_ID", is_inlined); - new_order_columns.push_back(no_o_id_column); - auto no_d_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "NO_D_ID", is_inlined); - new_order_columns.push_back(no_d_id_column); - auto no_w_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "NO_W_ID", is_inlined); - new_order_columns.push_back(no_w_id_column); - - catalog::Schema *table_schema = new catalog::Schema(new_order_columns); - std::string table_name("NEW_ORDER"); - - new_order_table = storage::TableFactory::GetDataTable( - tpcc_database_oid, new_order_table_oid, table_schema, table_name, - DEFAULT_TUPLES_PER_TILEGROUP, own_schema, adapt_table); - - tpcc_database->AddTable(new_order_table); - - // Primary index on NO_O_ID, NO_D_ID, NO_W_ID - std::vector key_attrs = {0, 1, 2}; - - auto tuple_schema = new_order_table->GetSchema(); - catalog::Schema *key_schema = - catalog::Schema::CopySchema(tuple_schema, key_attrs); - key_schema->SetIndexedColumns(key_attrs); - bool unique = true; - - index::IndexMetadata *index_metadata = new index::IndexMetadata( - "new_order_pkey", new_order_table_pkey_index_oid, new_order_table_oid, - tpcc_database_oid, state.index, IndexConstraintType::PRIMARY_KEY, - tuple_schema, key_schema, key_attrs, unique); - - std::shared_ptr pkey_index( - index::IndexFactory::GetIndex(index_metadata)); - new_order_table->AddIndex(pkey_index); -} - -void CreateOrderLineTable() { - /* - CREATE TABLE ORDER_LINE ( - OL_O_ID INTEGER DEFAULT '0' NOT NULL, - OL_D_ID TINYINT DEFAULT '0' NOT NULL, - OL_W_ID SMALLINT DEFAULT '0' NOT NULL, - OL_NUMBER INTEGER DEFAULT '0' NOT NULL, - OL_I_ID INTEGER DEFAULT NULL, - OL_SUPPLY_W_ID SMALLINT DEFAULT NULL, - OL_DELIVERY_D TIMESTAMP DEFAULT NULL, - OL_QUANTITY INTEGER DEFAULT NULL, - OL_AMOUNT FLOAT DEFAULT NULL, - OL_DIST_INFO VARCHAR(32) DEFAULT NULL, - PRIMARY KEY (OL_W_ID,OL_D_ID,OL_O_ID,OL_NUMBER), - CONSTRAINT OL_FKEY_O FOREIGN KEY (OL_O_ID, OL_D_ID, OL_W_ID) REFERENCES - ORDERS (O_ID, O_D_ID, O_W_ID), - CONSTRAINT OL_FKEY_S FOREIGN KEY (OL_I_ID, OL_SUPPLY_W_ID) REFERENCES STOCK - (S_I_ID, S_W_ID) - ); - CREATE INDEX IDX_ORDER_LINE_TREE ON ORDER_LINE (OL_W_ID,OL_D_ID,OL_O_ID); - */ - - // Create schema first - std::vector order_line_columns; - - auto ol_o_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "OL_O_ID", is_inlined); - order_line_columns.push_back(ol_o_id_column); - auto ol_d_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "OL_D_ID", is_inlined); - order_line_columns.push_back(ol_d_id_column); - auto ol_w_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "OL_W_ID", is_inlined); - order_line_columns.push_back(ol_w_id_column); - auto ol_number_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "OL_NUMBER", is_inlined); - order_line_columns.push_back(ol_number_column); - auto ol_i_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "OL_I_ID", is_inlined); - order_line_columns.push_back(ol_i_id_column); - auto ol_supply_w_id_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "OL_SUPPLY_W_ID", is_inlined); - order_line_columns.push_back(ol_supply_w_id_column); - auto ol_delivery_d_column = - catalog::Column(type::TypeId::TIMESTAMP, type::Type::GetTypeSize(type::TypeId::TIMESTAMP), - "OL_DELIVERY_D", is_inlined); - order_line_columns.push_back(ol_delivery_d_column); - auto ol_quantity_column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "OL_QUANTITY", is_inlined); - order_line_columns.push_back(ol_quantity_column); - auto ol_amount_column = - catalog::Column(type::TypeId::DECIMAL, type::Type::GetTypeSize(type::TypeId::DECIMAL), - "OL_AMOUNT", is_inlined); - order_line_columns.push_back(ol_amount_column); - auto ol_dist_info_column = - catalog::Column(type::TypeId::VARCHAR, order_line_dist_info_length, - "OL_DIST_INFO", is_inlined); - order_line_columns.push_back(ol_dist_info_column); - - catalog::Schema *table_schema = new catalog::Schema(order_line_columns); - std::string table_name("ORDER_LINE"); - - order_line_table = storage::TableFactory::GetDataTable( - tpcc_database_oid, order_line_table_oid, table_schema, table_name, - DEFAULT_TUPLES_PER_TILEGROUP, own_schema, adapt_table); - - tpcc_database->AddTable(order_line_table); - - auto tuple_schema = order_line_table->GetSchema(); - std::vector key_attrs; - catalog::Schema *key_schema = nullptr; - index::IndexMetadata *index_metadata = nullptr; - - // Primary index on OL_O_ID, OL_D_ID, OL_W_ID, OL_NUMBER - key_attrs = {0, 1, 2, 3}; - key_schema = catalog::Schema::CopySchema(tuple_schema, key_attrs); - key_schema->SetIndexedColumns(key_attrs); - - index_metadata = new index::IndexMetadata( - "order_line_pkey", order_line_table_pkey_index_oid, order_line_table_oid, - tpcc_database_oid, state.index, IndexConstraintType::PRIMARY_KEY, - tuple_schema, key_schema, key_attrs, true); - - - std::shared_ptr pkey_index( - index::IndexFactory::GetIndex(index_metadata)); - order_line_table->AddIndex(pkey_index); - - // Secondary index on OL_O_ID, OL_D_ID, OL_W_ID - key_attrs = {0, 1, 2}; - key_schema = catalog::Schema::CopySchema(tuple_schema, key_attrs); - key_schema->SetIndexedColumns(key_attrs); - - index_metadata = new index::IndexMetadata( - "order_line_skey", order_line_table_skey_index_oid, order_line_table_oid, - tpcc_database_oid, state.index, IndexConstraintType::INVALID, - tuple_schema, key_schema, key_attrs, false); - - std::shared_ptr skey_index( - index::IndexFactory::GetIndex(index_metadata)); - order_line_table->AddIndex(skey_index); -} - -void CreateTPCCDatabase() { - // Clean up - delete tpcc_database; - tpcc_database = nullptr; - warehouse_table = nullptr; - district_table = nullptr; - item_table = nullptr; - customer_table = nullptr; - history_table = nullptr; - stock_table = nullptr; - orders_table = nullptr; - new_order_table = nullptr; - order_line_table = nullptr; - - auto catalog = catalog::Catalog::GetInstance(); - tpcc_database = new storage::Database(tpcc_database_oid); - catalog->AddDatabase(tpcc_database); - - CreateWarehouseTable(); - CreateDistrictTable(); - CreateItemTable(); - CreateCustomerTable(); - CreateHistoryTable(); - CreateStockTable(); - CreateOrdersTable(); - CreateNewOrderTable(); - CreateOrderLineTable(); -} - -///////////////////////////////////////////////////////// -// Load in the tables -///////////////////////////////////////////////////////// - -std::random_device rd; -std::mt19937 rng(rd()); - -// Create random NURand constants, appropriate for loading the database. -NURandConstant::NURandConstant() { - c_last = GetRandomInteger(0, 255); - c_id = GetRandomInteger(0, 1023); - order_line_itme_id = GetRandomInteger(0, 8191); -} - -// A non-uniform random number, as defined by TPC-C 2.1.6. (page 20). -int GetNURand(int a, int x, int y) { - PELOTON_ASSERT(x <= y); - int c = nu_rand_const.c_last; - - if (a == 255) { - c = nu_rand_const.c_last; - } else if (a == 1023) { - c = nu_rand_const.c_id; - } else if (a == 8191) { - c = nu_rand_const.order_line_itme_id; - } else { - PELOTON_ASSERT(false); - } - - return (((GetRandomInteger(0, a) | GetRandomInteger(x, y)) + c) % - (y - x + 1)) + - x; -} - -// A last name as defined by TPC-C 4.3.2.3. Not actually random. -std::string GetLastName(int number) { - PELOTON_ASSERT(number >= 0 && number <= 999); - - int idx1 = number / 100; - int idx2 = (number / 10 % 10); - int idx3 = number % 10; - - char lastname_cstr[name_length]; - std::strcpy(lastname_cstr, syllables[idx1]); - std::strcat(lastname_cstr, syllables[idx2]); - std::strcat(lastname_cstr, syllables[idx3]); - - return std::string(lastname_cstr, name_length); -} - -// A non-uniform random last name, as defined by TPC-C 4.3.2.3. -// The name will be limited to maxCID -std::string GetRandomLastName(int max_cid) { - int min_cid = 999; - if (max_cid - 1 < min_cid) { - min_cid = max_cid - 1; - } - - return GetLastName(GetNURand(255, 0, min_cid)); -} - -std::string GetRandomAlphaNumericString(const size_t string_length) { - const char alphanumeric[] = - "0123456789" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz"; - - std::uniform_int_distribution<> dist(0, sizeof(alphanumeric) - 1); - - char repeated_char = alphanumeric[dist(rng)]; - std::string sample(string_length, repeated_char); - return sample; -} - -bool GetRandomBoolean(double ratio) { - double sample = (double)rand() / RAND_MAX; - return (sample < ratio) ? true : false; -} - -int GetRandomInteger(const int lower_bound, const int upper_bound) { - std::uniform_int_distribution<> dist(lower_bound, upper_bound); - - int sample = dist(rng); - return sample; -} - -int GetRandomIntegerExcluding(const int lower_bound, const int upper_bound, - const int exclude_sample) { - int sample; - if (lower_bound == upper_bound) return lower_bound; - - while (1) { - sample = GetRandomInteger(lower_bound, upper_bound); - if (sample != exclude_sample) break; - } - return sample; -} - -double GetRandomDouble(const double lower_bound, const double upper_bound) { - std::uniform_real_distribution<> dist(lower_bound, upper_bound); - - double sample = dist(rng); - return sample; -} - -double GetRandomFixedPoint(int decimal_places, double minimum, double maximum) { - PELOTON_ASSERT(decimal_places > 0); - PELOTON_ASSERT(minimum < maximum); - - int multiplier = 1; - for (int i = 0; i < decimal_places; ++i) { - multiplier *= 10; - } - - int int_min = (int)(minimum * multiplier + 0.5); - int int_max = (int)(maximum * multiplier + 0.5); - - return GetRandomDouble(int_min, int_max) / (double)(multiplier); -} - -std::string GetStreetName() { - std::vector street_names = { - "5835 Alderson St", "117 Ettwein St", "1400 Fairstead Ln", - "1501 Denniston St", "898 Flemington St", "2325 Eldridge St", - "924 Lilac St", "4299 Minnesota St", "5498 Northumberland St", - "5534 Phillips Ave"}; - - std::uniform_int_distribution<> dist(0, street_names.size() - 1); - return street_names[dist(rng)]; -} - -std::string GetZipCode() { - std::vector zip_codes = {"15215", "14155", "80284", "61845", - "23146", "21456", "12345", "21561", - "87752", "91095"}; - - std::uniform_int_distribution<> dist(0, zip_codes.size() - 1); - return zip_codes[dist(rng)]; -} - -std::string GetCityName() { - std::vector city_names = { - "Madison", "Pittsburgh", "New York", "Seattle", "San Francisco", - "Berkeley", "Palo Alto", "Los Angeles", "Boston", "Redwood Shores"}; - - std::uniform_int_distribution<> dist(0, city_names.size() - 1); - return city_names[dist(rng)]; -} - -std::string GetStateName() { - std::vector state_names = {"WI", "PA", "NY", "WA", "CA", "MA"}; - - std::uniform_int_distribution<> dist(0, state_names.size() - 1); - return state_names[dist(rng)]; -} - -int GetTimeStamp() { - auto time_stamp = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()).count(); - return time_stamp; -} - -std::unique_ptr BuildItemTuple( - const int item_id, const std::unique_ptr &pool) { - auto item_table_schema = item_table->GetSchema(); - std::unique_ptr item_tuple( - new storage::Tuple(item_table_schema, allocate)); - - // I_ID - item_tuple->SetValue(0, type::ValueFactory::GetIntegerValue(item_id), nullptr); - // I_IM_ID - item_tuple->SetValue(1, type::ValueFactory::GetIntegerValue(item_id * 10), nullptr); - // I_NAME - auto i_name = GetRandomAlphaNumericString(name_length); - item_tuple->SetValue(2, type::ValueFactory::GetVarcharValue(i_name), pool.get()); - // I_PRICE - double i_price = GetRandomDouble(item_min_price, item_max_price); - item_tuple->SetValue(3, type::ValueFactory::GetDecimalValue(i_price), nullptr); - // I_DATA - auto i_data = GetRandomAlphaNumericString(data_length); - item_tuple->SetValue(4, type::ValueFactory::GetVarcharValue(i_data), pool.get()); - - return item_tuple; -} - -std::unique_ptr BuildWarehouseTuple( - const int warehouse_id, const std::unique_ptr &pool) { - auto warehouse_table_schema = warehouse_table->GetSchema(); - std::unique_ptr warehouse_tuple( - new storage::Tuple(warehouse_table_schema, allocate)); - - // W_ID - warehouse_tuple->SetValue(0, type::ValueFactory::GetIntegerValue(warehouse_id), - nullptr); - // W_NAME - auto w_name = GetRandomAlphaNumericString(warehouse_name_length); - warehouse_tuple->SetValue(1, type::ValueFactory::GetVarcharValue(w_name), - pool.get()); - // W_STREET_1, W_STREET_2 - auto w_street = GetStreetName(); - warehouse_tuple->SetValue(2, type::ValueFactory::GetVarcharValue(w_street), - pool.get()); - warehouse_tuple->SetValue(3, type::ValueFactory::GetVarcharValue(w_street), - pool.get()); - // W_CITY - auto w_city = GetCityName(); - warehouse_tuple->SetValue(4, type::ValueFactory::GetVarcharValue(w_city), - pool.get()); - // W_STATE - auto w_state = GetStateName(); - warehouse_tuple->SetValue(5, type::ValueFactory::GetVarcharValue(w_state), - pool.get()); - // W_ZIP - auto w_zip = GetZipCode(); - warehouse_tuple->SetValue(6, type::ValueFactory::GetVarcharValue(w_zip), pool.get()); - // W_TAX - double w_tax = GetRandomDouble(warehouse_min_tax, warehouse_max_tax); - warehouse_tuple->SetValue(7, type::ValueFactory::GetDecimalValue(w_tax), nullptr); - // W_YTD - warehouse_tuple->SetValue( - 8, type::ValueFactory::GetDecimalValue(warehouse_initial_ytd), nullptr); - - return warehouse_tuple; -} - -std::unique_ptr BuildDistrictTuple( - const int district_id, const int warehouse_id, - const std::unique_ptr &pool) { - auto district_table_schema = district_table->GetSchema(); - std::unique_ptr district_tuple( - new storage::Tuple(district_table_schema, allocate)); - - // D_ID - district_tuple->SetValue(0, type::ValueFactory::GetIntegerValue(district_id), - nullptr); - // D_W_ID - district_tuple->SetValue(1, type::ValueFactory::GetIntegerValue(warehouse_id), - nullptr); - // D_NAME - auto d_name = GetRandomAlphaNumericString(district_name_length); - district_tuple->SetValue(2, type::ValueFactory::GetVarcharValue(d_name), pool.get()); - // D_STREET_1, D_STREET_2 - auto d_street = GetStreetName(); - district_tuple->SetValue(3, type::ValueFactory::GetVarcharValue(d_street), - pool.get()); - district_tuple->SetValue(4, type::ValueFactory::GetVarcharValue(d_street), - pool.get()); - // D_CITY - auto d_city = GetCityName(); - district_tuple->SetValue(5, type::ValueFactory::GetVarcharValue(d_city), pool.get()); - // D_STATE - auto d_state = GetStateName(); - district_tuple->SetValue(6, type::ValueFactory::GetVarcharValue(d_state), - pool.get()); - // D_ZIP - auto d_zip = GetZipCode(); - district_tuple->SetValue(7, type::ValueFactory::GetVarcharValue(d_zip), pool.get()); - // D_TAX - double d_tax = GetRandomDouble(district_min_tax, district_max_tax); - district_tuple->SetValue(8, type::ValueFactory::GetDecimalValue(d_tax), nullptr); - // D_YTD - district_tuple->SetValue( - 9, type::ValueFactory::GetDecimalValue(district_initial_ytd), nullptr); - // D_NEXT_O_ID - auto next_o_id = state.customers_per_district + 1; - district_tuple->SetValue(10, type::ValueFactory::GetIntegerValue(next_o_id), - nullptr); - - return district_tuple; -} - -std::unique_ptr BuildCustomerTuple( - const int customer_id, const int district_id, const int warehouse_id, - const std::unique_ptr &pool) { - // Customer id begins from 0 - PELOTON_ASSERT(customer_id >= 0 && customer_id < state.customers_per_district); - - auto customer_table_schema = customer_table->GetSchema(); - std::unique_ptr customer_tuple( - new storage::Tuple(customer_table_schema, allocate)); - - // C_ID - customer_tuple->SetValue(0, type::ValueFactory::GetIntegerValue(customer_id), - nullptr); - // C_D_ID - customer_tuple->SetValue(1, type::ValueFactory::GetIntegerValue(district_id), - nullptr); - // C_W_ID - customer_tuple->SetValue(2, type::ValueFactory::GetIntegerValue(warehouse_id), - nullptr); - // C_FIRST, C_MIDDLE, C_LAST - auto c_first = GetRandomAlphaNumericString(name_length); - - std::string c_last; - - // Here our customer id begins from 0 - if (customer_id <= 999) { - c_last = GetLastName(customer_id); - } else { - c_last = GetRandomLastName(state.customers_per_district); - } - - auto c_middle = GetRandomAlphaNumericString(middle_name_length); - customer_tuple->SetValue(3, type::ValueFactory::GetVarcharValue(c_first), - pool.get()); - customer_tuple->SetValue(4, type::ValueFactory::GetVarcharValue(c_middle), - pool.get()); - customer_tuple->SetValue(5, type::ValueFactory::GetVarcharValue(c_last), pool.get()); - // C_STREET_1, C_STREET_2 - auto c_street = GetStreetName(); - customer_tuple->SetValue(6, type::ValueFactory::GetVarcharValue(c_street), - pool.get()); - customer_tuple->SetValue(7, type::ValueFactory::GetVarcharValue(c_street), - pool.get()); - // C_CITY - auto c_city = GetCityName(); - customer_tuple->SetValue(8, type::ValueFactory::GetVarcharValue(c_city), pool.get()); - // C_STATE - auto c_state = GetStateName(); - customer_tuple->SetValue(9, type::ValueFactory::GetVarcharValue(c_state), - pool.get()); - // C_ZIP - auto c_zip = GetZipCode(); - customer_tuple->SetValue(10, type::ValueFactory::GetVarcharValue(c_zip), pool.get()); - // C_PHONE - auto c_phone = GetRandomAlphaNumericString(phone_length); - customer_tuple->SetValue(11, type::ValueFactory::GetVarcharValue(c_phone), - pool.get()); - // C_SINCE_TIMESTAMP - auto c_since_timestamp = GetTimeStamp(); - customer_tuple->SetValue( - 12, type::ValueFactory::GetTimestampValue(c_since_timestamp), nullptr); - // C_CREDIT - auto c_bad_credit = GetRandomBoolean(customers_bad_credit_ratio); - auto c_credit = c_bad_credit ? customers_bad_credit : customers_good_credit; - customer_tuple->SetValue(13, type::ValueFactory::GetVarcharValue(c_credit), - pool.get()); - // C_CREDIT_LIM - customer_tuple->SetValue( - 14, type::ValueFactory::GetDecimalValue(customers_init_credit_lim), nullptr); - // C_DISCOUNT - double c_discount = - GetRandomDouble(customers_min_discount, customers_max_discount); - customer_tuple->SetValue(15, type::ValueFactory::GetDecimalValue(c_discount), - nullptr); - // C_BALANCE - customer_tuple->SetValue( - 16, type::ValueFactory::GetDecimalValue(customers_init_balance), nullptr); - // C_YTD_PAYMENT - customer_tuple->SetValue(17, type::ValueFactory::GetDecimalValue(customers_init_ytd), - nullptr); - // C_PAYMENT_CNT - customer_tuple->SetValue( - 18, type::ValueFactory::GetDecimalValue(customers_init_payment_cnt), nullptr); - // C_DELIVERY_CNT - customer_tuple->SetValue( - 19, type::ValueFactory::GetDecimalValue(customers_init_delivery_cnt), nullptr); - // C_DATA - auto c_data = GetRandomAlphaNumericString(data_length); - customer_tuple->SetValue(20, type::ValueFactory::GetVarcharValue(c_data), - pool.get()); - - return customer_tuple; -} - -std::unique_ptr BuildHistoryTuple( - const int customer_id, const int district_id, const int warehouse_id, - const int history_district_id, const int history_warehouse_id, - const std::unique_ptr &pool) { - auto history_table_schema = history_table->GetSchema(); - std::unique_ptr history_tuple( - new storage::Tuple(history_table_schema, allocate)); - - // H_C_ID - history_tuple->SetValue(0, type::ValueFactory::GetIntegerValue(customer_id), - nullptr); - // H_C_D_ID - history_tuple->SetValue(1, type::ValueFactory::GetIntegerValue(district_id), - nullptr); - // H_C_W_ID - history_tuple->SetValue(2, type::ValueFactory::GetIntegerValue(warehouse_id), - nullptr); - // H_D_ID - history_tuple->SetValue(3, type::ValueFactory::GetIntegerValue(history_district_id), - nullptr); - // H_W_ID - history_tuple->SetValue( - 4, type::ValueFactory::GetIntegerValue(history_warehouse_id), nullptr); - // H_DATE - auto h_date = GetTimeStamp(); - history_tuple->SetValue(5, type::ValueFactory::GetTimestampValue(h_date), nullptr); - // H_AMOUNT - history_tuple->SetValue(6, type::ValueFactory::GetDecimalValue(history_init_amount), - nullptr); - // H_DATA - auto h_data = GetRandomAlphaNumericString(history_data_length); - history_tuple->SetValue(7, type::ValueFactory::GetVarcharValue(h_data), pool.get()); - - return history_tuple; -} - -std::unique_ptr BuildOrdersTuple(const int orders_id, - const int district_id, - const int warehouse_id, - const bool new_order, - const int o_ol_cnt) { - auto orders_table_schema = orders_table->GetSchema(); - std::unique_ptr orders_tuple( - new storage::Tuple(orders_table_schema, allocate)); - - // O_ID - orders_tuple->SetValue(0, type::ValueFactory::GetIntegerValue(orders_id), nullptr); - // O_C_ID - auto o_c_id = GetRandomInteger(0, state.customers_per_district); - orders_tuple->SetValue(1, type::ValueFactory::GetIntegerValue(o_c_id), nullptr); - // O_D_ID - orders_tuple->SetValue(2, type::ValueFactory::GetIntegerValue(district_id), - nullptr); - // O_W_ID - orders_tuple->SetValue(3, type::ValueFactory::GetIntegerValue(warehouse_id), - nullptr); - // O_ENTRY_D - auto o_entry_d = GetTimeStamp(); - orders_tuple->SetValue(4, type::ValueFactory::GetTimestampValue(o_entry_d), - nullptr); - // O_CARRIER_ID - auto o_carrier_id = orders_null_carrier_id; - if (new_order == false) { - o_carrier_id = - GetRandomInteger(orders_min_carrier_id, orders_max_carrier_id); - } - orders_tuple->SetValue(5, type::ValueFactory::GetIntegerValue(o_carrier_id), - nullptr); - // O_OL_CNT - orders_tuple->SetValue(6, type::ValueFactory::GetIntegerValue(o_ol_cnt), nullptr); - // O_ALL_LOCAL - orders_tuple->SetValue( - 7, type::ValueFactory::GetIntegerValue(orders_init_all_local), nullptr); - - return orders_tuple; -} - -std::unique_ptr BuildNewOrderTuple(const int orders_id, - const int district_id, - const int warehouse_id) { - auto new_order_table_schema = new_order_table->GetSchema(); - std::unique_ptr new_order_tuple( - new storage::Tuple(new_order_table_schema, allocate)); - - // NO_O_ID - new_order_tuple->SetValue(0, type::ValueFactory::GetIntegerValue(orders_id), - nullptr); - // NO_D_ID - new_order_tuple->SetValue(1, type::ValueFactory::GetIntegerValue(district_id), - nullptr); - // NO_W_ID - new_order_tuple->SetValue(2, type::ValueFactory::GetIntegerValue(warehouse_id), - nullptr); - - return new_order_tuple; -} - -std::unique_ptr BuildOrderLineTuple( - const int orders_id, const int district_id, const int warehouse_id, - const int order_line_id, const int ol_supply_w_id, const bool new_order, - const std::unique_ptr &pool) { - auto order_line_table_schema = order_line_table->GetSchema(); - std::unique_ptr order_line_tuple( - new storage::Tuple(order_line_table_schema, allocate)); - - // OL_O_ID - order_line_tuple->SetValue(0, type::ValueFactory::GetIntegerValue(orders_id), - nullptr); - // OL_D_ID - order_line_tuple->SetValue(1, type::ValueFactory::GetIntegerValue(district_id), - nullptr); - // OL_W_ID - order_line_tuple->SetValue(2, type::ValueFactory::GetIntegerValue(warehouse_id), - nullptr); - // OL_NUMBER - order_line_tuple->SetValue(3, type::ValueFactory::GetIntegerValue(order_line_id), - nullptr); - // OL_I_ID - auto ol_i_id = GetRandomInteger(0, state.item_count); - order_line_tuple->SetValue(4, type::ValueFactory::GetIntegerValue(ol_i_id), - nullptr); - // OL_SUPPLY_W_ID - order_line_tuple->SetValue(5, type::ValueFactory::GetIntegerValue(ol_supply_w_id), - nullptr); - // OL_DELIVERY_D - int64_t ol_delivery_d = GetTimeStamp(); - if (new_order == true) { - ol_delivery_d = type::PELOTON_INT64_MIN; - } - order_line_tuple->SetValue(6, type::ValueFactory::GetTimestampValue(ol_delivery_d), - nullptr); - // OL_QUANTITY - order_line_tuple->SetValue( - 7, type::ValueFactory::GetIntegerValue(order_line_init_quantity), nullptr); - // OL_AMOUNT - double ol_amount = 0; - if (new_order == true) { - ol_amount = GetRandomDouble(order_line_min_amount, - order_line_max_ol_quantity * item_max_price); - } - order_line_tuple->SetValue(8, type::ValueFactory::GetDecimalValue(ol_amount), - nullptr); - // OL_DIST_INFO - auto ol_dist_info = GetRandomAlphaNumericString(order_line_dist_info_length); - order_line_tuple->SetValue(9, type::ValueFactory::GetVarcharValue(ol_dist_info), - pool.get()); - - return order_line_tuple; -} - -std::unique_ptr BuildStockTuple( - const int stock_id, const int s_w_id, - const std::unique_ptr &pool) { - auto stock_table_schema = stock_table->GetSchema(); - std::unique_ptr stock_tuple( - new storage::Tuple(stock_table_schema, allocate)); - - // S_I_ID - stock_tuple->SetValue(0, type::ValueFactory::GetIntegerValue(stock_id), nullptr); - // S_W_ID - stock_tuple->SetValue(1, type::ValueFactory::GetIntegerValue(s_w_id), nullptr); - // S_QUANTITY - auto s_quantity = GetRandomInteger(stock_min_quantity, stock_max_quantity); - stock_tuple->SetValue(2, type::ValueFactory::GetIntegerValue(s_quantity), nullptr); - // S_DIST_01 .. S_DIST_10 - auto s_dist = GetRandomAlphaNumericString(name_length); - stock_tuple->SetValue(3, type::ValueFactory::GetVarcharValue(s_dist), pool.get()); - stock_tuple->SetValue(4, type::ValueFactory::GetVarcharValue(s_dist), pool.get()); - stock_tuple->SetValue(5, type::ValueFactory::GetVarcharValue(s_dist), pool.get()); - stock_tuple->SetValue(6, type::ValueFactory::GetVarcharValue(s_dist), pool.get()); - stock_tuple->SetValue(7, type::ValueFactory::GetVarcharValue(s_dist), pool.get()); - stock_tuple->SetValue(8, type::ValueFactory::GetVarcharValue(s_dist), pool.get()); - stock_tuple->SetValue(9, type::ValueFactory::GetVarcharValue(s_dist), pool.get()); - stock_tuple->SetValue(10, type::ValueFactory::GetVarcharValue(s_dist), pool.get()); - stock_tuple->SetValue(11, type::ValueFactory::GetVarcharValue(s_dist), pool.get()); - stock_tuple->SetValue(12, type::ValueFactory::GetVarcharValue(s_dist), pool.get()); - // S_YTD - auto s_ytd = 0; - stock_tuple->SetValue(13, type::ValueFactory::GetIntegerValue(s_ytd), nullptr); - // S_ORDER_CNT - auto s_order_cnt = 0; - stock_tuple->SetValue(14, type::ValueFactory::GetIntegerValue(s_order_cnt), - nullptr); - // S_REMOTE_CNT - auto s_remote_cnt = 0; - stock_tuple->SetValue(15, type::ValueFactory::GetIntegerValue(s_remote_cnt), - nullptr); - // S_DATA - auto s_data = GetRandomAlphaNumericString(data_length); - stock_tuple->SetValue(16, type::ValueFactory::GetVarcharValue(s_data), pool.get()); - - return stock_tuple; -} - -void LoadItems() { - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - auto txn = txn_manager.BeginTransaction(); - std::unique_ptr pool(new type::EphemeralPool()); - std::unique_ptr context( - new executor::ExecutorContext(txn)); - - for (auto item_itr = 0; item_itr < state.item_count; item_itr++) { - auto item_tuple = BuildItemTuple(item_itr, pool); - planner::InsertPlan node(item_table, std::move(item_tuple)); - executor::InsertExecutor executor(&node, context.get()); - executor.Execute(); - } - - txn_manager.CommitTransaction(txn); -} - -void LoadWarehouses(const int &warehouse_from, const int &warehouse_to) { - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - std::unique_ptr context; - - // WAREHOUSES - for (auto warehouse_itr = warehouse_from; warehouse_itr < warehouse_to; warehouse_itr++) { - - std::unique_ptr pool(new type::EphemeralPool()); - - auto txn = txn_manager.BeginTransaction(); - context.reset(new executor::ExecutorContext(txn)); - - auto warehouse_tuple = BuildWarehouseTuple(warehouse_itr, pool); - planner::InsertPlan warehouse_node(warehouse_table, - std::move(warehouse_tuple)); - executor::InsertExecutor warehouse_executor(&warehouse_node, context.get()); - warehouse_executor.Execute(); - - txn_manager.CommitTransaction(txn); - - // DISTRICTS - for (auto district_itr = 0; district_itr < state.districts_per_warehouse; - district_itr++) { - auto txn = txn_manager.BeginTransaction(); - context.reset(new executor::ExecutorContext(txn)); - - auto district_tuple = - BuildDistrictTuple(district_itr, warehouse_itr, pool); - planner::InsertPlan district_node(district_table, - std::move(district_tuple)); - executor::InsertExecutor district_executor(&district_node, context.get()); - district_executor.Execute(); - - txn_manager.CommitTransaction(txn); - - // CUSTOMERS - for (auto customer_itr = 0; customer_itr < state.customers_per_district; - customer_itr++) { - auto txn = txn_manager.BeginTransaction(); - context.reset(new executor::ExecutorContext(txn)); - - auto customer_tuple = - BuildCustomerTuple(customer_itr, district_itr, warehouse_itr, pool); - planner::InsertPlan customer_node(customer_table, - std::move(customer_tuple)); - executor::InsertExecutor customer_executor(&customer_node, - context.get()); - customer_executor.Execute(); - - // HISTORY - - int history_district_id = district_itr; - int history_warehouse_id = warehouse_itr; - auto history_tuple = - BuildHistoryTuple(customer_itr, district_itr, warehouse_itr, - history_district_id, history_warehouse_id, pool); - planner::InsertPlan history_node(history_table, - std::move(history_tuple)); - executor::InsertExecutor history_executor(&history_node, context.get()); - history_executor.Execute(); - - txn_manager.CommitTransaction(txn); - - } // END CUSTOMERS - - // ORDERS - for (auto orders_itr = 0; orders_itr < state.customers_per_district; - orders_itr++) { - auto txn = txn_manager.BeginTransaction(); - context.reset(new executor::ExecutorContext(txn)); - - // New order ? - auto new_order_threshold = - state.customers_per_district - new_orders_per_district; - bool new_order = (orders_itr > new_order_threshold); - auto o_ol_cnt = GetRandomInteger(orders_min_ol_cnt, orders_max_ol_cnt); - - auto orders_tuple = BuildOrdersTuple( - orders_itr, district_itr, warehouse_itr, new_order, o_ol_cnt); - planner::InsertPlan orders_node(orders_table, std::move(orders_tuple)); - executor::InsertExecutor orders_executor(&orders_node, context.get()); - orders_executor.Execute(); - - // NEW_ORDER - if (new_order) { - auto new_order_tuple = - BuildNewOrderTuple(orders_itr, district_itr, warehouse_itr); - planner::InsertPlan new_order_node(new_order_table, - std::move(new_order_tuple)); - executor::InsertExecutor new_order_executor(&new_order_node, - context.get()); - new_order_executor.Execute(); - } - - // ORDER_LINE - for (auto order_line_itr = 0; order_line_itr < o_ol_cnt; - order_line_itr++) { - int ol_supply_w_id = warehouse_itr; - auto order_line_tuple = BuildOrderLineTuple( - orders_itr, district_itr, warehouse_itr, order_line_itr, - ol_supply_w_id, new_order, pool); - planner::InsertPlan order_line_node(order_line_table, - std::move(order_line_tuple)); - executor::InsertExecutor order_line_executor(&order_line_node, - context.get()); - order_line_executor.Execute(); - } - - txn_manager.CommitTransaction(txn); - } - - } // END DISTRICTS - - // STOCK - for (auto stock_itr = 0; stock_itr < state.item_count; stock_itr++) { - auto txn = txn_manager.BeginTransaction(); - context.reset(new executor::ExecutorContext(txn)); - - int s_w_id = warehouse_itr; - auto stock_tuple = BuildStockTuple(stock_itr, s_w_id, pool); - planner::InsertPlan stock_node(stock_table, std::move(stock_tuple)); - executor::InsertExecutor stock_executor(&stock_node, context.get()); - stock_executor.Execute(); - - txn_manager.CommitTransaction(txn); - } - - } // END WAREHOUSES -} - -void LoadTPCCDatabase() { - - std::chrono::steady_clock::time_point start_time; - start_time = std::chrono::steady_clock::now(); - - LoadItems(); - - if (state.warehouse_count < state.loader_count) { - std::vector> load_threads(state.warehouse_count); - for (int thread_id = 0; thread_id < state.warehouse_count; ++thread_id) { - int warehouse_from = thread_id; - int warehouse_to = thread_id + 1; - load_threads[thread_id].reset(new std::thread(LoadWarehouses, warehouse_from, warehouse_to)); - } - - for (auto thread_id = 0; thread_id < state.warehouse_count; ++thread_id) { - load_threads[thread_id]->join(); - } - - } else { - std::vector> load_threads(state.loader_count); - int warehouse_per_thread = state.warehouse_count / state.loader_count; - for (int thread_id = 0; thread_id < state.loader_count - 1; ++thread_id) { - int warehouse_from = warehouse_per_thread * thread_id; - int warehouse_to = warehouse_per_thread * (thread_id + 1); - load_threads[thread_id].reset(new std::thread(LoadWarehouses, warehouse_from, warehouse_to)); - } - int thread_id = state.loader_count - 1; - int warehouse_from = warehouse_per_thread * thread_id; - int warehouse_to = state.warehouse_count; - load_threads[thread_id].reset(new std::thread(LoadWarehouses, warehouse_from, warehouse_to)); - - for (auto thread_id = 0; thread_id < state.loader_count; ++thread_id) { - load_threads[thread_id]->join(); - } - } - - std::chrono::steady_clock::time_point end_time = std::chrono::steady_clock::now(); - UNUSED_ATTRIBUTE double diff = std::chrono::duration_cast(end_time - start_time).count(); - LOG_INFO("database loading time = %lf ms", diff); - - LOG_INFO("%sTABLE SIZES%s", peloton::GETINFO_HALF_THICK_LINE.c_str(), peloton::GETINFO_HALF_THICK_LINE.c_str()); - LOG_INFO("warehouse count = %lu", warehouse_table->GetTupleCount()); - LOG_INFO("district count = %lu", district_table->GetTupleCount()); - LOG_INFO("item count = %lu", item_table->GetTupleCount()); - LOG_INFO("customer count = %lu", customer_table->GetTupleCount()); - LOG_INFO("history count = %lu", history_table->GetTupleCount()); - LOG_INFO("stock count = %lu", stock_table->GetTupleCount()); - LOG_INFO("orders count = %lu", orders_table->GetTupleCount()); - LOG_INFO("new order count = %lu", new_order_table->GetTupleCount()); - LOG_INFO("order line count = %lu", order_line_table->GetTupleCount()); -} - -} // namespace tpcc -} // namespace benchmark -} // namespace peloton - -#ifdef __APPLE__ -#pragma clang diagnostic pop -#endif \ No newline at end of file diff --git a/src/main/tpcc/tpcc_new_order.cpp b/src/main/tpcc/tpcc_new_order.cpp deleted file mode 100644 index 5ba66431355..00000000000 --- a/src/main/tpcc/tpcc_new_order.cpp +++ /dev/null @@ -1,732 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpcc_new_order.cpp -// -// Identification: src/main/tpcc/tpcc_new_order.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "benchmark/tpcc/tpcc_configuration.h" -#include "benchmark/tpcc/tpcc_loader.h" -#include "benchmark/tpcc/tpcc_workload.h" - -#include "catalog/manager.h" -#include "catalog/schema.h" - -#include "common/generator.h" -#include "common/internal_types.h" -#include "common/logger.h" -#include "common/timer.h" -#include "type/value.h" -#include "type/value_factory.h" - -#include "concurrency/transaction_context.h" -#include "concurrency/transaction_manager_factory.h" - -#include "executor/abstract_executor.h" -#include "executor/executor_context.h" -#include "executor/index_scan_executor.h" -#include "executor/insert_executor.h" -#include "executor/logical_tile.h" -#include "executor/logical_tile_factory.h" -#include "executor/materialization_executor.h" -#include "executor/update_executor.h" - -#include "common/container_tuple.h" -#include "expression/abstract_expression.h" -#include "expression/comparison_expression.h" -#include "expression/constant_value_expression.h" -#include "expression/expression_util.h" -#include "expression/tuple_value_expression.h" - -#include "index/index_factory.h" - -#include "logging/log_manager.h" - -#include "planner/abstract_plan.h" -#include "planner/index_scan_plan.h" -#include "planner/insert_plan.h" -#include "planner/materialization_plan.h" -#include "planner/update_plan.h" - -#include "storage/data_table.h" -#include "storage/table_factory.h" - -namespace peloton { -namespace benchmark { -namespace tpcc { - -bool RunNewOrder(const size_t &thread_id) { - /* - "NEW_ORDER": { - "getWarehouseTaxRate": "SELECT W_TAX FROM WAREHOUSE WHERE W_ID = ?", # w_id - "getDistrict": "SELECT D_TAX, D_NEXT_O_ID FROM DISTRICT WHERE D_ID = ? AND - D_W_ID = ?", # d_id, w_id - "getCustomer": "SELECT C_DISCOUNT, C_LAST, C_CREDIT FROM CUSTOMER WHERE - C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # w_id, d_id, c_id - "incrementNextOrderId": "UPDATE DISTRICT SET D_NEXT_O_ID = ? WHERE D_ID = ? - AND D_W_ID = ?", # d_next_o_id, d_id, w_id - "createOrder": "INSERT INTO ORDERS (O_ID, O_D_ID, O_W_ID, O_C_ID, - O_ENTRY_D, O_CARRIER_ID, O_OL_CNT, O_ALL_LOCAL) VALUES (?, ?, ?, ?, ?, ?, - ?, ?)", # d_next_o_id, d_id, w_id, c_id, o_entry_d, o_carrier_id, o_ol_cnt, - o_all_local - "createNewOrder": "INSERT INTO NEW_ORDER (NO_O_ID, NO_D_ID, NO_W_ID) VALUES - (?, ?, ?)", # o_id, d_id, w_id - "getItemInfo": "SELECT I_PRICE, I_NAME, I_DATA FROM ITEM WHERE I_ID = ?", # - ol_i_id - "getStockInfo": "SELECT S_QUANTITY, S_DATA, S_YTD, S_ORDER_CNT, - S_REMOTE_CNT, S_DIST_%02d FROM STOCK WHERE S_I_ID = ? AND S_W_ID = ?", # - d_id, ol_i_id, ol_supply_w_id - "updateStock": "UPDATE STOCK SET S_QUANTITY = ?, S_YTD = ?, S_ORDER_CNT = - ?, S_REMOTE_CNT = ? WHERE S_I_ID = ? AND S_W_ID = ?", # s_quantity, - s_order_cnt, s_remote_cnt, ol_i_id, ol_supply_w_id - "createOrderLine": "INSERT INTO ORDER_LINE (OL_O_ID, OL_D_ID, OL_W_ID, - OL_NUMBER, OL_I_ID, OL_SUPPLY_W_ID, OL_DELIVERY_D, OL_QUANTITY, OL_AMOUNT, - OL_DIST_INFO) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", # o_id, d_id, w_id, - ol_number, ol_i_id, ol_supply_w_id, ol_quantity, ol_amount, ol_dist_info - } - */ - - LOG_TRACE("-------------------------------------"); - - ///////////////////////////////////////////////////////// - // PREPARE ARGUMENTS - ///////////////////////////////////////////////////////// - int warehouse_id = GenerateWarehouseId(thread_id); - int district_id = GetRandomInteger(0, state.districts_per_warehouse - 1); - int customer_id = GetRandomInteger(0, state.customers_per_district - 1); - int o_ol_cnt = GetRandomInteger(orders_min_ol_cnt, orders_max_ol_cnt); - - std::vector i_ids, ol_w_ids, ol_qtys; - bool o_all_local = true; - - for (auto ol_itr = 0; ol_itr < o_ol_cnt; ol_itr++) { - // in the original TPC-C benchmark, it is possible to read an item that does - // not exist. - // for simplicity, we ignore this case. - // this essentially makes the processing of NewOrder transaction more - // time-consuming. - i_ids.push_back(GetRandomInteger(0, state.item_count - 1)); - bool remote = GetRandomBoolean(new_order_remote_txns); - ol_w_ids.push_back(warehouse_id); - - if (remote == true) { - ol_w_ids[ol_itr] = - GetRandomIntegerExcluding(0, state.warehouse_count - 1, warehouse_id); - o_all_local = false; - } - - ol_qtys.push_back(GetRandomInteger(0, order_line_max_ol_quantity)); - } - - std::vector runtime_keys; - - ///////////////////////////////////////////////////////// - // BEGIN TRANSACTION - ///////////////////////////////////////////////////////// - - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - - auto txn = txn_manager.BeginTransaction(thread_id); - - std::unique_ptr context( - new executor::ExecutorContext(txn)); - - std::vector item_key_column_ids; - std::vector item_expr_types; - item_key_column_ids.push_back(0); // I_ID - item_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - - std::vector item_column_ids = {2, 3, 4}; // I_NAME, I_PRICE, I_DATA - - for (auto item_id : i_ids) { - LOG_TRACE( - "getItemInfo: SELECT I_PRICE, I_NAME, I_DATA FROM ITEM WHERE I_ID = %d", - item_id); - - std::vector item_key_values; - - item_key_values.push_back( - type::ValueFactory::GetIntegerValue(item_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc item_index_scan_desc( - item_table_pkey_index_oid, item_key_column_ids, item_expr_types, - item_key_values, runtime_keys); - - planner::IndexScanPlan item_index_scan_node( - item_table, nullptr, item_column_ids, item_index_scan_desc); - - executor::IndexScanExecutor item_index_scan_executor(&item_index_scan_node, - context.get()); - - auto gii_lists_values = ExecuteRead(&item_index_scan_executor); - - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - if (gii_lists_values.size() != 1) { - LOG_ERROR("getItemInfo return size incorrect : %lu", - gii_lists_values.size()); - PELOTON_ASSERT(false); - } - } - - LOG_TRACE("getWarehouseTaxRate: SELECT W_TAX FROM WAREHOUSE WHERE W_ID = %d", - warehouse_id); - - std::vector warehouse_key_column_ids; - std::vector warehouse_expr_types; - warehouse_key_column_ids.push_back(0); // W_ID - warehouse_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - - std::vector warehouse_key_values; - - warehouse_key_values.push_back( - type::ValueFactory::GetIntegerValue(warehouse_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc warehouse_index_scan_desc( - warehouse_table_pkey_index_oid, warehouse_key_column_ids, - warehouse_expr_types, warehouse_key_values, runtime_keys); - - std::vector warehouse_column_ids = {7}; // W_TAX - - planner::IndexScanPlan warehouse_index_scan_node(warehouse_table, nullptr, - warehouse_column_ids, - warehouse_index_scan_desc); - - executor::IndexScanExecutor warehouse_index_scan_executor( - &warehouse_index_scan_node, context.get()); - - auto gwtr_lists_values = ExecuteRead(&warehouse_index_scan_executor); - - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - if (gwtr_lists_values.size() != 1) { - LOG_ERROR("getWarehouseTaxRate return size incorrect : %lu", - gwtr_lists_values.size()); - PELOTON_ASSERT(false); - } - - UNUSED_ATTRIBUTE auto w_tax = gwtr_lists_values[0][0]; - - LOG_TRACE("w_tax: %s", w_tax.GetInfo().c_str()); - - LOG_TRACE( - "getDistrict: SELECT D_TAX, D_NEXT_O_ID FROM DISTRICT WHERE D_ID = %d " - "AND D_W_ID = %d", - district_id, warehouse_id); - - std::vector district_key_column_ids; - std::vector district_expr_types; - - district_key_column_ids.push_back(0); // D_ID - district_key_column_ids.push_back(1); // D_W_ID - district_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - district_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - - std::vector district_key_values; - district_key_values.push_back( - type::ValueFactory::GetIntegerValue(district_id).Copy()); - district_key_values.push_back( - type::ValueFactory::GetIntegerValue(warehouse_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc district_index_scan_desc( - district_table_pkey_index_oid, district_key_column_ids, - district_expr_types, district_key_values, runtime_keys); - - std::vector district_column_ids = {8, 10}; // D_TAX, D_NEXT_O_ID - - // Create plan node. - planner::IndexScanPlan district_index_scan_node( - district_table, nullptr, district_column_ids, district_index_scan_desc); - - executor::IndexScanExecutor district_index_scan_executor( - &district_index_scan_node, context.get()); - - auto gd_lists_values = ExecuteRead(&district_index_scan_executor); - - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - if (gd_lists_values.size() != 1) { - LOG_ERROR("getDistrict return size incorrect : %lu", - gd_lists_values.size()); - PELOTON_ASSERT(false); - } - - UNUSED_ATTRIBUTE auto d_tax = gd_lists_values[0][0]; - UNUSED_ATTRIBUTE auto d_next_o_id = gd_lists_values[0][1]; - - LOG_TRACE("d_tax: %s, d_next_o_id: %s", d_tax.GetInfo().c_str(), - d_next_o_id.GetInfo().c_str()); - - LOG_TRACE( - "getCustomer: SELECT C_DISCOUNT, C_LAST, C_CREDIT FROM CUSTOMER WHERE " - "C_W_ID = %d AND C_D_ID = %d AND C_ID = %d", - warehouse_id, district_id, customer_id); - - std::vector customer_key_column_ids; - std::vector customer_expr_types; - - customer_key_column_ids.push_back(0); // C_ID - customer_key_column_ids.push_back(1); // C_D_ID - customer_key_column_ids.push_back(2); // C_W_ID - customer_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - customer_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - customer_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - - std::vector customer_key_values; - customer_key_values.push_back( - type::ValueFactory::GetIntegerValue(customer_id).Copy()); - customer_key_values.push_back( - type::ValueFactory::GetIntegerValue(district_id).Copy()); - customer_key_values.push_back( - type::ValueFactory::GetIntegerValue(warehouse_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc customer_index_scan_desc( - customer_table_pkey_index_oid, customer_key_column_ids, - customer_expr_types, customer_key_values, runtime_keys); - - std::vector customer_column_ids = { - 5, 13, 15}; // C_LAST, C_CREDIT, C_DISCOUNT - - // Create plan node. - planner::IndexScanPlan customer_index_scan_node( - customer_table, nullptr, customer_column_ids, customer_index_scan_desc); - - executor::IndexScanExecutor customer_index_scan_executor( - &customer_index_scan_node, context.get()); - - auto gc_lists_values = ExecuteRead(&customer_index_scan_executor); - - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - if (gc_lists_values.size() != 1) { - LOG_ERROR("getCustomer return size incorrect : %lu", - gc_lists_values.size()); - PELOTON_ASSERT(false); - } - - UNUSED_ATTRIBUTE auto c_last = gc_lists_values[0][0]; - UNUSED_ATTRIBUTE auto c_credit = gc_lists_values[0][1]; - UNUSED_ATTRIBUTE auto c_discount = gc_lists_values[0][2]; - - LOG_TRACE("c_last: %s, c_credit: %s, c_discount: %s", - c_last.GetInfo().c_str(), c_credit.GetInfo().c_str(), - c_discount.GetInfo().c_str()); - - int district_update_value = type::ValuePeeker::PeekInteger(d_next_o_id) + 1; - LOG_TRACE("district update value = %d", district_update_value); - - LOG_TRACE( - "incrementNextOrderId: UPDATE DISTRICT SET D_NEXT_O_ID = %d WHERE D_ID = " - "%d AND D_W_ID = %d", - district_update_value, district_id, warehouse_id); - - std::vector district_update_column_ids = {10}; // D_NEXT_O_ID - - std::vector district_update_key_values; - district_update_key_values.push_back( - type::ValueFactory::GetIntegerValue(district_id).Copy()); - district_update_key_values.push_back( - type::ValueFactory::GetIntegerValue(warehouse_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc district_update_index_scan_desc( - district_table_pkey_index_oid, district_key_column_ids, - district_expr_types, district_update_key_values, runtime_keys); - - // Create plan node. - planner::IndexScanPlan district_update_index_scan_node( - district_table, nullptr, district_update_column_ids, - district_update_index_scan_desc); - - executor::IndexScanExecutor district_update_index_scan_executor( - &district_update_index_scan_node, context.get()); - - TargetList district_target_list; - DirectMapList district_direct_map_list; - - // Update the last attribute - for (oid_t col_itr = 0; col_itr < 10; col_itr++) { - district_direct_map_list.emplace_back(col_itr, - std::pair(0, col_itr)); - } - type::Value district_update_val = - type::ValueFactory::GetIntegerValue(district_update_value).Copy(); - - planner::DerivedAttribute attribute{ - expression::ExpressionUtil::ConstantValueFactory(district_update_val)}; - district_target_list.emplace_back(10, attribute); - - std::unique_ptr district_project_info( - new planner::ProjectInfo(std::move(district_target_list), - std::move(district_direct_map_list))); - planner::UpdatePlan district_update_node(district_table, - std::move(district_project_info)); - - executor::UpdateExecutor district_update_executor(&district_update_node, - context.get()); - - district_update_executor.AddChild(&district_update_index_scan_executor); - - ExecuteUpdate(&district_update_executor); - - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - LOG_TRACE( - "createOrder: INSERT INTO ORDERS (O_ID, O_D_ID, O_W_ID, O_C_ID, " - "O_ENTRY_D, O_CARRIER_ID, O_OL_CNT, O_ALL_LOCAL)"); - - std::unique_ptr orders_tuple( - new storage::Tuple(orders_table->GetSchema(), true)); - - // O_ID - orders_tuple->SetValue(0, type::ValueFactory::GetIntegerValue( - type::ValuePeeker::PeekInteger(d_next_o_id)), - nullptr); - // O_C_ID - orders_tuple->SetValue(1, type::ValueFactory::GetIntegerValue(customer_id), - nullptr); - // O_D_ID - orders_tuple->SetValue(2, type::ValueFactory::GetIntegerValue(district_id), - nullptr); - // O_W_ID - orders_tuple->SetValue(3, type::ValueFactory::GetIntegerValue(warehouse_id), - nullptr); - // O_ENTRY_D - // auto o_entry_d = GetTimeStamp(); - orders_tuple->SetValue(4, type::ValueFactory::GetTimestampValue(1), nullptr); - // O_CARRIER_ID - orders_tuple->SetValue(5, type::ValueFactory::GetIntegerValue(0), nullptr); - // O_OL_CNT - orders_tuple->SetValue(6, type::ValueFactory::GetIntegerValue(o_ol_cnt), - nullptr); - // O_ALL_LOCAL - orders_tuple->SetValue(7, type::ValueFactory::GetIntegerValue(o_all_local), - nullptr); - - planner::InsertPlan orders_node(orders_table, std::move(orders_tuple)); - executor::InsertExecutor orders_executor(&orders_node, context.get()); - orders_executor.Execute(); - - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE( - "abort transaction when inserting order table, thread_id = %d, d_id = " - "%d, next_o_id = %d", - (int)thread_id, (int)district_id, - (int)type::ValuePeeker::PeekInteger(d_next_o_id)); - txn_manager.AbortTransaction(txn); - return false; - } else { - LOG_TRACE( - "successfully insert order table, thread_id = %d, d_id = %d, next_o_id " - "= %d", - (int)thread_id, (int)district_id, - (int)type::ValuePeeker::PeekInteger(d_next_o_id)); - } - - LOG_TRACE( - "createNewOrder: INSERT INTO NEW_ORDER (NO_O_ID, NO_D_ID, NO_W_ID) " - "VALUES (?, ?, ?)"); - std::unique_ptr new_order_tuple( - new storage::Tuple(new_order_table->GetSchema(), true)); - - // NO_O_ID - new_order_tuple->SetValue(0, type::ValueFactory::GetIntegerValue( - type::ValuePeeker::PeekInteger(d_next_o_id)), - nullptr); - // NO_D_ID - new_order_tuple->SetValue(1, type::ValueFactory::GetIntegerValue(district_id), - nullptr); - // NO_W_ID - new_order_tuple->SetValue( - 2, type::ValueFactory::GetIntegerValue(warehouse_id), nullptr); - - planner::InsertPlan new_order_node(new_order_table, - std::move(new_order_tuple)); - executor::InsertExecutor new_order_executor(&new_order_node, context.get()); - new_order_executor.Execute(); - - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction when inserting new order table"); - txn_manager.AbortTransaction(txn); - return false; - } - - std::vector stock_key_column_ids; - std::vector stock_expr_types; - - stock_key_column_ids.push_back(0); // S_I_ID - stock_key_column_ids.push_back(1); // S_W_ID - stock_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - stock_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - - auto stock_pkey_index = - stock_table->GetIndexWithOid(stock_table_pkey_index_oid); - - // S_QUANTITY, S_DIST_%02d, S_YTD, S_ORDER_CNT, S_REMOTE_CNT, S_DATA - std::vector stock_column_ids = {2, oid_t(3 + district_id), 13, 14, 15, - 16}; - - std::vector stock_update_column_ids = { - 2, 13, 14, 15}; // S_QUANTITY, S_YTD, S_ORDER_CNT, S_REMOTE_CNT - - for (size_t i = 0; i < i_ids.size(); ++i) { - int item_id = i_ids.at(i); - int ol_w_id = ol_w_ids.at(i); - int ol_qty = ol_qtys.at(i); - - LOG_TRACE( - "getStockInfo: SELECT S_QUANTITY, S_DATA, S_YTD, S_ORDER_CNT, " - "S_REMOTE_CNT, S_DIST_? FROM STOCK WHERE S_I_ID = %d AND S_W_ID = %d", - item_id, ol_w_id); - - std::vector stock_key_values; - - stock_key_values.push_back( - type::ValueFactory::GetIntegerValue(item_id).Copy()); - stock_key_values.push_back( - type::ValueFactory::GetIntegerValue(ol_w_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc stock_index_scan_desc( - stock_table_pkey_index_oid, stock_key_column_ids, stock_expr_types, - stock_key_values, runtime_keys); - - std::vector stock_update_key_values; - - stock_update_key_values.push_back( - type::ValueFactory::GetIntegerValue(item_id).Copy()); - stock_update_key_values.push_back( - type::ValueFactory::GetIntegerValue(ol_w_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc stock_update_index_scan_desc( - stock_table_pkey_index_oid, stock_key_column_ids, stock_expr_types, - stock_update_key_values, runtime_keys); - - // Create plan node. - planner::IndexScanPlan stock_index_scan_node( - stock_table, nullptr, stock_column_ids, stock_index_scan_desc); - - executor::IndexScanExecutor stock_index_scan_executor( - &stock_index_scan_node, context.get()); - - auto gsi_lists_values = ExecuteRead(&stock_index_scan_executor); - - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - if (gsi_lists_values.size() != 1) { - LOG_ERROR("getStockInfo return size incorrect : %lu", - gsi_lists_values.size()); - PELOTON_ASSERT(false); - } - - int s_quantity = type::ValuePeeker::PeekInteger(gsi_lists_values[0][0]); - - if (s_quantity >= ol_qty + 10) { - s_quantity = s_quantity - ol_qty; - } else { - s_quantity = s_quantity + 91 - ol_qty; - } - - type::Value s_data = gsi_lists_values[0][1]; - - int s_ytd = type::ValuePeeker::PeekInteger(gsi_lists_values[0][2]) + ol_qty; - - int s_order_cnt = - type::ValuePeeker::PeekInteger(gsi_lists_values[0][3]) + 1; - - int s_remote_cnt = type::ValuePeeker::PeekInteger(gsi_lists_values[0][4]); - - if (ol_w_id != warehouse_id) { - s_remote_cnt += 1; - } - - LOG_TRACE( - "updateStock: UPDATE STOCK SET S_QUANTITY = ?, S_YTD = ?, S_ORDER_CNT " - "= ?, S_REMOTE_CNT = ? WHERE S_I_ID = ? AND S_W_ID = ?"); - - // Create plan node. - planner::IndexScanPlan stock_update_index_scan_node( - stock_table, nullptr, stock_update_column_ids, - stock_update_index_scan_desc); - - executor::IndexScanExecutor stock_update_index_scan_executor( - &stock_update_index_scan_node, context.get()); - - TargetList stock_target_list; - DirectMapList stock_direct_map_list; - - // Update the last attribute - for (oid_t col_itr = 0; col_itr < 17; col_itr++) { - if (col_itr != 2 && col_itr != 13 && col_itr != 14 && col_itr != 15) { - stock_direct_map_list.emplace_back(col_itr, - std::pair(0, col_itr)); - } - } - planner::DerivedAttribute s_quantity_attr{ - expression::ExpressionUtil::ConstantValueFactory( - type::ValueFactory::GetIntegerValue(s_quantity))}; - - planner::DerivedAttribute s_ytd_attr{ - expression::ExpressionUtil::ConstantValueFactory( - type::ValueFactory::GetIntegerValue(s_ytd))}; - - planner::DerivedAttribute s_order_cnt_attr{ - expression::ExpressionUtil::ConstantValueFactory( - type::ValueFactory::GetIntegerValue(s_order_cnt))}; - - planner::DerivedAttribute s_remote_cnt_attr{ - expression::ExpressionUtil::ConstantValueFactory( - type::ValueFactory::GetIntegerValue(s_remote_cnt))}; - - stock_target_list.emplace_back(2, s_quantity_attr); - stock_target_list.emplace_back(13, s_ytd_attr); - stock_target_list.emplace_back(14, s_order_cnt_attr); - stock_target_list.emplace_back(15, s_remote_cnt_attr); - - std::unique_ptr stock_project_info( - new planner::ProjectInfo(std::move(stock_target_list), - std::move(stock_direct_map_list))); - planner::UpdatePlan stock_update_node(stock_table, - std::move(stock_project_info)); - - executor::UpdateExecutor stock_update_executor(&stock_update_node, - context.get()); - - stock_update_executor.AddChild(&stock_update_index_scan_executor); - - ExecuteUpdate(&stock_update_executor); - - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - // the original benchmark requires check constraints. - // however, we ignored here. - // it does not influence the performance. - // if i_data.find(constants.ORIGINAL_STRING) != -1 and - // s_data.find(constants.ORIGINAL_STRING) != -1: - // brand_generic = 'B' - // else: - // brand_generic = 'G' - - LOG_TRACE( - "createOrderLine: INSERT INTO ORDER_LINE (OL_O_ID, OL_D_ID, OL_W_ID, " - "OL_NUMBER, OL_I_ID, OL_SUPPLY_W_ID, OL_DELIVERY_D, OL_QUANTITY, " - "OL_AMOUNT, OL_DIST_INFO) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); - std::unique_ptr order_line_tuple( - new storage::Tuple(order_line_table->GetSchema(), true)); - - // OL_O_ID - order_line_tuple->SetValue(0, - type::ValueFactory::GetIntegerValue( - type::ValuePeeker::PeekInteger(d_next_o_id)), - nullptr); - // OL_D_ID - order_line_tuple->SetValue( - 1, type::ValueFactory::GetIntegerValue(district_id), nullptr); - // OL_W_ID - order_line_tuple->SetValue( - 2, type::ValueFactory::GetIntegerValue(warehouse_id), nullptr); - // OL_NUMBER - order_line_tuple->SetValue(3, type::ValueFactory::GetIntegerValue(i), - nullptr); - // OL_I_ID - order_line_tuple->SetValue(4, type::ValueFactory::GetIntegerValue(item_id), - nullptr); - // OL_SUPPLY_W_ID - order_line_tuple->SetValue(5, type::ValueFactory::GetIntegerValue(ol_w_id), - nullptr); - // OL_DELIVERY_D - order_line_tuple->SetValue(6, type::ValueFactory::GetTimestampValue(1), - nullptr); - // OL_QUANTITY - order_line_tuple->SetValue(7, type::ValueFactory::GetIntegerValue(ol_qty), - nullptr); - // OL_AMOUNT - // TODO: workaround!!! I don't know how to get float from Value. - order_line_tuple->SetValue(8, type::ValueFactory::GetDecimalValue(0), - nullptr); - // OL_DIST_INFO - order_line_tuple->SetValue(9, s_data, nullptr); - - planner::InsertPlan order_line_node(order_line_table, - std::move(order_line_tuple)); - executor::InsertExecutor order_line_executor(&order_line_node, - context.get()); - order_line_executor.Execute(); - - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction when inserting order line table"); - txn_manager.AbortTransaction(txn); - return false; - } - } - - // transaction passed execution. - PELOTON_ASSERT(txn->GetResult() == ResultType::SUCCESS); - - auto result = txn_manager.CommitTransaction(txn); - - if (result == ResultType::SUCCESS) { - // transaction passed commitment. - LOG_TRACE("commit txn, thread_id = %d, d_id = %d, next_o_id = %d", - (int)thread_id, (int)district_id, - (int)type::ValuePeeker::PeekInteger(d_next_o_id)); - return true; - - } else { - // transaction failed commitment. - PELOTON_ASSERT(result == ResultType::ABORTED || - result == ResultType::FAILURE); - LOG_TRACE("abort txn, thread_id = %d, d_id = %d, next_o_id = %d", - (int)thread_id, (int)district_id, - (int)type::ValuePeeker::PeekInteger(d_next_o_id)); - return false; - } -} -} -} -} diff --git a/src/main/tpcc/tpcc_order_status.cpp b/src/main/tpcc/tpcc_order_status.cpp deleted file mode 100644 index 8628f5e8345..00000000000 --- a/src/main/tpcc/tpcc_order_status.cpp +++ /dev/null @@ -1,354 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpcc_order_status.cpp -// -// Identification: src/main/tpcc/tpcc_order_status.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "benchmark/tpcc/tpcc_configuration.h" -#include "benchmark/tpcc/tpcc_loader.h" -#include "benchmark/tpcc/tpcc_workload.h" - -#include "catalog/manager.h" -#include "catalog/schema.h" - -#include "common/generator.h" -#include "common/internal_types.h" -#include "common/logger.h" -#include "common/timer.h" -#include "type/value.h" -#include "type/value_factory.h" - -#include "concurrency/transaction_context.h" -#include "concurrency/transaction_manager_factory.h" - -#include "executor/abstract_executor.h" -#include "executor/executor_context.h" -#include "executor/index_scan_executor.h" -#include "executor/insert_executor.h" -#include "executor/limit_executor.h" -#include "executor/logical_tile.h" -#include "executor/logical_tile_factory.h" -#include "executor/materialization_executor.h" -#include "executor/order_by_executor.h" -#include "executor/update_executor.h" - -#include "common/container_tuple.h" -#include "expression/abstract_expression.h" -#include "expression/comparison_expression.h" -#include "expression/constant_value_expression.h" -#include "expression/expression_util.h" -#include "expression/tuple_value_expression.h" - -#include "index/index_factory.h" - -#include "logging/log_manager.h" - -#include "planner/abstract_plan.h" -#include "planner/index_scan_plan.h" -#include "planner/insert_plan.h" -#include "planner/limit_plan.h" -#include "planner/materialization_plan.h" -#include "planner/order_by_plan.h" -#include "planner/update_plan.h" - -#include "storage/data_table.h" -#include "storage/table_factory.h" - -namespace peloton { -namespace benchmark { -namespace tpcc { - -bool RunOrderStatus(const size_t &thread_id) { - /* - "ORDER_STATUS": { - "getCustomerByCustomerId": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, - C_BALANCE FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # - w_id, d_id, c_id - "getCustomersByLastName": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_BALANCE - FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_LAST = ? ORDER BY - C_FIRST", # w_id, d_id, c_last - "getLastOrder": "SELECT O_ID, O_CARRIER_ID, O_ENTRY_D FROM ORDERS WHERE - O_W_ID = ? AND O_D_ID = ? AND O_C_ID = ? ORDER BY O_ID DESC LIMIT 1", # - w_id, d_id, c_id - "getOrderLines": "SELECT OL_SUPPLY_W_ID, OL_I_ID, OL_QUANTITY, OL_AMOUNT, - OL_DELIVERY_D FROM ORDER_LINE WHERE OL_W_ID = ? AND OL_D_ID = ? AND OL_O_ID - = ?", # w_id, d_id, o_id - } - */ - - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - auto txn = txn_manager.BeginTransaction(thread_id); - - std::unique_ptr context( - new executor::ExecutorContext(txn)); - - // Generate w_id, d_id, c_id, c_last - // int w_id = GetRandomInteger(0, state.warehouse_count - 1); - int w_id = GenerateWarehouseId(thread_id); - int d_id = GetRandomInteger(0, state.districts_per_warehouse - 1); - - int c_id = -1; - std::string c_last; - - // if (GetRandomInteger(1, 100) <= 60) { - // c_last = GetRandomLastName(state.customers_per_district); - // } else { - c_id = GetNURand(1023, 0, state.customers_per_district - 1); - // } - - // Run queries - if (c_id != -1) { - LOG_TRACE( - "getCustomerByCustomerId: SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, " - "C_BALANCE FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ? " - "# w_id, d_id, c_id"); - // Construct index scan executor - std::vector customer_column_ids = {COL_IDX_C_ID, COL_IDX_C_FIRST, - COL_IDX_C_MIDDLE, COL_IDX_C_LAST, - COL_IDX_C_BALANCE}; - std::vector customer_key_column_ids = {COL_IDX_C_W_ID, - COL_IDX_C_D_ID, COL_IDX_C_ID}; - std::vector customer_expr_types; - std::vector customer_key_values; - std::vector runtime_keys; - - customer_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - customer_key_values.push_back( - type::ValueFactory::GetIntegerValue(w_id).Copy()); - customer_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - customer_key_values.push_back( - type::ValueFactory::GetIntegerValue(d_id).Copy()); - customer_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - customer_key_values.push_back( - type::ValueFactory::GetIntegerValue(c_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc customer_index_scan_desc( - customer_table_pkey_index_oid, customer_key_column_ids, - customer_expr_types, customer_key_values, runtime_keys); - - auto predicate = nullptr; - planner::IndexScanPlan customer_index_scan_node(customer_table, predicate, - customer_column_ids, - customer_index_scan_desc); - - executor::IndexScanExecutor customer_index_scan_executor( - &customer_index_scan_node, context.get()); - - auto result = ExecuteRead(&customer_index_scan_executor); - if (txn->GetResult() != ResultType::SUCCESS) { - txn_manager.AbortTransaction(txn); - return false; - } - - if (result.size() == 0) { - LOG_ERROR("wrong result size : %lu", result.size()); - PELOTON_ASSERT(false); - } - if (result[0].size() == 0) { - LOG_ERROR("wrong result[0] size : %lu", result[0].size()); - PELOTON_ASSERT(false); - } - } else { - LOG_ERROR( - "getCustomersByLastName: SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, " - "C_BALANCE FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_LAST = " - "? ORDER BY C_FIRST, # w_id, d_id, c_last"); - // Construct index scan executor - std::vector customer_column_ids = {COL_IDX_C_ID, COL_IDX_C_FIRST, - COL_IDX_C_MIDDLE, COL_IDX_C_LAST, - COL_IDX_C_BALANCE}; - std::vector customer_key_column_ids = { - COL_IDX_C_W_ID, COL_IDX_C_D_ID, COL_IDX_C_LAST}; - std::vector customer_expr_types; - std::vector customer_key_values; - std::vector runtime_keys; - - customer_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - customer_key_values.push_back( - type::ValueFactory::GetIntegerValue(w_id).Copy()); - customer_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - customer_key_values.push_back( - type::ValueFactory::GetIntegerValue(d_id).Copy()); - customer_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - customer_key_values.push_back( - type::ValueFactory::GetVarcharValue(c_last).Copy()); - - planner::IndexScanPlan::IndexScanDesc customer_index_scan_desc( - customer_table_skey_index_oid, customer_key_column_ids, - customer_expr_types, customer_key_values, runtime_keys); - - auto predicate = nullptr; - planner::IndexScanPlan customer_index_scan_node(customer_table, predicate, - customer_column_ids, - customer_index_scan_desc); - - executor::IndexScanExecutor customer_index_scan_executor( - &customer_index_scan_node, context.get()); - - // Construct order by executor - std::vector sort_keys = {1}; - std::vector descend_flags = {false}; - std::vector output_columns = {0, 1, 2, 3, 4}; - - planner::OrderByPlan customer_order_by_node(sort_keys, descend_flags, - output_columns); - - executor::OrderByExecutor customer_order_by_executor( - &customer_order_by_node, context.get()); - - customer_order_by_executor.AddChild(&customer_index_scan_executor); - - auto result = ExecuteRead(&customer_order_by_executor); - if (txn->GetResult() != ResultType::SUCCESS) { - txn_manager.AbortTransaction(txn); - return false; - } - - PELOTON_ASSERT(result.size() > 0); - // Get the middle one - size_t name_count = result.size(); - auto &customer = result[name_count / 2]; - PELOTON_ASSERT(customer.size() > 0); - c_id = type::ValuePeeker::PeekInteger(customer[0]); - } - - if (c_id < 0) { - LOG_ERROR("wrong c_id"); - PELOTON_ASSERT(false); - } - - LOG_TRACE( - "getLastOrder: SELECT O_ID, O_CARRIER_ID, O_ENTRY_D FROM ORDERS WHERE " - "O_W_ID = ? AND O_D_ID = ? AND O_C_ID = ? ORDER BY O_ID DESC LIMIT 1, # " - "w_id, d_id, c_id"); - - // Construct index scan executor - std::vector orders_column_ids = {COL_IDX_O_ID, COL_IDX_O_CARRIER_ID, - COL_IDX_O_ENTRY_D}; - std::vector orders_key_column_ids = {COL_IDX_O_W_ID, COL_IDX_O_D_ID, - COL_IDX_O_C_ID}; - std::vector orders_expr_types; - std::vector orders_key_values; - std::vector runtime_keys; - - orders_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - orders_key_values.push_back(type::ValueFactory::GetIntegerValue(w_id).Copy()); - orders_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - orders_key_values.push_back(type::ValueFactory::GetIntegerValue(d_id).Copy()); - orders_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - orders_key_values.push_back(type::ValueFactory::GetIntegerValue(c_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc orders_index_scan_desc( - orders_table_skey_index_oid, orders_key_column_ids, orders_expr_types, - orders_key_values, runtime_keys); - - auto predicate = nullptr; - - planner::IndexScanPlan orders_index_scan_node( - orders_table, predicate, orders_column_ids, orders_index_scan_desc); - - executor::IndexScanExecutor orders_index_scan_executor( - &orders_index_scan_node, context.get()); - - // Construct order by executor - std::vector sort_keys = {0}; - std::vector descend_flags = {true}; - std::vector output_columns = {0, 1, 2}; - - planner::OrderByPlan orders_order_by_node(sort_keys, descend_flags, - output_columns); - - executor::OrderByExecutor orders_order_by_executor(&orders_order_by_node, - context.get()); - orders_order_by_executor.AddChild(&orders_index_scan_executor); - - // Construct limit executor - size_t limit = 1; - size_t offset = 0; - planner::LimitPlan limit_node(limit, offset); - executor::LimitExecutor limit_executor(&limit_node, context.get()); - limit_executor.AddChild(&orders_order_by_executor); - - auto orders = ExecuteRead(&orders_order_by_executor); - if (txn->GetResult() != ResultType::SUCCESS) { - txn_manager.AbortTransaction(txn); - return false; - } - - if (orders.size() != 0) { - LOG_TRACE( - "getOrderLines: SELECT OL_SUPPLY_W_ID, OL_I_ID, OL_QUANTITY, " - "OL_AMOUNT, OL_DELIVERY_D FROM ORDER_LINE WHERE OL_W_ID = ? AND " - "OL_D_ID = ? AND OL_O_ID = ?, # w_id, d_id, o_id"); - - // Construct index scan executor - std::vector order_line_column_ids = { - COL_IDX_OL_SUPPLY_W_ID, COL_IDX_OL_I_ID, COL_IDX_OL_QUANTITY, - COL_IDX_OL_AMOUNT, COL_IDX_OL_DELIVERY_D}; - std::vector order_line_key_column_ids = { - COL_IDX_OL_W_ID, COL_IDX_OL_D_ID, COL_IDX_OL_O_ID}; - std::vector order_line_expr_types; - std::vector order_line_key_values; - - order_line_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - order_line_key_values.push_back( - type::ValueFactory::GetIntegerValue(w_id).Copy()); - order_line_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - order_line_key_values.push_back( - type::ValueFactory::GetIntegerValue(d_id).Copy()); - order_line_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - order_line_key_values.push_back(orders[0][0]); - - planner::IndexScanPlan::IndexScanDesc order_line_index_scan_desc( - order_line_table_skey_index_oid, order_line_key_column_ids, - order_line_expr_types, order_line_key_values, runtime_keys); - - predicate = nullptr; - - planner::IndexScanPlan order_line_index_scan_node( - order_line_table, predicate, order_line_column_ids, - order_line_index_scan_desc); - - executor::IndexScanExecutor order_line_index_scan_executor( - &order_line_index_scan_node, context.get()); - - ExecuteRead(&order_line_index_scan_executor); - if (txn->GetResult() != ResultType::SUCCESS) { - txn_manager.AbortTransaction(txn); - return false; - } - } - - PELOTON_ASSERT(txn->GetResult() == ResultType::SUCCESS); - - auto result = txn_manager.CommitTransaction(txn); - - if (result == ResultType::SUCCESS) { - return true; - } else { - return false; - } -} -} -} -} diff --git a/src/main/tpcc/tpcc_payment.cpp b/src/main/tpcc/tpcc_payment.cpp deleted file mode 100644 index 79a586bc5e8..00000000000 --- a/src/main/tpcc/tpcc_payment.cpp +++ /dev/null @@ -1,743 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpcc_payment.cpp -// -// Identification: src/main/tpcc/tpcc_payment.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "benchmark/tpcc/tpcc_configuration.h" -#include "benchmark/tpcc/tpcc_loader.h" -#include "benchmark/tpcc/tpcc_workload.h" - -#include "catalog/manager.h" -#include "catalog/schema.h" - -#include "common/generator.h" -#include "common/internal_types.h" -#include "common/logger.h" -#include "common/timer.h" -#include "type/value.h" -#include "type/value_factory.h" - -#include "concurrency/transaction_context.h" -#include "concurrency/transaction_manager_factory.h" - -#include "executor/abstract_executor.h" -#include "executor/executor_context.h" -#include "executor/index_scan_executor.h" -#include "executor/insert_executor.h" -#include "executor/logical_tile.h" -#include "executor/logical_tile_factory.h" -#include "executor/materialization_executor.h" -#include "executor/update_executor.h" - -#include "common/container_tuple.h" -#include "expression/abstract_expression.h" -#include "expression/comparison_expression.h" -#include "expression/constant_value_expression.h" -#include "expression/expression_util.h" -#include "expression/tuple_value_expression.h" - -#include "index/index_factory.h" - -#include "logging/log_manager.h" - -#include "planner/abstract_plan.h" -#include "planner/index_scan_plan.h" -#include "planner/insert_plan.h" -#include "planner/materialization_plan.h" -#include "planner/update_plan.h" - -#include "storage/data_table.h" -#include "storage/table_factory.h" - -namespace peloton { -namespace benchmark { -namespace tpcc { - -bool RunPayment(const size_t &thread_id) { - /* - "PAYMENT": { - "getWarehouse": "SELECT W_NAME, W_STREET_1, W_STREET_2, W_CITY, W_STATE, - W_ZIP FROM WAREHOUSE WHERE W_ID = ?", # w_id - "updateWarehouseBalance": "UPDATE WAREHOUSE SET W_YTD = W_YTD + ? WHERE - W_ID = ?", # h_amount, w_id - "getDistrict": "SELECT D_NAME, D_STREET_1, D_STREET_2, D_CITY, D_STATE, - D_ZIP FROM DISTRICT WHERE D_W_ID = ? AND D_ID = ?", # w_id, d_id - "updateDistrictBalance": "UPDATE DISTRICT SET D_YTD = D_YTD + ? WHERE - D_W_ID = ? AND D_ID = ?", # h_amount, d_w_id, d_id - "getCustomerByCustomerId": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, - C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, - C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA - FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # w_id, d_id, - c_id - "getCustomersByLastName": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, - C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, - C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA - FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_LAST = ? ORDER BY - C_FIRST", # w_id, d_id, c_last - "updateBCCustomer": "UPDATE CUSTOMER SET C_BALANCE = ?, C_YTD_PAYMENT = ?, - C_PAYMENT_CNT = ?, C_DATA = ? WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = - ?", # c_balance, c_ytd_payment, c_payment_cnt, c_data, c_w_id, c_d_id, c_id - "updateGCCustomer": "UPDATE CUSTOMER SET C_BALANCE = ?, C_YTD_PAYMENT = ?, - C_PAYMENT_CNT = ? WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # - c_balance, c_ytd_payment, c_payment_cnt, c_w_id, c_d_id, c_id - "insertHistory": "INSERT INTO HISTORY VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - } - */ - - LOG_TRACE("-------------------------------------"); - - ///////////////////////////////////////////////////////// - // PREPARE ARGUMENTS - ///////////////////////////////////////////////////////// - int warehouse_id = GenerateWarehouseId(thread_id); - int district_id = GetRandomInteger(0, state.districts_per_warehouse - 1); - int customer_warehouse_id; - int customer_district_id; - int customer_id = -1; - std::string customer_lastname; - double h_amount = - GetRandomFixedPoint(2, payment_min_amount, payment_max_amount); - // WARN: Hard code the date as 0. may cause problem - int h_date = 0; - - int x = GetRandomInteger(1, 100); - // currently we only retrieve data by id. - int y = 100; // GetRandomInteger(1, 100); - - // 85%: paying through own warehouse ( or there is only 1 warehosue) - if (state.warehouse_count == 1 || x <= 85) { - customer_warehouse_id = warehouse_id; - customer_district_id = district_id; - } - // 15%: paying through another warehouse - else { - customer_warehouse_id = - GetRandomIntegerExcluding(0, state.warehouse_count - 1, warehouse_id); - PELOTON_ASSERT(customer_warehouse_id != warehouse_id); - customer_district_id = - GetRandomInteger(0, state.districts_per_warehouse - 1); - } - - // 60%: payment by last name - if (y <= 60) { - LOG_TRACE("By last name"); - customer_lastname = GetRandomLastName(state.customers_per_district); - } - // 40%: payment by id - else { - LOG_TRACE("By id"); - customer_id = GetRandomInteger(0, state.customers_per_district - 1); - } - - std::vector runtime_keys; - - ///////////////////////////////////////////////////////// - // BEGIN TRANSACTION - ///////////////////////////////////////////////////////// - - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - - auto txn = txn_manager.BeginTransaction(thread_id); - - std::unique_ptr context( - new executor::ExecutorContext(txn)); - - std::vector customer; - - if (customer_id >= 0) { - LOG_TRACE( - "getCustomerByCustomerId: WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = " - "? , # w_id = %d, d_id = %d, c_id = %d", - warehouse_id, district_id, customer_id); - - std::vector customer_column_ids = { - 0, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 20}; - - std::vector customer_pkey_column_ids = {0, 1, 2}; - std::vector customer_pexpr_types; - customer_pexpr_types.push_back(ExpressionType::COMPARE_EQUAL); - customer_pexpr_types.push_back(ExpressionType::COMPARE_EQUAL); - customer_pexpr_types.push_back(ExpressionType::COMPARE_EQUAL); - - std::vector customer_pkey_values; - - customer_pkey_values.push_back( - type::ValueFactory::GetIntegerValue(customer_id).Copy()); - customer_pkey_values.push_back( - type::ValueFactory::GetIntegerValue(district_id).Copy()); - customer_pkey_values.push_back( - type::ValueFactory::GetIntegerValue(warehouse_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc customer_pindex_scan_desc( - customer_table_pkey_index_oid, customer_pkey_column_ids, - customer_pexpr_types, customer_pkey_values, runtime_keys); - - planner::IndexScanPlan customer_pindex_scan_node(customer_table, nullptr, - customer_column_ids, - customer_pindex_scan_desc); - - executor::IndexScanExecutor customer_pindex_scan_executor( - &customer_pindex_scan_node, context.get()); - - auto customer_list = ExecuteRead(&customer_pindex_scan_executor); - - // Check if aborted - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - if (customer_list.size() != 1) { - PELOTON_ASSERT(false); - } - - customer = customer_list[0]; - - } else { - PELOTON_ASSERT(customer_lastname.empty() == false); - - LOG_TRACE( - "getCustomersByLastName: WHERE C_W_ID = ? AND C_D_ID = ? AND C_LAST = " - "? ORDER BY C_FIRST, # w_id = %d, d_id = %d, c_last = %s", - warehouse_id, district_id, customer_lastname.c_str()); - - std::vector customer_column_ids = { - 0, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 20}; - - std::vector customer_key_column_ids = {1, 2, 5}; - std::vector customer_expr_types; - customer_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - customer_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - customer_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - - std::vector customer_key_values; - - customer_key_values.push_back( - type::ValueFactory::GetIntegerValue(district_id).Copy()); - customer_key_values.push_back( - type::ValueFactory::GetIntegerValue(warehouse_id).Copy()); - customer_key_values.push_back( - type::ValueFactory::GetVarcharValue(customer_lastname).Copy()); - - planner::IndexScanPlan::IndexScanDesc customer_index_scan_desc( - customer_table_skey_index_oid, customer_key_column_ids, - customer_expr_types, customer_key_values, runtime_keys); - - planner::IndexScanPlan customer_index_scan_node( - customer_table, nullptr, customer_column_ids, customer_index_scan_desc); - - executor::IndexScanExecutor customer_index_scan_executor( - &customer_index_scan_node, context.get()); - - auto customer_list = ExecuteRead(&customer_index_scan_executor); - - // Check if aborted - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - if (customer_list.size() < 1) { - LOG_INFO("C_W_ID=%d, C_D_ID=%d", warehouse_id, district_id); - PELOTON_ASSERT(false); - } - - // Get the midpoint customer's id - auto mid_pos = (customer_list.size() - 1) / 2; - customer = customer_list[mid_pos]; - } - - LOG_TRACE("getWarehouse:WHERE W_ID = ? # w_id = %d", warehouse_id); - - std::vector warehouse_key_column_ids = {0}; - std::vector warehouse_expr_types; - warehouse_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - - std::vector warehouse_key_values; - - warehouse_key_values.push_back( - type::ValueFactory::GetIntegerValue(warehouse_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc warehouse_index_scan_desc( - warehouse_table_pkey_index_oid, warehouse_key_column_ids, - warehouse_expr_types, warehouse_key_values, runtime_keys); - - std::vector warehouse_column_ids = {1, 2, 3, 4, 5, 6, 8}; - - planner::IndexScanPlan warehouse_index_scan_node(warehouse_table, nullptr, - warehouse_column_ids, - warehouse_index_scan_desc); - - executor::IndexScanExecutor warehouse_index_scan_executor( - &warehouse_index_scan_node, context.get()); - - // Execute the query - auto warehouse_list = ExecuteRead(&warehouse_index_scan_executor); - - // Check if aborted - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - if (warehouse_list.size() != 1) { - PELOTON_ASSERT(false); - } - - LOG_TRACE( - "getDistrict: WHERE D_W_ID = ? AND D_ID = ?, # w_id = %d, d_id = %d", - warehouse_id, district_id); - // We also retrieve the original D_YTD from this query, - // which is not the standard TPCC approach - - std::vector district_key_column_ids = {0, 1}; - std::vector district_expr_types; - district_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - district_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - - std::vector district_key_values; - - district_key_values.push_back( - type::ValueFactory::GetIntegerValue(district_id).Copy()); - district_key_values.push_back( - type::ValueFactory::GetIntegerValue(warehouse_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc district_index_scan_desc( - district_table_pkey_index_oid, district_key_column_ids, - district_expr_types, district_key_values, runtime_keys); - - std::vector district_column_ids = {2, 3, 4, 5, 6, 7, 9}; - - planner::IndexScanPlan district_index_scan_node( - district_table, nullptr, district_column_ids, district_index_scan_desc); - - executor::IndexScanExecutor district_index_scan_executor( - &district_index_scan_node, context.get()); - - // Execute the query - auto district_list = ExecuteRead(&district_index_scan_executor); - - // Check if aborted - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - if (district_list.size() != 1) { - PELOTON_ASSERT(false); - } - - double warehouse_new_balance = - type::ValuePeeker::PeekDouble(warehouse_list[0][6]) + h_amount; - - LOG_TRACE( - "updateWarehouseBalance: UPDATE WAREHOUSE SET W_YTD = W_YTD + ? WHERE " - "W_ID = ?,# h_amount = %f, w_id = %d", - h_amount, warehouse_id); - - std::vector warehouse_update_column_ids = {8}; - - std::vector warehouse_update_key_values; - - warehouse_update_key_values.push_back( - type::ValueFactory::GetIntegerValue(warehouse_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc warehouse_update_index_scan_desc( - warehouse_table_pkey_index_oid, warehouse_key_column_ids, - warehouse_expr_types, warehouse_update_key_values, runtime_keys); - - planner::IndexScanPlan warehouse_update_index_scan_node( - warehouse_table, nullptr, warehouse_update_column_ids, - warehouse_update_index_scan_desc); - - executor::IndexScanExecutor warehouse_update_index_scan_executor( - &warehouse_update_index_scan_node, context.get()); - - TargetList warehouse_target_list; - DirectMapList warehouse_direct_map_list; - - // Keep the first 8 columns unchanged - for (oid_t col_itr = 0; col_itr < 8; ++col_itr) { - warehouse_direct_map_list.emplace_back(col_itr, - std::pair(0, col_itr)); - } - // Update the 9th column - type::Value warehouse_new_balance_value = - type::ValueFactory::GetDecimalValue(warehouse_new_balance).Copy(); - - planner::DerivedAttribute warehouse_bal{ - expression::ExpressionUtil::ConstantValueFactory( - warehouse_new_balance_value)}; - warehouse_target_list.emplace_back(8, warehouse_bal); - - std::unique_ptr warehouse_project_info( - new planner::ProjectInfo(std::move(warehouse_target_list), - std::move(warehouse_direct_map_list))); - planner::UpdatePlan warehouse_update_node(warehouse_table, - std::move(warehouse_project_info)); - - executor::UpdateExecutor warehouse_update_executor(&warehouse_update_node, - context.get()); - - warehouse_update_executor.AddChild(&warehouse_update_index_scan_executor); - - // Execute the query - ExecuteUpdate(&warehouse_update_executor); - - // Check if aborted - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - double district_new_balance = - type::ValuePeeker::PeekDouble(district_list[0][6]) + h_amount; - - LOG_TRACE( - "updateDistrictBalance: UPDATE DISTRICT SET D_YTD = D_YTD + ? WHERE " - "D_W_ID = ? AND D_ID = ?,# h_amount = %f, d_w_id = %d, d_id = %d", - h_amount, district_id, warehouse_id); - - std::vector district_update_column_ids = {9}; - - std::vector district_update_key_values; - - district_update_key_values.push_back( - type::ValueFactory::GetIntegerValue(district_id).Copy()); - district_update_key_values.push_back( - type::ValueFactory::GetIntegerValue(warehouse_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc district_update_index_scan_desc( - district_table_pkey_index_oid, district_key_column_ids, - district_expr_types, district_update_key_values, runtime_keys); - - planner::IndexScanPlan district_update_index_scan_node( - district_table, nullptr, district_update_column_ids, - district_update_index_scan_desc); - - executor::IndexScanExecutor district_update_index_scan_executor( - &district_update_index_scan_node, context.get()); - - TargetList district_target_list; - DirectMapList district_direct_map_list; - - // Keep all columns unchanged except for the - for (oid_t col_itr = 0; col_itr < 11; ++col_itr) { - if (col_itr != 9) { - district_direct_map_list.emplace_back( - col_itr, std::pair(0, col_itr)); - } - } - // Update the 10th column - type::Value district_new_balance_value = - type::ValueFactory::GetDecimalValue(district_new_balance).Copy(); - - planner::DerivedAttribute district_bal{ - expression::ExpressionUtil::ConstantValueFactory( - district_new_balance_value)}; - district_target_list.emplace_back(9, district_bal); - - std::unique_ptr district_project_info( - new planner::ProjectInfo(std::move(district_target_list), - std::move(district_direct_map_list))); - planner::UpdatePlan district_update_node(district_table, - std::move(district_project_info)); - - executor::UpdateExecutor district_update_executor(&district_update_node, - context.get()); - - district_update_executor.AddChild(&district_update_index_scan_executor); - - // Execute the query - ExecuteUpdate(&district_update_executor); - - // Check the result - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - std::string customer_credit = type::ValuePeeker::PeekVarchar(customer[11]); - - double customer_balance = - type::ValuePeeker::PeekDouble(customer[14]) - h_amount; - double customer_ytd_payment = - type::ValuePeeker::PeekDouble(customer[15]) + h_amount; - int customer_payment_cnt = type::ValuePeeker::PeekInteger(customer[16]) + 1; - - customer_id = type::ValuePeeker::PeekInteger(customer[0]); - - // NOTE: Workaround, we assign a constant to the customer's data field - - // Check the credit record of the user - if (customer_credit == customers_bad_credit) { - LOG_TRACE( - "updateBCCustomer:# c_balance = %f, c_ytd_payment = %f, c_payment_cnt " - "= %d, c_data = %s, c_w_id = %d, c_d_id = %d, c_id = %d", - customer_balance, customer_ytd_payment, customer_payment_cnt, - data_constant.c_str(), customer_warehouse_id, customer_district_id, - customer_id); - - std::vector customer_pkey_column_ids = {0, 1, 2}; - std::vector customer_pexpr_types; - customer_pexpr_types.push_back(ExpressionType::COMPARE_EQUAL); - customer_pexpr_types.push_back(ExpressionType::COMPARE_EQUAL); - customer_pexpr_types.push_back(ExpressionType::COMPARE_EQUAL); - - std::vector customer_pkey_values; - - customer_pkey_values.push_back( - type::ValueFactory::GetIntegerValue(customer_id).Copy()); - customer_pkey_values.push_back( - type::ValueFactory::GetIntegerValue(customer_district_id).Copy()); - customer_pkey_values.push_back( - type::ValueFactory::GetIntegerValue(customer_warehouse_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc customer_pindex_scan_desc( - customer_table_pkey_index_oid, customer_pkey_column_ids, - customer_pexpr_types, customer_pkey_values, runtime_keys); - - std::vector customer_update_bc_column_ids = {16, 17, 18, 20}; - - // Create update executor - planner::IndexScanPlan customer_update_bc_index_scan_node( - customer_table, nullptr, customer_update_bc_column_ids, - customer_pindex_scan_desc); - - executor::IndexScanExecutor customer_update_bc_index_scan_executor( - &customer_update_bc_index_scan_node, context.get()); - - TargetList customer_bc_target_list; - DirectMapList customer_bc_direct_map_list; - - // Only update the 17th to 19th and the 21th columns - for (oid_t col_itr = 0; col_itr < 21; ++col_itr) { - if ((col_itr >= 16 && col_itr <= 18) || (col_itr == 20)) { - continue; - } - customer_bc_direct_map_list.emplace_back( - col_itr, std::pair(0, col_itr)); - } - - type::Value customer_new_balance_value = - type::ValueFactory::GetDecimalValue(customer_balance).Copy(); - type::Value customer_new_ytd_value = - type::ValueFactory::GetDecimalValue(customer_ytd_payment).Copy(); - type::Value customer_new_paycnt_value = - type::ValueFactory::GetIntegerValue(customer_payment_cnt).Copy(); - type::Value customer_new_data_value = - type::ValueFactory::GetVarcharValue(data_constant.c_str()).Copy(); - - planner::DerivedAttribute c_new_bal{ - expression::ExpressionUtil::ConstantValueFactory( - customer_new_balance_value)}; - planner::DerivedAttribute c_new_ytd{ - expression::ExpressionUtil::ConstantValueFactory( - customer_new_ytd_value)}; - planner::DerivedAttribute c_new_paycnt{ - expression::ExpressionUtil::ConstantValueFactory( - customer_new_paycnt_value)}; - planner::DerivedAttribute c_new_data{ - expression::ExpressionUtil::ConstantValueFactory( - customer_new_data_value)}; - - customer_bc_target_list.emplace_back(16, c_new_bal); - customer_bc_target_list.emplace_back(17, c_new_ytd); - customer_bc_target_list.emplace_back(18, c_new_paycnt); - customer_bc_target_list.emplace_back(20, c_new_data); - - std::unique_ptr customer_bc_project_info( - new planner::ProjectInfo(std::move(customer_bc_target_list), - std::move(customer_bc_direct_map_list))); - - planner::UpdatePlan customer_update_bc_node( - customer_table, std::move(customer_bc_project_info)); - - executor::UpdateExecutor customer_update_bc_executor( - &customer_update_bc_node, context.get()); - - customer_update_bc_executor.AddChild( - &customer_update_bc_index_scan_executor); - - // Execute the query - ExecuteUpdate(&customer_update_bc_executor); - } else { - LOG_TRACE( - "updateGCCustomer: # c_balance = %f, c_ytd_payment = %f, c_payment_cnt " - "= %d, c_w_id = %d, c_d_id = %d, c_id = %d", - customer_balance, customer_ytd_payment, customer_payment_cnt, - customer_warehouse_id, customer_district_id, customer_id); - - std::vector customer_pkey_column_ids = {0, 1, 2}; - std::vector customer_pexpr_types; - customer_pexpr_types.push_back(ExpressionType::COMPARE_EQUAL); - customer_pexpr_types.push_back(ExpressionType::COMPARE_EQUAL); - customer_pexpr_types.push_back(ExpressionType::COMPARE_EQUAL); - - std::vector customer_pkey_values; - - customer_pkey_values.push_back( - type::ValueFactory::GetIntegerValue(customer_id).Copy()); - customer_pkey_values.push_back( - type::ValueFactory::GetIntegerValue(customer_district_id).Copy()); - customer_pkey_values.push_back( - type::ValueFactory::GetIntegerValue(customer_warehouse_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc customer_pindex_scan_desc( - customer_table_pkey_index_oid, customer_pkey_column_ids, - customer_pexpr_types, customer_pkey_values, runtime_keys); - - std::vector customer_update_gc_column_ids = {16, 17, 18}; - - // Create update executor - planner::IndexScanPlan customer_update_gc_index_scan_node( - customer_table, nullptr, customer_update_gc_column_ids, - customer_pindex_scan_desc); - - executor::IndexScanExecutor customer_update_gc_index_scan_executor( - &customer_update_gc_index_scan_node, context.get()); - - TargetList customer_gc_target_list; - DirectMapList customer_gc_direct_map_list; - - // Only update the 17th to 19th columns - for (oid_t col_itr = 0; col_itr < 21; ++col_itr) { - if (col_itr >= 16 && col_itr <= 18) { - continue; - } - customer_gc_direct_map_list.emplace_back( - col_itr, std::pair(0, col_itr)); - } - type::Value customer_new_balance_value = - type::ValueFactory::GetDecimalValue(customer_balance).Copy(); - type::Value customer_new_ytd_value = - type::ValueFactory::GetDecimalValue(customer_ytd_payment).Copy(); - type::Value customer_new_paycnt_value = - type::ValueFactory::GetIntegerValue(customer_payment_cnt).Copy(); - - planner::DerivedAttribute c_new_bal{ - expression::ExpressionUtil::ConstantValueFactory( - customer_new_balance_value)}; - planner::DerivedAttribute c_new_ytd{ - expression::ExpressionUtil::ConstantValueFactory( - customer_new_ytd_value)}; - planner::DerivedAttribute c_new_paycnt{ - expression::ExpressionUtil::ConstantValueFactory( - customer_new_paycnt_value)}; - - customer_gc_target_list.emplace_back(16, c_new_bal); - customer_gc_target_list.emplace_back(17, c_new_ytd); - customer_gc_target_list.emplace_back(18, c_new_paycnt); - - std::unique_ptr customer_gc_project_info( - new planner::ProjectInfo(std::move(customer_gc_target_list), - std::move(customer_gc_direct_map_list))); - - planner::UpdatePlan customer_update_gc_node( - customer_table, std::move(customer_gc_project_info)); - - executor::UpdateExecutor customer_update_gc_executor( - &customer_update_gc_node, context.get()); - - customer_update_gc_executor.AddChild( - &customer_update_gc_index_scan_executor); - - // Execute the query - ExecuteUpdate(&customer_update_gc_executor); - } - - // Check the result - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - LOG_TRACE( - "insertHistory: INSERT INTO HISTORY VALUES (?, ?, ?, ?, ?, ?, ?, ?)"); - std::unique_ptr history_tuple( - new storage::Tuple(history_table->GetSchema(), true)); - - // H_C_ID - history_tuple->SetValue(0, type::ValueFactory::GetIntegerValue(customer_id), - nullptr); - // H_C_D_ID - history_tuple->SetValue( - 1, type::ValueFactory::GetIntegerValue(customer_district_id), nullptr); - // H_C_W_ID - history_tuple->SetValue( - 2, type::ValueFactory::GetIntegerValue(customer_warehouse_id), nullptr); - // H_D_ID - history_tuple->SetValue(3, type::ValueFactory::GetIntegerValue(district_id), - nullptr); - // H_W_ID - history_tuple->SetValue(4, type::ValueFactory::GetIntegerValue(warehouse_id), - nullptr); - // H_DATE - history_tuple->SetValue(5, type::ValueFactory::GetTimestampValue(h_date), - nullptr); - // H_AMOUNT - history_tuple->SetValue(6, type::ValueFactory::GetDecimalValue(h_amount), - nullptr); - // H_DATA - // Note: workaround - history_tuple->SetValue(7, type::ValueFactory::GetVarcharValue(data_constant), - context.get()->GetPool()); - - planner::InsertPlan history_insert_node(history_table, - std::move(history_tuple)); - executor::InsertExecutor history_insert_executor(&history_insert_node, - context.get()); - - // Execute - history_insert_executor.Execute(); - - // Check result - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - PELOTON_ASSERT(txn->GetResult() == ResultType::SUCCESS); - - auto result = txn_manager.CommitTransaction(txn); - - if (result == ResultType::SUCCESS) { - return true; - } else { - PELOTON_ASSERT(result == ResultType::ABORTED || - result == ResultType::FAILURE); - return false; - } -} -} -} -} diff --git a/src/main/tpcc/tpcc_stock_level.cpp b/src/main/tpcc/tpcc_stock_level.cpp deleted file mode 100644 index 1cad0da3dd2..00000000000 --- a/src/main/tpcc/tpcc_stock_level.cpp +++ /dev/null @@ -1,273 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpcc_stock_level.cpp -// -// Identification: src/main/tpcc/tpcc_stock_level.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "benchmark/tpcc/tpcc_configuration.h" -#include "benchmark/tpcc/tpcc_loader.h" -#include "benchmark/tpcc/tpcc_workload.h" - -#include "catalog/manager.h" -#include "catalog/schema.h" - -#include "common/generator.h" -#include "common/internal_types.h" -#include "common/logger.h" -#include "common/timer.h" -#include "type/value.h" -#include "type/value_factory.h" - -#include "concurrency/transaction_context.h" -#include "concurrency/transaction_manager_factory.h" - -#include "executor/abstract_executor.h" -#include "executor/aggregate_executor.h" -#include "executor/executor_context.h" -#include "executor/index_scan_executor.h" -#include "executor/insert_executor.h" -#include "executor/logical_tile.h" -#include "executor/logical_tile_factory.h" -#include "executor/materialization_executor.h" -#include "executor/nested_loop_join_executor.h" -#include "executor/update_executor.h" - -#include "common/container_tuple.h" -#include "expression/abstract_expression.h" -#include "expression/comparison_expression.h" -#include "expression/constant_value_expression.h" -#include "expression/expression_util.h" -#include "expression/tuple_value_expression.h" - -#include "index/index_factory.h" - -#include "logging/log_manager.h" - -#include "planner/abstract_plan.h" -#include "planner/aggregate_plan.h" -#include "planner/index_scan_plan.h" -#include "planner/insert_plan.h" -#include "planner/materialization_plan.h" -#include "planner/nested_loop_join_plan.h" -#include "planner/project_info.h" -#include "planner/update_plan.h" - -#include "storage/data_table.h" -#include "storage/table_factory.h" - -namespace peloton { -namespace benchmark { -namespace tpcc { - -bool RunStockLevel(const size_t &thread_id) { - /* - "STOCK_LEVEL": { - "getOId": "SELECT D_NEXT_O_ID FROM DISTRICT WHERE D_W_ID = ? AND D_ID = ?", - "getStockCount": "SELECT COUNT(DISTINCT(OL_I_ID)) FROM ORDER_LINE, STOCK - WHERE OL_W_ID = ? AND OL_D_ID = ? AND OL_O_ID < ? AND OL_O_ID >= ? AND - S_W_ID = ? AND S_I_ID = OL_I_ID AND S_QUANTITY < ? - } - */ - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - auto txn = txn_manager.BeginTransaction(thread_id); - - std::unique_ptr context( - new executor::ExecutorContext(txn)); - - // Prepare random data - int w_id = GenerateWarehouseId(thread_id); - int d_id = GetRandomInteger(0, state.districts_per_warehouse - 1); - int threshold = GetRandomInteger(stock_min_threshold, stock_max_threshold); - - LOG_TRACE( - "getOId: SELECT D_NEXT_O_ID FROM DISTRICT WHERE D_W_ID = ? AND D_ID = ?"); - - // Construct index scan executor - std::vector district_column_ids = {COL_IDX_D_NEXT_O_ID}; - std::vector district_key_column_ids = {COL_IDX_D_W_ID, COL_IDX_D_ID}; - std::vector district_expr_types; - std::vector district_key_values; - std::vector runtime_keys; - - district_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - district_key_values.push_back( - type::ValueFactory::GetIntegerValue(w_id).Copy()); - district_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - district_key_values.push_back( - type::ValueFactory::GetIntegerValue(d_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc district_index_scan_desc( - district_table_pkey_index_oid, district_key_column_ids, - district_expr_types, district_key_values, runtime_keys); - - expression::AbstractExpression *predicate = nullptr; - planner::IndexScanPlan district_index_scan_node( - district_table, predicate, district_column_ids, district_index_scan_desc); - executor::IndexScanExecutor district_index_scan_executor( - &district_index_scan_node, context.get()); - - auto districts = ExecuteRead(&district_index_scan_executor); - if (txn->GetResult() != ResultType::SUCCESS) { - txn_manager.AbortTransaction(txn); - return false; - } - if (districts.size() != 1) { - LOG_ERROR("incorrect districts size : %lu", districts.size()); - PELOTON_ASSERT(false); - } - - type::Value o_id = districts[0][0]; - - LOG_TRACE( - "getStockCount: SELECT COUNT(DISTINCT(OL_I_ID)) FROM ORDER_LINE, STOCK " - "WHERE OL_W_ID = ? AND OL_D_ID = ? AND OL_O_ID < ? AND OL_O_ID >= ? AND " - "S_W_ID = ? AND S_I_ID = OL_I_ID AND S_QUANTITY < ?"); - - int max_o_id = type::ValuePeeker::PeekInteger(o_id); - int min_o_id = max_o_id - 20; - - ////////////////////////////////////////////////////////////// - std::vector order_line_column_ids = {COL_IDX_OL_I_ID}; - std::vector order_line_key_column_ids = { - COL_IDX_OL_W_ID, COL_IDX_OL_D_ID, COL_IDX_OL_O_ID}; - std::vector order_line_expr_types; - order_line_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - order_line_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - order_line_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - - auto order_line_skey_index = - order_line_table->GetIndexWithOid(order_line_table_skey_index_oid); - - ////////////////////////////////////////////////////////////// - std::vector stock_column_ids = {COL_IDX_S_QUANTITY}; - std::vector stock_key_column_ids = {COL_IDX_S_W_ID, COL_IDX_S_I_ID}; - std::vector stock_expr_types; - stock_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - stock_expr_types.push_back(ExpressionType::COMPARE_EQUAL); - - auto stock_pkey_index = - stock_table->GetIndexWithOid(stock_table_pkey_index_oid); - - ////////////////////////////////////////////////////////////// - std::unordered_set distinct_items; - - for (int curr_o_id = min_o_id; curr_o_id < max_o_id; ++curr_o_id) { - //////////////////////////////////////////////////////////////// - /////////// Construct left table index scan //////////////////// - //////////////////////////////////////////////////////////////// - - std::vector order_line_key_values; - - order_line_key_values.push_back( - type::ValueFactory::GetIntegerValue(w_id).Copy()); - order_line_key_values.push_back( - type::ValueFactory::GetIntegerValue(d_id).Copy()); - order_line_key_values.push_back( - type::ValueFactory::GetIntegerValue(curr_o_id).Copy()); - - planner::IndexScanPlan::IndexScanDesc order_line_index_scan_desc( - order_line_table_skey_index_oid, order_line_key_column_ids, - order_line_expr_types, order_line_key_values, runtime_keys); - - planner::IndexScanPlan order_line_index_scan_node( - order_line_table, nullptr, order_line_column_ids, - order_line_index_scan_desc); - - executor::IndexScanExecutor order_line_index_scan_executor( - &order_line_index_scan_node, context.get()); - - auto order_line_values = ExecuteRead(&order_line_index_scan_executor); - - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - if (order_line_values.size() == 0) { - LOG_TRACE("order line return size incorrect : %lu", - order_line_values.size()); - continue; - } - - auto item_id = order_line_values[0][0]; - - LOG_TRACE("item_id: %s", item_id.GetInfo().c_str()); - - ////////////////////////////////////////////////////////////////// - ///////////// Construct right table index scan /////////////////// - ////////////////////////////////////////////////////////////////// - - std::vector stock_key_values; - - stock_key_values.push_back( - type::ValueFactory::GetIntegerValue(w_id).Copy()); - stock_key_values.push_back(item_id); - - planner::IndexScanPlan::IndexScanDesc stock_index_scan_desc( - stock_table_pkey_index_oid, stock_key_column_ids, stock_expr_types, - stock_key_values, runtime_keys); - - // Add predicate S_QUANTITY < threshold - planner::IndexScanPlan stock_index_scan_node( - stock_table, nullptr, stock_column_ids, stock_index_scan_desc); - - executor::IndexScanExecutor stock_index_scan_executor( - &stock_index_scan_node, context.get()); - - auto stock_values = ExecuteRead(&stock_index_scan_executor); - - if (txn->GetResult() != ResultType::SUCCESS) { - LOG_TRACE("abort transaction"); - txn_manager.AbortTransaction(txn); - return false; - } - - if (stock_values.size() == 0) { - // LOG_ERROR("stock return size incorrect : %lu", - // order_line_values.size()); - continue; - } - - auto quantity = stock_values[0][0]; - if (type::ValuePeeker::PeekInteger(quantity) < threshold) { - distinct_items.insert(type::ValuePeeker::PeekInteger(item_id)); - } - } - LOG_TRACE("number of distinct items=%lu", distinct_items.size()); - - PELOTON_ASSERT(txn->GetResult() == ResultType::SUCCESS); - - auto result = txn_manager.CommitTransaction(txn); - - if (result == ResultType::SUCCESS) { - return true; - } else { - return false; - } - - return true; -} -} -} -} diff --git a/src/main/tpcc/tpcc_workload.cpp b/src/main/tpcc/tpcc_workload.cpp deleted file mode 100644 index 3e1705b4b42..00000000000 --- a/src/main/tpcc/tpcc_workload.cpp +++ /dev/null @@ -1,421 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpcc_workload.cpp -// -// Identification: src/main/tpcc/tpcc_workload.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "storage/storage_manager.h" - -#include "benchmark/benchmark_common.h" -#include "benchmark/tpcc/tpcc_workload.h" -#include "benchmark/tpcc/tpcc_configuration.h" -#include "benchmark/tpcc/tpcc_loader.h" - -#include "catalog/manager.h" -#include "catalog/schema.h" - -#include "common/internal_types.h" -#include "type/value.h" -#include "type/value_factory.h" -#include "common/logger.h" -#include "common/timer.h" -#include "common/generator.h" - -#include "concurrency/transaction_context.h" -#include "concurrency/transaction_manager_factory.h" -#include "concurrency/epoch_manager_factory.h" - -#include "executor/executor_context.h" -#include "executor/abstract_executor.h" -#include "executor/logical_tile.h" -#include "executor/logical_tile_factory.h" -#include "executor/materialization_executor.h" -#include "executor/update_executor.h" -#include "executor/index_scan_executor.h" - -#include "expression/abstract_expression.h" -#include "expression/constant_value_expression.h" -#include "expression/tuple_value_expression.h" -#include "expression/comparison_expression.h" -#include "expression/expression_util.h" -#include "common/container_tuple.h" - -#include "index/index_factory.h" - -#include "logging/log_manager.h" - -#include "planner/abstract_plan.h" -#include "planner/materialization_plan.h" -#include "planner/insert_plan.h" -#include "planner/update_plan.h" -#include "planner/index_scan_plan.h" -#include "planner/nested_loop_join_plan.h" -#include "planner/aggregate_plan.h" -#include "planner/order_by_plan.h" -#include "planner/limit_plan.h" - -#include "storage/data_table.h" -#include "storage/table_factory.h" -#include "storage/storage_manager.h" - -namespace peloton { -namespace benchmark { -namespace tpcc { - -///////////////////////////////////////////////////////// -// WORKLOAD -///////////////////////////////////////////////////////// - -#define STOCK_LEVEL_RATIO 0.04 -#define ORDER_STATUS_RATIO 0.0 -#define PAYMENT_RATIO 0.48 -#define NEW_ORDER_RATIO 0.48 - -volatile bool is_running = true; - -PadInt *abort_counts; -PadInt *commit_counts; - -size_t GenerateWarehouseId(const size_t &thread_id) { - if (state.affinity) { - if (state.warehouse_count <= state.backend_count) { - return thread_id % state.warehouse_count; - } else { - int warehouse_per_partition = state.warehouse_count / state.backend_count; - int start_warehouse = warehouse_per_partition * thread_id; - int end_warehouse = ((int)thread_id != (state.backend_count - 1)) ? - start_warehouse + warehouse_per_partition - 1 : state.warehouse_count - 1; - return GetRandomInteger(start_warehouse, end_warehouse); - } - } else { - return GetRandomInteger(0, state.warehouse_count - 1); - } -} - -#ifndef __APPLE__ -void PinToCore(size_t core) { - cpu_set_t cpuset; - CPU_ZERO(&cpuset); - CPU_SET(core, &cpuset); - pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); -#else -void PinToCore(size_t UNUSED_ATTRIBUTE core) { -// Mac OS X does not export interfaces that identify processors or control thread placement -// explicit thread to processor binding is not supported. -// Reference: https://superuser.com/questions/149312/how-to-set-processor-affinity-on-os-x -#endif -} - -void RunBackend(const size_t thread_id) { - - PinToCore(thread_id); - - if (concurrency::EpochManagerFactory::GetEpochType() == EpochType::DECENTRALIZED_EPOCH) { - // register thread to epoch manager - auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); - epoch_manager.RegisterThread(thread_id); - } - - PadInt &execution_count_ref = abort_counts[thread_id]; - PadInt &transaction_count_ref = commit_counts[thread_id]; - - // backoff - uint32_t backoff_shifts = 0; - - while (true) { - - if (is_running == false) { - break; - } - - FastRandom rng(rand()); - - auto rng_val = rng.NextUniform(); - if (rng_val <= STOCK_LEVEL_RATIO) { - while (RunStockLevel(thread_id) == false) { - if (is_running == false) { - break; - } - execution_count_ref.data++; - // backoff - if (state.exp_backoff) { - if (backoff_shifts < 13) { - ++backoff_shifts; - } - uint64_t sleep_duration = 1UL << backoff_shifts; - sleep_duration *= 100; - std::this_thread::sleep_for(std::chrono::microseconds(sleep_duration)); - } - } - } else if (rng_val <= ORDER_STATUS_RATIO + STOCK_LEVEL_RATIO) { - while (RunOrderStatus(thread_id) == false) { - if (is_running == false) { - break; - } - execution_count_ref.data++; - // backoff - if (state.exp_backoff) { - if (backoff_shifts < 13) { - ++backoff_shifts; - } - uint64_t sleep_duration = 1UL << backoff_shifts; - sleep_duration *= 100; - std::this_thread::sleep_for(std::chrono::microseconds(sleep_duration)); - } - } - } else if (rng_val <= PAYMENT_RATIO + ORDER_STATUS_RATIO + STOCK_LEVEL_RATIO) { - while (RunPayment(thread_id) == false) { - if (is_running == false) { - break; - } - execution_count_ref.data++; - // backoff - if (state.exp_backoff) { - if (backoff_shifts < 13) { - ++backoff_shifts; - } - uint64_t sleep_duration = 1UL << backoff_shifts; - sleep_duration *= 100; - std::this_thread::sleep_for(std::chrono::microseconds(sleep_duration)); - } - } - } else if (rng_val <= PAYMENT_RATIO + ORDER_STATUS_RATIO + STOCK_LEVEL_RATIO + NEW_ORDER_RATIO) { - while (RunNewOrder(thread_id) == false) { - if (is_running == false) { - break; - } - execution_count_ref.data++; - // backoff - if (state.exp_backoff) { - if (backoff_shifts < 13) { - ++backoff_shifts; - } - uint64_t sleep_duration = 1UL << backoff_shifts; - sleep_duration *= 100; - std::this_thread::sleep_for(std::chrono::microseconds(sleep_duration)); - } - } - } else { - while (RunDelivery(thread_id) == false) { - if (is_running == false) { - break; - } - execution_count_ref.data++; - // backoff - if (state.exp_backoff) { - if (backoff_shifts < 13) { - ++backoff_shifts; - } - uint64_t sleep_duration = 1UL << backoff_shifts; - sleep_duration *= 100; - std::this_thread::sleep_for(std::chrono::microseconds(sleep_duration)); - } - } - } - - backoff_shifts >>= 1; - transaction_count_ref.data++; - - } -} - -void RunWorkload() { - - // Execute the workload to build the log - std::vector thread_group; - size_t num_threads = state.backend_count; - - abort_counts = new PadInt[num_threads]; - PELOTON_MEMSET(abort_counts, 0, sizeof(PadInt) * num_threads); - - commit_counts = new PadInt[num_threads]; - PELOTON_MEMSET(commit_counts, 0, sizeof(PadInt) * num_threads); - - size_t profile_round = (size_t)(state.duration / state.profile_duration); - - PadInt **abort_counts_profiles = new PadInt *[profile_round]; - for (size_t round_id = 0; round_id < profile_round; ++round_id) { - abort_counts_profiles[round_id] = new PadInt[num_threads]; - } - - PadInt **commit_counts_profiles = new PadInt *[profile_round]; - for (size_t round_id = 0; round_id < profile_round; ++round_id) { - commit_counts_profiles[round_id] = new PadInt[num_threads]; - } - - for (size_t thread_itr = 0; thread_itr < num_threads; ++thread_itr) { - thread_group.push_back(std::thread(RunBackend, thread_itr)); - } - - ////////////////////////////////////// - oid_t last_tile_group_id = 0; - for (size_t round_id = 0; round_id < profile_round; ++round_id) { - std::this_thread::sleep_for( - std::chrono::milliseconds(int(state.profile_duration * 1000))); - PELOTON_MEMCPY(abort_counts_profiles[round_id], abort_counts, - sizeof(PadInt) * num_threads); - PELOTON_MEMCPY(commit_counts_profiles[round_id], commit_counts, - sizeof(PadInt) * num_threads); - - storage::StorageManager *storage_manager = storage::StorageManager::GetInstance(); - oid_t current_tile_group_id = storage_manager->GetCurrentTileGroupId(); - if (round_id != 0) { - state.profile_memory.push_back(current_tile_group_id - last_tile_group_id); - } - last_tile_group_id = current_tile_group_id; - - } - - state.profile_memory.push_back(state.profile_memory.at(state.profile_memory.size() - 1)); - - is_running = false; - - // Join the threads with the main thread - for (size_t thread_itr = 0; thread_itr < num_threads; ++thread_itr) { - thread_group[thread_itr].join(); - } - - // calculate the throughput and abort rate for the first round. - uint64_t total_commit_count = 0; - for (size_t i = 0; i < num_threads; ++i) { - total_commit_count += commit_counts_profiles[0][i].data; - } - - uint64_t total_abort_count = 0; - for (size_t i = 0; i < num_threads; ++i) { - total_abort_count += abort_counts_profiles[0][i].data; - } - - state.profile_throughput - .push_back(total_commit_count * 1.0 / state.profile_duration); - state.profile_abort_rate - .push_back(total_abort_count * 1.0 / total_commit_count); - - // calculate the throughput and abort rate for the remaining rounds. - for (size_t round_id = 0; round_id < profile_round - 1; ++round_id) { - total_commit_count = 0; - for (size_t i = 0; i < num_threads; ++i) { - total_commit_count += commit_counts_profiles[round_id + 1][i].data - - commit_counts_profiles[round_id][i].data; - } - - total_abort_count = 0; - for (size_t i = 0; i < num_threads; ++i) { - total_abort_count += abort_counts_profiles[round_id + 1][i].data - - abort_counts_profiles[round_id][i].data; - } - - state.profile_throughput - .push_back(total_commit_count * 1.0 / state.profile_duration); - state.profile_abort_rate - .push_back(total_abort_count * 1.0 / total_commit_count); - } - - // calculate the aggregated throughput and abort rate. - total_commit_count = 0; - for (size_t i = 0; i < num_threads; ++i) { - total_commit_count += commit_counts_profiles[profile_round - 1][i].data; - } - - total_abort_count = 0; - for (size_t i = 0; i < num_threads; ++i) { - total_abort_count += abort_counts_profiles[profile_round - 1][i].data; - } - - state.throughput = total_commit_count * 1.0 / state.duration; - state.abort_rate = total_abort_count * 1.0 / total_commit_count; - - // cleanup everything. - for (size_t round_id = 0; round_id < profile_round; ++round_id) { - delete[] abort_counts_profiles[round_id]; - abort_counts_profiles[round_id] = nullptr; - } - - for (size_t round_id = 0; round_id < profile_round; ++round_id) { - delete[] commit_counts_profiles[round_id]; - commit_counts_profiles[round_id] = nullptr; - } - delete[] abort_counts_profiles; - abort_counts_profiles = nullptr; - delete[] commit_counts_profiles; - commit_counts_profiles = nullptr; - - delete[] abort_counts; - abort_counts = nullptr; - delete[] commit_counts; - commit_counts = nullptr; -} - -///////////////////////////////////////////////////////// -// HARNESS -///////////////////////////////////////////////////////// - -std::vector> ExecuteRead(executor::AbstractExecutor* executor) { - executor->Init(); - - std::vector> logical_tile_values; - - // Execute stuff - while (executor->Execute() == true) { - std::unique_ptr result_tile(executor->GetOutput()); - - if(result_tile == nullptr) { - break; - } - - auto column_count = result_tile->GetColumnCount(); - LOG_TRACE("result column count = %d\n", (int)column_count); - - for (oid_t tuple_id : *result_tile) { - ContainerTuple cur_tuple(result_tile.get(), - tuple_id); - std::vector tuple_values; - for (oid_t column_itr = 0; column_itr < column_count; column_itr++){ - auto value = cur_tuple.GetValue(column_itr); - tuple_values.push_back(value); - } - - // Move the tuple list - logical_tile_values.push_back(std::move(tuple_values)); - } - } - - return logical_tile_values; -} - -void ExecuteUpdate(executor::AbstractExecutor* executor) { - executor->Init(); - // Execute stuff - while (executor->Execute() == true); -} - - -void ExecuteDelete(executor::AbstractExecutor* executor) { - executor->Init(); - // Execute stuff - while (executor->Execute() == true); -} - - -} // namespace tpcc -} // namespace benchmark -} // namespace peloton diff --git a/src/main/tpch/tpch.cpp b/src/main/tpch/tpch.cpp deleted file mode 100644 index 001acb53f4c..00000000000 --- a/src/main/tpch/tpch.cpp +++ /dev/null @@ -1,125 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpch.cpp -// -// Identification: src/main/tpch/tpch.cpp -// -// Copyright (c) 2015-17, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include -#include -#include - -#include "benchmark/tpch/tpch_configuration.h" -#include "benchmark/tpch/tpch_database.h" -#include "benchmark/tpch/tpch_workload.h" -#include "common/logger.h" - -namespace peloton { -namespace benchmark { -namespace tpch { - -void Usage(FILE *out) { - fprintf(out, - "Command line options : tpch \n" - " -h --help : print help message \n" - " -i --input-dir : location of data \n" - " -n --num-runs : the number of runs to execute for each query \n" - " -s --suffix : input file suffix \n" - " -d --dict-encode : dictionary encode \n" - " -q --queries : comma-separated list of queries to run (i.g., 1,14 for Q1 and Q14) \n"); -} - -static struct option opts[] = { - {"input-dir", required_argument, NULL, 'i'}, - {"dict-encode", optional_argument, NULL, 'd'}, - {"queries", optional_argument, NULL, 'q'}, - {NULL, 0, NULL, 0}}; - -void ParseArguments(int argc, char **argv, Configuration &config) { - config.suffix = "tbl"; - - // Parse args - while (1) { - int idx = 0; - int c = getopt_long(argc, argv, "hi:n:s:dq:", opts, &idx); - - if (c == -1) break; - - switch (c) { - case 'i': { - char *input = optarg; - config.data_dir = input; - break; - } - case 'n': { - char *input = optarg; - config.num_runs = static_cast(std::atoi(input)); - } - case 'd': { - config.dictionary_encode = true; - break; - } - case 'q': { - char *csv_queries = optarg; - config.SetRunnableQueries(csv_queries); - break; - } - case 'h': { - Usage(stderr); - exit(EXIT_FAILURE); - } - default: { - LOG_ERROR("Unknown option: -%c-", c); - Usage(stderr); - exit(EXIT_FAILURE); - } - } - } - - // Validate everything - if (!config.IsValid()) { - exit(EXIT_FAILURE); - } - - LOG_INFO("Input directory : '%s'", config.data_dir.c_str()); - LOG_INFO("Dictionary encode : %s", - config.dictionary_encode ? "true" : "false"); - for (uint32_t i = 0; i < 22; i++) { - LOG_INFO("Run query %u : %s", i + 1, - config.queries_to_run[i] ? "true" : "false"); - } -} - -void RunBenchmark(const Configuration &config) { - // Create the DB instance - TPCHDatabase tpch_db{config}; - - // Create the benchmark - TPCHBenchmark tpch_benchmark{config, tpch_db}; - - // Run the benchmark - tpch_benchmark.RunBenchmark(); -} - -} // namespace tpch -} // namespace benchmark -} // namespace peloton - -// Entry point -int main(int argc, char **argv) { - // The configuration - peloton::benchmark::tpch::Configuration config; - - // Parse arguments - peloton::benchmark::tpch::ParseArguments(argc, argv, config); - - // Run workload - peloton::benchmark::tpch::RunBenchmark(config); - - return 0; -} \ No newline at end of file diff --git a/src/main/tpch/tpch_configuration.cpp b/src/main/tpch/tpch_configuration.cpp deleted file mode 100644 index f7ad9a70361..00000000000 --- a/src/main/tpch/tpch_configuration.cpp +++ /dev/null @@ -1,117 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpch_configuration.cpp -// -// Identification: src/main/tpch/tpch_configuration.cpp -// -// Copyright (c) 2015-17, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "benchmark/tpch/tpch_configuration.h" - -#include - -#include "common/logger.h" - -namespace peloton { -namespace benchmark { -namespace tpch { - -oid_t kIntSize = - static_cast(type::Type::GetTypeSize(type::TypeId::INTEGER)); - -oid_t kDateSize = - static_cast(type::Type::GetTypeSize(type::TypeId::DATE)); - -oid_t kBigIntSize = - static_cast(type::Type::GetTypeSize(type::TypeId::BIGINT)); - -oid_t kDecimalSize = - static_cast(type::Type::GetTypeSize(type::TypeId::DECIMAL)); - -bool Configuration::IsValid() const { - struct stat info; - if (stat(data_dir.c_str(), &info) != 0) { - LOG_ERROR("Data directory [%s] isn't accessible", data_dir.c_str()); - return false; - } else if ((info.st_mode & S_IFDIR) == 0) { - LOG_ERROR("Data directory [%s] isn't a directory", data_dir.c_str()); - return false; - } - auto inputs = {GetCustomerPath(), GetLineitemPath(), - GetNationPath(), GetOrdersPath(), - GetPartSuppPath(), GetPartPath(), - GetSupplierPath(), GetRegionPath()}; - for (const auto &input : inputs) { - struct stat info; - if (stat(input.c_str(), &info) != 0) { - LOG_ERROR("Input file [%s] isn't accessible", input.c_str()); - return false; - } - } - - // All good - return true; -} - -std::string Configuration::GetInputPath(std::string file_name) const { - auto name = file_name + "." + suffix; - return data_dir + - (data_dir[data_dir.length() - 1] == '/' ? name : "/" + name); -} - -std::string Configuration::GetCustomerPath() const { - return GetInputPath("customer"); -} - -std::string Configuration::GetLineitemPath() const { - return GetInputPath("lineitem"); -} - -std::string Configuration::GetNationPath() const { - return GetInputPath("nation"); -} - -std::string Configuration::GetOrdersPath() const { - return GetInputPath("orders"); -} - -std::string Configuration::GetPartSuppPath() const { - return GetInputPath("partsupp"); -} - -std::string Configuration::GetPartPath() const { return GetInputPath("part"); } - -std::string Configuration::GetSupplierPath() const { - return GetInputPath("supplier"); -} - -std::string Configuration::GetRegionPath() const { - return GetInputPath("region"); -} - -void Configuration::SetRunnableQueries(char *query_list) { - // Disable all queries - for (uint32_t i = 0; i < 22; i++) queries_to_run[i] = false; - - // Now pull out the queries the user actually wants to run - char *ptr = strtok(query_list, ","); - while (ptr != nullptr) { - int query = atoi(ptr); - if (query >= 1 && query <= 22) { - queries_to_run[query - 1] = true; - } - ptr = strtok(nullptr, ","); - } -} - -bool Configuration::ShouldRunQuery(QueryId qid) const { - return queries_to_run[static_cast(qid)]; -} - -} // namespace tpch -} // namespace benchmark -} // namespace peloton \ No newline at end of file diff --git a/src/main/tpch/tpch_database.cpp b/src/main/tpch/tpch_database.cpp deleted file mode 100644 index 5252104cc88..00000000000 --- a/src/main/tpch/tpch_database.cpp +++ /dev/null @@ -1,690 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpch_database.cpp -// -// Identification: src/main/tpch/tpch_database.cpp -// -// Copyright (c) 2015-17, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "benchmark/tpch/tpch_database.h" - -#include "benchmark/tpch/tpch_workload.h" -#include "catalog/catalog.h" - -namespace peloton { -namespace benchmark { -namespace tpch { - -// The TPCH database ID and each of the tables it owns -static constexpr oid_t kTPCHDatabaseId = 44; - -//===----------------------------------------------------------------------===// -// Utility function to run a function over every line in the given file -//===----------------------------------------------------------------------===// -void ForEachLine(const std::string fname, std::function cb) { - // Open the file and advise the kernel of sequential access pattern - int input_fd = open(fname.c_str(), O_RDONLY); - - const uint32_t BUFFER_SIZE = 16 * 1024; - char buffer[BUFFER_SIZE] = {0}; - char *buf_pos = buffer; - - size_t bytes_to_read = BUFFER_SIZE; - ssize_t bytes_read; - while ((bytes_read = read(input_fd, buf_pos, bytes_to_read)) != 0) { - if (bytes_read == -1) { - perror("Error reading from input file"); - close(input_fd); - exit(errno); - } - - char *end = buf_pos + bytes_read; - - char *line_start = buffer; - char *next_line_start; - while ((next_line_start = strchr(line_start, '\n')) != nullptr && - next_line_start < end) { - // Invoke callback on start of line - cb(line_start); - - if (next_line_start == end - 1) { - line_start = end; - break; - } - - line_start = next_line_start + 1; - } - - bytes_to_read = BUFFER_SIZE; - buf_pos = buffer; - - size_t tail_size = end - line_start; - if (tail_size > 0) { - PELOTON_MEMCPY(buffer, line_start, tail_size); - buf_pos = buffer + tail_size; - bytes_to_read -= tail_size; - } - } - close(input_fd); -} - -// Convert the given string into a i32 date -uint32_t ConvertDate(char *p) { - std::tm t_shipdate; - PELOTON_MEMSET(&t_shipdate, 0, sizeof(std::tm)); - strptime(p, "%Y-%m-%d", &t_shipdate); - t_shipdate.tm_isdst = -1; - return static_cast(mktime(&t_shipdate)); -} - -//===----------------------------------------------------------------------===// -// TPCH DATABASE -//===----------------------------------------------------------------------===// - -TPCHDatabase::TPCHDatabase(const Configuration &c) : config_(c) { - // Create database instance - auto *database = new storage::Database(kTPCHDatabaseId); - - // Add databse instance to catalog - catalog::Catalog::GetInstance()->AddDatabase(database); - - // Create all the test table - CreateTables(); - - // Mark all tables as not loaded - for (uint32_t i = 0; i < 8; i++) { - loaded_tables_[i] = false; - } -} - -TPCHDatabase::~TPCHDatabase() { - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - auto *txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithOid(txn, kTPCHDatabaseId); - txn_manager.CommitTransaction(txn); -} - -// Create all the TPCH tables -void TPCHDatabase::CreateTables() const { - // Create all table instances - CreateCustomerTable(); - CreateLineitemTable(); - CreateNationTable(); - CreateOrdersTable(); - CreatePartTable(); - CreatePartSupplierTable(); - CreateRegionTable(); - CreateSupplierTable(); -} - -storage::Database &TPCHDatabase::GetDatabase() const { - auto *catalog = catalog::Catalog::GetInstance(); - return *catalog->GetDatabaseWithOid(kTPCHDatabaseId); -} - -storage::DataTable &TPCHDatabase::GetTable(TableId table_id) const { - return *GetDatabase().GetTableWithOid(static_cast(table_id)); -} - -uint32_t TPCHDatabase::DictionaryEncode(Dictionary &dict, - const std::string &val) { - auto iter = dict.find(val); - if (iter == dict.end()) { - uint32_t code = dict.size(); - dict.insert(std::make_pair(val, code)); - return code; - } else { - return iter->second; - } -} - -uint32_t TPCHDatabase::CodeForMktSegment(const std::string mktsegment) const { - auto iter = c_mktsegment_dict_.find(mktsegment); - return iter != c_mktsegment_dict_.end() ? iter->second : 0; -} - -//===----------------------------------------------------------------------===// -// TABLE CREATORS -//===----------------------------------------------------------------------===// - -void TPCHDatabase::CreateCustomerTable() const { - catalog::Column c_custkey = {type::TypeId::INTEGER, kIntSize, - "c_custkey", true}; - catalog::Column c_name = {type::TypeId::VARCHAR, 25, "c_name", false}; - catalog::Column c_address = {type::TypeId::VARCHAR, 40, "c_address", - false}; - catalog::Column c_nationkey = {type::TypeId::INTEGER, kIntSize, - "c_nationkey", true}; - catalog::Column c_phone = {type::TypeId::VARCHAR, 15, "c_phone", false}; - catalog::Column c_acctbal = {type::TypeId::DECIMAL, kDecimalSize, - "c_acctbal", true}; - catalog::Column c_mktsegment; - if (config_.dictionary_encode) { - c_mktsegment = {type::TypeId::INTEGER, kIntSize, "c_mktsegment", - true}; - } else { - c_mktsegment = {type::TypeId::VARCHAR, 10, "c_mktsegment", true}; - } - catalog::Column c_comment = {type::TypeId::VARCHAR, 117, "c_comment", - false}; - - auto customer_cols = {c_custkey, c_name, c_address, c_nationkey, - c_phone, c_acctbal, c_mktsegment, c_comment}; - - // Create the schema - std::unique_ptr customer_schema{ - new catalog::Schema{customer_cols}}; - - // Create the table! - bool owns_schema = true; - bool adapt_table = true; - storage::DataTable *customer_table = storage::TableFactory::GetDataTable( - kTPCHDatabaseId, (uint32_t)TableId::Customer, customer_schema.release(), - "Customer", config_.tuples_per_tile_group, owns_schema, adapt_table); - - // Add the table to the database (we're releasing ownership at this point) - GetDatabase().AddTable(customer_table); -} - -void TPCHDatabase::CreateLineitemTable() const { - // Define columns - catalog::Column l_orderkey = {type::TypeId::INTEGER, kIntSize, "l_orderkey"}; - catalog::Column l_partkey = {type::TypeId::INTEGER, kIntSize, "l_partkey"}; - catalog::Column l_suppkey = {type::TypeId::INTEGER, kIntSize, "l_suppkey"}; - catalog::Column l_linenumber = {type::TypeId::INTEGER, kIntSize, "l_linenumber"}; - catalog::Column l_quantity = {type::TypeId::INTEGER, kIntSize, "l_quantity"}; - catalog::Column l_extendedprice = {type::TypeId::DECIMAL, kDecimalSize, "l_extendedprice"}; - catalog::Column l_discount = {type::TypeId::DECIMAL, kDecimalSize, "l_discount"}; - catalog::Column l_tax = {type::TypeId::DECIMAL, kDecimalSize, "l_tax"}; - - catalog::Column l_returnflag; - if (config_.dictionary_encode) { - l_returnflag = {type::TypeId::INTEGER, kIntSize, "l_returnflag"}; - } else { - l_returnflag = {type::TypeId::VARCHAR, 1, "l_returnflag"}; - }; - - catalog::Column l_linestatus; - if (config_.dictionary_encode) { - l_linestatus = {type::TypeId::INTEGER, kIntSize, "l_linestatus"}; - } else { - l_linestatus = {type::TypeId::VARCHAR, 1, "l_returnflag"}; - } - - catalog::Column l_shipdate = {type::TypeId::DATE, kDateSize, "l_shipdate"}; - catalog::Column l_commitdate = {type::TypeId::DATE, kDateSize, "l_commitdate"}; - catalog::Column l_receiptdate = {type::TypeId::DATE, kDateSize, "l_receiptdate"}; - catalog::Column l_shipinstruct = {type::TypeId::INTEGER, kIntSize, "l_shipinstruct"}; - catalog::Column l_shipmode = {type::TypeId::INTEGER, kIntSize, "l_shipmode"}; - catalog::Column l_comment = {type::TypeId::VARCHAR, 44, "l_comment"}; - - auto lineitem_cols = { - l_orderkey, l_partkey, l_suppkey, l_linenumber, - l_quantity, l_extendedprice, l_discount, l_tax, - l_returnflag, l_linestatus, l_shipdate, l_commitdate, - l_receiptdate, l_shipinstruct, l_shipmode, l_comment}; - - // Create the schema - std::unique_ptr lineitem_schema{ - new catalog::Schema{lineitem_cols}}; - - // Create the table! - bool owns_schema = true; - bool adapt_table = true; - storage::DataTable *lineitem_table = storage::TableFactory::GetDataTable( - kTPCHDatabaseId, (uint32_t)TableId::Lineitem, lineitem_schema.release(), - "Line Item", config_.tuples_per_tile_group, owns_schema, adapt_table); - - // Add the table to the database (we're releasing ownership at this point) - GetDatabase().AddTable(lineitem_table); -} - -void TPCHDatabase::CreateNationTable() const { - catalog::Column n_nationkey = {type::TypeId::INTEGER, kIntSize, - "n_nationkey", true}; - catalog::Column n_name = {type::TypeId::VARCHAR, 25, "n_name", false}; - catalog::Column n_regionKey = {type::TypeId::INTEGER, kIntSize, "n_regionkey", - true}; - - catalog::Column n_comment = {type::TypeId::VARCHAR, 152, "n_comment", - false}; - - // Create the schema - auto nation_cols = {n_nationkey, n_name, n_regionKey, n_comment}; - std::unique_ptr nation_schema{new catalog::Schema{nation_cols}}; - - // Create the table! - bool owns_schema = true; - bool adapt_table = true; - storage::DataTable *nation_table = storage::TableFactory::GetDataTable( - kTPCHDatabaseId, (uint32_t)TableId::Nation, nation_schema.release(), "Nation", - config_.tuples_per_tile_group, owns_schema, adapt_table); - - // Add the table to the database (we're releasing ownership at this point) - GetDatabase().AddTable(nation_table); -} - -void TPCHDatabase::CreateOrdersTable() const { - catalog::Column o_orderkey = {type::TypeId::INTEGER, kIntSize, - "o_orderkey", true}; - catalog::Column o_custkey = {type::TypeId::INTEGER, kIntSize, - "o_custkey", true}; - catalog::Column o_orderstatus = {type::TypeId::VARCHAR, 1, - "o_orderstatus", true}; - catalog::Column o_totalprice = {type::TypeId::DECIMAL, kDecimalSize, - "o_totalprice", true}; - catalog::Column o_orderdate = {type::TypeId::DATE, kDateSize, - "o_orderdate", true}; - catalog::Column o_orderpriority = {type::TypeId::VARCHAR, 15, - "o_orderpriority", false}; - catalog::Column o_clerk = {type::TypeId::VARCHAR, 15, "o_clerk", false}; - catalog::Column o_shippriority = {type::TypeId::INTEGER, kIntSize, - "o_shippriority", true}; - catalog::Column o_comment = {type::TypeId::VARCHAR, 79, "o_comment", - true}; - - // Create the schema - auto orders_cols = {o_orderkey, o_custkey, o_orderstatus, - o_totalprice, o_orderdate, o_orderpriority, - o_clerk, o_shippriority, o_comment}; - std::unique_ptr order_schema{ - new catalog::Schema{orders_cols}}; - - // Create the table! - bool owns_schema = true; - bool adapt_table = true; - storage::DataTable *order_table = storage::TableFactory::GetDataTable( - kTPCHDatabaseId, (uint32_t)TableId::Orders, order_schema.release(), - "Orders", config_.tuples_per_tile_group, owns_schema, adapt_table); - - // Add the table to the database (we're releasing ownership at this point) - GetDatabase().AddTable(order_table); -} - -void TPCHDatabase::CreatePartTable() const { - catalog::Column p_parkey = {type::TypeId::INTEGER, kIntSize, - "p_partkey", true}; - catalog::Column p_name = {type::TypeId::VARCHAR, 55, "p_name", false}; - catalog::Column p_mfgr = {type::TypeId::VARCHAR, 25, "p_mfgr", false}; - - catalog::Column p_brand; - if (config_.dictionary_encode) { - p_brand = {type::TypeId::INTEGER, kIntSize, "p_brand", true}; - } else { - p_brand = {type::TypeId::VARCHAR, 10, "p_brand", true}; - } - - catalog::Column p_type = {type::TypeId::VARCHAR, 25, "p_type", true}; - catalog::Column p_size = {type::TypeId::INTEGER, kIntSize, "p_size", - true}; - - catalog::Column p_container; - if (config_.dictionary_encode) { - p_container = {type::TypeId::INTEGER, kIntSize, "p_container", true}; - } else { - p_container = {type::TypeId::VARCHAR, 10, "p_container", true}; - } - - catalog::Column p_retailprice = {type::TypeId::DECIMAL, kDecimalSize, - "p_retailprice", true}; - catalog::Column p_comment = {type::TypeId::VARCHAR, 23, "p_comment", - false}; - - // Create the schema - auto part_cols = {p_parkey, p_name, p_mfgr, p_brand, p_type, - p_size, p_container, p_retailprice, p_comment}; - std::unique_ptr part_schema{new catalog::Schema{part_cols}}; - - // Create the table! - bool owns_schema = true; - bool adapt_table = true; - storage::DataTable *part_table = storage::TableFactory::GetDataTable( - kTPCHDatabaseId, (uint32_t)TableId::Part, part_schema.release(), "Part", - config_.tuples_per_tile_group, owns_schema, adapt_table); - - // Add the table to the database (we're releasing ownership at this point) - GetDatabase().AddTable(part_table); -} - -void TPCHDatabase::CreatePartSupplierTable() const {} - -void TPCHDatabase::CreateRegionTable() const {} - -void TPCHDatabase::CreateSupplierTable() const {} - -//===----------------------------------------------------------------------===// -// TABLE LOADERS -//===----------------------------------------------------------------------===// - -void TPCHDatabase::LoadTable(TableId table_id) { - switch (table_id) { - case TableId::Customer: LoadCustomerTable(); break; - case TableId::Lineitem: LoadLineitemTable(); break; - case TableId::Nation: LoadNationTable(); break; - case TableId::Orders: LoadOrdersTable(); break; - case TableId::Part: LoadPartTable(); break; - case TableId::PartSupp: LoadPartSupplierTable(); break; - case TableId::Region: LoadRegionTable(); break; - case TableId::Supplier: LoadSupplierTable(); break; - } -} - -void TPCHDatabase::LoadPartTable() { - if (TableIsLoaded(TableId::Part)) { - return; - } - - const std::string filename = config_.GetPartPath(); - - LOG_INFO("Loading Part ['%s']\n", filename.c_str()); - - auto &table = GetTable(TableId::Part); - - Timer> timer; - timer.Start(); - - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - auto *txn = txn_manager.BeginTransaction(); - - uint64_t num_tuples = 0; - std::unique_ptr pool{new type::EphemeralPool()}; - - ForEachLine(filename, [&](char *p){ - storage::Tuple tuple{table.GetSchema(), true /* allocate */}; - - tuple.SetValue(0, type::ValueFactory::GetIntegerValue(std::atoi(p))); - - p = strchr(p, '|') + 1; - char *p_end = strchr(p, '|'); - tuple.SetValue(1, type::ValueFactory::GetVarcharValue(std::string{p, p_end}), pool.get()); - - p = p_end + 1; - p_end = strchr(p, '|'); - tuple.SetValue(2, type::ValueFactory::GetVarcharValue(std::string{p, p_end}), pool.get()); - - p = p_end + 1; - p_end = strchr(p, '|'); - std::string p_brand{p, p_end}; - if (config_.dictionary_encode) { - uint32_t code = DictionaryEncode(p_brand_dict_, p_brand); - tuple.SetValue(3, type::ValueFactory::GetIntegerValue(code)); - } else { - tuple.SetValue(3, type::ValueFactory::GetVarcharValue(p_brand), pool.get()); - } - - p = p_end + 1; - p_end = strchr(p, '|'); - tuple.SetValue(4, type::ValueFactory::GetVarcharValue(std::string{p, p_end}), pool.get()); - - p = p_end + 1; - tuple.SetValue(5, type::ValueFactory::GetIntegerValue(std::atoi(p))); - - p = p_end + 1; - p_end = strchr(p, '|'); - std::string p_container{p, p_end}; - if (config_.dictionary_encode) { - uint32_t code = DictionaryEncode(p_container_dict_, p_brand); - tuple.SetValue(6, type::ValueFactory::GetIntegerValue(code)); - } else { - tuple.SetValue(6, type::ValueFactory::GetVarcharValue(p_container), pool.get()); - } - - p = p_end + 1; - tuple.SetValue(7, type::ValueFactory::GetDecimalValue(std::atof(p))); - - p = p_end + 1; - p_end = strchr(p, '|'); - tuple.SetValue(8, type::ValueFactory::GetVarcharValue(std::string{p, p_end}), pool.get()); - - // Insert into table - ItemPointer tuple_slot_id = table.InsertTuple(&tuple); - PELOTON_ASSERT(tuple_slot_id.block != INVALID_OID); - PELOTON_ASSERT(tuple_slot_id.offset != INVALID_OID); - txn_manager.PerformInsert(txn, tuple_slot_id); - - num_tuples++; - - }); - - // Commit - PELOTON_ASSERT(txn_manager.CommitTransaction(txn) == ResultType::SUCCESS); - - timer.Stop(); - LOG_INFO("Loading Part finished: %.2f ms (%lu tuples)\n", - timer.GetDuration(), num_tuples); - - // Set table as loaded - SetTableIsLoaded(TableId::Part); -} - -void TPCHDatabase::LoadSupplierTable() {} - -void TPCHDatabase::LoadPartSupplierTable() {} - -void TPCHDatabase::LoadCustomerTable() { - if (TableIsLoaded(TableId::Customer)) { - return; - } - - const std::string filename = config_.GetPartPath(); - - LOG_INFO("Loading Customer ['%s']\n", filename.c_str()); - - auto &table = GetTable(TableId::Customer); - - Timer> timer; - timer.Start(); - - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - auto *txn = txn_manager.BeginTransaction(); - - uint64_t num_tuples = 0; - std::unique_ptr pool{new type::EphemeralPool()}; - - ForEachLine(filename, [&](char *p) { - storage::Tuple tuple{table.GetSchema(), true /* allocate */}; - - // C_CUSTKEY - int32_t c_custkey = std::atoi(p); - tuple.SetValue(0, type::ValueFactory::GetIntegerValue(c_custkey)); - - // C_NAME - p = strchr(p, '|') + 1; - char *p_end = strchr(p, '|'); - - std::string c_name{p, p_end}; - tuple.SetValue(1, type::ValueFactory::GetVarcharValue(c_name), pool.get()); - - // C_ADDRESS - p = p_end + 1; - p_end = strchr(p, '|'); - - std::string c_address{p, p_end}; - tuple.SetValue(2, type::ValueFactory::GetVarcharValue(c_address), pool.get()); - - // C_NATIONKEY - p = p_end + 1; - - int32_t c_nationkey = std::atoi(p); - tuple.SetValue(3, type::ValueFactory::GetIntegerValue(c_nationkey)); - - // C_PHONE - p = strchr(p, '|') + 1; - p_end = strchr(p, '|'); - std::string c_phone{p, p_end}; - tuple.SetValue(4, type::ValueFactory::GetVarcharValue(c_phone), pool.get()); - - // C_ACCTBA - p = p_end + 1; - double c_acctba = std::atof(p); - tuple.SetValue(5, type::ValueFactory::GetDecimalValue(c_acctba)); - - // C_MKTSEGMENT - p = strchr(p, '|') + 1; - p_end = strchr(p, '|'); - std::string c_mktsegment{p, p_end}; - if (config_.dictionary_encode) { - uint32_t code = DictionaryEncode(c_mktsegment_dict_, c_mktsegment); - tuple.SetValue(6, type::ValueFactory::GetIntegerValue(code)); - } else { - tuple.SetValue(6, type::ValueFactory::GetVarcharValue(c_mktsegment), pool.get()); - } - - // C_COMMENT - p = p_end + 1; - p_end = strchr(p, '|'); - std::string c_comment{p, p_end}; - tuple.SetValue(7, type::ValueFactory::GetVarcharValue(c_comment), pool.get()); - - // Insert into table - ItemPointer tuple_slot_id = table.InsertTuple(&tuple); - PELOTON_ASSERT(tuple_slot_id.block != INVALID_OID); - PELOTON_ASSERT(tuple_slot_id.offset != INVALID_OID); - txn_manager.PerformInsert(txn, tuple_slot_id); - - num_tuples++; - }); - - // Commit - PELOTON_ASSERT(txn_manager.CommitTransaction(txn) == ResultType::SUCCESS); - - timer.Stop(); - LOG_INFO("Loading Customer finished: %.2f ms (%lu tuples)\n", - timer.GetDuration(), num_tuples); - - // Set table as loaded - SetTableIsLoaded(TableId::Customer); -} - -void TPCHDatabase::LoadNationTable() {} - -void TPCHDatabase::LoadLineitemTable() { - // Short-circuit if table is already loaded - if (TableIsLoaded(TableId::Lineitem)) { - return; - } - - const std::string filename = config_.GetLineitemPath(); - - LOG_INFO("Loading Lineitem ['%s']\n", filename.c_str()); - - uint64_t num_tuples = 0; - - auto &table = GetTable(TableId::Lineitem); - - std::unique_ptr pool{new type::EphemeralPool()}; - - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - auto *txn = txn_manager.BeginTransaction(); - - Timer> timer; - timer.Start(); - - ForEachLine(filename, [&](char *p) { - // The input tuple - storage::Tuple tuple{table.GetSchema(), /*allocate*/ true}; - - tuple.SetValue(0, type::ValueFactory::GetIntegerValue(std::atoi(p))); - - p = strchr(p, '|') + 1; - tuple.SetValue(1, type::ValueFactory::GetIntegerValue(std::atoi(p))); - - p = strchr(p, '|') + 1; - tuple.SetValue(2, type::ValueFactory::GetIntegerValue(std::atoi(p))); - - p = strchr(p, '|') + 1; - tuple.SetValue(3, type::ValueFactory::GetIntegerValue(std::atoi(p))); - - p = strchr(p, '|') + 1; - tuple.SetValue(4, - type::ValueFactory::GetIntegerValue((int32_t)std::atof(p))); - - p = strchr(p, '|') + 1; - tuple.SetValue(5, type::ValueFactory::GetDecimalValue(std::atof(p))); - - p = strchr(p, '|') + 1; - tuple.SetValue(6, type::ValueFactory::GetDecimalValue(std::atof(p))); - - p = strchr(p, '|') + 1; - tuple.SetValue(7, type::ValueFactory::GetDecimalValue(std::atof(p))); - - p = strchr(p, '|') + 1; - char returnflag = *p; - tuple.SetValue(8, type::ValueFactory::GetIntegerValue(returnflag)); - - p = strchr(p, '|') + 1; - char linestatus = *p; - tuple.SetValue(9, type::ValueFactory::GetIntegerValue(linestatus)); - - p = strchr(p, '|') + 1; - tuple.SetValue(10, type::ValueFactory::GetDateValue(ConvertDate(p))); - - p = strchr(p, '|') + 1; - tuple.SetValue(11, type::ValueFactory::GetDateValue(ConvertDate(p))); - - p = strchr(p, '|') + 1; - tuple.SetValue(12, type::ValueFactory::GetDateValue(ConvertDate(p))); - - p = strchr(p, '|') + 1; - char *p_end = strchr(p, '|'); - std::string l_shipinstruct{p, p_end}; - if (config_.dictionary_encode) { - uint32_t code = DictionaryEncode(l_shipinstruct_dict_, l_shipinstruct); - tuple.SetValue(13, type::ValueFactory::GetIntegerValue(code), nullptr); - } else { - tuple.SetValue(13, type::ValueFactory::GetVarcharValue(l_shipinstruct), - pool.get()); - } - - p = p_end + 1; - p_end = strchr(p, '|'); - std::string l_shipmode{p, p_end}; - if (config_.dictionary_encode) { - uint32_t code = DictionaryEncode(l_shipmode_dict_, l_shipmode); - tuple.SetValue(14, type::ValueFactory::GetIntegerValue(code)); - } else { - tuple.SetValue(14, type::ValueFactory::GetVarcharValue(l_shipinstruct), - pool.get()); - } - - p = p_end + 1; - p_end = strchr(p, '|'); - tuple.SetValue(15, - type::ValueFactory::GetVarcharValue(std::string{p, p_end}), - pool.get()); - - // Insert into table - ItemPointer tuple_slot_id = table.InsertTuple(&tuple); - PELOTON_ASSERT(tuple_slot_id.block != INVALID_OID); - PELOTON_ASSERT(tuple_slot_id.offset != INVALID_OID); - txn_manager.PerformInsert(txn, tuple_slot_id); - - num_tuples++; - }); - - // Commit - auto res = txn_manager.CommitTransaction(txn); - PELOTON_ASSERT(res == ResultType::SUCCESS); - if (res != ResultType::SUCCESS) { - LOG_ERROR("Could not commit transaction during load!"); - } - - timer.Stop(); - LOG_INFO("Loading Lineitem finished: %.2f ms (%lu tuples)\n", - timer.GetDuration(), num_tuples); - - // Set table as loaded - SetTableIsLoaded(TableId::Lineitem); -} - -void TPCHDatabase::LoadRegionTable() {} - -void TPCHDatabase::LoadOrdersTable() {} - -} // namespace tpch -} // namespace benchmark -} // namespace peloton diff --git a/src/main/tpch/tpch_workload.cpp b/src/main/tpch/tpch_workload.cpp deleted file mode 100644 index d4721d9b080..00000000000 --- a/src/main/tpch/tpch_workload.cpp +++ /dev/null @@ -1,214 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpch_workload.cpp -// -// Identification: src/main/tpch/tpch_workload.cpp -// -// Copyright (c) 2015-17, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "benchmark/tpch/tpch_workload.h" - -#include "codegen/query.h" -#include "concurrency/transaction_manager_factory.h" -#include "executor/plan_executor.h" -#include "planner/abstract_plan.h" -#include "planner/binding_context.h" -#include "codegen/counting_consumer.h" - -namespace peloton { -namespace benchmark { -namespace tpch { - -TPCHBenchmark::TPCHBenchmark(const Configuration &config, TPCHDatabase &db) - : config_(config), db_(db) { - query_configs_ = { - {"Q1", - QueryId::Q1, - {TableId::Lineitem}, - [&]() { return ConstructQ1Plan(); }}, - - {"Q2", - QueryId::Q2, - {TableId::Lineitem}, - [&]() { return ConstructQ1Plan(); }}, - - {"Q3", - QueryId::Q3, - {TableId::Lineitem, TableId::Customer, TableId::Orders}, - [&]() { return ConstructQ3Plan(); }}, - - {"Q4", - QueryId::Q4, - {TableId::Lineitem}, - [&]() { return ConstructQ1Plan(); }}, - - {"Q5", - QueryId::Q5, - {TableId::Lineitem}, - [&]() { return ConstructQ1Plan(); }}, - - {"Q6", - QueryId::Q6, - {TableId::Lineitem}, - [&]() { return ConstructQ6Plan(); }}, - - {"Q7", - QueryId::Q7, - {TableId::Lineitem}, - [&]() { return ConstructQ1Plan(); }}, - - {"Q8", - QueryId::Q8, - {TableId::Lineitem}, - [&]() { return ConstructQ1Plan(); }}, - - {"Q9", - QueryId::Q9, - {TableId::Lineitem}, - [&]() { return ConstructQ1Plan(); }}, - - {"Q10", - QueryId::Q10, - {TableId::Lineitem}, - [&]() { return ConstructQ1Plan(); }}, - - {"Q11", - QueryId::Q11, - {TableId::Lineitem}, - [&]() { return ConstructQ1Plan(); }}, - - {"Q12", - QueryId::Q12, - {TableId::Lineitem}, - [&]() { return ConstructQ1Plan(); }}, - - {"Q13", - QueryId::Q13, - {TableId::Lineitem}, - [&]() { return ConstructQ1Plan(); }}, - - {"Q14", - QueryId::Q14, - {TableId::Lineitem}, - [&]() { return ConstructQ1Plan(); }}, - - {"Q15", - QueryId::Q15, - {TableId::Lineitem}, - [&]() { return ConstructQ1Plan(); }}, - - {"Q16", - QueryId::Q16, - {TableId::Lineitem}, - [&]() { return ConstructQ1Plan(); }}, - - {"Q17", - QueryId::Q17, - {TableId::Lineitem}, - [&]() { return ConstructQ1Plan(); }}, - - {"Q18", - QueryId::Q18, - {TableId::Lineitem}, - [&]() { return ConstructQ1Plan(); }}, - - {"Q19", - QueryId::Q19, - {TableId::Lineitem}, - [&]() { return ConstructQ1Plan(); }}, - - {"Q20", - QueryId::Q20, - {TableId::Lineitem}, - [&]() { return ConstructQ1Plan(); }}, - - {"Q21", - QueryId::Q21, - {TableId::Lineitem}, - [&]() { return ConstructQ1Plan(); }}, - - {"Q22", - QueryId::Q22, - {TableId::Lineitem}, - [&]() { return ConstructQ1Plan(); }}, - }; -} - -void TPCHBenchmark::RunBenchmark() { - for (uint32_t i = 0; i < 22; i++) { - const auto &query_config = query_configs_[i]; - if (config_.ShouldRunQuery(query_config.query_id)) { - RunQuery(query_config); - } - } -} - -void TPCHBenchmark::RunQuery(const TPCHBenchmark::QueryConfig &query_config) { - LOG_INFO("Running TPCH %s", query_config.query_name.c_str()); - - // Load all the necessary tables - for (auto tid : query_config.required_tables) { - db_.LoadTable(tid); - } - - // Construct the plan for Q1 - std::unique_ptr plan = query_config.PlanConstructor(); - - // Do attribute binding - planner::BindingContext binding_context; - plan->PerformBinding(binding_context); - - // The consumer - codegen::CountingConsumer counter; - - // Compile - codegen::QueryCompiler::CompileStats compile_stats; - codegen::QueryCompiler compiler; - auto compiled_query = compiler.Compile(*plan, counter, &compile_stats); - - codegen::Query::RuntimeStats overall_stats; - overall_stats.init_ms = 0.0; - overall_stats.plan_ms = 0.0; - overall_stats.tear_down_ms = 0.0; - for (uint32_t i = 0; i < config_.num_runs; i++) { - // Reset the counter for this run - counter.ResetCount(); - - // Begin a transaction - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - auto *txn = txn_manager.BeginTransaction(); - - // Execute query in a transaction - codegen::Query::RuntimeStats runtime_stats; - compiled_query->Execute(*txn, counter.GetCountAsState(), &runtime_stats); - - // Commit transaction - txn_manager.CommitTransaction(txn); - - // Collect stats - overall_stats.init_ms += runtime_stats.init_ms; - overall_stats.plan_ms += runtime_stats.plan_ms; - overall_stats.tear_down_ms += runtime_stats.tear_down_ms; - } - - LOG_INFO("%s: %s", - query_config.query_name.c_str(), peloton::GETINFO_THICK_LINE.c_str()); - LOG_INFO("# Runs: %u, # Result tuples: %lu", config_.num_runs, - counter.GetCount()); - LOG_INFO("Setup: %.2lf, IR Gen: %.2lf, Compile: %.2lf", - compile_stats.setup_ms, compile_stats.ir_gen_ms, - compile_stats.jit_ms); - LOG_INFO("Init: %.2lf ms, Plan: %.2lf ms, TearDown: %.2lf ms", - overall_stats.init_ms / config_.num_runs, - overall_stats.plan_ms / config_.num_runs, - overall_stats.tear_down_ms / config_.num_runs); -} - - -} // namespace tpch -} // namespace benchmark -} // namespace peloton \ No newline at end of file diff --git a/src/main/tpch/tpch_workload_q1.cpp b/src/main/tpch/tpch_workload_q1.cpp deleted file mode 100644 index 2dd0e2c0f54..00000000000 --- a/src/main/tpch/tpch_workload_q1.cpp +++ /dev/null @@ -1,170 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpch_workload_q1.cpp -// -// Identification: src/main/tpch/tpch_workload_q1.cpp -// -// Copyright (c) 2015-17, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "benchmark/tpch/tpch_workload.h" - -#include "concurrency/transaction_manager_factory.h" -#include "expression/comparison_expression.h" -#include "expression/constant_value_expression.h" -#include "expression/operator_expression.h" -#include "expression/tuple_value_expression.h" -#include "planner/aggregate_plan.h" -#include "planner/order_by_plan.h" -#include "planner/seq_scan_plan.h" - - -namespace peloton { -namespace benchmark { -namespace tpch { - -static constexpr int32_t _1998_08_28 = 904276800; - -std::unique_ptr TPCHBenchmark::ConstructQ1Plan() const { - auto &lineitem = db_.GetTable(TableId::Lineitem); - - ////////////////////////////////////////////////////////////////////////////// - /// THE PREDICATE FOR THE SCAN OVER LINEITEM - ////////////////////////////////////////////////////////////////////////////// - - auto shipdate_predicate = std::unique_ptr{ - new expression::ComparisonExpression( - ExpressionType::COMPARE_LESSTHANOREQUALTO, - new expression::TupleValueExpression(type::TypeId::INTEGER, 0, - 10), - new expression::ConstantValueExpression( - type::ValueFactory::GetDateValue(_1998_08_28)))}; - - ////////////////////////////////////////////////////////////////////////////// - /// THE SCAN PLAN - ////////////////////////////////////////////////////////////////////////////// - - // Lineitem scan - std::unique_ptr lineitem_scan{new planner::SeqScanPlan( - &lineitem, shipdate_predicate.release(), {8, 9, 4, 5, 6, 7})}; - - ////////////////////////////////////////////////////////////////////////////// - /// THE AGGREGATION PLAN - ///////////////////////////////////////////////////////////////////////////// - - // sum(l_quantity) as sum_qty - planner::AggregatePlan::AggTerm agg1{ - ExpressionType::AGGREGATE_SUM, - new expression::TupleValueExpression(type::TypeId::INTEGER, 0, 2)}; - agg1.agg_ai.type = type::TypeId::INTEGER; - - // sum(l_extendedprice) as sum_base_price - planner::AggregatePlan::AggTerm agg2{ - ExpressionType::AGGREGATE_SUM, - new expression::TupleValueExpression(type::TypeId::DECIMAL, 0, 3)}; - agg2.agg_ai.type = type::TypeId::DECIMAL; - - // sum(l_extendedprice * (1 - l_discount)) as sum_disc_price - planner::AggregatePlan::AggTerm agg3{ - ExpressionType::AGGREGATE_SUM, - new expression::OperatorExpression( - ExpressionType::OPERATOR_MULTIPLY, type::TypeId::DECIMAL, - new expression::TupleValueExpression(type::TypeId::DECIMAL, 0, - 3), - new expression::OperatorExpression( - ExpressionType::OPERATOR_MINUS, type::TypeId::DECIMAL, - new expression::ConstantValueExpression( - type::ValueFactory::GetDecimalValue(1.0)), - new expression::TupleValueExpression(type::TypeId::DECIMAL, - 0, 4)))}; - agg3.agg_ai.type = type::TypeId::DECIMAL; - - // sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) - planner::AggregatePlan::AggTerm agg4{ - ExpressionType::AGGREGATE_SUM, - new expression::OperatorExpression( - ExpressionType::OPERATOR_MULTIPLY, type::TypeId::DECIMAL, - // l_extendedprice - new expression::TupleValueExpression(type::TypeId::DECIMAL, 0, - 3), - // (1 - l_discount) * (1 + l_tax) - new expression::OperatorExpression( - ExpressionType::OPERATOR_MULTIPLY, type::TypeId::DECIMAL, - // 1 - l_discount - new expression::OperatorExpression( - ExpressionType::OPERATOR_MINUS, type::TypeId::DECIMAL, - new expression::ConstantValueExpression( - type::ValueFactory::GetDecimalValue(1.0)), - new expression::TupleValueExpression( - type::TypeId::DECIMAL, 0, 4)), - // 1 + l_tax - new expression::OperatorExpression( - ExpressionType::OPERATOR_PLUS, type::TypeId::DECIMAL, - new expression::ConstantValueExpression( - type::ValueFactory::GetDecimalValue(1.0)), - new expression::TupleValueExpression( - type::TypeId::DECIMAL, 0, 5))))}; - agg4.agg_ai.type = type::TypeId::DECIMAL; - - // avg(l_quantity) - planner::AggregatePlan::AggTerm agg5{ - ExpressionType::AGGREGATE_AVG, - new expression::TupleValueExpression(type::TypeId::INTEGER, 0, 2)}; - agg5.agg_ai.type = type::TypeId::DECIMAL; - - // avg(l_extendedprice) - planner::AggregatePlan::AggTerm agg6{ - ExpressionType::AGGREGATE_AVG, - new expression::TupleValueExpression(type::TypeId::DECIMAL, 0, 3)}; - agg6.agg_ai.type = type::TypeId::DECIMAL; - - // avg(l_discount) - planner::AggregatePlan::AggTerm agg7{ - ExpressionType::AGGREGATE_AVG, - new expression::TupleValueExpression(type::TypeId::DECIMAL, 0, 4)}; - agg7.agg_ai.type = type::TypeId::DECIMAL; - - // count(*) - planner::AggregatePlan::AggTerm agg8{ExpressionType::AGGREGATE_COUNT_STAR, - nullptr}; - agg8.agg_ai.type = type::TypeId::BIGINT; - - auto output_schema = - std::shared_ptr{new catalog::Schema( - {{type::TypeId::INTEGER, kIntSize, "l_returnflag"}, - {type::TypeId::INTEGER, kIntSize, "l_linestatus"}, - {type::TypeId::INTEGER, kIntSize, "sum_qty"}, - {type::TypeId::DECIMAL, kDecimalSize, "sum_base_price"}, - {type::TypeId::DECIMAL, kDecimalSize, "sum_disc_price"}, - {type::TypeId::DECIMAL, kDecimalSize, "sum_charge"}, - {type::TypeId::DECIMAL, kDecimalSize, "avg_qty"}, - {type::TypeId::DECIMAL, kDecimalSize, "avg_price"}, - {type::TypeId::DECIMAL, kDecimalSize, "avg_disc"}, - {type::TypeId::BIGINT, kBigIntSize, "count_order"}})}; - - DirectMapList dml = {{0, {0, 0}}, {1, {0, 1}}, {2, {1, 0}}, {3, {1, 1}}, - {4, {1, 2}}, {5, {1, 3}}, {6, {1, 4}}, {7, {1, 5}}, - {8, {1, 6}}, {9, {1, 7}}}; - - std::unique_ptr agg_project{ - new planner::ProjectInfo(TargetList{}, std::move(dml))}; - auto agg_terms = {agg1, agg2, agg3, agg4, agg5, agg6, agg7, agg8}; - std::unique_ptr agg_plan{new planner::AggregatePlan( - std::move(agg_project), nullptr, std::move(agg_terms), {0, 1}, - output_schema, AggregateType::HASH)}; - - std::unique_ptr sort_plan{new planner::OrderByPlan{ - {0, 1}, {false, false}, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}}}; - - agg_plan->AddChild(std::move(lineitem_scan)); - sort_plan->AddChild(std::move(agg_plan)); - - return sort_plan; -} - -} // namespace tpch -} // namespace benchmark -} // namespace peloton \ No newline at end of file diff --git a/src/main/tpch/tpch_workload_q3.cpp b/src/main/tpch/tpch_workload_q3.cpp deleted file mode 100644 index 98a41e01b1e..00000000000 --- a/src/main/tpch/tpch_workload_q3.cpp +++ /dev/null @@ -1,253 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpch_workload_q3.cpp -// -// Identification: src/main/tpch/tpch_workload_q3.cpp -// -// Copyright (c) 2015-17, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "benchmark/tpch/tpch_workload.h" - -#include "concurrency/transaction_manager_factory.h" -#include "expression/comparison_expression.h" -#include "expression/constant_value_expression.h" -#include "expression/operator_expression.h" -#include "expression/tuple_value_expression.h" -#include "planner/aggregate_plan.h" -#include "planner/hash_join_plan.h" -#include "planner/hash_plan.h" -#include "planner/order_by_plan.h" -#include "planner/projection_plan.h" -#include "planner/seq_scan_plan.h" - -namespace peloton { -namespace benchmark { -namespace tpch { - -static int32_t _1995_03_10 = 794811600; - -std::unique_ptr TPCHBenchmark::ConstructQ3Plan() const { - auto &lineitem = db_.GetTable(TableId::Lineitem); - auto &customer = db_.GetTable(TableId::Customer); - auto &orders = db_.GetTable(TableId::Orders); - - ////////////////////////////////////////////////////////////////////////////// [3136/4535] - /// THE PREDICATE FOR THE SCAN OVER LINEITEM - ////////////////////////////////////////////////////////////////////////////// - - uint32_t machinery = db_.CodeForMktSegment("MACHINERY"); - - auto orderdate_pred = std::unique_ptr{ - new expression::ComparisonExpression( - ExpressionType::COMPARE_LESSTHANOREQUALTO, - new expression::TupleValueExpression(type::TypeId::INTEGER, 0, 4), - new expression::ConstantValueExpression(type::ValueFactory::GetDateValue(_1995_03_10)))}; - - auto mktsegment_pred = std::unique_ptr{ - new expression::ComparisonExpression( - ExpressionType::COMPARE_EQUAL, - new expression::TupleValueExpression(type::TypeId::INTEGER, 0, 6), - new expression::ConstantValueExpression(type::ValueFactory::GetIntegerValue(machinery)))}; - - auto shipdate_pred = std::unique_ptr{ - new expression::ComparisonExpression( - ExpressionType::COMPARE_GREATERTHAN, - new expression::TupleValueExpression(type::TypeId::INTEGER, 0, 10), - new expression::ConstantValueExpression(type::ValueFactory::GetDateValue(_1995_03_10)))}; - - ////////////////////////////////////////////////////////////////////////////// - /// THE SCAN PLANS - ////////////////////////////////////////////////////////////////////////////// - - // The table scans - std::unique_ptr lineitem_scan{ - new planner::SeqScanPlan(&lineitem, shipdate_pred.release(), {0,5,6})}; - std::unique_ptr order_scan{ - new planner::SeqScanPlan(&orders, orderdate_pred.release(), {0,1,4,7})}; - std::unique_ptr customer_scan{ - new planner::SeqScanPlan(&customer, mktsegment_pred.release(), {0})}; - - ////////////////////////////////////////////////////////////////////////////// - /// REARRANGE ORDERS COLUMNS FOR JOIN - ////////////////////////////////////////////////////////////////////////////// - - DirectMap dmo1 = std::make_pair(0, std::make_pair(0, 1)); - DirectMap dmo2 = std::make_pair(1, std::make_pair(0, 0)); - DirectMap dmo3 = std::make_pair(2, std::make_pair(0, 2)); - DirectMap dmo4 = std::make_pair(3, std::make_pair(0, 3)); - DirectMapList order_dm = {dmo1, dmo2, dmo3, dmo4}; - - std::unique_ptr order_project_info{ - new planner::ProjectInfo(TargetList{}, std::move(order_dm))}; - auto order_schema = std::shared_ptr(new catalog::Schema( - {{type::TypeId::INTEGER, kIntSize, "o_custkey", true}, - {type::TypeId::INTEGER, kIntSize, "o_orderkey", true}, - {type::TypeId::DATE, kDateSize, "o_orderdate", true}, - {type::TypeId::INTEGER, kIntSize, "o_shippriority", true}})); - std::unique_ptr order_projection{ - new planner::ProjectionPlan(std::move(order_project_info), order_schema)}; - - ////////////////////////////////////////////////////////////////////////////// - /// Customer - Order HASH PLAN - ////////////////////////////////////////////////////////////////////////////// - - std::vector> hash_keys; - hash_keys.emplace_back( - new expression::TupleValueExpression(type::TypeId::INTEGER, 0, 0)); - - std::unique_ptr customer_hash_plan{new planner::HashPlan(hash_keys)}; - - ////////////////////////////////////////////////////////////////////////////// - /// Customer - Order JOIN - ////////////////////////////////////////////////////////////////////////////// - - DirectMap dm1 = std::make_pair(0, std::make_pair(0, 1)); - DirectMap dm2 = std::make_pair(1, std::make_pair(0, 2)); - DirectMap dm3 = std::make_pair(2, std::make_pair(0, 3)); - DirectMapList direct_map_list = {dm1, dm2, dm3}; - - std::unique_ptr projection{new planner::ProjectInfo( - TargetList{}, std::move(direct_map_list))}; - - auto schema = std::shared_ptr(new catalog::Schema( - {{type::TypeId::INTEGER, kIntSize, "o_orderkey", true}, - {type::TypeId::DATE, kDateSize, "o_orderdate", true}, - {type::TypeId::INTEGER, kIntSize, "o_shippriority", true}})); - - std::vector> left_hash_keys; - left_hash_keys.emplace_back( - new expression::TupleValueExpression(type::TypeId::INTEGER, 0, 1)); - - std::vector> right_hash_keys; - right_hash_keys.emplace_back( - new expression::TupleValueExpression(type::TypeId::INTEGER, 1, 0)); - - std::unique_ptr cust_order_hj_plan{new planner::HashJoinPlan( - JoinType::INNER, nullptr, std::move(projection), - schema, left_hash_keys, right_hash_keys)}; - - ////////////////////////////////////////////////////////////////////////////// - /// ORDER - LINITEM HASH PLAN - ////////////////////////////////////////////////////////////////////////////// - - std::vector> hash_keys2; - hash_keys2.emplace_back( - new expression::TupleValueExpression(type::TypeId::INTEGER, 0, 0)); - - std::unique_ptr cust_order_hash_plan{new planner::HashPlan(hash_keys2)}; - - ////////////////////////////////////////////////////////////////////////////// - /// ORDER - LINEITEM JOIN - ////////////////////////////////////////////////////////////////////////////// - - DirectMap dm12 = std::make_pair(0, std::make_pair(0, 0)); - DirectMap dm22 = std::make_pair(1, std::make_pair(0, 1)); - DirectMap dm32 = std::make_pair(2, std::make_pair(0, 2)); - DirectMap dm42 = std::make_pair(3, std::make_pair(1, 0)); - DirectMap dm52 = std::make_pair(4, std::make_pair(1, 1)); - DirectMap dm62 = std::make_pair(5, std::make_pair(1, 2)); - DirectMapList dm_ol = {dm12, dm22, dm32, dm42, dm52, dm62}; - - std::unique_ptr projection2{new planner::ProjectInfo( - TargetList{}, std::move(dm_ol))}; - - auto ol_schema = std::shared_ptr(new catalog::Schema( - {{type::TypeId::INTEGER, kIntSize, "l_orderkey", true}, - {type::TypeId::DECIMAL, kDecimalSize, "l_extendedprice", true}, - {type::TypeId::DECIMAL, kDecimalSize, "l_discount", true}, - {type::TypeId::INTEGER, kIntSize, "o_orderkey", true}, - {type::TypeId::DATE, kDateSize, "o_shipddate", true}, - {type::TypeId::INTEGER, kIntSize, "o_shippriority", true}})); - - std::vector> left_hash_keys2; - left_hash_keys2.emplace_back( - new expression::TupleValueExpression(type::TypeId::INTEGER, 0, 1)); - - std::vector> right_hash_keys2; - right_hash_keys2.emplace_back( - new expression::TupleValueExpression(type::TypeId::INTEGER, 1, 0)); - - std::unique_ptr hj_plan2{new planner::HashJoinPlan( - JoinType::INNER, nullptr, std::move(projection2), - ol_schema, left_hash_keys2, right_hash_keys2)}; - - ////////////////////////////////////////////////////////////////////////////// - /// AGGREGATION - ////////////////////////////////////////////////////////////////////////////// - planner::AggregatePlan::AggTerm agg1{ - ExpressionType::AGGREGATE_SUM, - new expression::OperatorExpression{ - ExpressionType::OPERATOR_MULTIPLY, type::TypeId::DECIMAL, - new expression::TupleValueExpression{type::TypeId::DECIMAL, 0, 1}, - new expression::OperatorExpression{ - ExpressionType::OPERATOR_MINUS, type::TypeId::DECIMAL, - new expression::ConstantValueExpression{type::ValueFactory::GetDecimalValue(1.0)}, - new expression::TupleValueExpression{type::TypeId::DECIMAL, 0, 2} - } - } - }; - auto agg_out_schema = std::shared_ptr{ - new catalog::Schema{ - {{type::TypeId::INTEGER, kIntSize, "l_orderkey", true}, - {type::TypeId::DATE, kDateSize, "o_orderdate", true}, - {type::TypeId::INTEGER, kIntSize, "o_shippriority", true}, - {type::TypeId::DECIMAL, kDecimalSize, "revenue", true}} - } - }; - - DirectMapList dml = { - {0, {0, 0}}, {1, {0, 4}}, {2, {0, 5}}, {3, {1, 0}} - }; - std::unique_ptr agg_project{ - new planner::ProjectInfo(TargetList{}, std::move(dml))}; - std::unique_ptr agg_plan{ - new planner::AggregatePlan(std::move(agg_project), nullptr, - {agg1}, {0, 4, 5}, agg_out_schema, - AggregateType::HASH)}; - - ////////////////////////////////////////////////////////////////////////////// - /// SORT - ////////////////////////////////////////////////////////////////////////////// - - std::unique_ptr sort_plan{ - new planner::OrderByPlan{{3,1}, {false, false}, {0,1,2,3}} - }; - - ////////////////////////////////////////////////////////////////////////////// - /// TIE THE SHIT UP - ////////////////////////////////////////////////////////////////////////////// - - // Build hash on customer - customer_hash_plan->AddChild(std::move(customer_scan)); - - // Project order - order_projection->AddChild(std::move(order_scan)); - - // Customer x Orders hash join (from convention, build goes on right) - cust_order_hj_plan->AddChild(std::move(order_projection)); - cust_order_hj_plan->AddChild(std::move(customer_hash_plan)); - - // Build hash on output of cust x order - cust_order_hash_plan->AddChild(std::move(cust_order_hj_plan)); - - // Final join - hj_plan2->AddChild(std::move(lineitem_scan)); - hj_plan2->AddChild(std::move(cust_order_hash_plan)); - - // Aggregation on output of join - agg_plan->AddChild(std::move(hj_plan2)); - - // Sort final results - sort_plan->AddChild(std::move(agg_plan)); - - return sort_plan; - -} - -} // namespace tpch -} // namespace benchmark -} // namespace peloton \ No newline at end of file diff --git a/src/main/tpch/tpch_workload_q6.cpp b/src/main/tpch/tpch_workload_q6.cpp deleted file mode 100644 index 8a1008d7567..00000000000 --- a/src/main/tpch/tpch_workload_q6.cpp +++ /dev/null @@ -1,138 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tpch_workload_q1.cpp -// -// Identification: src/main/tpch/tpch_workload_q1.cpp -// -// Copyright (c) 2015-17, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "benchmark/tpch/tpch_workload.h" - -#include "concurrency/transaction_manager_factory.h" -#include "expression/comparison_expression.h" -#include "expression/conjunction_expression.h" -#include "expression/constant_value_expression.h" -#include "expression/operator_expression.h" -#include "expression/tuple_value_expression.h" -#include "planner/aggregate_plan.h" -#include "planner/order_by_plan.h" -#include "planner/seq_scan_plan.h" - - -namespace peloton { -namespace benchmark { -namespace tpch { - -static constexpr int32_t _1997_01_01 = 852094800; -static constexpr int32_t _1998_01_01 = 883630800; - -std::unique_ptr TPCHBenchmark::ConstructQ6Plan() const { - auto &lineitem = db_.GetTable(TableId::Lineitem); - - ////////////////////////////////////////////////////////////////////////////// - /// THE PREDICATE FOR THE SCAN OVER LINEITEM - ////////////////////////////////////////////////////////////////////////////// - - auto shipdate_gte = std::unique_ptr{ - new expression::ComparisonExpression( - ExpressionType::COMPARE_GREATERTHANOREQUALTO, - new expression::TupleValueExpression(type::TypeId::DATE, 0, 10), - new expression::ConstantValueExpression( - type::ValueFactory::GetDateValue(_1997_01_01)))}; - - auto shipdate_lt = std::unique_ptr{ - new expression::ComparisonExpression( - ExpressionType::COMPARE_LESSTHAN, - new expression::TupleValueExpression(type::TypeId::DATE, 0, 10), - new expression::ConstantValueExpression( - type::ValueFactory::GetDateValue(_1998_01_01)))}; - - auto discount_gt = std::unique_ptr{ - new expression::ComparisonExpression( - ExpressionType::COMPARE_GREATERTHAN, - new expression::TupleValueExpression(type::TypeId::DECIMAL, 0, 6), - new expression::OperatorExpression( - ExpressionType::OPERATOR_MINUS, type::TypeId::DECIMAL, - new expression::ConstantValueExpression(type::ValueFactory::GetDecimalValue(0.07)), - new expression::ConstantValueExpression(type::ValueFactory::GetDecimalValue(0.01)) - ) - ) - }; - - auto discount_lt = std::unique_ptr{ - new expression::ComparisonExpression( - ExpressionType::COMPARE_LESSTHAN, - new expression::TupleValueExpression(type::TypeId::DECIMAL, 0, 6), - new expression::OperatorExpression( - ExpressionType::OPERATOR_PLUS, type::TypeId::DECIMAL, - new expression::ConstantValueExpression(type::ValueFactory::GetDecimalValue(0.07)), - new expression::ConstantValueExpression(type::ValueFactory::GetDecimalValue(0.01)) - ) - ) - }; - - auto quantity_lt = std::unique_ptr{ - new expression::ComparisonExpression( - ExpressionType::COMPARE_LESSTHAN, - new expression::TupleValueExpression(type::TypeId::INTEGER, 0, 4), - new expression::ConstantValueExpression(type::ValueFactory::GetIntegerValue(24)) - ) - }; - - auto lineitem_pred = std::unique_ptr{ - new expression::ConjunctionExpression(ExpressionType::CONJUNCTION_AND, - quantity_lt.release(), - new expression::ConjunctionExpression(ExpressionType::CONJUNCTION_AND, - new expression::ConjunctionExpression(ExpressionType::CONJUNCTION_AND, shipdate_gte.release(), shipdate_lt.release()), - new expression::ConjunctionExpression(ExpressionType::CONJUNCTION_AND, discount_gt.release(), discount_lt.release()) - ) - ) - }; - - ////////////////////////////////////////////////////////////////////////////// - /// THE SCAN PLAN - ////////////////////////////////////////////////////////////////////////////// - - // Lineitem scan - auto lineitem_scan = std::unique_ptr{ - new planner::SeqScanPlan(&lineitem, lineitem_pred.release(), {5,6})}; - - ////////////////////////////////////////////////////////////////////////////// - /// THE GLOBAL AGGREGATION - ////////////////////////////////////////////////////////////////////////////// - - // sum(l_extendedprice * l_discount) as revenue - planner::AggregatePlan::AggTerm revenue_agg{ - ExpressionType::AGGREGATE_SUM, - new expression::OperatorExpression( - ExpressionType::OPERATOR_MULTIPLY, type::TypeId::DECIMAL, - new expression::TupleValueExpression(type::TypeId::DECIMAL, 0, 0), - new expression::TupleValueExpression(type::TypeId::DECIMAL, 0, 1))}; - revenue_agg.agg_ai.type = type::TypeId::DECIMAL; - - auto output_schema = - std::shared_ptr{new catalog::Schema( - {{type::TypeId::DECIMAL, kDecimalSize, "revenue"}})}; - - DirectMapList dml = {{0, {1, 0}}}; - - std::unique_ptr agg_project{ - new planner::ProjectInfo(TargetList{}, std::move(dml))}; - auto agg_terms = {revenue_agg}; - auto aggregation_plan = std::unique_ptr{ - new planner::AggregatePlan(std::move(agg_project), nullptr, - std::move(agg_terms), {}, output_schema, - AggregateType::HASH)}; - - aggregation_plan->AddChild(std::move(lineitem_scan)); - - return aggregation_plan; -} - -} // namespace tpch -} // namespace benchmark -} // namespace peloton \ No newline at end of file diff --git a/src/main/ycsb/ycsb.cpp b/src/main/ycsb/ycsb.cpp deleted file mode 100644 index 9b748da8ce7..00000000000 --- a/src/main/ycsb/ycsb.cpp +++ /dev/null @@ -1,102 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// ycsb.cpp -// -// Identification: src/main/ycsb/ycsb.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===//w - -#include -#include -#include - -#include "common/logger.h" -#include "benchmark/ycsb/ycsb_configuration.h" -#include "benchmark/ycsb/ycsb_loader.h" -#include "benchmark/ycsb/ycsb_workload.h" - -#include "gc/gc_manager_factory.h" -#include "concurrency/epoch_manager_factory.h" - -namespace peloton { -namespace benchmark { -namespace ycsb { - -configuration state; - -// Main Entry Point -void RunBenchmark() { - - if (state.gc_mode == false) { - gc::GCManagerFactory::Configure(0); - } else { - gc::GCManagerFactory::Configure(state.gc_backend_count); - } - - concurrency::EpochManagerFactory::Configure(state.epoch); - - std::unique_ptr epoch_thread; - std::vector> gc_threads; - - concurrency::EpochManager &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); - - if (concurrency::EpochManagerFactory::GetEpochType() == EpochType::DECENTRALIZED_EPOCH) { - for (size_t i = 0; i < (size_t) state.backend_count; ++i) { - // register thread to epoch manager - epoch_manager.RegisterThread(i); - } - } - - // start epoch. - epoch_manager.StartEpoch(epoch_thread); - - gc::GCManager &gc_manager = gc::GCManagerFactory::GetInstance(); - - // start GC. - gc_manager.StartGC(gc_threads); - - // Create the database - CreateYCSBDatabase(); - - // Load the databases - LoadYCSBDatabase(); - - // Run the workload - RunWorkload(); - - // stop GC. - gc_manager.StopGC(); - - // stop epoch. - epoch_manager.StopEpoch(); - - // join all gc threads - for (auto &gc_thread : gc_threads) { - PELOTON_ASSERT(gc_thread != nullptr); - gc_thread->join(); - } - - // join epoch thread - PELOTON_ASSERT(epoch_thread != nullptr); - epoch_thread->join(); - - // Emit throughput - WriteOutput(); -} - -} // namespace ycsb -} // namespace benchmark -} // namespace peloton - -int main(int argc, char **argv) { - peloton::benchmark::ycsb::ParseArguments(argc, argv, - peloton::benchmark::ycsb::state); - - peloton::benchmark::ycsb::RunBenchmark(); - - return 0; -} diff --git a/src/main/ycsb/ycsb_configuration.cpp b/src/main/ycsb/ycsb_configuration.cpp deleted file mode 100644 index 06be5414d55..00000000000 --- a/src/main/ycsb/ycsb_configuration.cpp +++ /dev/null @@ -1,318 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// ycsb_configuration.cpp -// -// Identification: src/main/ycsb/ycsb_configuration.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#include -#include -#include -#include - -#include "benchmark/ycsb/ycsb_configuration.h" -#include "common/logger.h" - -namespace peloton { -namespace benchmark { -namespace ycsb { - -void Usage(FILE *out) { - fprintf(out, - "Command line options : ycsb \n" - " -h --help : print help message \n" - " -i --index : index type: bwtree (default) \n" - " -k --scale_factor : # of K tuples \n" - " -d --duration : execution duration \n" - " -p --profile_duration : profile duration \n" - " -b --backend_count : # of backends \n" - " -c --column_count : # of columns \n" - " -o --operation_count : # of operations \n" - " -u --update_ratio : fraction of updates \n" - " -z --zipf_theta : theta to control skewness \n" - " -e --exp_backoff : enable exponential backoff \n" - " -m --string_mode : store strings \n" - " -g --gc_mode : enable garbage collection \n" - " -n --gc_backend_count : # of gc backends \n" - " -l --loader_count : # of loaders \n" - " -y --epoch : epoch type: centralized or decentralized \n" - ); -} - -static struct option opts[] = { - { "index", optional_argument, NULL, 'i' }, - { "scale_factor", optional_argument, NULL, 'k' }, - { "duration", optional_argument, NULL, 'd' }, - { "profile_duration", optional_argument, NULL, 'p' }, - { "backend_count", optional_argument, NULL, 'b' }, - { "column_count", optional_argument, NULL, 'c' }, - { "operation_count", optional_argument, NULL, 'o' }, - { "update_ratio", optional_argument, NULL, 'u' }, - { "zipf_theta", optional_argument, NULL, 'z' }, - { "exp_backoff", no_argument, NULL, 'e' }, - { "string_mode", no_argument, NULL, 'm' }, - { "gc_mode", no_argument, NULL, 'g' }, - { "gc_backend_count", optional_argument, NULL, 'n' }, - { "loader_count", optional_argument, NULL, 'n' }, - { "epoch", optional_argument, NULL, 'y' }, - { NULL, 0, NULL, 0 } -}; - -void ValidateIndex(const configuration &state) { - if (state.index != IndexType::BWTREE && state.index != IndexType::BWTREE) { - LOG_ERROR("Invalid index"); - exit(EXIT_FAILURE); - } -} - -void ValidateScaleFactor(const configuration &state) { - if (state.scale_factor <= 0) { - LOG_ERROR("Invalid scale_factor :: %d", state.scale_factor); - exit(EXIT_FAILURE); - } - - LOG_TRACE("%s : %d", "scale_factor", state.scale_factor); -} - -void ValidateDuration(const configuration &state) { - if (state.duration <= 0) { - LOG_ERROR("Invalid duration :: %lf", state.duration); - exit(EXIT_FAILURE); - } - - LOG_TRACE("%s : %lf", "duration", state.duration); -} - -void ValidateProfileDuration(const configuration &state) { - if (state.profile_duration <= 0) { - LOG_ERROR("Invalid profile_duration :: %lf", state.profile_duration); - exit(EXIT_FAILURE); - } - - LOG_TRACE("%s : %lf", "profile_duration", state.profile_duration); -} - -void ValidateBackendCount(const configuration &state) { - if (state.backend_count <= 0) { - LOG_ERROR("Invalid backend_count :: %d", state.backend_count); - exit(EXIT_FAILURE); - } - - LOG_TRACE("%s : %d", "backend_count", state.backend_count); -} - -void ValidateColumnCount(const configuration &state) { - if (state.column_count <= 0) { - LOG_ERROR("Invalid column_count :: %d", state.column_count); - exit(EXIT_FAILURE); - } - - LOG_TRACE("%s : %d", "column_count", state.column_count); -} - -void ValidateOperationCount(const configuration &state) { - if (state.operation_count <= 0) { - LOG_ERROR("Invalid operation_count :: %d", state.operation_count); - exit(EXIT_FAILURE); - } - - LOG_TRACE("%s : %d", "operation_count", state.operation_count); -} - -void ValidateUpdateRatio(const configuration &state) { - if (state.update_ratio < 0 || state.update_ratio > 1) { - LOG_ERROR("Invalid update_ratio :: %lf", state.update_ratio); - exit(EXIT_FAILURE); - } - - LOG_TRACE("%s : %lf", "update_ratio", state.update_ratio); -} - -void ValidateZipfTheta(const configuration &state) { - if (state.zipf_theta < 0 || state.zipf_theta > 1.0) { - LOG_ERROR("Invalid zipf_theta :: %lf", state.zipf_theta); - exit(EXIT_FAILURE); - } - - LOG_TRACE("%s : %lf", "zipf_theta", state.zipf_theta); -} - -void ValidateGCBackendCount(const configuration &state) { - if (state.gc_backend_count <= 0) { - LOG_ERROR("Invalid gc_backend_count :: %d", state.gc_backend_count); - exit(EXIT_FAILURE); - } - - LOG_TRACE("%s : %d", "gc_backend_count", state.gc_backend_count); -} - -void ParseArguments(int argc, char *argv[], configuration &state) { - // Default Values - state.index = IndexType::BWTREE; - state.epoch = EpochType::DECENTRALIZED_EPOCH; - state.scale_factor = 1; - state.duration = 10; - state.profile_duration = 1; - state.backend_count = 2; - state.column_count = 10; - state.operation_count = 10; - state.update_ratio = 0.5; - state.zipf_theta = 0.0; - state.exp_backoff = false; - state.string_mode = false; - state.gc_mode = false; - state.gc_backend_count = 1; - state.loader_count = 1; - - // Parse args - while (1) { - int idx = 0; - int c = getopt_long(argc, argv, "hemgi:k:d:p:b:c:o:u:z:n:l:y:", opts, &idx); - - if (c == -1) break; - - switch (c) { - case 'i': { - char *index = optarg; - if (strcmp(index, "bwtree") == 0) { - state.index = IndexType::BWTREE; - } else { - LOG_ERROR("Unknown index: %s", index); - exit(EXIT_FAILURE); - } - break; - } - case 'y': { - char *epoch = optarg; - if (strcmp(epoch, "decentralized") == 0) { - state.epoch = EpochType::DECENTRALIZED_EPOCH; - } else { - LOG_ERROR("Unknown epoch: %s", epoch); - exit(EXIT_FAILURE); - } - break; - } - case 'l': - state.loader_count = atoi(optarg); - break; - case 'k': - state.scale_factor = atoi(optarg); - break; - case 'd': - state.duration = atof(optarg); - break; - case 'p': - state.profile_duration = atof(optarg); - break; - case 'b': - state.backend_count = atoi(optarg); - break; - case 'c': - state.column_count = atoi(optarg); - break; - case 'o': - state.operation_count = atoi(optarg); - break; - case 'u': - state.update_ratio = atof(optarg); - break; - case 'z': - state.zipf_theta = atof(optarg); - break; - case 'e': - state.exp_backoff = true; - break; - case 'm': - state.string_mode = true; - break; - case 'g': - state.gc_mode = true; - break; - case 'n': - state.gc_backend_count = atoi(optarg); - break; - - case 'h': - Usage(stderr); - exit(EXIT_FAILURE); - break; - - default: - LOG_ERROR("Unknown option: -%c-", c); - Usage(stderr); - exit(EXIT_FAILURE); - break; - } - } - - // Print configuration - ValidateIndex(state); - ValidateScaleFactor(state); - ValidateDuration(state); - ValidateProfileDuration(state); - ValidateBackendCount(state); - ValidateColumnCount(state); - ValidateOperationCount(state); - ValidateUpdateRatio(state); - ValidateZipfTheta(state); - ValidateGCBackendCount(state); - - LOG_TRACE("%s : %d", "Run exponential backoff", state.exp_backoff); - LOG_TRACE("%s : %d", "Run string mode", state.string_mode); - LOG_TRACE("%s : %d", "Run garbage collection", state.gc_mode); - -} - - -void WriteOutput() { - std::ofstream out("outputfile.summary"); - - oid_t total_profile_memory = 0; - for (auto &entry : state.profile_memory) { - total_profile_memory += entry; - } - - LOG_INFO("----------------------------------------------------------"); - LOG_INFO("%d %d %d %d %lf %lf :: %lf %lf %d", - state.scale_factor, - state.backend_count, - state.column_count, - state.operation_count, - state.update_ratio, - state.zipf_theta, - state.throughput, - state.abort_rate, - total_profile_memory); - - out << state.scale_factor << " "; - out << state.backend_count << " "; - out << state.column_count << " "; - out << state.operation_count << " "; - out << state.update_ratio << " "; - out << state.zipf_theta << " "; - out << state.throughput << " "; - out << state.abort_rate << " "; - out << total_profile_memory << "\n"; - - for (size_t round_id = 0; round_id < state.profile_throughput.size(); - ++round_id) { - out << "[" << std::setw(3) << std::left - << state.profile_duration * round_id << " - " << std::setw(3) - << std::left << state.profile_duration * (round_id + 1) - << " s]: " << state.profile_throughput[round_id] << " " - << state.profile_abort_rate[round_id] << " " - << state.profile_memory[round_id] << "\n"; - } - out.flush(); - out.close(); -} - -} // namespace ycsb -} // namespace benchmark -} // namespace peloton diff --git a/src/main/ycsb/ycsb_loader.cpp b/src/main/ycsb/ycsb_loader.cpp deleted file mode 100644 index 50e2192ba9d..00000000000 --- a/src/main/ycsb/ycsb_loader.cpp +++ /dev/null @@ -1,209 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// ycsb_loader.cpp -// -// Identification: src/main/ycsb/ycsb_loader.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include -#include -#include -#include -#include -#include -#include - -#include "benchmark/ycsb/ycsb_loader.h" -#include "benchmark/ycsb/ycsb_configuration.h" -#include "catalog/catalog.h" -#include "catalog/schema.h" -#include "concurrency/transaction_context.h" -#include "concurrency/transaction_manager_factory.h" -#include "executor/abstract_executor.h" -#include "executor/insert_executor.h" -#include "executor/executor_context.h" -#include "expression/constant_value_expression.h" -#include "expression/expression_util.h" -#include "index/index_factory.h" -#include "planner/insert_plan.h" -#include "storage/tile.h" -#include "storage/tile_group.h" -#include "storage/data_table.h" -#include "storage/table_factory.h" -#include "storage/database.h" - -// Logging mode -// extern peloton::LoggingType peloton_logging_mode; - -namespace peloton { -namespace benchmark { -namespace ycsb { - -storage::Database *ycsb_database = nullptr; - -storage::DataTable *user_table = nullptr; - -void CreateYCSBDatabase() { - const oid_t col_count = state.column_count + 1; - const bool is_inlined = false; - - ///////////////////////////////////////////////////////// - // Create tables - ///////////////////////////////////////////////////////// - // Clean up - delete ycsb_database; - ycsb_database = nullptr; - user_table = nullptr; - - auto catalog = catalog::Catalog::GetInstance(); - ycsb_database = new storage::Database(ycsb_database_oid); - catalog->AddDatabase(ycsb_database); - - bool own_schema = true; - bool adapt_table = false; - // Create schema first - std::vector columns; - - auto column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "YCSB_KEY", is_inlined); - columns.push_back(column); - - if (state.string_mode == true) { - for (oid_t col_itr = 1; col_itr < col_count; col_itr++) { - auto column = - catalog::Column(type::TypeId::VARCHAR, 100, - "FIELD" + std::to_string(col_itr), is_inlined); - columns.push_back(column); - } - } else { - for (oid_t col_itr = 1; col_itr < col_count; col_itr++) { - auto column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "FIELD" + std::to_string(col_itr), is_inlined); - columns.push_back(column); - } - } - - catalog::Schema *table_schema = new catalog::Schema(columns); - std::string table_name("USERTABLE"); - - user_table = storage::TableFactory::GetDataTable( - ycsb_database_oid, user_table_oid, table_schema, table_name, - DEFAULT_TUPLES_PER_TILEGROUP, own_schema, adapt_table); - - ycsb_database->AddTable(user_table); - - // Primary index on user key - std::vector key_attrs; - - auto tuple_schema = user_table->GetSchema(); - catalog::Schema *key_schema; - index::IndexMetadata *index_metadata; - bool unique; - - key_attrs = {0}; - key_schema = catalog::Schema::CopySchema(tuple_schema, key_attrs); - key_schema->SetIndexedColumns(key_attrs); - - unique = true; - - index_metadata = new index::IndexMetadata( - "primary_index", user_table_pkey_index_oid, user_table_oid, - ycsb_database_oid, state.index, IndexConstraintType::PRIMARY_KEY, - tuple_schema, key_schema, key_attrs, unique); - - std::shared_ptr pkey_index( - index::IndexFactory::GetIndex(index_metadata)); - user_table->AddIndex(pkey_index); -} - -void LoadYCSBRows(const int begin_rowid, const int end_rowid) { - const oid_t col_count = state.column_count + 1; - - // Pick the user table - auto table_schema = user_table->GetSchema(); - - ///////////////////////////////////////////////////////// - // Load in the data - ///////////////////////////////////////////////////////// - - std::unique_ptr pool(new type::EphemeralPool()); - - // Insert tuples into tile_group. - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - const bool allocate = true; - auto txn = txn_manager.BeginTransaction(); - std::unique_ptr context( - new executor::ExecutorContext(txn)); - - for (int rowid = begin_rowid; rowid < end_rowid; rowid++) { - std::unique_ptr tuple( - new storage::Tuple(table_schema, allocate)); - - auto primary_key_value = type::ValueFactory::GetIntegerValue(rowid); - tuple->SetValue(0, primary_key_value, nullptr); - - - if (state.string_mode == true) { - auto key_value = type::ValueFactory::GetVarcharValue(std::string(100, 'z')); - for (oid_t col_itr = 1; col_itr < col_count; col_itr++) { - tuple->SetValue(col_itr, key_value, pool.get()); - } - } else { - auto key_value = type::ValueFactory::GetIntegerValue(rowid); - for (oid_t col_itr = 1; col_itr < col_count; col_itr++) { - tuple->SetValue(col_itr, key_value, nullptr); - } - } - - planner::InsertPlan node(user_table, std::move(tuple)); - executor::InsertExecutor executor(&node, context.get()); - executor.Execute(); - } - - txn_manager.CommitTransaction(txn); -} - -void LoadYCSBDatabase() { - - std::chrono::steady_clock::time_point start_time; - start_time = std::chrono::steady_clock::now(); - - const int tuple_count = state.scale_factor * 1000; - int row_per_thread = tuple_count / state.loader_count; - - std::vector> load_threads(state.loader_count); - - for (int thread_id = 0; thread_id < state.loader_count - 1; ++thread_id) { - int begin_rowid = row_per_thread * thread_id; - int end_rowid = row_per_thread * (thread_id + 1); - load_threads[thread_id].reset(new std::thread(LoadYCSBRows, begin_rowid, end_rowid)); - } - - int thread_id = state.loader_count - 1; - int begin_rowid = row_per_thread * thread_id; - int end_rowid = tuple_count; - load_threads[thread_id].reset(new std::thread(LoadYCSBRows, begin_rowid, end_rowid)); - - for (int thread_id = 0; thread_id < state.loader_count; ++thread_id) { - load_threads[thread_id]->join(); - } - - std::chrono::steady_clock::time_point end_time = std::chrono::steady_clock::now(); - double diff = std::chrono::duration_cast(end_time - start_time).count(); - LOG_INFO("database table loading time = %lf ms", diff); - - LOG_INFO("%sTABLE SIZES%s", peloton::GETINFO_HALF_THICK_LINE.c_str(), peloton::GETINFO_HALF_THICK_LINE.c_str()); - LOG_INFO("user count = %lu", user_table->GetTupleCount()); - -} - -} // namespace ycsb -} // namespace benchmark -} // namespace peloton diff --git a/src/main/ycsb/ycsb_mixed.cpp b/src/main/ycsb/ycsb_mixed.cpp deleted file mode 100644 index 769b630f6cf..00000000000 --- a/src/main/ycsb/ycsb_mixed.cpp +++ /dev/null @@ -1,231 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// ycsb_mixed.cpp -// -// Identification: src/main/ycsb/ycsb_mixed.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "benchmark/ycsb/ycsb_configuration.h" -#include "benchmark/ycsb/ycsb_loader.h" -#include "benchmark/ycsb/ycsb_workload.h" - -#include "catalog/manager.h" -#include "catalog/schema.h" - -#include "common/generator.h" -#include "common/internal_types.h" -#include "common/logger.h" -#include "common/timer.h" -#include "type/value.h" -#include "type/value_factory.h" - -#include "concurrency/transaction_context.h" -#include "concurrency/transaction_manager_factory.h" - -#include "executor/abstract_executor.h" -#include "executor/executor_context.h" -#include "executor/index_scan_executor.h" -#include "executor/insert_executor.h" -#include "executor/logical_tile.h" -#include "executor/logical_tile_factory.h" -#include "executor/materialization_executor.h" -#include "executor/update_executor.h" - -#include "common/container_tuple.h" -#include "expression/abstract_expression.h" -#include "expression/comparison_expression.h" -#include "expression/constant_value_expression.h" -#include "expression/expression_util.h" -#include "expression/tuple_value_expression.h" - -#include "index/index_factory.h" - -#include "logging/log_manager.h" - -#include "planner/abstract_plan.h" -#include "planner/index_scan_plan.h" -#include "planner/insert_plan.h" -#include "planner/materialization_plan.h" -#include "planner/update_plan.h" - -#include "storage/data_table.h" -#include "storage/table_factory.h" - -namespace peloton { -namespace benchmark { -namespace ycsb { - -bool RunMixed(const size_t thread_id, ZipfDistribution &zipf, FastRandom &rng) { - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - - concurrency::TransactionContext *txn = - txn_manager.BeginTransaction(thread_id); - - std::unique_ptr context( - new executor::ExecutorContext(txn)); - - // Column ids to be added to logical tile. - std::vector column_ids; - oid_t column_count = state.column_count + 1; - - // read all the attributes in a tuple. - for (oid_t col_itr = 0; col_itr < column_count; col_itr++) { - column_ids.push_back(col_itr); - } - - // Create and set up index scan executor - std::vector key_column_ids; - std::vector expr_types; - - key_column_ids.push_back(0); - expr_types.push_back(ExpressionType::COMPARE_EQUAL); - - std::vector runtime_keys; - - for (int i = 0; i < state.operation_count; i++) { - auto rng_val = rng.NextUniform(); - - if (rng_val < state.update_ratio) { - ///////////////////////////////////////////////////////// - // PERFORM UPDATE - ///////////////////////////////////////////////////////// - - // set up parameter values - std::vector values; - - auto lookup_key = zipf.GetNextNumber(); - - values.push_back(type::ValueFactory::GetIntegerValue(lookup_key).Copy()); - - planner::IndexScanPlan::IndexScanDesc index_scan_desc( - user_table_pkey_index_oid, key_column_ids, expr_types, values, - runtime_keys); - - // Create plan node. - auto predicate = nullptr; - - planner::IndexScanPlan index_scan_node(user_table, predicate, column_ids, - index_scan_desc); - - // Run the executor - executor::IndexScanExecutor index_scan_executor(&index_scan_node, - context.get()); - - TargetList target_list; - DirectMapList direct_map_list; - - // update multiple attributes - for (oid_t col_itr = 0; col_itr < column_count; col_itr++) { - if (col_itr == 1) { - if (state.string_mode == true) { - std::string update_raw_value(100, 'a'); - type::Value update_val = - type::ValueFactory::GetVarcharValue(update_raw_value).Copy(); - - planner::DerivedAttribute attr{ - expression::ExpressionUtil::ConstantValueFactory(update_val)}; - target_list.emplace_back(col_itr, attr); - - } else { - int update_raw_value = 1; - type::Value update_val = - type::ValueFactory::GetIntegerValue(update_raw_value).Copy(); - - planner::DerivedAttribute attr{ - expression::ExpressionUtil::ConstantValueFactory(update_val)}; - target_list.emplace_back(col_itr, attr); - } - } else { - direct_map_list.emplace_back(col_itr, - std::pair(0, col_itr)); - } - } - - std::unique_ptr project_info( - new planner::ProjectInfo(std::move(target_list), - std::move(direct_map_list))); - planner::UpdatePlan update_node(user_table, std::move(project_info)); - - executor::UpdateExecutor update_executor(&update_node, context.get()); - - update_executor.AddChild(&index_scan_executor); - - ExecuteUpdate(&update_executor); - - if (txn->GetResult() != ResultType::SUCCESS) { - txn_manager.AbortTransaction(txn); - return false; - } - - } else { - ///////////////////////////////////////////////////////// - // PERFORM READ - ///////////////////////////////////////////////////////// - - // set up parameter values - std::vector values; - - auto lookup_key = zipf.GetNextNumber(); - - values.push_back(type::ValueFactory::GetIntegerValue(lookup_key).Copy()); - - planner::IndexScanPlan::IndexScanDesc index_scan_desc( - user_table_pkey_index_oid, key_column_ids, expr_types, values, - runtime_keys); - - // Create plan node. - auto predicate = nullptr; - - planner::IndexScanPlan index_scan_node(user_table, predicate, column_ids, - index_scan_desc); - - // Run the executor - executor::IndexScanExecutor index_scan_executor(&index_scan_node, - context.get()); - - ExecuteRead(&index_scan_executor); - - if (txn->GetResult() != ResultType::SUCCESS) { - txn_manager.AbortTransaction(txn); - return false; - } - } - } - - // transaction passed execution. - PELOTON_ASSERT(txn->GetResult() == ResultType::SUCCESS); - - auto result = txn_manager.CommitTransaction(txn); - - if (result == ResultType::SUCCESS) { - return true; - - } else { - // transaction failed commitment. - PELOTON_ASSERT(result == ResultType::ABORTED || - result == ResultType::FAILURE); - return false; - } -} -} -} -} diff --git a/src/main/ycsb/ycsb_workload.cpp b/src/main/ycsb/ycsb_workload.cpp deleted file mode 100644 index 994f1fbf10c..00000000000 --- a/src/main/ycsb/ycsb_workload.cpp +++ /dev/null @@ -1,321 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// ycsb_workload.cpp -// -// Identification: src/main/ycsb/ycsb_workload.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "storage/storage_manager.h" - -#include "benchmark/ycsb/ycsb_workload.h" -#include "benchmark/ycsb/ycsb_configuration.h" -#include "benchmark/ycsb/ycsb_loader.h" - -#include "catalog/manager.h" -#include "catalog/schema.h" - -#include "common/internal_types.h" -#include "type/value.h" -#include "type/value_factory.h" -#include "common/logger.h" -#include "common/timer.h" -#include "common/generator.h" -#include "common/platform.h" -#include "common/container_tuple.h" - -#include "concurrency/transaction_context.h" -#include "concurrency/transaction_manager_factory.h" - -#include "executor/executor_context.h" -#include "executor/abstract_executor.h" -#include "executor/logical_tile.h" -#include "executor/logical_tile_factory.h" -#include "executor/materialization_executor.h" -#include "executor/update_executor.h" -#include "executor/index_scan_executor.h" - -#include "expression/abstract_expression.h" -#include "expression/constant_value_expression.h" -#include "expression/tuple_value_expression.h" -#include "expression/comparison_expression.h" -#include "expression/expression_util.h" - -#include "index/index_factory.h" - -#include "logging/log_manager.h" - -#include "planner/abstract_plan.h" -#include "planner/materialization_plan.h" -#include "planner/insert_plan.h" -#include "planner/update_plan.h" -#include "planner/index_scan_plan.h" - -#include "storage/data_table.h" -#include "storage/table_factory.h" - -namespace peloton { -namespace benchmark { -namespace ycsb { - - -///////////////////////////////////////////////////////// -// WORKLOAD -///////////////////////////////////////////////////////// - -volatile bool is_running = true; - -PadInt *abort_counts; -PadInt *commit_counts; - -#ifndef __APPLE__ -void PinToCore(size_t core) { - cpu_set_t cpuset; - CPU_ZERO(&cpuset); - CPU_SET(core, &cpuset); - pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); -#else -void PinToCore(size_t UNUSED_ATTRIBUTE core) { -// Mac OS X does not export interfaces that identify processors or control thread placement -// explicit thread to processor binding is not supported. -// Reference: https://superuser.com/questions/149312/how-to-set-processor-affinity-on-os-x -#endif -} - -void RunBackend(const size_t thread_id) { - - PinToCore(thread_id); - - PadInt &execution_count_ref = abort_counts[thread_id]; - PadInt &transaction_count_ref = commit_counts[thread_id]; - - ZipfDistribution zipf((state.scale_factor * 1000) - 1, - state.zipf_theta); - - FastRandom rng(rand()); - - // backoff - uint32_t backoff_shifts = 0; - - while (true) { - if (is_running == false) { - break; - } - while (RunMixed(thread_id, zipf, rng) == false) { - if (is_running == false) { - break; - } - execution_count_ref.data++; - // backoff - if (state.exp_backoff) { - if (backoff_shifts < 13) { - ++backoff_shifts; - } - uint64_t sleep_duration = 1UL << backoff_shifts; - sleep_duration *= 100; - std::this_thread::sleep_for(std::chrono::microseconds(sleep_duration)); - } - } - backoff_shifts >>= 1; - transaction_count_ref.data++; - } -} - -void RunWorkload() { - // Execute the workload to build the log - std::vector thread_group; - size_t num_threads = state.backend_count; - - abort_counts = new PadInt[num_threads]; - PELOTON_MEMSET(abort_counts, 0, sizeof(PadInt) * num_threads); - - commit_counts = new PadInt[num_threads]; - PELOTON_MEMSET(commit_counts, 0, sizeof(PadInt) * num_threads); - - size_t profile_round = (size_t)(state.duration / state.profile_duration); - - PadInt **abort_counts_profiles = new PadInt *[profile_round]; - for (size_t round_id = 0; round_id < profile_round; ++round_id) { - abort_counts_profiles[round_id] = new PadInt[num_threads]; - } - - PadInt **commit_counts_profiles = new PadInt *[profile_round]; - for (size_t round_id = 0; round_id < profile_round; ++round_id) { - commit_counts_profiles[round_id] = new PadInt[num_threads]; - } - - // Launch a group of threads - for (size_t thread_itr = 0; thread_itr < num_threads; ++thread_itr) { - thread_group.push_back(std::thread(RunBackend, thread_itr)); - } - - ////////////////////////////////////// - oid_t last_tile_group_id = 0; - for (size_t round_id = 0; round_id < profile_round; ++round_id) { - std::this_thread::sleep_for( - std::chrono::milliseconds(int(state.profile_duration * 1000))); - PELOTON_MEMCPY(abort_counts_profiles[round_id], abort_counts, - sizeof(PadInt) * num_threads); - PELOTON_MEMCPY(commit_counts_profiles[round_id], commit_counts, - sizeof(PadInt) * num_threads); - - storage::StorageManager *storage_manager = storage::StorageManager::GetInstance(); - oid_t current_tile_group_id = storage_manager->GetCurrentTileGroupId(); - if (round_id != 0) { - state.profile_memory.push_back(current_tile_group_id - last_tile_group_id); - } - last_tile_group_id = current_tile_group_id; - - } - - state.profile_memory.push_back(state.profile_memory.at(state.profile_memory.size() - 1)); - - is_running = false; - - // Join the threads with the main thread - for (size_t thread_itr = 0; thread_itr < num_threads; ++thread_itr) { - thread_group[thread_itr].join(); - } - - // calculate the throughput and abort rate for the first round. - uint64_t total_commit_count = 0; - for (size_t i = 0; i < num_threads; ++i) { - total_commit_count += commit_counts_profiles[0][i].data; - } - - uint64_t total_abort_count = 0; - for (size_t i = 0; i < num_threads; ++i) { - total_abort_count += abort_counts_profiles[0][i].data; - } - - state.profile_throughput.push_back(total_commit_count * 1.0 / - state.profile_duration); - state.profile_abort_rate.push_back(total_abort_count * 1.0 / - total_commit_count); - - // calculate the throughput and abort rate for the remaining rounds. - for (size_t round_id = 0; round_id < profile_round - 1; ++round_id) { - total_commit_count = 0; - for (size_t i = 0; i < num_threads; ++i) { - total_commit_count += commit_counts_profiles[round_id + 1][i].data - - commit_counts_profiles[round_id][i].data; - } - - total_abort_count = 0; - for (size_t i = 0; i < num_threads; ++i) { - total_abort_count += abort_counts_profiles[round_id + 1][i].data - - abort_counts_profiles[round_id][i].data; - } - - state.profile_throughput.push_back(total_commit_count * 1.0 / - state.profile_duration); - state.profile_abort_rate.push_back(total_abort_count * 1.0 / - total_commit_count); - } - - ////////////////////////////////////////////////// - // calculate the aggregated throughput and abort rate. - total_commit_count = 0; - for (size_t i = 0; i < num_threads; ++i) { - total_commit_count += commit_counts_profiles[profile_round - 1][i].data; - } - - total_abort_count = 0; - for (size_t i = 0; i < num_threads; ++i) { - total_abort_count += abort_counts_profiles[profile_round - 1][i].data; - } - - state.throughput = total_commit_count * 1.0 / state.duration; - state.abort_rate = total_abort_count * 1.0 / total_commit_count; - - ////////////////////////////////////////////////// - - // cleanup everything. - for (size_t round_id = 0; round_id < profile_round; ++round_id) { - delete[] abort_counts_profiles[round_id]; - abort_counts_profiles[round_id] = nullptr; - } - - for (size_t round_id = 0; round_id < profile_round; ++round_id) { - delete[] commit_counts_profiles[round_id]; - commit_counts_profiles[round_id] = nullptr; - } - - delete[] abort_counts_profiles; - abort_counts_profiles = nullptr; - delete[] commit_counts_profiles; - commit_counts_profiles = nullptr; - - delete[] abort_counts; - abort_counts = nullptr; - delete[] commit_counts; - commit_counts = nullptr; - -} - - -///////////////////////////////////////////////////////// -// HARNESS -///////////////////////////////////////////////////////// - -std::vector> ExecuteRead(executor::AbstractExecutor* executor) { - executor->Init(); - - std::vector> logical_tile_values; - - // Execute stuff - while (executor->Execute() == true) { - std::unique_ptr result_tile(executor->GetOutput()); - - if(result_tile == nullptr) { - break; - } - - auto column_count = result_tile->GetColumnCount(); - LOG_TRACE("result column count = %d\n", (int)column_count); - - for (oid_t tuple_id : *result_tile) { - ContainerTuple cur_tuple(result_tile.get(), - tuple_id); - std::vector tuple_values; - for (oid_t column_itr = 0; column_itr < column_count; column_itr++){ - auto value = cur_tuple.GetValue(column_itr); - tuple_values.push_back(value); - } - - // Move the tuple list - logical_tile_values.push_back(std::move(tuple_values)); - } - } - - return logical_tile_values; -} - -void ExecuteUpdate(executor::AbstractExecutor* executor) { - executor->Init(); - // Execute stuff - while (executor->Execute() == true); -} - - - -} // namespace ycsb -} // namespace benchmark -} // namespace peloton From c949481ecb6b347fa116669cd50fc802962d3b5f Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Mon, 2 Jul 2018 21:30:06 -0400 Subject: [PATCH 15/19] Disable TensorFlow tests to allow Travis macOS builds to succeed. (#1449) --- test/brain/tensorflow_test.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/brain/tensorflow_test.cpp b/test/brain/tensorflow_test.cpp index c0c659f5bb9..ef2607a3c57 100644 --- a/test/brain/tensorflow_test.cpp +++ b/test/brain/tensorflow_test.cpp @@ -28,7 +28,8 @@ namespace test { class TensorflowTests : public PelotonTest {}; -TEST_F(TensorflowTests, BasicTFTest) { +// TODO: Enable this test once tensorflow package supports Python 3.7 (#1448) +TEST_F(TensorflowTests, DISABLED_BasicTFTest) { // Check that the tensorflow library imports and prints version info correctly EXPECT_TRUE(brain::TFUtil::GetTFVersion()); } @@ -40,7 +41,8 @@ TEST_F(TensorflowTests, BasicEigenTest) { EXPECT_TRUE(m.IsRowMajor); } -TEST_F(TensorflowTests, SineWavePredictionTest) { +// TODO: Enable this test once tensorflow package supports Python 3.7 (#1448) +TEST_F(TensorflowTests, DISABLED_SineWavePredictionTest) { // Sine Wave prediction test works here int NUM_SAMPLES = 1000; int NUM_WAVES = 3; From 898219f9fe6e1d2b6901ea3797caac5c93850213 Mon Sep 17 00:00:00 2001 From: Kazuhiro Saito <32720872+ksaito7@users.noreply.github.com> Date: Tue, 3 Jul 2018 13:29:59 -0400 Subject: [PATCH 16/19] Constraint refactoring (#1415) * Add pg_constraint catalog table * Reconstruct constraints --- src/catalog/abstract_catalog.cpp | 31 +- src/catalog/catalog.cpp | 707 +++++++++++++--- src/catalog/column.cpp | 26 +- src/catalog/column_catalog.cpp | 218 +++-- src/catalog/constraint.cpp | 33 +- src/catalog/constraint_catalog.cpp | 428 ++++++++++ src/catalog/database_catalog.cpp | 22 +- src/catalog/index_catalog.cpp | 49 +- src/catalog/layout_catalog.cpp | 28 +- src/catalog/manager.cpp | 1 - src/catalog/multi_constraint.cpp | 39 - src/catalog/schema.cpp | 24 +- src/catalog/schema_catalog.cpp | 26 +- src/catalog/system_catalogs.cpp | 26 +- src/catalog/table_catalog.cpp | 129 ++- src/common/internal_types.cpp | 88 +- src/executor/create_executor.cpp | 201 +++-- src/executor/update_executor.cpp | 3 +- src/include/catalog/catalog.h | 83 +- src/include/catalog/catalog_defaults.h | 36 +- src/include/catalog/column.h | 72 +- src/include/catalog/column_catalog.h | 31 +- src/include/catalog/constraint.h | 115 ++- src/include/catalog/constraint_catalog.h | 156 ++++ src/include/catalog/foreign_key.h | 76 -- src/include/catalog/layout_catalog.h | 3 +- src/include/catalog/multi_constraint.h | 70 -- src/include/catalog/schema.h | 176 +++- src/include/catalog/system_catalogs.h | 10 + src/include/catalog/table_catalog.h | 40 +- src/include/common/internal_types.h | 27 +- src/include/planner/create_plan.h | 40 +- src/include/storage/abstract_table.h | 10 - src/include/storage/data_table.h | 57 +- src/include/storage/temp_table.h | 6 - .../stats/column_stats_collector.cpp | 9 +- src/optimizer/stats/stats_storage.cpp | 2 +- src/planner/create_plan.cpp | 145 ++-- src/planner/update_plan.cpp | 12 +- src/storage/data_table.cpp | 248 ++---- src/storage/database.cpp | 11 +- test/brain/query_logger_test.cpp | 6 +- test/catalog/catalog_test.cpp | 413 +++++++++- test/catalog/constraints_test.cpp | 773 +++++++++--------- test/codegen/testing_codegen_util.cpp | 33 +- test/common/internal_types_test.cpp | 8 +- test/concurrency/testing_transaction_util.cpp | 43 +- test/executor/drop_test.cpp | 2 +- test/executor/testing_executor_util.cpp | 21 +- test/executor/update_test.cpp | 27 +- test/gc/garbage_collection_test.cpp | 13 +- test/gc/transaction_level_gc_manager_test.cpp | 18 +- .../catalog/testing_constraints_util.h | 55 +- test/include/codegen/testing_codegen_util.h | 3 +- .../concurrency/testing_transaction_util.h | 2 +- test/optimizer/stats_storage_test.cpp | 2 +- test/planner/plan_util_test.cpp | 16 +- test/planner/planner_test.cpp | 6 +- test/sql/optimizer_sql_test.cpp | 2 +- test/statistics/stats_test.cpp | 15 +- test/statistics/testing_stats_util.cpp | 34 +- 61 files changed, 3415 insertions(+), 1591 deletions(-) create mode 100644 src/catalog/constraint_catalog.cpp delete mode 100644 src/catalog/multi_constraint.cpp create mode 100644 src/include/catalog/constraint_catalog.h delete mode 100644 src/include/catalog/foreign_key.h delete mode 100644 src/include/catalog/multi_constraint.h diff --git a/src/catalog/abstract_catalog.cpp b/src/catalog/abstract_catalog.cpp index 5ce2bceaedc..4ddad1c61ca 100644 --- a/src/catalog/abstract_catalog.cpp +++ b/src/catalog/abstract_catalog.cpp @@ -29,6 +29,7 @@ #include "planner/seq_scan_plan.h" #include "executor/executor_context.h" +#include "executor/create_executor.h" #include "executor/delete_executor.h" #include "executor/index_scan_executor.h" #include "executor/insert_executor.h" @@ -60,30 +61,24 @@ AbstractCatalog::AbstractCatalog(storage::Database *pg_catalog, AbstractCatalog::AbstractCatalog(concurrency::TransactionContext *txn, const std::string &catalog_table_ddl) { - // get catalog table schema + // Execute create catalog table auto &peloton_parser = parser::PostgresParser::GetInstance(); + std::unique_ptr context( + new executor::ExecutorContext(txn)); auto create_plan = std::dynamic_pointer_cast( optimizer::Optimizer().BuildPelotonPlanTree( peloton_parser.BuildParseTree(catalog_table_ddl), txn)); - auto catalog_table_schema = create_plan->GetSchema(); - auto catalog_table_name = create_plan->GetTableName(); - auto catalog_schema_name = create_plan->GetSchemaName(); - auto catalog_database_name = create_plan->GetDatabaseName(); - PELOTON_ASSERT(catalog_schema_name == std::string(CATALOG_SCHEMA_NAME)); - // create catalog table - Catalog::GetInstance()->CreateTable(txn, - catalog_database_name, - catalog_schema_name, - std::unique_ptr( - catalog_table_schema), - catalog_table_name, - true); + executor::CreateExecutor executor(create_plan.get(), context.get()); + + executor.Init(); + executor.Execute(); // get catalog table oid - auto catalog_table_object = Catalog::GetInstance()->GetTableCatalogEntry(txn, - catalog_database_name, - catalog_schema_name, - catalog_table_name); + auto catalog_table_object = + Catalog::GetInstance()->GetTableCatalogEntry(txn, + create_plan->GetDatabaseName(), + create_plan->GetSchemaName(), + create_plan->GetTableName()); // set catalog_table_ try { diff --git a/src/catalog/catalog.cpp b/src/catalog/catalog.cpp index b3dcd815ac3..99f58aeb53a 100644 --- a/src/catalog/catalog.cpp +++ b/src/catalog/catalog.cpp @@ -13,6 +13,7 @@ #include "catalog/catalog.h" #include "catalog/column_catalog.h" +#include "catalog/constraint_catalog.h" #include "catalog/database_catalog.h" #include "catalog/database_metrics_catalog.h" #include "catalog/index_catalog.h" @@ -232,6 +233,49 @@ void Catalog::BootstrapSystemCatalogs(concurrency::TransactionContext *txn, {TableCatalog::ColumnId::DATABASE_OID}, pool_.get()); + system_catalogs->GetIndexCatalog()->InsertIndex(txn, + CATALOG_SCHEMA_NAME, + LAYOUT_CATALOG_OID, + LAYOUT_CATALOG_PKEY_OID, + LAYOUT_CATALOG_NAME "_pkey", + IndexType::BWTREE, + IndexConstraintType::PRIMARY_KEY, + true, + {LayoutCatalog::ColumnId::TABLE_OID, + LayoutCatalog::ColumnId::LAYOUT_OID}, + pool_.get()); + system_catalogs->GetIndexCatalog()->InsertIndex(txn, + CATALOG_SCHEMA_NAME, + LAYOUT_CATALOG_OID, + LAYOUT_CATALOG_SKEY0_OID, + LAYOUT_CATALOG_NAME "_skey0", + IndexType::BWTREE, + IndexConstraintType::DEFAULT, + true, + {LayoutCatalog::ColumnId::TABLE_OID}, + pool_.get()); + + system_catalogs->GetIndexCatalog()->InsertIndex(txn, + CATALOG_SCHEMA_NAME, + CONSTRAINT_CATALOG_OID, + CONSTRAINT_CATALOG_PKEY_OID, + CONSTRAINT_CATALOG_NAME "_pkey", + IndexType::BWTREE, + IndexConstraintType::PRIMARY_KEY, + true, + {ConstraintCatalog::ColumnId::CONSTRAINT_OID}, + pool_.get()); + system_catalogs->GetIndexCatalog()->InsertIndex(txn, + CATALOG_SCHEMA_NAME, + CONSTRAINT_CATALOG_OID, + CONSTRAINT_CATALOG_SKEY0_OID, + CONSTRAINT_CATALOG_NAME "_skey0", + IndexType::BWTREE, + IndexConstraintType::DEFAULT, + true, + {ConstraintCatalog::ColumnId::TABLE_OID}, + pool_.get()); + // Insert records(default + pg_catalog namespace) into pg_namespace system_catalogs->GetSchemaCatalog()->InsertSchema(txn, CATALOG_SCHEMA_OID, @@ -286,6 +330,13 @@ void Catalog::BootstrapSystemCatalogs(concurrency::TransactionContext *txn, LAYOUT_CATALOG_NAME, ROW_STORE_LAYOUT_OID, pool_.get()); + system_catalogs->GetTableCatalog()->InsertTable(txn, + database_oid, + CATALOG_SCHEMA_NAME, + CONSTRAINT_CATALOG_OID, + CONSTRAINT_CATALOG_NAME, + ROW_STORE_LAYOUT_OID, + pool_.get()); } void Catalog::Bootstrap() { @@ -403,11 +454,13 @@ ResultType Catalog::CreateSchema(concurrency::TransactionContext *txn, } /*@brief create table + * @param txn TransactionContext * @param database_name the database which the table belongs to * @param schema_name name of schema the table belongs to - * @param table_name name of the table * @param schema schema, a.k.a metadata of the table - * @param txn TransactionContext + * @param table_name name of the table + * @param is_catalog table is built as catalog or not(useful in + * catalog table Initialization) * @return TransactionContext ResultType(SUCCESS or FAILURE) */ ResultType Catalog::CreateTable(concurrency::TransactionContext *txn, @@ -488,40 +541,24 @@ ResultType Catalog::CreateTable(concurrency::TransactionContext *txn, table_name, table->GetDefaultLayout()->GetOid(), pool_.get()); + + // Insert column info into each catalog oid_t column_id = 0; for (const auto &column : table->GetSchema()->GetColumns()) { pg_attribute->InsertColumn(txn, table_oid, - column_id, column.GetName(), + column_id, column.GetOffset(), column.GetType(), column.GetLength(), - column.GetConstraints(), column.IsInlined(), + column.IsNotNull(), + column.HasDefault(), + column.GetDefaultValue(), pool_.get()); - - // Create index on unique single column - if (column.IsUnique()) { - std::string col_name = column.GetName(); - std::string index_name = table->GetName() + "_" + col_name + "_UNIQ"; - CreateIndex(txn, - database_name, - schema_name, - table_name, - index_name, - {column_id}, - true, - IndexType::BWTREE); - LOG_DEBUG("Added a UNIQUE index on %s in %s.", col_name.c_str(), - table_name.c_str()); - } column_id++; } - CreatePrimaryIndex(txn, - database_object->GetDatabaseOid(), - schema_name, - table_oid); // Create layout as default layout auto pg_layout = @@ -534,93 +571,15 @@ ResultType Catalog::CreateTable(concurrency::TransactionContext *txn, return ResultType::SUCCESS; } -/*@brief create primary index on table - * Note that this is a catalog helper function only called within catalog.cpp - * If you want to create index on table outside, call CreateIndex() instead - * @param database_oid the database which the indexed table belongs to - * @param table_oid oid of the table to add index on - * @param schema_name the schema which the indexed table belongs to - * @param txn TransactionContext - * @return TransactionContext ResultType(SUCCESS or FAILURE) - */ -ResultType Catalog::CreatePrimaryIndex(concurrency::TransactionContext *txn, - oid_t database_oid, - const std::string &schema_name, - oid_t table_oid) { - LOG_TRACE("Trying to create primary index for table %d", table_oid); - - auto storage_manager = storage::StorageManager::GetInstance(); - - auto database = storage_manager->GetDatabaseWithOid(database_oid); - - auto table = database->GetTableWithOid(table_oid); - - std::vector key_attrs; - catalog::Schema *key_schema = nullptr; - index::IndexMetadata *index_metadata = nullptr; - auto schema = table->GetSchema(); - - // Find primary index attributes - int column_idx = 0; - auto &schema_columns = schema->GetColumns(); - for (auto &column : schema_columns) { - if (column.IsPrimary()) { - key_attrs.push_back(column_idx); - } - column_idx++; - } - - if (key_attrs.empty()) return ResultType::FAILURE; - - key_schema = catalog::Schema::CopySchema(schema, key_attrs); - key_schema->SetIndexedColumns(key_attrs); - - std::string index_name = table->GetName() + "_pkey"; - - bool unique_keys = true; - auto pg_index = catalog_map_[database_oid]->GetIndexCatalog(); - oid_t index_oid = pg_index->GetNextOid(); - - index_metadata = new index::IndexMetadata( - index_name, index_oid, table_oid, database_oid, IndexType::BWTREE, - IndexConstraintType::PRIMARY_KEY, schema, key_schema, key_attrs, - unique_keys); - - std::shared_ptr pkey_index( - index::IndexFactory::GetIndex(index_metadata)); - table->AddIndex(pkey_index); - - // put index object into rw_object_set - txn->RecordCreate(database_oid, table_oid, index_oid); - // insert index record into index_catalog(pg_index) table - pg_index->InsertIndex(txn, - schema_name, - table_oid, - index_oid, - index_name, - IndexType::BWTREE, - IndexConstraintType::PRIMARY_KEY, - unique_keys, - key_attrs, - pool_.get()); - - LOG_TRACE("Successfully created primary key index '%s' for table '%s'", - index_name.c_str(), table->GetName().c_str()); - - return ResultType::SUCCESS; -} - /*@brief create index on table + * @param txn TransactionContext * @param database_name the database which the indexed table belongs to * @param schema_name the namespace which the indexed table belongs to * @param table_name name of the table to add index on - * @param index_attr collection of the indexed attribute(column) name * @param index_name name of the table to add index on + * @param key_attrs collection of the indexed attribute(column) name * @param unique_keys index supports duplicate key or not * @param index_type the type of index(default value is BWTREE) - * @param txn TransactionContext - * @param is_catalog index is built on catalog table or not(useful in - * catalog table Initialization) * @return TransactionContext ResultType(SUCCESS or FAILURE) */ ResultType Catalog::CreateIndex(concurrency::TransactionContext *txn, @@ -655,6 +614,9 @@ ResultType Catalog::CreateIndex(concurrency::TransactionContext *txn, throw CatalogException("Can't find table " + schema_name + "." + table_name + " to create index"); + auto pg_index = + catalog_map_[database_object->GetDatabaseOid()]->GetIndexCatalog(); + oid_t index_oid = pg_index->GetNextOid(); IndexConstraintType index_constraint = unique_keys ? IndexConstraintType::UNIQUE : IndexConstraintType::DEFAULT; @@ -663,6 +625,7 @@ ResultType Catalog::CreateIndex(concurrency::TransactionContext *txn, schema_name, table_object->GetTableOid(), false, + index_oid, index_name, key_attrs, unique_keys, @@ -672,11 +635,27 @@ ResultType Catalog::CreateIndex(concurrency::TransactionContext *txn, return success; } +/*@brief create index on table + * @param txn TransactionContext + * @param database_oid the database which the indexed table belongs to + * @param schema_name the namespace which the indexed table belongs to + * @param table_oid name of the table to add index on + * @param is_catalog index is built on catalog table or not(useful in + * catalog table Initialization) + * @param index_oid oid of the index to be added + * @param index_name index name to be added + * @param key_attrs collection of the indexed attribute(column) name + * @param unique_keys index supports duplicate key or not + * @param index_type the type of index + * @param index_constraint the constraint type of index + * @return TransactionContext ResultType(SUCCESS or FAILURE) + */ ResultType Catalog::CreateIndex(concurrency::TransactionContext *txn, oid_t database_oid, const std::string &schema_name, oid_t table_oid, bool is_catalog, + oid_t index_oid, const std::string &index_name, const std::vector &key_attrs, bool unique_keys, @@ -711,8 +690,6 @@ ResultType Catalog::CreateIndex(concurrency::TransactionContext *txn, // Passed all checks, now get all index metadata LOG_TRACE("Trying to create index %s on table %d", index_name.c_str(), table_oid); - auto pg_index = catalog_map_[database_oid]->GetIndexCatalog(); - oid_t index_oid = pg_index->GetNextOid(); auto key_schema = catalog::Schema::CopySchema(schema, key_attrs); key_schema->SetIndexedColumns(key_attrs); @@ -729,6 +706,7 @@ ResultType Catalog::CreateIndex(concurrency::TransactionContext *txn, // Put index object into rw_object_set txn->RecordCreate(database_oid, table_oid, index_oid); // Insert index record into pg_index + auto pg_index = catalog_map_[database_oid]->GetIndexCatalog(); pg_index->InsertIndex(txn, schema_name, table_oid, @@ -789,6 +767,371 @@ std::shared_ptr Catalog::CreateDefaultLayout(concurrency: return new_layout; } +//===--------------------------------------------------------------------===// +// SET FUNCTIONS FOR COLUMN CONSTRAINT +//===--------------------------------------------------------------------===// + +/** + * @brief Set not null constraint for a column + * @param txn TransactionContext + * @param database_oid Database to which the table belongs to + * @param table_oid Table to which the constraint has to be set + * @param column_id Column that the constraint affects + * @return ResultType(SUCCESS or FAILURE) + */ +ResultType Catalog::SetNotNullConstraint(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + oid_t column_id) { + auto table_object = catalog_map_[database_oid] + ->GetTableCatalog() + ->GetTableCatalogEntry(txn, table_oid); + auto schema = storage::StorageManager::GetInstance() + ->GetTableWithOid(database_oid, table_oid) + ->GetSchema(); + auto column = schema->GetColumn(column_id); + + // Check not null + if (column.IsNotNull()) { + throw CatalogException("Column " + column.GetName() + " in table " + + table_object->GetTableName() + + " is already NOT NULL."); + } + + // Update pg_column to set constraint of the column + auto pg_column = catalog_map_[database_oid]->GetColumnCatalog(); + pg_column->UpdateNotNullConstraint(txn, + table_oid, + column.GetName(), + true); + + // Set not null constraint in the schema + { + std::lock_guard lock(catalog_mutex); + schema->SetNotNull(column_id); + } + + LOG_TRACE("Added a NOT NULL constraint to column %s in %s.", + column.GetName().c_str(), table_object->GetTableName().c_str()); + + return ResultType::SUCCESS; +} + +/** + * @brief Set default constraint for a column + * @param txn TransactionContext + * @param database_oid Database to which the table belongs to + * @param table_oid Table to which the constraint has to be set + * @param column_id Column that the constraint affects + * @param default_value Value for the default constraint + * @return ResultType(SUCCESS or FAILURE) + */ +ResultType Catalog::SetDefaultConstraint(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + oid_t column_id, + const type::Value &default_value) { + auto table_object = catalog_map_[database_oid]->GetTableCatalog() + ->GetTableCatalogEntry(txn, table_oid); + auto schema = storage::StorageManager::GetInstance() + ->GetTableWithOid(database_oid, table_oid) + ->GetSchema(); + auto column = schema->GetColumn(column_id); + + // Check default + if (column.HasDefault()) { + throw CatalogException("Column " + column.GetName() + " in table" + + table_object->GetTableName() + + " is already set default value '" + + column.GetDefaultValue()->ToString() + "'."); + } + + // Update pg_column to set constraint of the column + auto pg_column = catalog_map_[database_oid]->GetColumnCatalog(); + pg_column->UpdateDefaultConstraint(txn, + table_oid, + column.GetName(), + true, + &default_value); + + // Set default constraint in the schema + { + std::lock_guard lock(catalog_mutex); + schema->SetDefaultValue(column_id, default_value); + } + + LOG_TRACE("Added a DEFAULT constraint to column %s in %s.", + column.GetName().c_str(), table_object->GetTableName().c_str()); + + return ResultType::SUCCESS; +} + +//===--------------------------------------------------------------------===// +// ADD FUNCTIONS FOR TABLE CONSTRAINT +//===--------------------------------------------------------------------===// + +/** + * @brief Add a new primary constraint for a table + * @param txn TransactionContext + * @param database_oid Database to which the table belongs to + * @param table_oid Table to which the constraint has to be added + * @param column_ids Columns that the constraint affects + * @param constraint_name constraint name + * @return ResultType(SUCCESS or FAILURE) + */ +ResultType Catalog::AddPrimaryKeyConstraint(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + const std::vector &column_ids, + const std::string &constraint_name) { + auto table_object = + catalog_map_[database_oid]->GetTableCatalog() + ->GetTableCatalogEntry(txn,table_oid); + auto schema = storage::StorageManager::GetInstance() + ->GetTableWithOid(database_oid, table_oid) + ->GetSchema(); + + // Check primary key in the table + if (schema->HasPrimary()) { + throw CatalogException("Table " + table_object->GetTableName() + + " already has primary key."); + } + + // Create index + std::string index_name = table_object->GetTableName() + "_pkey"; + auto index_oid = catalog_map_[database_oid]->GetIndexCatalog()->GetNextOid(); + CreateIndex(txn, + database_oid, + table_object->GetSchemaName(), + table_oid, + false, + index_oid, + index_name, + column_ids, + true, + IndexType::BWTREE, + IndexConstraintType::PRIMARY_KEY); + + // Insert constraint into pg_constraint + auto pg_constraint = catalog_map_[database_oid]->GetConstraintCatalog(); + std::shared_ptr constraint( + new Constraint(pg_constraint->GetNextOid(), + ConstraintType::PRIMARY, + constraint_name, + table_oid, + column_ids, + index_oid)); + pg_constraint->InsertConstraint(txn, constraint, pool_.get()); + + // Add constraint into the schema + { + std::lock_guard lock(catalog_mutex); + schema->AddConstraint(constraint); + } + + LOG_TRACE("Added a PRIMARY KEY constraint in %s.", + table_object->GetTableName().c_str()); + + return ResultType::SUCCESS; +} + +/** + * @brief Add a new unique constraint for a table + * @param txn TransactionContext + * @param database_oid Database to which the table belongs to + * @param table_oid Table to which the constraint has to be added + * @param column_ids Columns that the constraint affects + * @param constraint_name constraint name + * @return ResultType(SUCCESS or FAILURE) + * note: if add a new foreign key constraint, use AddForeignKeyConstraint + */ +ResultType Catalog::AddUniqueConstraint(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + const std::vector &column_ids, + const std::string &constraint_name) { + auto table_object = + catalog_map_[database_oid]->GetTableCatalog() + ->GetTableCatalogEntry(txn, table_oid); + auto schema = storage::StorageManager::GetInstance() + ->GetTableWithOid(database_oid, table_oid) + ->GetSchema(); + + // Create index + std::stringstream index_name(table_object->GetTableName()); + for (auto column_id : column_ids) + index_name << "_" + schema->GetColumn(column_id).GetName(); + index_name << "_UNIQ"; + auto index_oid = catalog_map_[database_oid]->GetIndexCatalog()->GetNextOid(); + CreateIndex(txn, + database_oid, + table_object->GetSchemaName(), + table_oid, + false, + index_oid, + index_name.str(), + column_ids, + true, + IndexType::BWTREE, + IndexConstraintType::UNIQUE); + + // Insert constraint into pg_constraint + auto pg_constraint = catalog_map_[database_oid]->GetConstraintCatalog(); + std::shared_ptr constraint( + new Constraint(pg_constraint->GetNextOid(), + ConstraintType::UNIQUE, + constraint_name, + table_oid, + column_ids, + index_oid)); + pg_constraint->InsertConstraint(txn, constraint, pool_.get()); + + // Add constraint into the schema + { + std::lock_guard lock(catalog_mutex); + schema->AddConstraint(constraint); + } + + LOG_TRACE("Added a UNIQUE constraint in %s.", + table_object->GetTableName().c_str()); + + return ResultType::SUCCESS; +} + +/** + * @brief Add a new foreign key constraint for a table + * @param txn TransactionContext + * @param database_oid database to which the table belongs to + * @param src_table_oid table to which the constraint has to be added + * @param src_col_ids Columns that the constraint affects + * @param sink_table_oid sink table + * @param sink_col_ids Columns that limit the source columns + * @param upd_action foreign key constraint action when update + * @param del_action foreign key constraint action when delete + * @param constraint_name constraint name + * @return ResultType(SUCCESS or FAILURE) + */ +ResultType Catalog::AddForeignKeyConstraint(concurrency::TransactionContext *txn, + oid_t database_oid, oid_t src_table_oid, + const std::vector &src_col_ids, + oid_t sink_table_oid, + const std::vector &sink_col_ids, + FKConstrActionType upd_action, + FKConstrActionType del_action, + const std::string &constraint_name) { + auto pg_constraint = catalog_map_[database_oid]->GetConstraintCatalog(); + auto constraint_oid = pg_constraint->GetNextOid(); + auto storage_manager = storage::StorageManager::GetInstance(); + auto src_table = + storage_manager->GetTableWithOid(database_oid, src_table_oid); + auto sink_table = + storage_manager->GetTableWithOid(database_oid, sink_table_oid); + + // Add a non-unique index on the source table if needed + auto src_table_object = + catalog_map_[database_oid]->GetTableCatalog() + ->GetTableCatalogEntry(txn, src_table_oid); + auto src_schema = src_table->GetSchema(); + + std::stringstream index_name(src_table_object->GetTableName()); + for (auto col_id : src_col_ids) + index_name << "_" << src_schema->GetColumn(col_id).GetName(); + index_name << "_fkey"; + oid_t index_oid = catalog_map_[database_oid]->GetIndexCatalog()->GetNextOid(); + CreateIndex(txn, + database_oid, + src_table_object->GetSchemaName(), + src_table_oid, + false, + index_oid, + index_name.str(), + src_col_ids, + false, + IndexType::BWTREE, + IndexConstraintType::DEFAULT); + + // Insert constraint into pg_constraint + std::shared_ptr constraint( + new Constraint(constraint_oid, + ConstraintType::FOREIGN, + constraint_name, + src_table_oid, + src_col_ids, + index_oid, + sink_table_oid, + sink_col_ids, + upd_action, + del_action)); + pg_constraint->InsertConstraint(txn, constraint, pool_.get()); + + // add constraint into schema in source table and sink table + { + std::lock_guard lock(catalog_mutex); + src_schema->AddConstraint(constraint); + sink_table->GetSchema()->RegisterForeignKeySource(constraint); + } + + LOG_TRACE("Added a FOREIGN KEY constraint in %s to %s.", + src_table_object->GetTableName().c_str(), + catalog_map_[database_oid] + ->GetTableCatalog() + ->GetTableCatalogEntry(txn, sink_table_oid) + ->GetTableName() + .c_str()); + + return ResultType::SUCCESS; +} + +/** + * @brief Add a new check constraint for a table + * @param txn TransactionContext + * @param database_oid Database to which the table belongs to + * @param table_oid Table to which the constraint has to be added + * @param column_ids Columns that the constraint affects + * @param cmd Command string for check constraint + * @param exp Expression for check constraint + * @param constraint_name constraint name + * @return ResultType(SUCCESS or FAILURE) + * note: if add a new foreign key constraint, use AddForeignKeyConstraint + */ +ResultType Catalog::AddCheckConstraint(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + const std::vector &column_ids, + const std::pair &exp, + const std::string &constraint_name) { + auto table_object = + catalog_map_[database_oid]->GetTableCatalog() + ->GetTableCatalogEntry(txn, table_oid); + auto schema = storage::StorageManager::GetInstance() + ->GetTableWithOid(database_oid, table_oid) + ->GetSchema(); + + // Insert constraint into pg_constraint + auto pg_constraint = catalog_map_[database_oid]->GetConstraintCatalog(); + std::shared_ptr constraint( + new Constraint(pg_constraint->GetNextOid(), + ConstraintType::CHECK, + constraint_name, + table_oid, + column_ids, + INVALID_OID, + exp)); + pg_constraint->InsertConstraint(txn, constraint, pool_.get()); + + // Add constraint into the schema + { + std::lock_guard lock(catalog_mutex); + schema->AddConstraint(constraint); + } + + LOG_TRACE("Added a CHECK constraint in %s.", + table_object->GetTableName().c_str()); + + return ResultType::SUCCESS; +} + //===----------------------------------------------------------------------===// // DROP FUNCTIONS //===----------------------------------------------------------------------===// @@ -963,6 +1306,11 @@ ResultType Catalog::DropTable(concurrency::TransactionContext *txn, table_oid, trigger_lists->Get(i)->GetTriggerName()); + // delete record in pg_constraint + auto pg_constraint = + catalog_map_[database_object->GetDatabaseOid()]->GetConstraintCatalog(); + pg_constraint->DeleteConstraints(txn, table_oid); + // delete index and records pg_index for (auto it : index_objects) DropIndex(txn, database_oid, it.second->GetIndexOid()); @@ -1063,6 +1411,153 @@ ResultType Catalog::DropLayout(concurrency::TransactionContext *txn, return ResultType::SUCCESS; } +/** + * @brief Drop not null constraint for a column + * @param txn TransactionContext + * @param database_oid the database to which the table belongs to + * @param table_oid the table to which the column belongs to + * @param column_id the column which has the not null constraint + * @return ResultType(SUCCESS or FAILURE) + */ +ResultType Catalog::DropNotNullConstraint(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + oid_t column_id) { + auto table_object = + catalog_map_[database_oid]->GetTableCatalog() + ->GetTableCatalogEntry(txn, table_oid); + auto schema = storage::StorageManager::GetInstance() + ->GetTableWithOid(database_oid, table_oid) + ->GetSchema(); + auto column = schema->GetColumn(column_id); + + // Check not null + if (!column.IsNotNull()) { + throw CatalogException("Column " + column.GetName() + " in table " + + table_object->GetTableName() + " isn't NOT NULL."); + } + + // Update pg_column to set constraint of the column + auto pg_column = catalog_map_[database_oid]->GetColumnCatalog(); + pg_column->UpdateNotNullConstraint(txn, + table_oid, + column.GetName(), + false); + + // Set not null constraint in the schema + { + std::lock_guard lock(catalog_mutex); + schema->DropNotNull(column_id); + } + + return ResultType::SUCCESS; +} + +/** + * @brief Drop default constraint for a column + * @param txn TransactionContext + * @param database_oid the database to which the table belongs to + * @param table_oid the table to which the column belongs to + * @param column_id the column which has the default constraint + * @return ResultType(SUCCESS or FAILURE) + */ +ResultType Catalog::DropDefaultConstraint(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + oid_t column_id) { + auto table_object = + catalog_map_[database_oid]->GetTableCatalog() + ->GetTableCatalogEntry(txn, table_oid); + auto schema = storage::StorageManager::GetInstance() + ->GetTableWithOid(database_oid, table_oid) + ->GetSchema(); + auto column = schema->GetColumn(column_id); + + // Check default + if (!column.HasDefault()) { + throw CatalogException("Column " + column.GetName() + " in table" + + table_object->GetTableName() + + " doesn't have default value."); + } + + // Update pg_column to set constraint of the column + auto pg_column = catalog_map_[database_oid]->GetColumnCatalog(); + pg_column->UpdateDefaultConstraint(txn, + table_oid, + column.GetName(), + false, + nullptr); + + // Set default constraint in the schema + { + std::lock_guard lock(catalog_mutex); + schema->DropDefaultValue(column_id); + } + + return ResultType::SUCCESS; +} + +/** + * @brief Drop a constraint for a table + * @param txn TransactionContext + * @param database_oid the database to which the table belongs to + * @param table_oid the table which has the constraint + * @param constraint_oid the constraint to be dropped + * @return ResultType(SUCCESS or FAILURE) + */ +ResultType Catalog::DropConstraint(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + oid_t constraint_oid) { + auto pg_constraint = catalog_map_[database_oid]->GetConstraintCatalog(); + auto constraint_object = + pg_constraint->GetConstraintCatalogEntry(txn, + table_oid, + constraint_oid); + + // delete constraint object from pg_constraint + if (!pg_constraint->DeleteConstraint(txn, table_oid, constraint_oid)) { + throw CatalogException("Failed to delete the constraint: " + + std::to_string(constraint_oid)); + } + + // delete index if exists + if (constraint_object->GetIndexOid() != INVALID_OID) { + DropIndex(txn, database_oid, constraint_object->GetIndexOid()); + } + + // delete constraint from table + auto storage_manager = storage::StorageManager::GetInstance(); + auto table = storage_manager->GetTableWithOid(database_oid, table_oid); + { + std::lock_guard lock(catalog_mutex); + table->GetSchema()->DropConstraint(constraint_oid); + } + + // delete foreign key info from sink table + if (constraint_object->GetConstraintType() == ConstraintType::FOREIGN) { + auto sink_table = + storage_manager->GetTableWithOid(database_oid, + constraint_object->GetFKSinkTableOid()); + { + std::lock_guard lock(catalog_mutex); + sink_table->GetSchema() + ->DeleteForeignKeySource(constraint_object->GetConstraintOid()); + } + } + + LOG_TRACE( + "Drop a %s constraint in %s.", + ConstraintTypeToString(constraint_object->GetConstraintType()).c_str(), + catalog_map_[database_oid] + ->GetTableCatalog() + ->GetTableCatalogEntry(txn, table_oid) + ->GetTableName() + .c_str()); + + return ResultType::SUCCESS; +} + //===--------------------------------------------------------------------===// // GET WITH NAME - CHECK FROM CATALOG TABLES, USING TRANSACTION //===--------------------------------------------------------------------===// diff --git a/src/catalog/column.cpp b/src/catalog/column.cpp index 3195de231d4..a4c8e2f5acf 100644 --- a/src/catalog/column.cpp +++ b/src/catalog/column.cpp @@ -45,7 +45,9 @@ void Column::SetInlined() { const std::string Column::GetInfo() const { std::ostringstream os; - os << "Column[" << column_name << ", " << TypeIdToString(column_type_) << ", " + os << "Column[" << column_name_ << ", " + << TypeIdToString(column_type_) << ", " + << "Offset:" << column_offset_ << ", "; if (is_inlined_) { @@ -54,19 +56,17 @@ const std::string Column::GetInfo() const { os << "VarLength:" << variable_length_; } - if (constraints_.empty() == false) { - os << ", {"; - bool first = true; - for (auto constraint : constraints_) { - if (first) { - first = false; - } else { - os << ", "; - } - os << constraint.GetInfo(); - } - os << "}"; + if (is_not_null_ && has_default_) { + os << ", {NOT NULL, DEFAULT:" + << default_value_->ToString() << "}"; + } else if (is_not_null_) { + os << ", {NOT NULL}"; + } else if (has_default_) { + os << ", {DEFAULT:" + << default_value_->ToString() << "}"; + } + os << "]"; return (os.str()); diff --git a/src/catalog/column_catalog.cpp b/src/catalog/column_catalog.cpp index 6d4e7b717c1..5bc3a1d7902 100644 --- a/src/catalog/column_catalog.cpp +++ b/src/catalog/column_catalog.cpp @@ -18,6 +18,7 @@ #include "concurrency/transaction_context.h" #include "storage/data_table.h" #include "storage/database.h" +#include "type/ephemeral_pool.h" #include "type/value_factory.h" namespace peloton { @@ -38,14 +39,22 @@ ColumnCatalogEntry::ColumnCatalogEntry(executor::LogicalTile *tile, tile->GetValue(tupleId, ColumnCatalog::ColumnId::COLUMN_TYPE) .ToString())), column_length_( - tile->GetValue(tupleId, ColumnCatalog::ColumnId::COLUMN_LENGTH) - .GetAs()), + tile->GetValue(tupleId, ColumnCatalog::ColumnId::COLUMN_LENGTH) + .GetAs()), is_inlined_(tile->GetValue(tupleId, ColumnCatalog::ColumnId::IS_INLINED) .GetAs()), - is_primary_(tile->GetValue(tupleId, ColumnCatalog::ColumnId::IS_PRIMARY) - .GetAs()), is_not_null_(tile->GetValue(tupleId, ColumnCatalog::ColumnId::IS_NOT_NULL) - .GetAs()) {} + .GetAs()), + has_default_(tile->GetValue(tupleId, ColumnCatalog::ColumnId::HAS_DEFAULT) + .GetAs()) { + // deserialize default value if the column has default constraint + if (has_default_) { + auto dv_val = + tile->GetValue(tupleId, ColumnCatalog::ColumnId::DEFAULT_VALUE_BIN); + CopySerializeInput input_buffer(dv_val.GetData(), dv_val.GetLength()); + default_value_ = type::Value::DeserializeFrom(input_buffer, column_type_); + } +} ColumnCatalog::ColumnCatalog(concurrency::TransactionContext *txn, storage::Database *pg_catalog, @@ -73,13 +82,15 @@ ColumnCatalog::ColumnCatalog(concurrency::TransactionContext *txn, for (auto column : catalog_table_->GetSchema()->GetColumns()) { InsertColumn(txn, COLUMN_CATALOG_OID, - column_id, column.GetName(), + column_id, column.GetOffset(), column.GetType(), column.GetLength(), - column.GetConstraints(), column.IsInlined(), + column.IsNotNull(), + column.HasDefault(), + column.GetDefaultValue(), pool); column_id++; } @@ -91,82 +102,90 @@ ColumnCatalog::~ColumnCatalog() {} * @return unqiue pointer to schema */ std::unique_ptr ColumnCatalog::InitializeSchema() { - const std::string primary_key_constraint_name = "primary_key"; - const std::string not_null_constraint_name = "not_null"; - auto table_id_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "table_oid", true); - table_id_column.AddConstraint(catalog::Constraint( - ConstraintType::PRIMARY, primary_key_constraint_name)); - table_id_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + table_id_column.SetNotNull(); auto column_name_column = catalog::Column( type::TypeId::VARCHAR, max_name_size_, "column_name", false); - column_name_column.AddConstraint(catalog::Constraint( - ConstraintType::PRIMARY, primary_key_constraint_name)); - column_name_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + column_name_column.SetNotNull(); auto column_id_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "column_id", true); - column_id_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + column_id_column.SetNotNull(); auto column_offset_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "column_offset", true); - column_offset_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + column_offset_column.SetNotNull(); auto column_type_column = catalog::Column( type::TypeId::VARCHAR, max_name_size_, "column_type", false); - column_type_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + column_type_column.SetNotNull(); + auto column_length_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "column_length", true); - column_length_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + column_length_column.SetNotNull(); auto is_inlined_column = catalog::Column( type::TypeId::BOOLEAN, type::Type::GetTypeSize(type::TypeId::BOOLEAN), "is_inlined", true); - is_inlined_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); - - auto is_primary_column = catalog::Column( - type::TypeId::BOOLEAN, type::Type::GetTypeSize(type::TypeId::BOOLEAN), - "is_primary", true); - is_primary_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + is_inlined_column.SetNotNull(); auto is_not_null_column = catalog::Column( type::TypeId::BOOLEAN, type::Type::GetTypeSize(type::TypeId::BOOLEAN), "is_not_null", true); - is_not_null_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + is_not_null_column.SetNotNull(); + + auto has_default_column = catalog::Column( + type::TypeId::BOOLEAN, type::Type::GetTypeSize(type::TypeId::BOOLEAN), + "has_default", true); + has_default_column.SetNotNull(); + + auto default_value_src_column = catalog::Column( + type::TypeId::VARCHAR, type::Type::GetTypeSize(type::TypeId::VARCHAR), + "default_value_src", false); + + auto default_value_bin_column = catalog::Column( + type::TypeId::VARBINARY, type::Type::GetTypeSize(type::TypeId::VARBINARY), + "default_value_bin", false); std::unique_ptr column_catalog_schema(new catalog::Schema( {table_id_column, column_name_column, column_id_column, column_offset_column, column_type_column, column_length_column, - is_inlined_column, is_primary_column, is_not_null_column})); + is_inlined_column, is_not_null_column, has_default_column, + default_value_src_column, default_value_bin_column})); + + column_catalog_schema->AddConstraint(std::make_shared( + COLUMN_CATALOG_CON_PKEY_OID, ConstraintType::PRIMARY, "con_primary", + COLUMN_CATALOG_OID, + std::vector{ColumnId::TABLE_OID, ColumnId::COLUMN_NAME}, + COLUMN_CATALOG_PKEY_OID)); + + column_catalog_schema->AddConstraint(std::make_shared( + COLUMN_CATALOG_CON_UNI0_OID, ConstraintType::UNIQUE, "con_unique", + COLUMN_CATALOG_OID, + std::vector{ColumnId::TABLE_OID, ColumnId::COLUMN_ID}, + COLUMN_CATALOG_SKEY0_OID)); return column_catalog_schema; } bool ColumnCatalog::InsertColumn(concurrency::TransactionContext *txn, oid_t table_oid, - oid_t column_id, const std::string &column_name, + oid_t column_id, oid_t column_offset, type::TypeId column_type, size_t column_length, - const std::vector &constraints, bool is_inlined, + bool is_not_null, + bool is_default, + const std::shared_ptr default_value, type::AbstractPool *pool) { // Create the tuple first std::unique_ptr tuple( @@ -180,17 +199,8 @@ bool ColumnCatalog::InsertColumn(concurrency::TransactionContext *txn, type::ValueFactory::GetVarcharValue(TypeIdToString(column_type), nullptr); auto val5 = type::ValueFactory::GetIntegerValue(column_length); auto val6 = type::ValueFactory::GetBooleanValue(is_inlined); - bool is_primary = false, is_not_null = false; - for (auto constraint : constraints) { - if (constraint.GetType() == ConstraintType::PRIMARY) { - is_primary = true; - } else if (constraint.GetType() == ConstraintType::NOT_NULL || - constraint.GetType() == ConstraintType::NOTNULL) { - is_not_null = true; - } - } - auto val7 = type::ValueFactory::GetBooleanValue(is_primary); - auto val8 = type::ValueFactory::GetBooleanValue(is_not_null); + auto val7 = type::ValueFactory::GetBooleanValue(is_not_null); + auto val8 = type::ValueFactory::GetBooleanValue(is_default); tuple->SetValue(ColumnId::TABLE_OID, val0, pool); tuple->SetValue(ColumnId::COLUMN_NAME, val1, pool); @@ -199,8 +209,22 @@ bool ColumnCatalog::InsertColumn(concurrency::TransactionContext *txn, tuple->SetValue(ColumnId::COLUMN_TYPE, val4, pool); tuple->SetValue(ColumnId::COLUMN_LENGTH, val5, pool); tuple->SetValue(ColumnId::IS_INLINED, val6, pool); - tuple->SetValue(ColumnId::IS_PRIMARY, val7, pool); - tuple->SetValue(ColumnId::IS_NOT_NULL, val8, pool); + tuple->SetValue(ColumnId::IS_NOT_NULL, val7, pool); + tuple->SetValue(ColumnId::HAS_DEFAULT, val8, pool); + + // set default value if the column has default constraint + if (is_default) { + auto val9 = + type::ValueFactory::GetVarcharValue(default_value->ToString(), nullptr); + CopySerializeOutput output_buffer; + default_value->SerializeTo(output_buffer); + auto val10 = type::ValueFactory::GetVarbinaryValue( + (unsigned char *)output_buffer.Data(), output_buffer.Size(), true, + pool); + + tuple->SetValue(ColumnId::DEFAULT_VALUE_SRC, val9, pool); + tuple->SetValue(ColumnId::DEFAULT_VALUE_BIN, val10, pool); + } // Insert the tuple return InsertTuple(txn, std::move(tuple)); @@ -247,11 +271,91 @@ bool ColumnCatalog::DeleteColumns(concurrency::TransactionContext *txn, oid_t ta return DeleteWithIndexScan(txn, index_offset, values); } -const std::unordered_map> -ColumnCatalog::GetColumnCatalogEntries( - concurrency::TransactionContext *txn, - oid_t table_oid) { +bool ColumnCatalog::UpdateNotNullConstraint(concurrency::TransactionContext *txn, + oid_t table_oid, + const std::string &column_name, + bool is_not_null) { + std::vector update_columns({ColumnId::IS_NOT_NULL}); + oid_t index_offset = + IndexId::PRIMARY_KEY; // Index of table_oid & column_name + // values to execute index scan + std::vector scan_values; + scan_values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); + scan_values.push_back( + type::ValueFactory::GetVarcharValue(column_name, nullptr).Copy()); + + // values to update + std::vector update_values; + update_values.push_back( + type::ValueFactory::GetBooleanValue(is_not_null).Copy()); + + // delete column from cache + auto pg_table = Catalog::GetInstance() + ->GetSystemCatalogs(database_oid_) + ->GetTableCatalog(); + auto table_object = pg_table->GetTableCatalogEntry(txn, table_oid); + table_object->EvictColumnCatalogEntry(column_name); + + return UpdateWithIndexScan(txn, + index_offset, + scan_values, + update_columns, + update_values); +} + +bool ColumnCatalog::UpdateDefaultConstraint(concurrency::TransactionContext *txn, + oid_t table_oid, + const std::string &column_name, + bool has_default, + const type::Value *default_value) { + std::vector update_columns({ColumnId::HAS_DEFAULT, + ColumnId::DEFAULT_VALUE_SRC, + ColumnId::DEFAULT_VALUE_BIN}); + oid_t index_offset = + IndexId::PRIMARY_KEY; // Index of table_oid & column_name + // values to execute index scan + std::vector scan_values; + scan_values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); + scan_values.push_back( + type::ValueFactory::GetVarcharValue(column_name, nullptr).Copy()); + + // values to update + std::vector update_values; + update_values.push_back( + type::ValueFactory::GetBooleanValue(has_default).Copy()); + if (has_default) { + PELOTON_ASSERT(default_value != nullptr); + update_values.push_back( + type::ValueFactory::GetVarcharValue(default_value->ToString()).Copy()); + CopySerializeOutput output_buffer; + default_value->SerializeTo(output_buffer); + update_values.push_back(type::ValueFactory::GetVarbinaryValue( + (unsigned char *)output_buffer.Data(), + output_buffer.Size(), true).Copy()); + } else { + update_values.push_back( + type::ValueFactory::GetNullValueByType(type::TypeId::VARCHAR)); + update_values.push_back( + type::ValueFactory::GetNullValueByType(type::TypeId::VARBINARY)); + } + + // delete column from cache + auto pg_table = Catalog::GetInstance() + ->GetSystemCatalogs(database_oid_) + ->GetTableCatalog(); + auto table_object = pg_table->GetTableCatalogEntry(txn, table_oid); + table_object->EvictColumnCatalogEntry(column_name); + + return UpdateWithIndexScan(txn, + index_offset, + scan_values, + update_columns, + update_values); +} + +const std::unordered_map> +ColumnCatalog::GetColumnCatalogEntries(concurrency::TransactionContext *txn, + oid_t table_oid) { // try get from cache auto pg_table = Catalog::GetInstance() ->GetSystemCatalogs(database_oid_) diff --git a/src/catalog/constraint.cpp b/src/catalog/constraint.cpp index f9c2e025a9e..f1de24cfba6 100644 --- a/src/catalog/constraint.cpp +++ b/src/catalog/constraint.cpp @@ -20,10 +20,37 @@ namespace catalog { const std::string Constraint::GetInfo() const { std::ostringstream os; os << "Constraint[" << GetName() << ", " - << ConstraintTypeToString(constraint_type_); + << "OID=" << constraint_oid_ << ", " + << ConstraintTypeToString(constraint_type_) << ", "; - if (GetType() == ConstraintType::CHECK) { - os << ", " << exp_.first << " " << exp_.second.GetInfo(); + os << "Column: ("; + bool first = true; + for (auto col_id : column_ids_) { + if(first) first = false; + else os << ", "; + os << std::to_string(col_id); + } + os << "), "; + + os << "index_oid:" << std::to_string(index_oid_); + + if (constraint_type_ == ConstraintType::FOREIGN) { + os << ", Foreign key: (Sink table:" + << std::to_string(fk_sink_table_oid_) << ", " + << "Column:("; + bool first = true; + for (auto col_id : fk_sink_col_ids_) { + if(first) first = false; + else os << ", "; + os << std::to_string(col_id); + } + os << "), " << FKConstrActionTypeToString(fk_update_action_) << ", " + << FKConstrActionTypeToString(fk_delete_action_) << ")"; + } + + if (constraint_type_ == ConstraintType::CHECK) { + os << ", Check: (" << check_exp_.first << " " + << check_exp_.second.GetInfo() << ")"; } os << "]"; return os.str(); diff --git a/src/catalog/constraint_catalog.cpp b/src/catalog/constraint_catalog.cpp new file mode 100644 index 00000000000..f780333e700 --- /dev/null +++ b/src/catalog/constraint_catalog.cpp @@ -0,0 +1,428 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// constraint_catalog.cpp +// +// Identification: src/catalog/constraint_catalog.cpp +// +// Copyright (c) 2015-17, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "catalog/constraint_catalog.h" + +#include +#include + +#include "catalog/catalog.h" +#include "catalog/system_catalogs.h" +#include "catalog/table_catalog.h" +#include "concurrency/transaction_context.h" +#include "storage/data_table.h" +#include "storage/database.h" +#include "storage/storage_manager.h" +#include "type/value_factory.h" + +namespace peloton { +namespace catalog { + +ConstraintCatalogEntry::ConstraintCatalogEntry(executor::LogicalTile *tile, + int tupleId) + : constraint_oid_( + tile->GetValue(tupleId, ConstraintCatalog::ColumnId::CONSTRAINT_OID) + .GetAs()), + constraint_name_( + tile->GetValue(tupleId, ConstraintCatalog::ColumnId::CONSTRAINT_NAME) + .ToString()), + constraint_type_(StringToConstraintType( + tile->GetValue(tupleId, ConstraintCatalog::ColumnId::CONSTRAINT_TYPE) + .ToString())), + table_oid_(tile->GetValue(tupleId, ConstraintCatalog::ColumnId::TABLE_OID) + .GetAs()), + index_oid_(tile->GetValue(tupleId, ConstraintCatalog::ColumnId::INDEX_OID) + .GetAs()) { + std::string src_column_ids_str = + tile->GetValue(tupleId, ConstraintCatalog::ColumnId::COLUMN_IDS) + .ToString(); + std::stringstream src_ss(src_column_ids_str.c_str()); + std::string src_tok; + while (std::getline(src_ss, src_tok, ' ')) { + column_ids_.push_back(std::stoi(src_tok)); + } + + // create values by type of constraint + switch (constraint_type_) { + case ConstraintType::PRIMARY: + case ConstraintType::UNIQUE: + // nothing to do more + break; + + case ConstraintType::FOREIGN: { + fk_sink_table_oid_ = + tile->GetValue(tupleId, + ConstraintCatalog::ColumnId::FK_SINK_TABLE_OID) + .GetAs(); + std::string snk_column_ids_str = + tile->GetValue(tupleId, ConstraintCatalog::ColumnId::FK_SINK_COL_IDS) + .ToString(); + std::stringstream snk_ss(snk_column_ids_str.c_str()); + std::string snk_tok; + while (std::getline(snk_ss, snk_tok, ' ')) { + fk_sink_col_ids_.push_back(std::stoi(snk_tok)); + } + fk_update_action_ = StringToFKConstrActionType( + tile->GetValue(tupleId, ConstraintCatalog::ColumnId::FK_UPDATE_ACTION) + .ToString()); + fk_delete_action_ = StringToFKConstrActionType( + tile->GetValue(tupleId, ConstraintCatalog::ColumnId::FK_DELETE_ACTION) + .ToString()); + break; + } + + case ConstraintType::CHECK: { + auto dv_val = + tile->GetValue(tupleId, ConstraintCatalog::ColumnId::CHECK_EXP_BIN); + CopySerializeInput input_buffer(dv_val.GetData(), dv_val.GetLength()); + ExpressionType exp_type = (ExpressionType)input_buffer.ReadInt(); + type::TypeId value_type = (type::TypeId)input_buffer.ReadInt(); + auto exp_value = type::Value::DeserializeFrom(input_buffer, value_type); + check_exp_ = std::make_pair(exp_type, exp_value); + break; + } + + case ConstraintType::EXCLUSION: + default: + LOG_ERROR("Invalid Constraint type from pg_constraint: %s", + ConstraintTypeToString(constraint_type_).c_str()); + break; + } +} + +ConstraintCatalog::ConstraintCatalog( + UNUSED_ATTRIBUTE concurrency::TransactionContext *txn, + storage::Database *pg_catalog, UNUSED_ATTRIBUTE type::AbstractPool *pool) + : AbstractCatalog(pg_catalog, + InitializeSchema().release(), + CONSTRAINT_CATALOG_OID, + CONSTRAINT_CATALOG_NAME) { + // Add indexes for pg_constraint + AddIndex(CONSTRAINT_CATALOG_NAME "_pkey", + CONSTRAINT_CATALOG_PKEY_OID, + {ColumnId::CONSTRAINT_OID}, + IndexConstraintType::PRIMARY_KEY); + AddIndex(CONSTRAINT_CATALOG_NAME "_skey0", + CONSTRAINT_CATALOG_SKEY0_OID, + {ColumnId::TABLE_OID}, + IndexConstraintType::DEFAULT); +} + +ConstraintCatalog::~ConstraintCatalog() {} + +/*@brief private function for initialize schema of pg_constraint + * @return unqiue pointer to schema + */ +std::unique_ptr ConstraintCatalog::InitializeSchema() { + auto constraint_oid_column = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "constraint_oid", true); + constraint_oid_column.SetNotNull(); + + auto constraint_name_column = catalog::Column( + type::TypeId::VARCHAR, max_name_size_, "constraint_name_", false); + constraint_name_column.SetNotNull(); + + auto constraint_type_column = catalog::Column( + type::TypeId::VARCHAR, max_name_size_, "constraint_type", false); + constraint_type_column.SetNotNull(); + + auto table_oid_column = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "table_oid", true); + table_oid_column.SetNotNull(); + + auto column_ids_column = catalog::Column( + type::TypeId::VARCHAR, type::Type::GetTypeSize(type::TypeId::VARCHAR), + "column_ids", false); + column_ids_column.SetNotNull(); + + auto index_oid_column = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "index_oid", true); + index_oid_column.SetNotNull(); + + auto fk_sink_table_oid_column = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "fk_sink_table_oid", true); + + auto fk_sink_col_ids_column = catalog::Column( + type::TypeId::VARCHAR, type::Type::GetTypeSize(type::TypeId::VARCHAR), + "fk_sink_col_ids", false); + + auto fk_update_action_column = catalog::Column( + type::TypeId::VARCHAR, max_name_size_, "fk_update_action", false); + + auto fk_delete_action_column = catalog::Column( + type::TypeId::VARCHAR, max_name_size_, "fk_delete_action", false); + + auto check_exp_src_column = catalog::Column( + type::TypeId::VARCHAR, type::Type::GetTypeSize(type::TypeId::VARCHAR), + "check_exp_src", false); + + auto check_exp_bin_column = catalog::Column( + type::TypeId::VARBINARY, type::Type::GetTypeSize(type::TypeId::VARBINARY), + "check_exp_bin", false); + + std::unique_ptr constraint_catalog_schema( + new catalog::Schema({constraint_oid_column, constraint_name_column, + constraint_type_column, table_oid_column, + column_ids_column, index_oid_column, + fk_sink_table_oid_column, fk_sink_col_ids_column, + fk_update_action_column, fk_delete_action_column, + check_exp_src_column, check_exp_bin_column})); + + constraint_catalog_schema->AddConstraint(std::make_shared( + CONSTRAINT_CATALOG_CON_PKEY_OID, ConstraintType::PRIMARY, "con_primary", + CONSTRAINT_CATALOG_OID, std::vector{ColumnId::CONSTRAINT_OID}, + CONSTRAINT_CATALOG_PKEY_OID)); + + return constraint_catalog_schema; +} + +/*@brief Insert a constraint into the pg_constraint table + * This targets PRIMARY KEY, FOREIGN KEY, UNIQUE or CHECK constraint + * @param txn TransactionContext for adding the constraint. + * @param constraint to be inserted into pg_constraint + * @param pool to allocate memory for the column_map column. + * @return true on success. + */ +bool ConstraintCatalog::InsertConstraint(concurrency::TransactionContext *txn, + const std::shared_ptr constraint, + type::AbstractPool *pool) { + // Create the tuple first + std::unique_ptr tuple( + new storage::Tuple(catalog_table_->GetSchema(), true)); + + // Common information of constraint + auto val0 = + type::ValueFactory::GetIntegerValue(constraint->GetConstraintOid()); + auto val1 = + type::ValueFactory::GetVarcharValue(constraint->GetName(), nullptr); + auto val2 = type::ValueFactory::GetVarcharValue( + ConstraintTypeToString(constraint->GetType()), nullptr); + auto val3 = type::ValueFactory::GetIntegerValue(constraint->GetTableOid()); + std::stringstream ss; + for (auto column_oid : constraint->GetColumnIds()) + ss << std::to_string(column_oid) << " "; + auto val4 = type::ValueFactory::GetVarcharValue(ss.str(), nullptr); + auto val5 = type::ValueFactory::GetIntegerValue(constraint->GetIndexOid()); + + tuple->SetValue(ColumnId::CONSTRAINT_OID, val0, pool); + tuple->SetValue(ColumnId::CONSTRAINT_NAME, val1, pool); + tuple->SetValue(ColumnId::CONSTRAINT_TYPE, val2, pool); + tuple->SetValue(ColumnId::TABLE_OID, val3, pool); + tuple->SetValue(ColumnId::COLUMN_IDS, val4, pool); + tuple->SetValue(ColumnId::INDEX_OID, val5, pool); + + // create values by type of constraint + switch (constraint->GetType()) { + case ConstraintType::PRIMARY: + case ConstraintType::UNIQUE: + // nothing to do more + // need to set a valid index oid + PELOTON_ASSERT(constraint->GetIndexOid() != INVALID_OID); + break; + + case ConstraintType::FOREIGN: { + // need to set a valid index oid + PELOTON_ASSERT(constraint->GetIndexOid() != INVALID_OID); + + auto val6 = + type::ValueFactory::GetIntegerValue(constraint->GetFKSinkTableOid()); + std::stringstream snk_ss; + for (auto column_oid : constraint->GetFKSinkColumnIds()) + snk_ss << std::to_string(column_oid) << " "; + auto val7 = type::ValueFactory::GetVarcharValue(snk_ss.str(), nullptr); + auto val8 = type::ValueFactory::GetVarcharValue( + FKConstrActionTypeToString(constraint->GetFKUpdateAction()), nullptr); + auto val9 = type::ValueFactory::GetVarcharValue( + FKConstrActionTypeToString(constraint->GetFKDeleteAction()), nullptr); + + tuple->SetValue(ColumnId::FK_SINK_TABLE_OID, val6, pool); + tuple->SetValue(ColumnId::FK_SINK_COL_IDS, val7, pool); + tuple->SetValue(ColumnId::FK_UPDATE_ACTION, val8, pool); + tuple->SetValue(ColumnId::FK_DELETE_ACTION, val9, pool); + break; + } + + case ConstraintType::CHECK: { + // set value of check expression + PELOTON_ASSERT(constraint->GetColumnIds().size() == 1); + auto exp = constraint->GetCheckExpression(); + auto column = + storage::StorageManager::GetInstance() + ->GetTableWithOid(database_oid_, constraint->GetTableOid()) + ->GetSchema() + ->GetColumn(constraint->GetColumnIds().at(0)); + + std::stringstream exp_ss; + exp_ss << column.GetName() << " " << ExpressionTypeToString(exp.first) + << " " << exp.second.ToString(); + auto val6 = type::ValueFactory::GetVarcharValue(exp_ss.str(), nullptr); + + CopySerializeOutput output_buffer; + output_buffer.WriteInt((int)exp.first); + output_buffer.WriteInt((int)column.GetType()); + exp.second.SerializeTo(output_buffer); + auto val7 = type::ValueFactory::GetVarbinaryValue( + (unsigned char *)output_buffer.Data(), output_buffer.Size(), true, + pool); + + tuple->SetValue(ColumnId::CHECK_EXP_SRC, val6, pool); + tuple->SetValue(ColumnId::CHECK_EXP_BIN, val7, pool); + break; + } + + case ConstraintType::EXCLUSION: + default: + // unexpected constraint type + throw CatalogException("Unexpected constraint type '" + + ConstraintTypeToString(constraint->GetType()) + + "' appears in insertion into pg_constraint "); + return false; + } + + // Insert the tuple + return InsertTuple(txn, std::move(tuple)); +} + +/* @brief delete all constraint records from the same table + * this function is useful when calling DropTable + * @param txn TransactionContext + * @param table_oid + * @return a vector of table oid + */ +bool ConstraintCatalog::DeleteConstraints(concurrency::TransactionContext *txn, + oid_t table_oid) { + oid_t index_offset = IndexId::SKEY_TABLE_OID; // Index of table_oid + std::vector values; + values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); + + // delete constraints from cache + auto pg_table = Catalog::GetInstance() + ->GetSystemCatalogs(database_oid_) + ->GetTableCatalog(); + auto table_object = pg_table->GetTableCatalogEntry(txn, table_oid); + table_object->EvictAllConstraintCatalogEntries(); + + return DeleteWithIndexScan(txn, index_offset, values); +} + +/** @brief Delete a constraint from the pg_constraint table. + * @param txn TransactionContext for deleting the constraint. + * @param table_oid oid of the table to which the old constraint belongs. + * @param constraint_oid oid of the constraint to be deleted. + * @return true on success. + */ +bool ConstraintCatalog::DeleteConstraint(concurrency::TransactionContext *txn, + oid_t table_oid, + oid_t constraint_oid) { + oid_t index_offset = IndexId::PRIMARY_KEY; // Index of constraint_oid + std::vector values; + values.push_back(type::ValueFactory::GetIntegerValue(constraint_oid).Copy()); + + // delete constraint from cache + auto pg_table = Catalog::GetInstance() + ->GetSystemCatalogs(database_oid_) + ->GetTableCatalog(); + auto table_object = pg_table->GetTableCatalogEntry(txn, table_oid); + table_object->EvictConstraintCatalogEntry(constraint_oid); + + return DeleteWithIndexScan(txn, index_offset, values); +} + +/** @brief Get all constraint objects correponding to a table + * from the pg_constraint. + * @param txn TransactionContext for getting the constraints. + * @param table_oid oid of the table to fetch all constraints. + * @return unordered_map containing a constraint_oid -> + * constraint object mapping. + */ +const std::unordered_map> +ConstraintCatalog::GetConstraintCatalogEntries(concurrency::TransactionContext *txn, + oid_t table_oid) { + // try get from cache + auto pg_table = Catalog::GetInstance() + ->GetSystemCatalogs(database_oid_) + ->GetTableCatalog(); + auto table_object = pg_table->GetTableCatalogEntry(txn, table_oid); + PELOTON_ASSERT(table_object && table_object->GetTableOid() == table_oid); + + auto constraint_objects = table_object->GetConstraintCatalogEntries(true); + if (constraint_objects.size() != 0) return constraint_objects; + + // cache miss, get from pg_constraint + std::vector column_ids(all_column_ids_); + oid_t index_offset = IndexId::SKEY_TABLE_OID; // Index of table_oid + std::vector values; + values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); + + auto result_tiles = + GetResultWithIndexScan(txn, column_ids, index_offset, values); + + for (auto &tile : (*result_tiles)) { + for (auto tuple_id : *tile) { + auto constraint_object = + std::make_shared(tile.get(), tuple_id); + table_object->InsertConstraintCatalogEntry(constraint_object); + } + } + + table_object->SetValidConstraintCatalogEntries(true); + return table_object->GetConstraintCatalogEntries(); +} + +/** @brief Get the constraint object by constraint_oid from + * the pg_constraint. + * @param txn TransactionContext for getting the constraint. + * @param table_oid oid of the table to fetch the constraint. + * @param constraint_oid oid of the constraint being queried. + * @return shared_ptr constraint object to the constraint_oid if found. + * nullptr otherwise. + */ +const std::shared_ptr +ConstraintCatalog::GetConstraintCatalogEntry(concurrency::TransactionContext *txn, + oid_t table_oid, + oid_t constraint_oid) { + // try get from cache + auto pg_table = Catalog::GetInstance() + ->GetSystemCatalogs(database_oid_) + ->GetTableCatalog(); + auto table_object = pg_table->GetTableCatalogEntry(txn, table_oid); + PELOTON_ASSERT(table_object && table_object->GetTableOid() == table_oid); + + auto constraint_object = + table_object->GetConstraintCatalogEntry(constraint_oid, true); + if (constraint_object != nullptr) return constraint_object; + + // cache miss, get from pg_constraint + std::vector column_ids(all_column_ids_); + oid_t index_offset = IndexId::PRIMARY_KEY; // Index of table_oid + std::vector values; + values.push_back(type::ValueFactory::GetIntegerValue(constraint_oid).Copy()); + + auto result_tiles = + GetResultWithIndexScan(txn, column_ids, index_offset, values); + + if (result_tiles->size() == 1 && (*result_tiles)[0]->GetTupleCount() == 1) { + auto constraint_object = + std::make_shared((*result_tiles)[0].get()); + table_object->InsertConstraintCatalogEntry(constraint_object); + return constraint_object; + } + + return nullptr; +} + +} // namespace catalog +} // namespace peloton diff --git a/src/catalog/database_catalog.cpp b/src/catalog/database_catalog.cpp index 555e8c9d7d6..d869b20654d 100644 --- a/src/catalog/database_catalog.cpp +++ b/src/catalog/database_catalog.cpp @@ -276,24 +276,28 @@ DatabaseCatalog::~DatabaseCatalog() {} * @return unqiue pointer to schema */ std::unique_ptr DatabaseCatalog::InitializeSchema() { - const std::string not_null_constraint_name = "not_null"; - const std::string primary_key_constraint_name = "primary_key"; - auto database_id_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "database_oid", true); - database_id_column.AddConstraint(catalog::Constraint( - ConstraintType::PRIMARY, primary_key_constraint_name)); - database_id_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + database_id_column.SetNotNull(); auto database_name_column = catalog::Column( type::TypeId::VARCHAR, max_name_size_, "database_name", false); - database_name_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + database_name_column.SetNotNull(); std::unique_ptr database_catalog_schema( new catalog::Schema({database_id_column, database_name_column})); + + database_catalog_schema->AddConstraint(std::make_shared( + DATABASE_CATALOG_CON_PKEY_OID, ConstraintType::PRIMARY, "con_primary", + DATABASE_CATALOG_OID, std::vector{ColumnId::DATABASE_OID}, + DATABASE_CATALOG_PKEY_OID)); + + database_catalog_schema->AddConstraint(std::make_shared( + DATABASE_CATALOG_CON_UNI0_OID, ConstraintType::UNIQUE, "con_unique", + DATABASE_CATALOG_OID, std::vector{ColumnId::DATABASE_NAME}, + DATABASE_CATALOG_SKEY0_OID)); + return database_catalog_schema; } diff --git a/src/catalog/index_catalog.cpp b/src/catalog/index_catalog.cpp index eaab5f3cfa7..69a2633ed44 100644 --- a/src/catalog/index_catalog.cpp +++ b/src/catalog/index_catalog.cpp @@ -64,15 +64,15 @@ IndexCatalog::IndexCatalog(concurrency::TransactionContext *, // Add indexes for pg_index AddIndex(INDEX_CATALOG_NAME "_pkey", INDEX_CATALOG_PKEY_OID, - {0}, + {ColumnId::INDEX_OID}, IndexConstraintType::PRIMARY_KEY); AddIndex(INDEX_CATALOG_NAME "_skey0", INDEX_CATALOG_SKEY0_OID, - {1, 3}, + {ColumnId::INDEX_NAME, ColumnId::SCHEMA_NAME}, IndexConstraintType::UNIQUE); AddIndex(INDEX_CATALOG_NAME "_skey1", INDEX_CATALOG_SKEY1_OID, - {2}, + {ColumnId::TABLE_OID}, IndexConstraintType::DEFAULT); } @@ -82,59 +82,60 @@ IndexCatalog::~IndexCatalog() {} * @return unqiue pointer to schema */ std::unique_ptr IndexCatalog::InitializeSchema() { - const std::string not_null_constraint_name = "not_null"; - const std::string primary_key_constraint_name = "primary_key"; - auto index_id_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "index_oid", true); - index_id_column.AddConstraint(catalog::Constraint( - ConstraintType::PRIMARY, primary_key_constraint_name)); - index_id_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + index_id_column.SetNotNull(); auto index_name_column = catalog::Column(type::TypeId::VARCHAR, max_name_size_, "index_name", false); - index_name_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + index_name_column.SetNotNull(); auto table_id_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "table_oid", true); - table_id_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + table_id_column.SetNotNull(); auto schema_name_column = catalog::Column( type::TypeId::VARCHAR, max_name_size_, "schema_name", false); - schema_name_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + schema_name_column.SetNotNull(); + auto index_type_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "index_type", true); - index_type_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + index_type_column.SetNotNull(); auto index_constraint_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "index_constraint", true); - index_constraint_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + index_constraint_column.SetNotNull(); auto unique_keys = catalog::Column( type::TypeId::BOOLEAN, type::Type::GetTypeSize(type::TypeId::BOOLEAN), "unique_keys", true); - unique_keys.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + unique_keys.SetNotNull(); auto indexed_attributes_column = catalog::Column( type::TypeId::VARCHAR, max_name_size_, "indexed_attributes", false); - indexed_attributes_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + indexed_attributes_column.SetNotNull(); + + std::unique_ptr index_schema(new catalog::Schema( {index_id_column, index_name_column, table_id_column, schema_name_column, index_type_column, index_constraint_column, unique_keys, indexed_attributes_column})); + + index_schema->AddConstraint(std::make_shared( + INDEX_CATALOG_CON_PKEY_OID, ConstraintType::PRIMARY, "con_primary", + INDEX_CATALOG_OID, std::vector{ColumnId::INDEX_OID}, + INDEX_CATALOG_PKEY_OID)); + + index_schema->AddConstraint(std::make_shared( + INDEX_CATALOG_CON_UNI0_OID, ConstraintType::UNIQUE, "con_unique", + INDEX_CATALOG_OID, std::vector{ColumnId::INDEX_NAME, ColumnId::SCHEMA_NAME}, + INDEX_CATALOG_SKEY0_OID)); + return index_schema; } diff --git a/src/catalog/layout_catalog.cpp b/src/catalog/layout_catalog.cpp index 8db442a03e0..4d5ac563dde 100644 --- a/src/catalog/layout_catalog.cpp +++ b/src/catalog/layout_catalog.cpp @@ -50,42 +50,36 @@ LayoutCatalog::~LayoutCatalog() {} * @return unique_ptr of the schema for pg_layout. */ std::unique_ptr LayoutCatalog::InitializeSchema() { - const std::string primary_key_constraint_name = "primary_key"; - const std::string not_null_constraint_name = "not_null"; - auto table_id_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "table_oid", true); - table_id_column.AddConstraint(catalog::Constraint( - ConstraintType::PRIMARY, primary_key_constraint_name)); - table_id_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + table_id_column.SetNotNull(); auto layout_oid_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "layout_oid", true); - layout_oid_column.AddConstraint(catalog::Constraint( - ConstraintType::PRIMARY, primary_key_constraint_name)); - layout_oid_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + layout_oid_column.SetNotNull(); auto num_columns_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "num_columns", true); - num_columns_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + num_columns_column.SetNotNull(); auto column_map_column = catalog::Column( type::TypeId::VARCHAR, type::Type::GetTypeSize(type::TypeId::VARCHAR), "column_map", false); - column_map_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + column_map_column.SetNotNull(); - std::unique_ptr column_catalog_schema( + std::unique_ptr layout_catalog_schema( new catalog::Schema({table_id_column, layout_oid_column, num_columns_column, column_map_column})); - return column_catalog_schema; + layout_catalog_schema->AddConstraint(std::make_shared( + LAYOUT_CATALOG_CON_PKEY_OID, ConstraintType::PRIMARY, "con_primary", + LAYOUT_CATALOG_OID, std::vector{ColumnId::TABLE_OID, ColumnId::LAYOUT_OID}, + LAYOUT_CATALOG_PKEY_OID)); + + return layout_catalog_schema; } /** @brief Insert a layout into the pg_layout table. diff --git a/src/catalog/manager.cpp b/src/catalog/manager.cpp index 7c17c53d7bc..a683d36843e 100644 --- a/src/catalog/manager.cpp +++ b/src/catalog/manager.cpp @@ -13,7 +13,6 @@ #include "common/exception.h" #include "common/logger.h" #include "catalog/manager.h" -#include "catalog/foreign_key.h" #include "storage/database.h" #include "storage/data_table.h" #include "concurrency/transaction_manager_factory.h" diff --git a/src/catalog/multi_constraint.cpp b/src/catalog/multi_constraint.cpp deleted file mode 100644 index d1e4e3d9817..00000000000 --- a/src/catalog/multi_constraint.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// multi_constraint.cpp -// -// Identification: src/catalog/multi_constraint.cpp -// -// Copyright (c) 2015-2017, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "catalog/multi_constraint.h" -#include "common/internal_types.h" - -#include - -namespace peloton { -namespace catalog { - -const std::string MultiConstraint::GetInfo() const { - std::ostringstream os; - os << "Constraint[" << GetName() << ", " - << ConstraintTypeToString(constraint_type_) << " , related columns: ("; - bool first = true; - for (auto id : column_ids_) { - if (first) { - os << id; - first = false; - } else { - os << " ," << id; - } - } - os << ")]"; - return os.str(); -} - -} // namespace catalog -} // namespace peloton diff --git a/src/catalog/schema.cpp b/src/catalog/schema.cpp index 66f557f2532..3024ef0a1c3 100644 --- a/src/catalog/schema.cpp +++ b/src/catalog/schema.cpp @@ -76,10 +76,14 @@ Schema::Schema(const std::vector &columns) CreateTupleSchema(column_types, column_lengths, column_names, is_inlined); - // Add constraints + // Set constraints for (oid_t column_itr = 0; column_itr < column_count; column_itr++) { - for (auto constraint : columns[column_itr].GetConstraints()) - AddConstraint(column_itr, constraint); + if (columns[column_itr].IsNotNull()) { + SetNotNull(column_itr); + } + if (columns[column_itr].HasDefault()) { + SetDefaultValue(column_itr, *(columns[column_itr].GetDefaultValue())); + } } } @@ -292,6 +296,20 @@ const std::string Schema::GetInfo() const { } os << ")"; + if (constraints.empty() == false) { + os << ", {"; + bool first = true; + for (auto constraint : constraints) { + if (first) { + first = false; + } else { + os << ", "; + } + os << constraint.second->GetInfo(); + } + os << "}"; + } + return os.str(); } diff --git a/src/catalog/schema_catalog.cpp b/src/catalog/schema_catalog.cpp index 815c6e467a6..d93f2c0f4b7 100644 --- a/src/catalog/schema_catalog.cpp +++ b/src/catalog/schema_catalog.cpp @@ -42,11 +42,11 @@ SchemaCatalog::SchemaCatalog(concurrency::TransactionContext *, // Add indexes for pg_namespace AddIndex(SCHEMA_CATALOG_NAME "_pkey", SCHEMA_CATALOG_PKEY_OID, - {0}, + {ColumnId::SCHEMA_OID}, IndexConstraintType::PRIMARY_KEY); AddIndex(SCHEMA_CATALOG_NAME "_skey0", SCHEMA_CATALOG_SKEY0_OID, - {1}, + {ColumnId::SCHEMA_NAME}, IndexConstraintType::UNIQUE); } @@ -56,24 +56,28 @@ SchemaCatalog::~SchemaCatalog() {} * @return unqiue pointer to schema */ std::unique_ptr SchemaCatalog::InitializeSchema() { - const std::string not_null_constraint_name = "not_null"; - const std::string primary_key_constraint_name = "primary_key"; - auto schema_id_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "schema_oid", true); - schema_id_column.AddConstraint(catalog::Constraint( - ConstraintType::PRIMARY, primary_key_constraint_name)); - schema_id_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + schema_id_column.SetNotNull(); auto schema_name_column = catalog::Column( type::TypeId::VARCHAR, max_name_size_, "schema_name", false); - schema_name_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + schema_name_column.SetNotNull(); std::unique_ptr schema( new catalog::Schema({schema_id_column, schema_name_column})); + + schema->AddConstraint(std::make_shared( + SCHEMA_CATALOG_CON_PKEY_OID, ConstraintType::PRIMARY, "con_primary", + SCHEMA_CATALOG_OID, std::vector{ColumnId::SCHEMA_OID}, + SCHEMA_CATALOG_PKEY_OID)); + + schema->AddConstraint(std::make_shared( + SCHEMA_CATALOG_CON_UNI0_OID, ConstraintType::UNIQUE, "con_unique", + SCHEMA_CATALOG_OID, std::vector{ColumnId::SCHEMA_NAME}, + SCHEMA_CATALOG_SKEY0_OID)); + return schema; } diff --git a/src/catalog/system_catalogs.cpp b/src/catalog/system_catalogs.cpp index 7e92755c001..7d9267abfa9 100644 --- a/src/catalog/system_catalogs.cpp +++ b/src/catalog/system_catalogs.cpp @@ -35,40 +35,48 @@ SystemCatalogs::SystemCatalogs(concurrency::TransactionContext *txn, pg_index_metrics_(nullptr), pg_query_metrics_(nullptr) { oid_t database_oid = database->GetOid(); + pg_attribute_ = new ColumnCatalog(txn, database, pool); pg_namespace_ = new SchemaCatalog(txn, database, pool); pg_table_ = new TableCatalog(txn, database, pool); pg_index_ = new IndexCatalog(txn, database, pool); pg_layout_ = new LayoutCatalog(txn, database, pool); + pg_constraint_ = new ConstraintCatalog(txn, database, pool); // TODO: can we move this to BootstrapSystemCatalogs()? // insert column information into pg_attribute + // and insert constraint information into pg_constraint std::vector> shared_tables = { {CATALOG_DATABASE_OID, DATABASE_CATALOG_OID}, {database_oid, TABLE_CATALOG_OID}, {database_oid, SCHEMA_CATALOG_OID}, {database_oid, INDEX_CATALOG_OID}, - {database_oid, LAYOUT_CATALOG_OID}}; + {database_oid, LAYOUT_CATALOG_OID}, + {database_oid, CONSTRAINT_CATALOG_OID}}; for (int i = 0; i < (int)shared_tables.size(); i++) { + auto schema = storage::StorageManager::GetInstance() + ->GetTableWithOid(shared_tables[i].first, shared_tables[i].second) + ->GetSchema(); oid_t column_id = 0; - for (auto column : - storage::StorageManager::GetInstance() - ->GetTableWithOid(shared_tables[i].first, shared_tables[i].second) - ->GetSchema() - ->GetColumns()) { + for (auto column : schema->GetColumns()) { pg_attribute_->InsertColumn(txn, shared_tables[i].second, - column_id, column.GetName(), + column_id, column.GetOffset(), column.GetType(), column.GetLength(), - column.GetConstraints(), column.IsInlined(), + column.IsNotNull(), + column.HasDefault(), + column.GetDefaultValue(), pool); column_id++; } + for (auto constraint : schema->GetConstraints()) { + pg_constraint_->InsertConstraint(txn, constraint.second, pool); + } } } @@ -78,6 +86,7 @@ SystemCatalogs::~SystemCatalogs() { delete pg_table_; delete pg_attribute_; delete pg_namespace_; + delete pg_constraint_; if (pg_trigger_) delete pg_trigger_; // if (pg_proc) delete pg_proc; if (pg_table_metrics_) delete pg_table_metrics_; @@ -119,6 +128,7 @@ void SystemCatalogs::Bootstrap(concurrency::TransactionContext *txn, pg_namespace_->UpdateOid(OID_FOR_USER_OFFSET); pg_table_->UpdateOid(OID_FOR_USER_OFFSET); pg_index_->UpdateOid(OID_FOR_USER_OFFSET); + pg_constraint_->UpdateOid(OID_FOR_USER_OFFSET); pg_trigger_->UpdateOid(OID_FOR_USER_OFFSET); // pg_proc->UpdateOid(OID_FOR_USER_OFFSET); } diff --git a/src/catalog/table_catalog.cpp b/src/catalog/table_catalog.cpp index 8031fdd961e..14cdd46c7ef 100644 --- a/src/catalog/table_catalog.cpp +++ b/src/catalog/table_catalog.cpp @@ -16,6 +16,7 @@ #include "catalog/catalog.h" #include "catalog/column_catalog.h" +#include "catalog/constraint_catalog.h" #include "catalog/database_catalog.h" #include "catalog/index_catalog.h" #include "catalog/layout_catalog.h" @@ -50,6 +51,8 @@ TableCatalogEntry::TableCatalogEntry(concurrency::TransactionContext *txn, column_names_(), valid_column_catalog_entries_(false), valid_layout_catalog_entries_(false), + constraint_catalog_entries_(), + valid_constraint_catalog_entries_(false), txn_(txn) {} /* @brief insert index catalog object into cache @@ -330,15 +333,15 @@ TableCatalog::TableCatalog(concurrency::TransactionContext *, // Add indexes for pg_namespace AddIndex(TABLE_CATALOG_NAME "_pkey", TABLE_CATALOG_PKEY_OID, - {0}, + {ColumnId::TABLE_OID}, IndexConstraintType::PRIMARY_KEY); AddIndex(TABLE_CATALOG_NAME "_skey0", TABLE_CATALOG_SKEY0_OID, - {1, 2}, + {ColumnId::TABLE_NAME, ColumnId::SCHEMA_NAME}, IndexConstraintType::UNIQUE); AddIndex(TABLE_CATALOG_NAME "_skey1", TABLE_CATALOG_SKEY1_OID, - {3}, + {ColumnId::DATABASE_OID}, IndexConstraintType::DEFAULT); } @@ -423,55 +426,141 @@ bool TableCatalogEntry::EvictLayout(oid_t layout_id) { return true; } + +/** @brief Insert a constraint catalog entry into the cache. + * @param constraint_object Constraint to be inserted + * @return false if the constraint already exists in cache + */ +bool TableCatalogEntry::InsertConstraintCatalogEntry( + std::shared_ptr constraint_catalog_entry) { + // Invalid object + if (!constraint_catalog_entry + || (constraint_catalog_entry->GetConstraintOid() == INVALID_OID)) { + return false; + } + + oid_t constraint_oid = constraint_catalog_entry->GetConstraintOid(); + // layout is already present in the cache. + if (constraint_catalog_entries_.find(constraint_oid) != + constraint_catalog_entries_.end()) { + LOG_DEBUG("Constraint Object %u already exists in cache!", constraint_oid); + return false; + } + + constraint_catalog_entries_.insert(std::make_pair(constraint_oid, + constraint_catalog_entry)); + return true; +} + + +/** @brief Evict a constraint catalog entry from the cache. + * @param constraint_oid Id of the constraint to be deleted. + * @return true if constraint_oid is found and evicted; false if not found. + */ +bool TableCatalogEntry::EvictConstraintCatalogEntry(oid_t constraint_oid) { + if (!valid_constraint_catalog_entries_) return false; + + // find the constraint catalog entry from the cache + auto it = constraint_catalog_entries_.find(constraint_oid); + if (it == constraint_catalog_entries_.end()) { + return false; // constraint_oid not found in cache + } + + auto constraint_object = it->second; + PELOTON_ASSERT(constraint_object); + constraint_catalog_entries_.erase(it); + return true; +} + +/** @brief evict all constraint catalog entries from cache. */ +void TableCatalogEntry::EvictAllConstraintCatalogEntries() { + constraint_catalog_entries_.clear(); + valid_constraint_catalog_entries_ = false; +} + +/** @brief Get all constraint catalog entries of this table. + * Add it to the cache if necessary. + * @param cached_only If set to true, don't fetch the constraints. + * @return Map from constraint_oid to cached constraint object. + */ +std::unordered_map> +TableCatalogEntry::GetConstraintCatalogEntries(bool cached_only) { + if (!valid_constraint_catalog_entries_ && !cached_only) { + // get constraint catalog objects from pg_constraint + auto pg_constraint = Catalog::GetInstance() + ->GetSystemCatalogs(database_oid) + ->GetConstraintCatalog(); + pg_constraint->GetConstraintCatalogEntries(txn_, table_oid); + valid_constraint_catalog_entries_ = true; + } + return constraint_catalog_entries_; +} + +/** @brief Get a constraint catalog entry of the given constraint_oid. + * @param constraint_oid The id of the constraint to be fetched. + * @param cached_only If set to true, don't fetch the constraint. + * @return Constraint catalog object of corresponding to the oid if present. + */ +std::shared_ptr +TableCatalogEntry::GetConstraintCatalogEntry(oid_t constraint_oid, bool cached_only){ + GetConstraintCatalogEntries(cached_only); // fetch constraint in case we have not + auto it = constraint_catalog_entries_.find(constraint_oid); + if (it != constraint_catalog_entries_.end()) { + return it->second; + } + return nullptr; +} + + TableCatalog::~TableCatalog() {} /*@brief private function for initialize schema of pg_table * @return unqiue pointer to schema */ std::unique_ptr TableCatalog::InitializeSchema() { - const std::string primary_key_constraint_name = "primary_key"; - const std::string not_null_constraint_name = "not_null"; - auto table_id_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "table_oid", true); - table_id_column.AddConstraint(catalog::Constraint( - ConstraintType::PRIMARY, primary_key_constraint_name)); - table_id_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + table_id_column.SetNotNull(); auto table_name_column = catalog::Column(type::TypeId::VARCHAR, max_name_size_, "table_name", false); - table_name_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + table_name_column.SetNotNull(); auto schema_name_column = catalog::Column( type::TypeId::VARCHAR, max_name_size_, "schema_name", false); - schema_name_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + schema_name_column.SetNotNull(); + auto database_id_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "database_oid", true); - database_id_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + database_id_column.SetNotNull(); auto version_id_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "version_id", true); - version_id_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + version_id_column.SetNotNull(); auto default_layout_id_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "default_layout_oid", true); - default_layout_id_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + default_layout_id_column.SetNotNull(); std::unique_ptr table_catalog_schema(new catalog::Schema( {table_id_column, table_name_column, schema_name_column, database_id_column, version_id_column, default_layout_id_column})); + table_catalog_schema->AddConstraint(std::make_shared( + TABLE_CATALOG_CON_PKEY_OID, ConstraintType::PRIMARY, "con_primary", + TABLE_CATALOG_OID, std::vector{ColumnId::TABLE_OID}, + TABLE_CATALOG_PKEY_OID)); + + table_catalog_schema->AddConstraint(std::make_shared( + TABLE_CATALOG_CON_UNI0_OID, ConstraintType::UNIQUE, "con_unique", + TABLE_CATALOG_OID, std::vector{ColumnId::TABLE_NAME, ColumnId::SCHEMA_NAME}, + TABLE_CATALOG_SKEY0_OID)); + return table_catalog_schema; } diff --git a/src/common/internal_types.cpp b/src/common/internal_types.cpp index 855f7ef2d9b..3b0c20900df 100644 --- a/src/common/internal_types.cpp +++ b/src/common/internal_types.cpp @@ -2082,15 +2082,6 @@ std::string ConstraintTypeToString(ConstraintType type) { case ConstraintType::INVALID: { return ("INVALID"); } - case ConstraintType::NOT_NULL: { - return ("NOT_NULL"); - } - case ConstraintType::NOTNULL: { - return ("NOTNULL"); - } - case ConstraintType::DEFAULT: { - return ("DEFAULT"); - } case ConstraintType::CHECK: { return ("CHECK"); } @@ -2119,12 +2110,6 @@ ConstraintType StringToConstraintType(const std::string &str) { std::string upper_str = StringUtil::Upper(str); if (upper_str == "INVALID") { return ConstraintType::INVALID; - } else if (upper_str == "NOT_NULL") { - return ConstraintType::NOT_NULL; - } else if (upper_str == "NOTNULL") { - return ConstraintType::NOTNULL; - } else if (upper_str == "DEFAULT") { - return ConstraintType::DEFAULT; } else if (upper_str == "CHECK") { return ConstraintType::CHECK; } else if (upper_str == "PRIMARY") { @@ -2147,6 +2132,67 @@ std::ostream &operator<<(std::ostream &os, const ConstraintType &type) { return os; } +//===--------------------------------------------------------------------===// +// Foreign Key Action Type - String Utilities +//===--------------------------------------------------------------------===// + +std::string FKConstrActionTypeToString(FKConstrActionType type) { + switch (type) { + case FKConstrActionType::INVALID: { + return ("INVALID"); + } + case FKConstrActionType::NOACTION: { + return ("NOACTION"); + } + case FKConstrActionType::RESTRICT: { + return ("RESTRICT"); + } + case FKConstrActionType::CASCADE: { + return ("CASCADE"); + } + case FKConstrActionType::SETNULL: { + return ("SETNULL"); + } + case FKConstrActionType::SETDEFAULT: { + return ("SETDEFAULT"); + } + default: { + throw ConversionException(StringUtil::Format( + "No string conversion for FKConstrActionType value '%d'", + static_cast(type))); + } + } +return "INVALID"; +} + +FKConstrActionType StringToFKConstrActionType(const std::string &str){ + std::string upper_str = StringUtil::Upper(str); + if (upper_str == "INVALID") { + return FKConstrActionType::INVALID; + } else if (upper_str == "NOACTION") { + return FKConstrActionType::NOACTION; + } else if (upper_str == "RESTRICT") { + return FKConstrActionType::RESTRICT; + } else if (upper_str == "CASCADE") { + return FKConstrActionType::CASCADE; + } else if (upper_str == "SETNULL") { + return FKConstrActionType::SETNULL; + } else if (upper_str == "SETDEFAULT") { + return FKConstrActionType::SETDEFAULT; + } else { + throw ConversionException(StringUtil::Format( + "No FKConstrActionType conversion from string '%s'", + upper_str.c_str())); + } + return FKConstrActionType::INVALID; +} + + +std::ostream &operator<<(std::ostream &os, const FKConstrActionType &type) { + os << FKConstrActionTypeToString(type); + return os; +} + //===--------------------------------------------------------------------===// // SetOpType - String Utilities //===--------------------------------------------------------------------===// @@ -2782,18 +2828,6 @@ ConstraintType PostgresConstraintTypeToPelotonConstraintType( ConstraintType constraintType = ConstraintType::INVALID; switch (type) { - case PostgresConstraintType::NOT_NULL: - constraintType = ConstraintType::NOT_NULL; - break; - - case PostgresConstraintType::NOTNULL: - constraintType = ConstraintType::NOTNULL; - break; - - case PostgresConstraintType::DEFAULT: - constraintType = ConstraintType::DEFAULT; - break; - case PostgresConstraintType::CHECK: constraintType = ConstraintType::CHECK; break; diff --git a/src/executor/create_executor.cpp b/src/executor/create_executor.cpp index bbd3e773d43..9bd21ddaceb 100644 --- a/src/executor/create_executor.cpp +++ b/src/executor/create_executor.cpp @@ -13,7 +13,6 @@ #include "executor/create_executor.h" #include "catalog/catalog.h" -#include "catalog/foreign_key.h" #include "catalog/system_catalogs.h" #include "concurrency/transaction_context.h" #include "executor/executor_context.h" @@ -130,89 +129,130 @@ bool CreateExecutor::CreateTable(const planner::CreatePlan &node) { if (current_txn->GetResult() == ResultType::SUCCESS) { LOG_TRACE("Creating table succeeded!"); + auto catalog = catalog::Catalog::GetInstance(); + auto source_table = catalog->GetTableWithName(current_txn, + database_name, + schema_name, + table_name); + // Add the primary key constraint + if (node.HasPrimaryKey()) { + auto pk = node.GetPrimaryKey(); + std::vector col_ids; + for (auto col_name : pk.primary_key_cols) { + oid_t col_id = source_table->GetSchema()->GetColumnID(col_name); + if (col_id == INVALID_OID) { + std::string error = StringUtil::Format( + "Invalid key column name '%s.%s' for primary key '%s'", + table_name.c_str(), col_name.c_str(), pk.constraint_name.c_str()); + throw ExecutorException(error); + } + col_ids.push_back(col_id); + } + PELOTON_ASSERT(col_ids.size() == pk.primary_key_cols.size()); + + // Create the catalog object and shove it into the table + catalog->AddPrimaryKeyConstraint(current_txn, + source_table->GetDatabaseOid(), + source_table->GetOid(), + col_ids, + pk.constraint_name); + } - // Add the foreign key constraint (or other multi-column constraints) - if (node.GetForeignKeys().empty() == false) { - int count = 1; - auto catalog = catalog::Catalog::GetInstance(); - auto source_table = catalog->GetTableWithName(current_txn, - database_name, - schema_name, - table_name); - - for (auto fk : node.GetForeignKeys()) { - auto sink_table = catalog->GetTableWithName(current_txn, - database_name, - schema_name, - fk.sink_table_name); - // Source Column Offsets - std::vector source_col_ids; - for (auto col_name : fk.foreign_key_sources) { - oid_t col_id = source_table->GetSchema()->GetColumnID(col_name); - if (col_id == INVALID_OID) { - std::string error = StringUtil::Format( - "Invalid source column name '%s.%s' for foreign key '%s'", - table_name.c_str(), col_name.c_str(), - fk.constraint_name.c_str()); - throw ExecutorException(error); - } - source_col_ids.push_back(col_id); - } // FOR - PELOTON_ASSERT(source_col_ids.size() == fk.foreign_key_sources.size()); - - // Sink Column Offsets - std::vector sink_col_ids; - for (auto col_name : fk.foreign_key_sinks) { - oid_t col_id = sink_table->GetSchema()->GetColumnID(col_name); - if (col_id == INVALID_OID) { - std::string error = StringUtil::Format( - "Invalid sink column name '%s.%s' for foreign key '%s'", - sink_table->GetName().c_str(), col_name.c_str(), - fk.constraint_name.c_str()); - throw ExecutorException(error); - } - sink_col_ids.push_back(col_id); - } // FOR - PELOTON_ASSERT(sink_col_ids.size() == fk.foreign_key_sinks.size()); - - // Create the catalog object and shove it into the table - auto catalog_fk = new catalog::ForeignKey( - INVALID_OID, sink_table->GetOid(), sink_col_ids, source_col_ids, - fk.upd_action, fk.del_action, fk.constraint_name); - source_table->AddForeignKey(catalog_fk); - - // Register FK with the sink table for delete/update actions - catalog_fk = new catalog::ForeignKey( - source_table->GetOid(), INVALID_OID, sink_col_ids, source_col_ids, - fk.upd_action, fk.del_action, fk.constraint_name); - sink_table->RegisterForeignKeySource(catalog_fk); - - // Add a non-unique index on the source table if needed - std::vector source_col_names = fk.foreign_key_sources; - std::string index_name = table_name + "_FK_" + sink_table->GetName() + - "_" + std::to_string(count); - catalog->CreateIndex(current_txn, - database_name, - schema_name, - table_name, - index_name, - source_col_ids, - false, - IndexType::BWTREE); - count++; - -#ifdef LOG_DEBUG_ENABLED - LOG_DEBUG("Added a FOREIGN index on in %s.\n", table_name.c_str()); - LOG_DEBUG("Foreign key column names: \n"); - for (auto c : source_col_names) { - LOG_DEBUG("FK col name: %s\n", c.c_str()); + // Add the unique constraint + for (auto unique : node.GetUniques()) { + std::vector col_ids; + for (auto col_name : unique.unique_cols) { + oid_t col_id = source_table->GetSchema()->GetColumnID(col_name); + if (col_id == INVALID_OID) { + std::string error = StringUtil::Format( + "Invalid key column name '%s.%s' for unique '%s'", + table_name.c_str(), col_name.c_str(), + unique.constraint_name.c_str()); + throw ExecutorException(error); } - for (auto c : fk.foreign_key_sinks) { - LOG_DEBUG("FK sink col name: %s\n", c.c_str()); + col_ids.push_back(col_id); + } + PELOTON_ASSERT(col_ids.size() == unique.unique_cols.size()); + + // Create the catalog object and shove it into the table + catalog->AddUniqueConstraint(current_txn, + source_table->GetDatabaseOid(), + source_table->GetOid(), + col_ids, + unique.constraint_name); + } + + // Add the foreign key constraint + for (auto fk : node.GetForeignKeys()) { + auto sink_table = catalog->GetTableWithName(current_txn, + database_name, + schema_name, + fk.sink_table_name); + // Source Column Offsets + std::vector source_col_ids; + for (auto col_name : fk.foreign_key_sources) { + oid_t col_id = source_table->GetSchema()->GetColumnID(col_name); + if (col_id == INVALID_OID) { + std::string error = StringUtil::Format( + "Invalid source column name '%s.%s' for foreign key '%s'", + table_name.c_str(), col_name.c_str(), fk.constraint_name.c_str()); + throw ExecutorException(error); } -#endif + source_col_ids.push_back(col_id); + } // FOR + PELOTON_ASSERT(source_col_ids.size() == fk.foreign_key_sources.size()); + + // Sink Column Offsets + std::vector sink_col_ids; + for (auto col_name : fk.foreign_key_sinks) { + oid_t col_id = sink_table->GetSchema()->GetColumnID(col_name); + if (col_id == INVALID_OID) { + std::string error = StringUtil::Format( + "Invalid sink column name '%s.%s' for foreign key '%s'", + sink_table->GetName().c_str(), col_name.c_str(), + fk.constraint_name.c_str()); + throw ExecutorException(error); + } + sink_col_ids.push_back(col_id); + } // FOR + PELOTON_ASSERT(sink_col_ids.size() == fk.foreign_key_sinks.size()); + + // Create the catalog object and shove it into the table + catalog->AddForeignKeyConstraint(current_txn, + source_table->GetDatabaseOid(), + source_table->GetOid(), + source_col_ids, + sink_table->GetOid(), + sink_col_ids, + fk.upd_action, + fk.del_action, + fk.constraint_name); + } + + // Add the check constraint + for (auto check : node.GetChecks()) { + std::vector col_ids; + for (auto col_name : check.check_cols) { + oid_t col_id = source_table->GetSchema()->GetColumnID(col_name); + if (col_id == INVALID_OID) { + std::string error = StringUtil::Format( + "Invalid key column name '%s.%s' for unique '%s'", + table_name.c_str(), col_name.c_str(), + check.constraint_name.c_str()); + throw ExecutorException(error); + } + col_ids.push_back(col_id); } + PELOTON_ASSERT(col_ids.size() == check.check_cols.size()); + + // Create the catalog object and shove it into the table + catalog->AddCheckConstraint(current_txn, + source_table->GetDatabaseOid(), + source_table->GetOid(), + col_ids, check.exp, + check.constraint_name); } + } else if (current_txn->GetResult() == ResultType::FAILURE) { LOG_TRACE("Creating table failed!"); } else { @@ -271,8 +311,7 @@ bool CreateExecutor::CreateTrigger(const planner::CreatePlan &node) { // catalog table auto time_stamp = type::ValueFactory::GetTimestampValue( std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count()); + std::chrono::system_clock::now().time_since_epoch()).count()); CopySerializeOutput output; newTrigger.SerializeWhen(output, table_object->GetDatabaseOid(), diff --git a/src/executor/update_executor.cpp b/src/executor/update_executor.cpp index f387e674974..0c80039fc54 100644 --- a/src/executor/update_executor.cpp +++ b/src/executor/update_executor.cpp @@ -23,7 +23,6 @@ #include "storage/tile_group_header.h" #include "storage/tile.h" #include "storage/storage_manager.h" -#include "catalog/foreign_key.h" namespace peloton { namespace executor { @@ -119,7 +118,7 @@ bool UpdateExecutor::PerformUpdatePrimaryKey( } // Check the source table of any foreign key constraint - if (target_table_->GetForeignKeySrcCount() > 0) { + if (target_table_->GetSchema()->HasForeignKeySources()) { storage::Tuple prev_tuple(target_table_schema, true); // Get a copy of the old tuple for (oid_t column_itr = 0; column_itr < target_table_schema->GetColumnCount(); column_itr++) { diff --git a/src/include/catalog/catalog.h b/src/include/catalog/catalog.h index 5940f162864..03cc2a1abbb 100644 --- a/src/include/catalog/catalog.h +++ b/src/include/catalog/catalog.h @@ -20,6 +20,7 @@ namespace peloton { namespace catalog { +class Constraint; class Schema; class DatabaseCatalogEntry; class TableCatalogEntry; @@ -124,12 +125,14 @@ class Catalog { const std::string &schema_name, oid_t table_oid, bool is_catalog, + oid_t index_oid, const std::string &index_name, const std::vector &key_attrs, bool unique_keys, IndexType index_type, IndexConstraintType index_constraint); + /** * @brief create a new layout for a table * @param database_oid database to which the table belongs to @@ -159,6 +162,60 @@ class Catalog { oid_t table_oid, const column_map_type &column_map); + //===--------------------------------------------------------------------===// + // SET FUNCTIONS FOR COLUMN CONSTRAINT + //===--------------------------------------------------------------------===// + + // Set not null constraint for a column + ResultType SetNotNullConstraint(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + oid_t column_id); + + // Set default constraint for a column + ResultType SetDefaultConstraint(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + oid_t column_id, + const type::Value &default_value); + + //===--------------------------------------------------------------------===// + // ADD FUNCTIONS FOR TABLE CONSTRAINT + //===--------------------------------------------------------------------===// + + // Add a new primary constraint for a table + ResultType AddPrimaryKeyConstraint(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + const std::vector &column_ids, + const std::string &constraint_name); + + // Add a new unique constraint for a table + ResultType AddUniqueConstraint(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + const std::vector &column_ids, + const std::string &constraint_name); + + // Add a new foreign key constraint for a table + ResultType AddForeignKeyConstraint(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t src_table_oid, + const std::vector &src_col_ids, + oid_t sink_table_oid, + const std::vector &sink_col_ids, + FKConstrActionType upd_action, + FKConstrActionType del_action, + const std::string &constraint_name); + + // Add a new check constraint for a table + ResultType AddCheckConstraint(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + const std::vector &column_ids, + const std::pair &exp, + const std::string &constraint_name); + //===--------------------------------------------------------------------===// // DROP FUNCTIONS //===--------------------------------------------------------------------===// @@ -203,6 +260,25 @@ class Catalog { oid_t database_oid, oid_t table_oid, oid_t layout_oid); + + // Drop not null constraint for a column + ResultType DropNotNullConstraint(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + oid_t column_id); + + // Drop default constraint for a column + ResultType DropDefaultConstraint(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + oid_t column_id); + + // Drop constraint for a table + ResultType DropConstraint(concurrency::TransactionContext *txn, + oid_t database_oid, + oid_t table_oid, + oid_t constraint_oid); + //===--------------------------------------------------------------------===// // GET WITH NAME - CHECK FROM CATALOG TABLES, USING TRANSACTION //===--------------------------------------------------------------------===// @@ -296,13 +372,6 @@ class Catalog { void BootstrapSystemCatalogs(concurrency::TransactionContext *txn, storage::Database *database); - // Create the primary key index for a table, don't call this function outside - // catalog.cpp - ResultType CreatePrimaryIndex(concurrency::TransactionContext *txn, - oid_t database_oid, - const std::string &schema_name, - oid_t table_oid); - // The pool for new varlen tuple fields std::unique_ptr pool_; std::mutex catalog_mutex; diff --git a/src/include/catalog/catalog_defaults.h b/src/include/catalog/catalog_defaults.h index d8afd1a16d6..4a3a5bcd1b2 100644 --- a/src/include/catalog/catalog_defaults.h +++ b/src/include/catalog/catalog_defaults.h @@ -33,11 +33,12 @@ namespace catalog { #define INDEX_CATALOG_NAME "pg_index" #define COLUMN_CATALOG_NAME "pg_attribute" #define LAYOUT_CATALOG_NAME "pg_layout" +#define CONSTRAINT_CATALOG_NAME "pg_constraint" // Local oids from START_OID = 0 to START_OID + OID_OFFSET are reserved #define OID_OFFSET 100 #define OID_FOR_USER_OFFSET 10000 -#define CATALOG_TABLES_COUNT 9 +#define CATALOG_TABLES_COUNT 10 // Oid mask for each type #define DATABASE_OID_MASK (static_cast(catalog::CatalogType::DATABASE)) @@ -47,6 +48,7 @@ namespace catalog { #define TRIGGER_OID_MASK (static_cast(catalog::CatalogType::TRIGGER)) #define LANGUAGE_OID_MASK (static_cast(catalog::CatalogType::LANGUAGE)) #define PROC_OID_MASK (static_cast(catalog::CatalogType::PROC)) +#define CONSTRAINT_OID_MASK (static_cast(catalog::CatalogType::CONSTRAINT)) // Reserved peloton database oid #define CATALOG_DATABASE_OID (0 | DATABASE_OID_MASK) @@ -69,6 +71,7 @@ namespace catalog { #define INDEX_CATALOG_OID (3 | TABLE_OID_MASK) #define COLUMN_CATALOG_OID (4 | TABLE_OID_MASK) #define LAYOUT_CATALOG_OID (5 | TABLE_OID_MASK) +#define CONSTRAINT_CATALOG_OID (6 | TABLE_OID_MASK) // Reserved pg_column index oid #define COLUMN_CATALOG_PKEY_OID (0 | INDEX_OID_MASK) @@ -97,6 +100,36 @@ namespace catalog { #define LAYOUT_CATALOG_PKEY_OID (13 | INDEX_OID_MASK) #define LAYOUT_CATALOG_SKEY0_OID (14 | INDEX_OID_MASK) +// Reserve pg_constraint index oid +#define CONSTRAINT_CATALOG_PKEY_OID (15 | INDEX_OID_MASK) +#define CONSTRAINT_CATALOG_SKEY0_OID (16 | INDEX_OID_MASK) + +// Reserved pg_column constraint oid +#define COLUMN_CATALOG_CON_PKEY_OID (0 | CONSTRAINT_OID_MASK) +#define COLUMN_CATALOG_CON_UNI0_OID (1 | CONSTRAINT_OID_MASK) + +// Reserved pg_index index oid +#define INDEX_CATALOG_CON_PKEY_OID (2 | CONSTRAINT_OID_MASK) +#define INDEX_CATALOG_CON_UNI0_OID (3 | CONSTRAINT_OID_MASK) + +// Reserved pg_database index oid +#define DATABASE_CATALOG_CON_PKEY_OID (4 | CONSTRAINT_OID_MASK) +#define DATABASE_CATALOG_CON_UNI0_OID (5 | CONSTRAINT_OID_MASK) + +// Reserved pg_namespace index oid +#define SCHEMA_CATALOG_CON_PKEY_OID (6 | CONSTRAINT_OID_MASK) +#define SCHEMA_CATALOG_CON_UNI0_OID (7 | CONSTRAINT_OID_MASK) + +// Reserved pg_table index oid +#define TABLE_CATALOG_CON_PKEY_OID (8 | CONSTRAINT_OID_MASK) +#define TABLE_CATALOG_CON_UNI0_OID (9 | CONSTRAINT_OID_MASK) + +// Reserve pg_layout index oid +#define LAYOUT_CATALOG_CON_PKEY_OID (10 | CONSTRAINT_OID_MASK) + +// Reserve pg_constraint index oid +#define CONSTRAINT_CATALOG_CON_PKEY_OID (11 | CONSTRAINT_OID_MASK) + // Use upper 8 bits indicating catalog type #define CATALOG_TYPE_OFFSET 24 @@ -110,6 +143,7 @@ enum class CatalogType : uint32_t { TRIGGER = 6 << CATALOG_TYPE_OFFSET, LANGUAGE = 7 << CATALOG_TYPE_OFFSET, PROC = 8 << CATALOG_TYPE_OFFSET, + CONSTRAINT = 9 << CATALOG_TYPE_OFFSET, // To be added }; diff --git a/src/include/catalog/column.h b/src/include/catalog/column.h index a8741247648..7aa0fd13192 100644 --- a/src/include/catalog/column.h +++ b/src/include/catalog/column.h @@ -36,7 +36,7 @@ class Column : public Printable { Column(type::TypeId value_type, size_t column_length, std::string column_name, bool is_inlined = false, oid_t column_offset = INVALID_OID) - : column_name(column_name), + : column_name_(column_name), column_type_(value_type), fixed_length_(INVALID_OID), is_inlined_(is_inlined), @@ -61,47 +61,52 @@ class Column : public Printable { // Set the appropriate column length void SetLength(size_t column_length); - oid_t GetOffset() const { return column_offset_; } + inline oid_t GetOffset() const { return column_offset_; } - std::string GetName() const { return column_name; } + inline std::string GetName() const { return column_name_; } - size_t GetLength() const { + inline size_t GetLength() const { if (is_inlined_) return fixed_length_; else return variable_length_; } - size_t GetFixedLength() const { return fixed_length_; } + inline size_t GetFixedLength() const { return fixed_length_; } - size_t GetVariableLength() const { return variable_length_; } + inline size_t GetVariableLength() const { return variable_length_; } inline type::TypeId GetType() const { return column_type_; } inline bool IsInlined() const { return is_inlined_; } - inline bool IsPrimary() const { return is_primary_; } + // Constraint check functions for NOT NULL and DEFAULT + inline bool IsNotNull() const { return is_not_null_; } - inline bool IsUnique() const { return is_unique_; } + inline bool HasDefault() const { return has_default_; } - // Add a constraint to the column - void AddConstraint(const catalog::Constraint &constraint) { - if (constraint.GetType() == ConstraintType::DEFAULT) { - // Add the default constraint to the front - constraints_.insert(constraints_.begin(), constraint); - } else { - constraints_.push_back(constraint); - } + // Manage NOT NULL constraint + inline void SetNotNull() { is_not_null_ = true; } - if (constraint.GetType() == ConstraintType::PRIMARY) { - is_primary_ = true; - } - if (constraint.GetType() == ConstraintType::UNIQUE) { - is_unique_ = true; + inline void ClearNotNull() { is_not_null_ = false; } + + // Manage DEFAULT constraint + inline void SetDefaultValue(const type::Value &value) { + if (default_value_.get() != nullptr) { + return; } + default_value_.reset(new type::Value(value)); + has_default_ = true; } - const std::vector &GetConstraints() const { return constraints_; } + inline std::shared_ptr GetDefaultValue() const { + return default_value_; + } + + inline void ClearDefaultValue() { + default_value_.reset(); + has_default_ = false; + } hash_t Hash() const { hash_t hash = HashUtil::Hash(&column_type_); @@ -110,7 +115,8 @@ class Column : public Printable { // Compare two column objects bool operator==(const Column &other) const { - if (other.column_type_ != column_type_ || other.is_inlined_ != is_inlined_) { + if (other.column_type_ != column_type_ || + other.is_inlined_ != is_inlined_) { return false; } return true; @@ -121,14 +127,14 @@ class Column : public Printable { // Get a string representation for debugging const std::string GetInfo() const; - // name of the column - std::string column_name; - //===--------------------------------------------------------------------===// // MEMBERS //===--------------------------------------------------------------------===// private: + // name of the column + std::string column_name_; + // value type of column type::TypeId column_type_; // = type::TypeId::INVALID; @@ -143,17 +149,17 @@ class Column : public Printable { // is the column inlined ? bool is_inlined_ = false; - // is the column contained the primary key? - bool is_primary_ = false; + // is the column allowed null + bool is_not_null_ = false; + + // does the column have the default value + bool has_default_ = false; - // is the column unique - bool is_unique_ = false; + // default value + std::shared_ptr default_value_; // offset of column in tuple oid_t column_offset_ = INVALID_OID; - - // Constraints - std::vector constraints_; }; } // namespace catalog diff --git a/src/include/catalog/column_catalog.h b/src/include/catalog/column_catalog.h index 490855822f1..72b14a052f8 100644 --- a/src/include/catalog/column_catalog.h +++ b/src/include/catalog/column_catalog.h @@ -49,8 +49,9 @@ class ColumnCatalogEntry { inline type::TypeId GetColumnType() { return column_type_; } inline size_t GetColumnLength() { return column_length_; } inline bool IsInlined() { return is_inlined_; } - inline bool IsPrimary() { return is_primary_; } inline bool IsNotNull() { return is_not_null_; } + inline bool HasDefault() { return has_default_; } + inline const type::Value &GetDefaultValue() { return default_value_; } private: // member variables @@ -61,8 +62,9 @@ class ColumnCatalogEntry { type::TypeId column_type_; size_t column_length_; bool is_inlined_; - bool is_primary_; bool is_not_null_; + bool has_default_; + type::Value default_value_; }; class ColumnCatalog : public AbstractCatalog { @@ -87,13 +89,15 @@ class ColumnCatalog : public AbstractCatalog { //===--------------------------------------------------------------------===// bool InsertColumn(concurrency::TransactionContext *txn, oid_t table_oid, - oid_t column_id, const std::string &column_name, + oid_t column_id, oid_t column_offset, type::TypeId column_type, size_t column_length, - const std::vector &constraints, bool is_inlined, + bool is_not_null, + bool is_default, + const std::shared_ptr default_value, type::AbstractPool *pool); bool DeleteColumn(concurrency::TransactionContext *txn, @@ -102,6 +106,17 @@ class ColumnCatalog : public AbstractCatalog { bool DeleteColumns(concurrency::TransactionContext *txn, oid_t table_oid); + bool UpdateNotNullConstraint(concurrency::TransactionContext *txn, + oid_t table_oid, + const std::string &column_name, + bool is_not_null); + + bool UpdateDefaultConstraint(concurrency::TransactionContext *txn, + oid_t table_oid, + const std::string &column_name, + bool has_default, + const type::Value *default_value); + private: //===--------------------------------------------------------------------===// // Read Related API(only called within table catalog object) @@ -120,11 +135,13 @@ class ColumnCatalog : public AbstractCatalog { COLUMN_TYPE = 4, COLUMN_LENGTH = 5, IS_INLINED = 6, - IS_PRIMARY = 7, - IS_NOT_NULL = 8, + IS_NOT_NULL = 7, + HAS_DEFAULT = 8, + DEFAULT_VALUE_SRC = 9, + DEFAULT_VALUE_BIN = 10, // Add new columns here in creation order }; - std::vector all_column_ids_ = {0, 1, 2, 3, 4, 5, 6, 7, 8}; + std::vector all_column_ids_ = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; enum IndexId { PRIMARY_KEY = 0, diff --git a/src/include/catalog/constraint.h b/src/include/catalog/constraint.h index 7f67b1f78b6..72c5076a1ba 100644 --- a/src/include/catalog/constraint.h +++ b/src/include/catalog/constraint.h @@ -30,79 +30,108 @@ namespace catalog { class Constraint : public Printable { public: - Constraint(ConstraintType type, std::string constraint_name) - : constraint_type_(type), constraint_name_(std::move(constraint_name)) {} - Constraint(ConstraintType type, std::string constraint_name, - std::string check_cmd) - : constraint_type_(type), - constraint_name_(std::move(constraint_name)), - check_cmd_(std::move(check_cmd)) {} + // Constructor for primary key or unique + Constraint(oid_t constraint_oid, ConstraintType type, + std::string constraint_name, oid_t table_oid, + std::vector column_ids, oid_t index_oid) + : constraint_oid_(constraint_oid), constraint_name_(constraint_name), + constraint_type_(type), table_oid_(table_oid), + column_ids_(column_ids), index_oid_(index_oid) {} + + // Constructor for foreign key constraint + Constraint(oid_t constraint_oid, ConstraintType type, + std::string constraint_name, oid_t table_oid, + std::vector column_ids,oid_t index_oid, oid_t sink_table_oid, + std::vector sink_col_ids, FKConstrActionType update_action, + FKConstrActionType delete_action) + : constraint_oid_(constraint_oid), constraint_name_(constraint_name), + constraint_type_(type), table_oid_(table_oid), + column_ids_(column_ids), index_oid_(index_oid), + fk_sink_table_oid_(sink_table_oid), fk_sink_col_ids_(sink_col_ids), + fk_update_action_(update_action), fk_delete_action_(delete_action) {} + + // Constructor for check constraint + Constraint(oid_t constraint_oid, ConstraintType type, + std::string constraint_name, oid_t table_oid, + std::vector column_ids, oid_t index_oid, + std::pair exp) + : constraint_oid_(constraint_oid), constraint_name_(constraint_name), + constraint_type_(type), table_oid_(table_oid), column_ids_(column_ids), + index_oid_(index_oid), check_exp_(exp) {} //===--------------------------------------------------------------------===// // ACCESSORS //===--------------------------------------------------------------------===// - ConstraintType GetType() const { return constraint_type_; } + // Set oid for catalog + void SetConstraintOid(oid_t oid) { constraint_oid_ = oid; } - std::pair GetCheckExpression() { return exp_; } + inline oid_t GetConstraintOid() const { return constraint_oid_; } - // Offset into the list of "reference tables" in the Table. - void SetForeignKeyListOffset(oid_t offset) { fk_list_offset_ = offset; } + inline ConstraintType GetType() const { return constraint_type_; } - // Offset into the list of "unique indices" in the Table. - void SetUniqueIndexOffset(oid_t offset) { - unique_index_list_offset_ = offset; - } + // Get the table oid + inline oid_t GetTableOid() const { return table_oid_; } - // Get the offset - oid_t GetForeignKeyListOffset() const { return fk_list_offset_; } + // Get the column ids + inline const std::vector &GetColumnIds() const { return column_ids_; } - // Get the offset - oid_t GetUniqueIndexOffset() const { return unique_index_list_offset_; } + // Set index oid indicating the index constructing the constraint + void SetIndexOid(oid_t oid) { index_oid_ = oid; } - std::string GetName() const { return constraint_name_; } + // Get the index oid + inline oid_t GetIndexOid() const { return index_oid_; } + + inline std::string GetName() const { return constraint_name_; } // Get a string representation for debugging const std::string GetInfo() const override; - // Todo: default union data structure, - // For default constraint - void addDefaultValue(const type::Value &value) { - if (constraint_type_ != ConstraintType::DEFAULT - || default_value_.get() != nullptr) return; - default_value_.reset(new peloton::type::Value(value)); - } + inline oid_t GetFKSinkTableOid() const { return fk_sink_table_oid_; } + + inline const std::vector &GetFKSinkColumnIds() const { return fk_sink_col_ids_; } + + inline FKConstrActionType GetFKUpdateAction() const { return fk_update_action_; } - type::Value *getDefaultValue() { - return default_value_.get(); - } + inline FKConstrActionType GetFKDeleteAction() const { return fk_delete_action_; } - // Add check constrain - void AddCheck(ExpressionType op, peloton::type::Value val) { - exp_ = std::pair(op, val); - }; + inline std::pair GetCheckExpression() const { return check_exp_; } private: //===--------------------------------------------------------------------===// // MEMBERS //===--------------------------------------------------------------------===// + // constraint oid created by catalog + oid_t constraint_oid_; + + std::string constraint_name_; + // The type of constraint - ConstraintType constraint_type_ = ConstraintType::INVALID; + ConstraintType constraint_type_; - // Offsets into the Unique index and reference table lists in Table - oid_t fk_list_offset_ = INVALID_OID; + // Table having this constraints + oid_t table_oid_; - oid_t unique_index_list_offset_ = INVALID_OID; + // Column ids related the constraint + std::vector column_ids_; - std::string constraint_name_; + // Index constructing the constraint + oid_t index_oid_; + + // foreign key constraint information + // The reference table (sink) + oid_t fk_sink_table_oid_ = INVALID_OID; + + // Column ids in the reference table (sink) + std::vector fk_sink_col_ids_; - std::shared_ptr default_value_; + FKConstrActionType fk_update_action_ = FKConstrActionType::NOACTION; - std::string check_cmd_ = ""; + FKConstrActionType fk_delete_action_ = FKConstrActionType::NOACTION; - // key string is column name - std::pair exp_; + // key string is column name for check constraint + std::pair check_exp_; }; } // namespace catalog diff --git a/src/include/catalog/constraint_catalog.h b/src/include/catalog/constraint_catalog.h new file mode 100644 index 00000000000..a64ec5e3a75 --- /dev/null +++ b/src/include/catalog/constraint_catalog.h @@ -0,0 +1,156 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// constraint_catalog.h +// +// Identification: src/include/catalog/constraint_catalog.h +// +// Copyright (c) 2015-17, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// pg_constraint +// +// Schema: (column offset: column_name) +// 0: constraint_oid (pkey) +// 1: constraint_name (name of the constraint, not unique!) +// 2: constraint_type (type of the constraint) +// 3: table_oid (table created for the constraint) +// 4: column_ids (list of column id related to the constraint) +// 5: index_oid (index created for the constraint) +// 6: fk_sink_table_oid (for FOREIGN KEY) +// 7: fk_sink_col_ids (for FOREIGN KEY) +// 8: fk_update_action (for FOREIGN KEY) +// 9: fk_delete_action (for FOREIGN KEY) +// 10: default_value (for DEFAULT) +// 11: check_cmd (for CHECK) +// 12: check_exp (for CHECK) +// +// Indexes: (index offset: indexed columns) +// 0: constraint_oid (unique & primary key) +// 1: table_oid (non-unique) +// +// Note: Exclusive constraint is not supported yet +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "catalog/abstract_catalog.h" +#include "executor/logical_tile.h" + +namespace peloton { +namespace catalog { + +class Constraint; + +class ConstraintCatalogEntry { + friend class ConstraintCatalog; + + public: + ConstraintCatalogEntry(executor::LogicalTile *tile, int tupleId = 0); + + inline oid_t GetConstraintOid() { return constraint_oid_; } + inline const std::string &GetConstraintName() { return constraint_name_; } + inline ConstraintType GetConstraintType() { return constraint_type_; } + inline oid_t GetTableOid() { return table_oid_; } + inline const std::vector &GetColumnIds() { return column_ids_; } + inline oid_t GetIndexOid() { return index_oid_; } + inline oid_t GetFKSinkTableOid() { return fk_sink_table_oid_; } + inline const std::vector &GetFKSinkColumnIds() { + return fk_sink_col_ids_; + } + inline FKConstrActionType GetFKUpdateAction() { return fk_update_action_; } + inline FKConstrActionType GetFKDeleteAction() { return fk_delete_action_; } + inline const std::pair &GetCheckExp() { + return check_exp_; + } + + private: + // member variables + oid_t constraint_oid_; + std::string constraint_name_; + ConstraintType constraint_type_; + oid_t table_oid_; + std::vector column_ids_; + oid_t index_oid_; + oid_t fk_sink_table_oid_; + std::vector fk_sink_col_ids_; + FKConstrActionType fk_update_action_; + FKConstrActionType fk_delete_action_; + std::pair check_exp_; +}; + +class ConstraintCatalog : public AbstractCatalog { + friend class ConstraintCatalogEntry; + friend class TableCatalogEntry; + friend class Catalog; + + public: + ConstraintCatalog(concurrency::TransactionContext *txn, + storage::Database *pg_catalog, + type::AbstractPool *pool); + + ~ConstraintCatalog(); + + inline oid_t GetNextOid() { return oid_++ | CONSTRAINT_OID_MASK; } + + void UpdateOid(oid_t add_value) { oid_ += add_value; } + + //===--------------------------------------------------------------------===// + // write Related API + //===--------------------------------------------------------------------===// + bool InsertConstraint(concurrency::TransactionContext *txn, + const std::shared_ptr constraint, + type::AbstractPool *pool); + + bool DeleteConstraints(concurrency::TransactionContext *txn, + oid_t table_oid); + + bool DeleteConstraint(concurrency::TransactionContext *txn, + oid_t table_oid, + oid_t constraint_oid); + + private: + //===--------------------------------------------------------------------===// + // Read Related API(only called within table catalog object) + //===--------------------------------------------------------------------===// + const std::unordered_map> + GetConstraintCatalogEntries(concurrency::TransactionContext *txn, + oid_t table_oid); + + const std::shared_ptr + GetConstraintCatalogEntry(concurrency::TransactionContext *txn, + oid_t table_oid, + oid_t constraint_oid); + + std::unique_ptr InitializeSchema(); + + enum ColumnId { + CONSTRAINT_OID = 0, + CONSTRAINT_NAME = 1, + CONSTRAINT_TYPE = 2, + TABLE_OID = 3, + COLUMN_IDS = 4, + INDEX_OID = 5, + FK_SINK_TABLE_OID = 6, + FK_SINK_COL_IDS = 7, + FK_UPDATE_ACTION = 8, + FK_DELETE_ACTION = 9, + CHECK_EXP_SRC = 10, + CHECK_EXP_BIN = 11, + // Add new columns here in creation order + }; + std::vector all_column_ids_ = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + + enum IndexId { + PRIMARY_KEY = 0, + SKEY_TABLE_OID = 1, + // Add new indexes here in creation order + }; +}; + +} // namespace catalog +} // namespace peloton diff --git a/src/include/catalog/foreign_key.h b/src/include/catalog/foreign_key.h deleted file mode 100644 index 1fb745560e7..00000000000 --- a/src/include/catalog/foreign_key.h +++ /dev/null @@ -1,76 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// foreign_key.h -// -// Identification: src/include/catalog/foreign_key.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#pragma once - -#include -#include - -#include "common/internal_types.h" - -namespace peloton { -namespace catalog { - -//===--------------------------------------------------------------------===// -// Foreign Key Class -//===--------------------------------------------------------------------===// - -// Stores info about foreign key constraints, like the sink table id etc. -class ForeignKey { - public: - ForeignKey(oid_t source_table_id, - oid_t sink_table_id, - std::vector sink_col_ids, - std::vector source_col_ids, - FKConstrActionType update_action, - FKConstrActionType delete_action, - std::string constraint_name) - - : source_table_id_(source_table_id), - sink_table_id_(sink_table_id), - sink_col_ids_(sink_col_ids), - source_col_ids_(source_col_ids), - update_action_(update_action), - delete_action_(delete_action), - fk_name_(constraint_name) {} - - oid_t GetSourceTableOid() const { return source_table_id_; } - oid_t GetSinkTableOid() const { return sink_table_id_; } - - std::vector GetSinkColumnIds() const { return sink_col_ids_; } - std::vector GetSourceColumnIds() const { return source_col_ids_; } - - FKConstrActionType GetUpdateAction() const { return update_action_; } - FKConstrActionType GetDeleteAction() const { return delete_action_; } - std::string &GetConstraintName() { return fk_name_; } - - private: - oid_t source_table_id_ = INVALID_OID; - oid_t sink_table_id_ = INVALID_OID; - - // Columns in the reference table (sink) - std::vector sink_col_ids_; - - // Columns in the current table (source) - // Can be a single column or multiple columns depending - // on the constraint - std::vector source_col_ids_; - - FKConstrActionType update_action_; - FKConstrActionType delete_action_; - - std::string fk_name_; -}; - -} // namespace catalog -} // namespace peloton diff --git a/src/include/catalog/layout_catalog.h b/src/include/catalog/layout_catalog.h index babdb346583..616d0335b55 100644 --- a/src/include/catalog/layout_catalog.h +++ b/src/include/catalog/layout_catalog.h @@ -23,6 +23,7 @@ class Layout; namespace catalog { class LayoutCatalog : public AbstractCatalog { + friend class Catalog; public: LayoutCatalog(concurrency::TransactionContext *txn, @@ -75,4 +76,4 @@ class LayoutCatalog : public AbstractCatalog { }; } // namespace catalog -} // namespace peloton \ No newline at end of file +} // namespace peloton diff --git a/src/include/catalog/multi_constraint.h b/src/include/catalog/multi_constraint.h deleted file mode 100644 index 46b254ed90e..00000000000 --- a/src/include/catalog/multi_constraint.h +++ /dev/null @@ -1,70 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// multi_constraint.h -// -// Identification: src/include/catalog/multi_constraint.h -// -// Copyright (c) 2015-2017, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include -#include -#include - -#include "common/printable.h" -#include "common/internal_types.h" -#include "type/value.h" - -namespace peloton { -namespace catalog { - -//===--------------------------------------------------------------------===// -// MultiConstraint Class -//===--------------------------------------------------------------------===// - -class MultiConstraint : public Printable { - public: - MultiConstraint(ConstraintType type, std::string constraint_name) - : constraint_type_(type), constraint_name_(constraint_name){}; - - MultiConstraint(ConstraintType type, std::string constraint_name, - std::vector column_ids) - : constraint_type_(type), constraint_name_(constraint_name) { - this->column_ids_ = column_ids; - }; - - //===--------------------------------------------------------------------===// - // ACCESSORS - //===--------------------------------------------------------------------===// - - ConstraintType GetType() const { return constraint_type_; } - - std::string GetName() const { return constraint_name_; } - - // Get a string representation for debugging - const std::string GetInfo() const; - - std::vector GetCols() const { return column_ids_; } - - private: - //===--------------------------------------------------------------------===// - // MEMBERS - //===--------------------------------------------------------------------===// - - // The type of constraint - ConstraintType constraint_type_ = ConstraintType::INVALID; - - // constraints on column set - std::vector column_ids_; - - // we do not allow duplicate constraint name in single table - std::string constraint_name_; -}; - -} // namespace catalog -} // namespace peloton diff --git a/src/include/catalog/schema.h b/src/include/catalog/schema.h index e70c7636d5d..ead95e673a5 100644 --- a/src/include/catalog/schema.h +++ b/src/include/catalog/schema.h @@ -14,7 +14,7 @@ #include #include "catalog/column.h" -#include "catalog/multi_constraint.h" +#include "catalog/constraint.h" #include "common/printable.h" #include "type/type.h" #include "boost/algorithm/string.hpp" @@ -174,58 +174,146 @@ class Schema : public Printable { return indexed_columns_; } + //===--------------------------------------------------------------------===// + // Single column constraint accessors + //===--------------------------------------------------------------------===// + // Get the nullability of the column at a given index. inline bool AllowNull(const oid_t column_id) const { - for (auto constraint : columns_[column_id].GetConstraints()) { - if (constraint.GetType() == ConstraintType::NOTNULL) return false; + if (columns_[column_id].IsNotNull()) { + return false; } return true; } - // For single column default - inline bool AllowDefault(const oid_t column_id) const { - for (auto constraint : columns_[column_id].GetConstraints()) { - if (constraint.GetType() == ConstraintType::DEFAULT) { - return true; + // Set the not null for the column + inline void SetNotNull(const oid_t column_id) { + columns_[column_id].SetNotNull(); + not_null_columns_.push_back(column_id); + } + + // Drop the not null for the column + inline void DropNotNull(const oid_t column_id) { + columns_[column_id].ClearNotNull(); + for (auto itr = not_null_columns_.begin(); itr < not_null_columns_.end(); itr++) { + if (*itr == column_id) { + not_null_columns_.erase(itr); + break; } } + } - return false; + // Get not null column list + inline std::vector GetNotNullColumns() const { + return not_null_columns_; } + // For single column default + inline bool AllowDefault(const oid_t column_id) const { + return columns_[column_id].HasDefault(); + } // Get the default value for the column - inline type::Value *GetDefaultValue(const oid_t column_id) const { - for (auto constraint : columns_[column_id].GetConstraints()) { - if (constraint.GetType() == ConstraintType::DEFAULT) { - return constraint.getDefaultValue(); - } + inline type::Value* GetDefaultValue(const oid_t column_id) const { + if (columns_[column_id].HasDefault()) { + return columns_[column_id].GetDefaultValue().get(); } - return nullptr; } - // Add constraint for column by id - inline void AddConstraint(oid_t column_id, - const catalog::Constraint &constraint) { - columns_[column_id].AddConstraint(constraint); + // Set the default value for the column + inline void SetDefaultValue(const oid_t column_id, + const type::Value &default_value) { + if (columns_[column_id].HasDefault()) { + columns_[column_id].ClearDefaultValue(); + } + columns_[column_id].SetDefaultValue(default_value); + } + + // Drop the default value for the column + inline void DropDefaultValue(const oid_t column_id) { + if (columns_[column_id].HasDefault()) { + columns_[column_id].ClearDefaultValue(); + } + } + + //===--------------------------------------------------------------------===// + // Multi-column constraint accessors + //===--------------------------------------------------------------------===// + + // Add a constraint for the table + inline void AddConstraint(const std::shared_ptr constraint) { + constraints[constraint->GetConstraintOid()] = constraint; + + if (constraint->GetType() == ConstraintType::PRIMARY) { + has_primary_key_ = true; + } else if (constraint->GetType() == ConstraintType::UNIQUE) { + unique_constraint_count_++; + } else if (constraint->GetType() == ConstraintType::FOREIGN) { + fk_constraints_.push_back(constraint->GetConstraintOid()); + } } - // Add constraint for column by name - inline void AddConstraint(std::string column_name, - const catalog::Constraint &constraint) { - for (size_t column_itr = 0; column_itr < columns_.size(); column_itr++) { - if (columns_[column_itr].GetName() == column_name) { - columns_[column_itr].AddConstraint(constraint); + // Delete a constraint by id from the table + inline void DropConstraint(oid_t constraint_oid) { + if (constraints[constraint_oid]->GetType() == ConstraintType::PRIMARY) { + has_primary_key_ = false; + } else if (constraints[constraint_oid]->GetType() == ConstraintType::UNIQUE) { + unique_constraint_count_--; + } else if (constraints[constraint_oid]->GetType() == ConstraintType::FOREIGN) { + for (auto itr = fk_constraints_.begin(); itr < fk_constraints_.end(); itr++) { + if (*itr == constraint_oid) { + fk_constraints_.erase(itr); + break; + } } } + + constraints.erase(constraint_oid); } - inline void AddMultiConstraints(const catalog::MultiConstraint &mc) { - multi_constraints_.push_back(mc); + inline std::unordered_map> GetConstraints() const { + return constraints; } - inline std::vector GetMultiConstraints() { - return multi_constraints_; + inline std::shared_ptr GetConstraint(oid_t constraint_oid) const { + return constraints.at(constraint_oid); + } + + // For primary key constraints + inline bool HasPrimary() { return has_primary_key_; } + + // For unique constraints + inline bool HasUniqueConstraints() const { return (unique_constraint_count_ > 0); } + + // For foreign key constraints + inline std::vector> GetForeignKeyConstraints() { + std::vector> fks; + for (auto oid : fk_constraints_) { + PELOTON_ASSERT(constraints[oid]->GetType() == ConstraintType::FOREIGN); + fks.push_back(constraints[oid]); + } + return fks; + } + + inline bool HasForeignKeys() const { return (fk_constraints_.size() > 0); } + + inline void RegisterForeignKeySource(const std::shared_ptr constraint) { + fk_sources_.push_back(constraint); + } + + inline void DeleteForeignKeySource(const oid_t constraint_oid) { + for (auto itr = fk_sources_.begin(); itr < fk_sources_.end(); itr++) { + if ((*itr)->GetConstraintOid() == constraint_oid) { + fk_sources_.erase(itr); + break; + } + } + } + + inline bool HasForeignKeySources() const { return (fk_sources_.size() > 0); } + + inline std::vector> GetForeignKeySources() { + return fk_sources_; } // Get a string representation for debugging @@ -238,15 +326,12 @@ class Schema : public Printable { // all inlined and uninlined columns in the tuple std::vector columns_; - // keeps track of unlined columns - std::vector uninlined_columns_; - - // keeps multi_constraints - std::vector multi_constraints_; - // keep these in sync with the vectors above oid_t column_count_ = INVALID_OID; + // keeps track of unlined columns + std::vector uninlined_columns_; + oid_t uninlined_column_count_ = INVALID_OID; // are all columns inlined @@ -254,6 +339,27 @@ class Schema : public Printable { // keeps track of indexed columns in original table std::vector indexed_columns_; + + // Constraint Information + // keeps constraints + std::unordered_map> constraints; + + // not null column list for fast constraint checking + std::vector not_null_columns_; + + // has a primary key ? + bool has_primary_key_ = false; + + // # of unique constraints + oid_t unique_constraint_count_ = START_OID; + + // list of foreign key constraints + std::vector fk_constraints_; + + // fk constraints for which this table is the sink + // The complete information is stored so no need to lookup the table + // everytime there is a constraint check + std::vector> fk_sources_; }; } // namespace catalog diff --git a/src/include/catalog/system_catalogs.h b/src/include/catalog/system_catalogs.h index 1d3e8c081dc..e143d4c227d 100644 --- a/src/include/catalog/system_catalogs.h +++ b/src/include/catalog/system_catalogs.h @@ -14,6 +14,7 @@ #include +#include "catalog/constraint_catalog.h" #include "catalog/database_catalog.h" #include "catalog/index_metrics_catalog.h" #include "catalog/query_metrics_catalog.h" @@ -35,6 +36,7 @@ class TableCatalog; class IndexCatalog; class ColumnCatalog; class LayoutCatalog; +class ConstraintCatalog; class SystemCatalogs { public: @@ -88,6 +90,13 @@ class SystemCatalogs { return pg_layout_; } + ConstraintCatalog *GetConstraintCatalog() { + if (!pg_constraint_) { + throw CatalogException("Layout catalog has not been initialized"); + } + return pg_constraint_; + } + TriggerCatalog *GetTriggerCatalog() { if (!pg_trigger_) { throw CatalogException("Trigger catalog has not been initialized"); @@ -122,6 +131,7 @@ class SystemCatalogs { TableCatalog *pg_table_; IndexCatalog *pg_index_; LayoutCatalog *pg_layout_; + ConstraintCatalog *pg_constraint_; TriggerCatalog *pg_trigger_; // ProcCatalog *pg_proc; diff --git a/src/include/catalog/table_catalog.h b/src/include/catalog/table_catalog.h index e22af23b262..f7d373c4d3e 100644 --- a/src/include/catalog/table_catalog.h +++ b/src/include/catalog/table_catalog.h @@ -45,12 +45,14 @@ namespace catalog { class IndexCatalogEntry; class ColumnCatalogEntry; +class ConstraintCatalogEntry; class TableCatalogEntry { friend class TableCatalog; friend class IndexCatalog; friend class ColumnCatalog; friend class LayoutCatalog; + friend class ConstraintCatalog; public: TableCatalogEntry(concurrency::TransactionContext *txn, @@ -95,6 +97,15 @@ class TableCatalogEntry { std::shared_ptr GetLayout(oid_t layout_id, bool cached_entry = false); + // Evict all constraints from the cache + void EvictAllConstraintCatalogEntries(); + + // Get constraints + std::unordered_map> + GetConstraintCatalogEntries(bool cached_only = false); + std::shared_ptr + GetConstraintCatalogEntry(oid_t constraint_oid, bool cached_entry = false); + inline oid_t GetTableOid() { return table_oid; } inline const std::string &GetTableName() { return table_name; } inline const std::string &GetSchemaName() { return schema_name; } @@ -111,39 +122,53 @@ class TableCatalogEntry { uint32_t version_id; oid_t default_layout_oid; - // Get index objects + // Insert/Evict index catalog entries bool InsertIndexCatalogEntry(std::shared_ptr index_catalog_entry); bool EvictIndexCatalogEntry(oid_t index_oid); bool EvictIndexCatalogEntry(const std::string &index_name); - // Get column objects + // Insert/Evict column catalog entries bool InsertColumnCatalogEntry(std::shared_ptr column_catalog_entry); bool EvictColumnCatalogEntry(oid_t column_id); bool EvictColumnCatalogEntry(const std::string &column_name); - // Insert layout into table object + // Insert layout catalog entry into table catalog entry bool InsertLayout(std::shared_ptr layout); - // Evict layout_id from the table object + // Evict layout_id from the table catalog entry bool EvictLayout(oid_t layout_id); - // cache for *all* index catalog objects in this table + // Insert constraint catalog entry into table catalog entry + bool InsertConstraintCatalogEntry( + std::shared_ptr constraint_catalog_entry); + // Evict constraint_oid from the table catalog entry + bool EvictConstraintCatalogEntry(oid_t constraint_oid); + void SetValidConstraintCatalogEntries(bool valid = true) { + valid_constraint_catalog_entries_ = valid; + } + + // cache for *all* index catalog entries in this table std::unordered_map> index_catalog_entries; std::unordered_map> index_catalog_entries_by_name_; bool valid_index_catalog_entries_; - // cache for *all* column catalog objects in this table + // cache for *all* column catalog entries in this table std::unordered_map> column_catalog_entries_; std::unordered_map> column_names_; bool valid_column_catalog_entries_; - // cache for *all* layout objects in the table + // cache for *all* layout catalog entries in the table std::unordered_map> layout_catalog_entries_; bool valid_layout_catalog_entries_; + // cache for *all* constraint catalog entries in the table + std::unordered_map> + constraint_catalog_entries_; + bool valid_constraint_catalog_entries_; + // Pointer to its corresponding transaction concurrency::TransactionContext *txn_; }; @@ -154,6 +179,7 @@ class TableCatalog : public AbstractCatalog { friend class ColumnCatalog; friend class IndexCatalog; friend class LayoutCatalog; + friend class ConstraintCatalog; friend class Catalog; public: diff --git a/src/include/common/internal_types.h b/src/include/common/internal_types.h index 22598226407..b34d2971c70 100644 --- a/src/include/common/internal_types.h +++ b/src/include/common/internal_types.h @@ -892,26 +892,27 @@ enum class PostgresConstraintType { enum class ConstraintType { INVALID = INVALID_TYPE_ID, // invalid - NOT_NULL = 1, // notnull - NOTNULL = 2, // notnull - DEFAULT = 3, // default - CHECK = 4, // check - PRIMARY = 5, // primary key - UNIQUE = 6, // unique - FOREIGN = 7, // foreign key - EXCLUSION = 8 // foreign key + CHECK = 1, // check + PRIMARY = 2, // primary key + UNIQUE = 3, // unique + FOREIGN = 4, // foreign key + EXCLUSION = 5 // foreign key }; std::string ConstraintTypeToString(ConstraintType type); ConstraintType StringToConstraintType(const std::string &str); std::ostream &operator<<(std::ostream &os, const ConstraintType &type); enum class FKConstrActionType { - NOACTION = 0, - RESTRICT = 1, - CASCADE = 2, - SETNULL = 3, - SETDEFAULT = 4 + INVALID = INVALID_TYPE_ID, // invalid + NOACTION = 1, + RESTRICT = 2, + CASCADE = 3, + SETNULL = 4, + SETDEFAULT = 5 }; +std::string FKConstrActionTypeToString(FKConstrActionType type); +FKConstrActionType StringToFKConstrActionType(const std::string &str); +std::ostream &operator<<(std::ostream &os, const FKConstrActionType &type); enum class FKConstrMatchType { SIMPLE = 0, PARTIAL = 1, FULL = 2 }; diff --git a/src/include/planner/create_plan.h b/src/include/planner/create_plan.h index ecf6a0524fe..5f76c0f194b 100644 --- a/src/include/planner/create_plan.h +++ b/src/include/planner/create_plan.h @@ -33,11 +33,16 @@ class AbstractExpression; namespace planner { /** - * The meta-data for a foreign key reference. + * The meta-data for a constraint reference. * This is meant to be a bridge from the parser to the * catalog. It only has table names and not OIDs, whereas * the catalog only wants OIDs. */ +struct PrimaryKeyInfo { + std::vector primary_key_cols; + std::string constraint_name; +}; + struct ForeignKeyInfo { std::vector foreign_key_sources; std::vector foreign_key_sinks; @@ -47,6 +52,17 @@ struct ForeignKeyInfo { FKConstrActionType del_action; }; +struct UniqueInfo { + std::vector unique_cols; + std::string constraint_name; +}; + +struct CheckInfo { + std::vector check_cols; + std::string constraint_name; + std::pair exp; +}; + class CreatePlan : public AbstractPlan { public: CreatePlan() = delete; @@ -90,9 +106,18 @@ class CreatePlan : public AbstractPlan { std::vector GetIndexAttributes() const { return index_attrs; } + inline bool HasPrimaryKey() const { return has_primary_key; } + + inline PrimaryKeyInfo GetPrimaryKey() const { return primary_key; } + inline std::vector GetForeignKeys() const { return foreign_keys; } + + inline std::vector GetUniques() const { return con_uniques; } + + inline std::vector GetChecks() const { return con_checks; } + std::vector GetKeyAttrs() const { return key_attrs; } void SetKeyAttrs(std::vector p_key_attrs) { key_attrs = p_key_attrs; } @@ -115,11 +140,16 @@ class CreatePlan : public AbstractPlan { int16_t GetTriggerType() const { return trigger_type; } protected: - // This is a helper method for extracting foreign key information - // and storing it in an internal struct. + // These following protected function are a helper method for extracting + // Multi-column constraint information and storing it in an internal struct. void ProcessForeignKeyConstraint(const std::string &table_name, const parser::ColumnDefinition *col); + void ProcessUniqueConstraint(const parser::ColumnDefinition *col); + + void ProcessCheckConstraint(const parser::ColumnDefinition *col); + + private: // Table Name std::string table_name; @@ -150,7 +180,11 @@ class CreatePlan : public AbstractPlan { bool unique; // ColumnDefinition for multi-column constraints (including foreign key) + bool has_primary_key = false; + PrimaryKeyInfo primary_key; std::vector foreign_keys; + std::vector con_uniques; + std::vector con_checks; std::string trigger_name; std::vector trigger_funcname; std::vector trigger_args; diff --git a/src/include/storage/abstract_table.h b/src/include/storage/abstract_table.h index 6b49cf9e431..b9e5cdafa10 100644 --- a/src/include/storage/abstract_table.h +++ b/src/include/storage/abstract_table.h @@ -111,16 +111,6 @@ class AbstractTable : public Printable { // Get a string representation for debugging const std::string GetInfo() const; - //===--------------------------------------------------------------------===// - // UTILITIES - //===--------------------------------------------------------------------===// - - virtual bool HasPrimaryKey() const = 0; - - virtual bool HasUniqueConstraints() const = 0; - - virtual bool HasForeignKeys() const = 0; - //===--------------------------------------------------------------------===// // STATS //===--------------------------------------------------------------------===// diff --git a/src/include/storage/data_table.h b/src/include/storage/data_table.h index 1f0d4be29c4..48708b9edb5 100644 --- a/src/include/storage/data_table.h +++ b/src/include/storage/data_table.h @@ -134,8 +134,8 @@ class DataTable : public AbstractTable { // Insert tuple with ItemPointer provided explicitly bool InsertTuple(const AbstractTuple *tuple, ItemPointer location, - concurrency::TransactionContext *transaction, ItemPointer **index_entry_ptr, - bool check_fk = true); + concurrency::TransactionContext *transaction, + ItemPointer **index_entry_ptr, bool check_fk = true); //===--------------------------------------------------------------------===// // TILE GROUP @@ -167,13 +167,12 @@ class DataTable : public AbstractTable { int GetTriggerNumber(); - trigger::Trigger* GetTriggerByIndex(int n); + trigger::Trigger *GetTriggerByIndex(int n); - trigger::TriggerList* GetTriggerList(); + trigger::TriggerList *GetTriggerList(); void UpdateTriggerListFromCatalog(concurrency::TransactionContext *txn); - //===--------------------------------------------------------------------===// // INDEX //===--------------------------------------------------------------------===// @@ -198,29 +197,15 @@ class DataTable : public AbstractTable { const std::vector> &GetIndexColumns() const { return indexes_columns_; } + //===--------------------------------------------------------------------===// // FOREIGN KEYS //===--------------------------------------------------------------------===// - bool CheckForeignKeySrcAndCascade(storage::Tuple *prev_tuple, - storage::Tuple *new_tuple, - concurrency::TransactionContext *transaction, - executor::ExecutorContext *context, - bool is_update); - - void AddForeignKey(catalog::ForeignKey *key); - - catalog::ForeignKey *GetForeignKey(const oid_t &key_offset) const; - - void DropForeignKey(const oid_t &key_offset); - - size_t GetForeignKeyCount() const; - - void RegisterForeignKeySource(catalog::ForeignKey *key); - - size_t GetForeignKeySrcCount() const; - - catalog::ForeignKey *GetForeignKeySrc(const size_t) const; + bool CheckForeignKeySrcAndCascade( + storage::Tuple *prev_tuple, storage::Tuple *new_tuple, + concurrency::TransactionContext *transaction, + executor::ExecutorContext *context, bool is_update); //===--------------------------------------------------------------------===// // TRANSFORMERS @@ -290,12 +275,6 @@ class DataTable : public AbstractTable { // deprecated, use catalog::TableCatalog::GetInstance()->GetDatabaseOid() inline oid_t GetDatabaseOid() const { return (database_oid); } - bool HasPrimaryKey() const { return (has_primary_key_); } - - bool HasUniqueConstraints() const { return (unique_constraint_count_ > 0); } - - bool HasForeignKeys() const { return (foreign_keys_.empty() == false); } - // try to insert into all indexes. // the last argument is the index entry in primary index holding the new // tuple. @@ -335,8 +314,8 @@ class DataTable : public AbstractTable { //===--------------------------------------------------------------------===// bool CheckNotNulls(const AbstractTuple *tuple, oid_t column_idx) const; -// bool MultiCheckNotNulls(const storage::Tuple *tuple, -// std::vector cols) const; + // bool MultiCheckNotNulls(const storage::Tuple *tuple, + // std::vector cols) const; // bool CheckExp(const storage::Tuple *tuple, oid_t column_idx, // std::pair exp) const; @@ -425,20 +404,6 @@ class DataTable : public AbstractTable { // columns present in the indexes std::vector> indexes_columns_; - // CONSTRAINTS - // fk constraints for which this table is the source - std::vector foreign_keys_; - // fk constraints for which this table is the sink - // The complete information is stored so no need to lookup the table - // everytime there is a constraint check - std::vector foreign_key_sources_; - - // has a primary key ? - std::atomic has_primary_key_ = ATOMIC_VAR_INIT(false); - - // # of unique constraints - std::atomic unique_constraint_count_ = ATOMIC_VAR_INIT(START_OID); - // # of tuples. must be atomic as multiple transactions can perform insert // concurrently. std::atomic number_of_tuples_ = ATOMIC_VAR_INIT(0); diff --git a/src/include/storage/temp_table.h b/src/include/storage/temp_table.h index f6b94640750..cc45b541890 100644 --- a/src/include/storage/temp_table.h +++ b/src/include/storage/temp_table.h @@ -91,12 +91,6 @@ class TempTable : public AbstractTable { std::string GetName() const override; - inline bool HasPrimaryKey() const override { return (false); } - - inline bool HasUniqueConstraints() const override { return (false); } - - inline bool HasForeignKeys() const override { return (false); } - //===--------------------------------------------------------------------===// // STATS //===--------------------------------------------------------------------===// diff --git a/src/optimizer/stats/column_stats_collector.cpp b/src/optimizer/stats/column_stats_collector.cpp index f474504f954..f805258538d 100644 --- a/src/optimizer/stats/column_stats_collector.cpp +++ b/src/optimizer/stats/column_stats_collector.cpp @@ -44,11 +44,12 @@ void ColumnStatsCollector::AddValue(const type::Value &value) { total_count_++; if (value.IsNull()) { null_count_++; + } else { + // Update all stats + hll_.Update(value); + hist_.Update(value); + topk_.Add(value); } - // Update all stats - hll_.Update(value); - hist_.Update(value); - topk_.Add(value); } double ColumnStatsCollector::GetFracNull() { diff --git a/src/optimizer/stats/stats_storage.cpp b/src/optimizer/stats/stats_storage.cpp index 4231e95ee9e..0824a978b21 100644 --- a/src/optimizer/stats/stats_storage.cpp +++ b/src/optimizer/stats/stats_storage.cpp @@ -296,7 +296,7 @@ ResultType StatsStorage::AnalyzeStatsForAllTables( oid_t table_count = database->GetTableCount(); for (oid_t table_offset = 0; table_offset < table_count; table_offset++) { auto table = database->GetTable(table_offset); - LOG_TRACE("Analyzing table: %s", table->GetName().c_str()); + LOG_DEBUG("Analyzing table: %s", table->GetName().c_str()); std::unique_ptr table_stats_collector( new TableStatsCollector(table)); table_stats_collector->CollectColumnStats(); diff --git a/src/planner/create_plan.cpp b/src/planner/create_plan.cpp index 2a23a75abb4..47b415175b7 100644 --- a/src/planner/create_plan.cpp +++ b/src/planner/create_plan.cpp @@ -53,49 +53,31 @@ CreatePlan::CreatePlan(parser::CreateStatement *parse_tree) { schema_name = std::string(parse_tree->GetSchemaName()); database_name = std::string(parse_tree->GetDatabaseName()); std::vector columns; - std::vector column_constraints; + std::vector pri_cols; create_type = CreateType::TABLE; - // The parser puts the Foreign Key information into an artificial - // ColumnDefinition. - for (auto &fk : parse_tree->foreign_keys) { - this->ProcessForeignKeyConstraint(table_name, fk.get()); - } - for (auto &col : parse_tree->columns) { type::TypeId val = col->GetValueType(col->type); LOG_TRACE("Column name: %s.%s; Is primary key: %d", table_name.c_str(), col->name.c_str(), col->primary); - // Check main constraints - if (col->primary) { - catalog::Constraint constraint(ConstraintType::PRIMARY, - "con_primary"); - column_constraints.push_back(constraint); - LOG_TRACE("Added a primary key constraint on column \"%s.%s\"", - table_name.c_str(), col->name.c_str()); + // Create column + auto column = catalog::Column(val, type::Type::GetTypeSize(val), + std::string(col->name), false); + if (!column.IsInlined()) { + column.SetLength(col->varlen); } + // Add NOT NULL constraints to the column if (col->not_null) { - catalog::Constraint constraint(ConstraintType::NOTNULL, - "con_not_null"); - column_constraints.push_back(constraint); + column.SetNotNull(); LOG_TRACE("Added a not-null constraint on column \"%s.%s\"", table_name.c_str(), col->name.c_str()); } - if (col->unique) { - catalog::Constraint constraint(ConstraintType::UNIQUE, "con_unique"); - column_constraints.push_back(constraint); - LOG_TRACE("Added a unique constraint on column \"%s.%s\"", - table_name.c_str(), col->name.c_str()); - } - - /* **************** */ - - // Add the default value + // Add DEFAULT constraints to the column if (col->default_value != nullptr) { // Referenced from insert_plan.cpp if (col->default_value->GetExpressionType() != @@ -103,53 +85,58 @@ CreatePlan::CreatePlan(parser::CreateStatement *parse_tree) { expression::ConstantValueExpression *const_expr_elem = dynamic_cast( col->default_value.get()); - - catalog::Constraint constraint(ConstraintType::DEFAULT, - "con_default"); - type::Value v = const_expr_elem->GetValue(); - constraint.addDefaultValue(v); - column_constraints.push_back(constraint); + column.SetDefaultValue(const_expr_elem->GetValue()); LOG_TRACE("Added a default constraint %s on column \"%s.%s\"", - v.ToString().c_str(), table_name.c_str(), - col->name.c_str()); + const_expr_elem->GetValue().ToString().c_str(), + table_name.c_str(), col->name.c_str()); } } + columns.push_back(column); + + // Collect Multi-column constraints information + // TODO: Following constraints info in ColumnDefinition should be + // independent + // for multi-column constraints like foreign key. + + // Primary key + if (col->primary) { + pri_cols.push_back(col->name); + } + + // Unique constraint + // Currently only supports for single column + if (col->unique) { + ProcessUniqueConstraint(col.get()); + } + // Check expression constraint // Currently only supports simple boolean forms like (a > 0) if (col->check_expression != nullptr) { - // TODO: more expression types need to be supported - if (col->check_expression->GetValueType() == type::TypeId::BOOLEAN) { - catalog::Constraint constraint(ConstraintType::CHECK, "con_check"); - - const expression::ConstantValueExpression *const_expr_elem = - dynamic_cast( - col->check_expression->GetChild(1)); - - type::Value tmp_value = const_expr_elem->GetValue(); - constraint.AddCheck( - std::move(col->check_expression->GetExpressionType()), - std::move(tmp_value)); - column_constraints.push_back(constraint); - LOG_TRACE("Added a check constraint on column \"%s.%s\"", - table_name.c_str(), col->name.c_str()); - } + ProcessCheckConstraint(col.get()); } + } - auto column = catalog::Column(val, type::Type::GetTypeSize(val), - std::string(col->name), false); - if (!column.IsInlined()) { - column.SetLength(col->varlen); - } + catalog::Schema *schema = new catalog::Schema(columns); - for (auto con : column_constraints) { - column.AddConstraint(con); - } + // The parser puts the multi-column constraint information + // into an artificial ColumnDefinition. + // primary key constraint + if (pri_cols.size() > 0) { + primary_key.primary_key_cols = pri_cols; + primary_key.constraint_name = "con_primary"; + has_primary_key = true; + LOG_TRACE("Added a primary key constraint on column \"%s\"", + table_name.c_str()); + } - column_constraints.clear(); - columns.push_back(column); + // foreign key + for (auto &fk : parse_tree->foreign_keys) { + ProcessForeignKeyConstraint(table_name, fk.get()); } - catalog::Schema *schema = new catalog::Schema(columns); + + // TODO: UNIQUE and CHECK constraints + table_schema = schema; break; } @@ -241,6 +228,40 @@ void CreatePlan::ProcessForeignKeyConstraint( foreign_keys.push_back(fkey_info); } +void CreatePlan::ProcessUniqueConstraint(const parser::ColumnDefinition *col) { + UniqueInfo unique_info; + + unique_info.unique_cols = {col->name}; + unique_info.constraint_name = "con_unique"; + + LOG_TRACE("Added a unique constraint on column \"%s.%s\"", table_name.c_str(), + col->name.c_str()); + con_uniques.push_back(unique_info); +} + +void CreatePlan::ProcessCheckConstraint(const parser::ColumnDefinition *col) { + CheckInfo check_info; + + // TODO: more expression types need to be supported + if (col->check_expression->GetValueType() == type::TypeId::BOOLEAN) { + check_info.check_cols.push_back(col->name); + + const expression::ConstantValueExpression *const_expr_elem = + dynamic_cast( + col->check_expression->GetChild(1)); + type::Value tmp_value = const_expr_elem->GetValue(); + + check_info.exp = + std::make_pair(std::move(col->check_expression->GetExpressionType()), + std::move(tmp_value)); + + check_info.constraint_name = "con_check"; + + LOG_TRACE("Added a check constraint on column \"%s\"", table_name.c_str()); + con_checks.push_back(check_info); + } +} + expression::AbstractExpression *CreatePlan::GetTriggerWhen() const { if (trigger_when) { return trigger_when->Copy(); diff --git a/src/planner/update_plan.cpp b/src/planner/update_plan.cpp index 8b6bccc4c8a..0d1593738e3 100644 --- a/src/planner/update_plan.cpp +++ b/src/planner/update_plan.cpp @@ -23,18 +23,8 @@ UpdatePlan::UpdatePlan(storage::DataTable *table, std::unique_ptr project_info) : target_table_(table), project_info_(std::move(project_info)), - update_primary_key_(false) { + update_primary_key_(table->GetSchema()->HasPrimary()) { LOG_TRACE("Creating an Update Plan"); - - if (project_info_ != nullptr) { - for (const auto target : project_info_->GetTargetList()) { - auto col_id = target.first; - update_primary_key_ = - target_table_->GetSchema()->GetColumn(col_id).IsPrimary(); - if (update_primary_key_) - break; - } - } } void UpdatePlan::SetParameterValues(std::vector *values) { diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index c292e8d3718..85240e196e1 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -14,7 +14,6 @@ #include #include "catalog/catalog.h" -#include "catalog/foreign_key.h" #include "catalog/layout_catalog.h" #include "catalog/system_catalogs.h" #include "catalog/table_catalog.h" @@ -110,17 +109,6 @@ DataTable::~DataTable() { } } - // clean up foreign keys - for (auto foreign_key : foreign_keys_) { - delete foreign_key; - } - foreign_keys_.clear(); - - for (auto foreign_key_src : foreign_key_sources_) { - delete foreign_key_src; - } - foreign_key_sources_.clear(); - // drop all indirection arrays for (auto indirection_array : active_indirection_arrays_) { auto oid = indirection_array->GetOid(); @@ -146,71 +134,61 @@ bool DataTable::CheckNotNulls(const AbstractTuple *tuple, } bool DataTable::CheckConstraints(const AbstractTuple *tuple) const { - // For each column in the table, check to see whether they have - // any constraints. Then if they do, make sure that the - // given tuple does not violate them. - // - // TODO: PAVLO 2017-07-15 - // We should create a faster way of check the constraints for each - // column. Like maybe can store a list of just columns that - // even have constraints defined so that we don't have to - // look at each column individually. - size_t column_count = schema->GetColumnCount(); - for (oid_t column_itr = 0; column_itr < column_count; column_itr++) { - const std::vector &column_constraints = - schema->GetColumn(column_itr).GetConstraints(); - for (const auto &constraint : column_constraints) { - ConstraintType type = constraint.GetType(); - switch (type) { - case ConstraintType::NOTNULL: { - if (CheckNotNulls(tuple, column_itr) == false) { - std::string error = StringUtil::Format( - "%s constraint violated on column '%s' : %s", - ConstraintTypeToString(type).c_str(), - schema->GetColumn(column_itr).GetName().c_str(), - tuple->GetInfo().c_str()); - throw ConstraintException(error); - } - break; - } - case ConstraintType::CHECK: { - // std::pair exp = - // cons.GetCheckExpression(); - // if (CheckExp(tuple, column_itr, exp) == false) { - // LOG_TRACE("CHECK EXPRESSION constraint violated"); - // throw ConstraintException( - // "CHECK EXPRESSION constraint violated : " + - // std::string(tuple->GetInfo())); - // } - break; - } - case ConstraintType::UNIQUE: { - break; - } - case ConstraintType::DEFAULT: { - // Should not be handled here - // Handled in higher hierarchy - break; - } - case ConstraintType::PRIMARY: { - break; - } - case ConstraintType::FOREIGN: { - break; - } - case ConstraintType::EXCLUSION: { - break; - } - default: { - std::string error = - StringUtil::Format("ConstraintType '%s' is not supported", - ConstraintTypeToString(type).c_str()); - LOG_TRACE("%s", error.c_str()); - throw ConstraintException(error); - } - } + // make sure that the given tuple does not violate constraints. + + // NOT NULL constraint + for (oid_t column_id : schema->GetNotNullColumns()) { + if (schema->AllowNull(column_id) == false && + CheckNotNulls(tuple, column_id) == false) { + std::string error = + StringUtil::Format("NOT NULL constraint violated on column '%s' : %s", + schema->GetColumn(column_id).GetName().c_str(), + tuple->GetInfo().c_str()); + throw ConstraintException(error); } } + + // DEFAULT constraint should not be handled here + // Handled in higher hierarchy + + // multi-column constraints + for (auto cons_pair : schema->GetConstraints()) { + auto cons = cons_pair.second; + ConstraintType type = cons->GetType(); + switch (type) { + case ConstraintType::CHECK: { + // std::pair exp = + // cons.GetCheckExpression(); + // if (CheckExp(tuple, column_itr, exp) == false) { + // LOG_TRACE("CHECK EXPRESSION constraint violated"); + // throw ConstraintException( + // "CHECK EXPRESSION constraint violated : " + + // std::string(tuple->GetInfo())); + // } + break; + } + case ConstraintType::UNIQUE: { + break; + } + case ConstraintType::PRIMARY: { + break; + } + case ConstraintType::FOREIGN: { + break; + } + case ConstraintType::EXCLUSION: { + break; + } + default: { + std::string error = + StringUtil::Format("ConstraintType '%s' is not supported", + ConstraintTypeToString(type).c_str()); + LOG_TRACE("%s", error.c_str()); + throw ConstraintException(error); + } + } // SWITCH + } // FOR (constraints) + return true; } @@ -236,8 +214,8 @@ ItemPointer DataTable::GetEmptyTupleSlot(const storage::Tuple *tuple) { if (free_item_pointer.IsNull() == false) { // when inserting a tuple if (tuple != nullptr) { - auto tile_group = - storage::StorageManager::GetInstance()->GetTileGroup(free_item_pointer.block); + auto tile_group = storage::StorageManager::GetInstance()->GetTileGroup( + free_item_pointer.block); tile_group->CopyTuple(tuple, free_item_pointer.offset); } return free_item_pointer; @@ -386,7 +364,7 @@ bool DataTable::InsertTuple(const AbstractTuple *tuple, ItemPointer location, } PELOTON_ASSERT((*index_entry_ptr)->block == location.block && - (*index_entry_ptr)->offset == location.offset); + (*index_entry_ptr)->offset == location.offset); // Increase the table's number of tuples by 1 IncreaseTupleCount(1); @@ -591,18 +569,14 @@ bool DataTable::CheckForeignKeySrcAndCascade( storage::Tuple *prev_tuple, storage::Tuple *new_tuple, concurrency::TransactionContext *current_txn, executor::ExecutorContext *context, bool is_update) { - size_t fk_count = GetForeignKeySrcCount(); - - if (fk_count == 0) return true; + if (!schema->HasForeignKeySources()) return true; auto &transaction_manager = concurrency::TransactionManagerFactory::GetInstance(); - for (size_t iter = 0; iter < fk_count; iter++) { - catalog::ForeignKey *fk = GetForeignKeySrc(iter); - + for (auto cons : schema->GetForeignKeySources()) { // Check if any row in the source table references the current tuple - oid_t source_table_id = fk->GetSourceTableOid(); + oid_t source_table_id = cons->GetTableOid(); storage::DataTable *src_table = nullptr; try { src_table = (storage::DataTable *)storage::StorageManager::GetInstance() @@ -618,17 +592,18 @@ bool DataTable::CheckForeignKeySrcAndCascade( if (index == nullptr) continue; // Make sure this is the right index to search in - if (index->GetMetadata()->GetName().find("_FK_") != std::string::npos && - index->GetMetadata()->GetKeyAttrs() == fk->GetSourceColumnIds()) { + if (index->GetOid() == cons->GetIndexOid() && + index->GetMetadata()->GetKeyAttrs() == cons->GetColumnIds()) { LOG_DEBUG("Searching in source tables's fk index...\n"); - std::vector key_attrs = fk->GetSourceColumnIds(); + std::vector key_attrs = cons->GetColumnIds(); std::unique_ptr fk_schema( catalog::Schema::CopySchema(src_table->GetSchema(), key_attrs)); std::unique_ptr key( new storage::Tuple(fk_schema.get(), true)); - key->SetFromTuple(prev_tuple, fk->GetSinkColumnIds(), index->GetPool()); + key->SetFromTuple(prev_tuple, cons->GetFKSinkColumnIds(), + index->GetPool()); std::vector location_ptrs; index->ScanKey(key.get(), location_ptrs); @@ -646,7 +621,7 @@ bool DataTable::CheckForeignKeySrcAndCascade( if (visibility != VisibilityType::OK) continue; - switch (fk->GetUpdateAction()) { + switch (cons->GetFKUpdateAction()) { // Currently NOACTION is the same as RESTRICT case FKConstrActionType::NOACTION: case FKConstrActionType::RESTRICT: { @@ -661,11 +636,8 @@ bool DataTable::CheckForeignKeySrcAndCascade( // Read the referencing tuple, update the read timestamp so that // we can // delete it later - bool ret = - transaction_manager.PerformRead(current_txn, - *ptr, - src_tile_group_header, - true); + bool ret = transaction_manager.PerformRead( + current_txn, *ptr, src_tile_group_header, true); if (ret == false) { if (src_is_owner) { @@ -690,7 +662,7 @@ bool DataTable::CheckForeignKeySrcAndCascade( // Set the primary key fields for (oid_t k = 0; k < key_attrs.size(); k++) { auto src_col_index = key_attrs[k]; - auto sink_col_index = fk->GetSinkColumnIds()[k]; + auto sink_col_index = cons->GetFKSinkColumnIds()[k]; src_new_tuple.SetValue(src_col_index, new_tuple->GetValue(sink_col_index), context->GetPool()); @@ -756,14 +728,14 @@ bool DataTable::CheckForeignKeySrcAndCascade( */ bool DataTable::CheckForeignKeyConstraints( const AbstractTuple *tuple, concurrency::TransactionContext *transaction) { - for (auto foreign_key : foreign_keys_) { - oid_t sink_table_id = foreign_key->GetSinkTableOid(); + for (auto foreign_key : schema->GetForeignKeyConstraints()) { + oid_t sink_table_id = foreign_key->GetFKSinkTableOid(); storage::DataTable *ref_table = nullptr; try { ref_table = (storage::DataTable *)storage::StorageManager::GetInstance() ->GetTableWithOid(database_oid, sink_table_id); } catch (CatalogException &e) { - LOG_TRACE("Can't find table %d! Return false", sink_table_id); + LOG_ERROR("Can't find table %d! Return false", sink_table_id); return false; } int ref_table_index_count = ref_table->GetIndexCount(); @@ -775,13 +747,12 @@ bool DataTable::CheckForeignKeyConstraints( // The foreign key constraints only refer to the primary key if (index->GetIndexType() == IndexConstraintType::PRIMARY_KEY) { - std::vector key_attrs = foreign_key->GetSinkColumnIds(); + std::vector key_attrs = foreign_key->GetFKSinkColumnIds(); std::unique_ptr foreign_key_schema( catalog::Schema::CopySchema(ref_table->schema, key_attrs)); std::unique_ptr key( new storage::Tuple(foreign_key_schema.get(), true)); - key->SetFromTuple(tuple, foreign_key->GetSourceColumnIds(), - index->GetPool()); + key->SetFromTuple(tuple, foreign_key->GetColumnIds(), index->GetPool()); LOG_TRACE("check key: %s", key->GetInfo().c_str()); std::vector location_ptrs; @@ -790,7 +761,7 @@ bool DataTable::CheckForeignKeyConstraints( // if this key doesn't exist in the referred column if (location_ptrs.size() == 0) { LOG_DEBUG("The key: %s does not exist in table %s\n", - key->GetInfo().c_str(), ref_table->GetInfo().c_str()); + key->GetInfo().c_str(), ref_table->GetName().c_str()); return false; } @@ -808,7 +779,7 @@ bool DataTable::CheckForeignKeyConstraints( LOG_DEBUG( "The key: %s is not yet visible in table %s, visibility " "type: %s.\n", - key->GetInfo().c_str(), ref_table->GetInfo().c_str(), + key->GetInfo().c_str(), ref_table->GetName().c_str(), VisibilityTypeToString(visibility).c_str()); return false; } @@ -875,7 +846,8 @@ void DataTable::ResetDirty() { dirty_ = false; } TileGroup *DataTable::GetTileGroupWithLayout( std::shared_ptr layout) { - oid_t tile_group_id = storage::StorageManager::GetInstance()->GetNextTileGroupId(); + oid_t tile_group_id = + storage::StorageManager::GetInstance()->GetNextTileGroupId(); return (AbstractTable::GetTileGroupWithLayout(database_oid, tile_group_id, layout, tuples_per_tilegroup_)); } @@ -915,7 +887,8 @@ oid_t DataTable::AddDefaultTileGroup(const size_t &active_tile_group_id) { tile_groups_.Append(tile_group_id); // add tile group metadata in locator - storage::StorageManager::GetInstance()->AddTileGroup(tile_group_id, tile_group); + storage::StorageManager::GetInstance()->AddTileGroup(tile_group_id, + tile_group); COMPILER_MEMORY_FENCE; @@ -961,7 +934,8 @@ void DataTable::AddTileGroupWithOidForRecovery(const oid_t &tile_group_id) { LOG_TRACE("Added a tile group "); // add tile group metadata in locator - storage::StorageManager::GetInstance()->AddTileGroup(tile_group_id, tile_group); + storage::StorageManager::GetInstance()->AddTileGroup(tile_group_id, + tile_group); // we must guarantee that the compiler always add tile group before adding // tile_group_count_. @@ -984,7 +958,8 @@ void DataTable::AddTileGroup(const std::shared_ptr &tile_group) { tile_groups_.Append(tile_group_id); // add tile group in catalog - storage::StorageManager::GetInstance()->AddTileGroup(tile_group_id, tile_group); + storage::StorageManager::GetInstance()->AddTileGroup(tile_group_id, + tile_group); // we must guarantee that the compiler always add tile group before adding // tile_group_count_. @@ -1048,14 +1023,6 @@ void DataTable::AddIndex(std::shared_ptr index) { index_columns_.end()); indexes_columns_.push_back(index_columns_set); - - // Update index stats - auto index_type = index->GetIndexType(); - if (index_type == IndexConstraintType::PRIMARY_KEY) { - has_primary_key_ = true; - } else if (index_type == IndexConstraintType::UNIQUE) { - unique_constraint_count_++; - } } std::shared_ptr DataTable::GetIndexWithOid( @@ -1147,55 +1114,6 @@ oid_t DataTable::GetValidIndexCount() const { return valid_index_count; } -//===--------------------------------------------------------------------===// -// FOREIGN KEYS -//===--------------------------------------------------------------------===// - -void DataTable::AddForeignKey(catalog::ForeignKey *key) { - { - std::lock_guard lock(data_table_mutex_); - catalog::Constraint constraint(ConstraintType::FOREIGN, - key->GetConstraintName()); - constraint.SetForeignKeyListOffset(GetForeignKeyCount()); - for (auto fk_column : key->GetSourceColumnIds()) { - schema->AddConstraint(fk_column, constraint); - } - foreign_keys_.push_back(key); - } -} - -catalog::ForeignKey *DataTable::GetForeignKey(const oid_t &key_offset) const { - catalog::ForeignKey *key = nullptr; - key = foreign_keys_.at(key_offset); - return key; -} - -void DataTable::DropForeignKey(const oid_t &key_offset) { - { - std::lock_guard lock(data_table_mutex_); - PELOTON_ASSERT(key_offset < foreign_keys_.size()); - foreign_keys_.erase(foreign_keys_.begin() + key_offset); - } -} - -size_t DataTable::GetForeignKeyCount() const { return foreign_keys_.size(); } - -// Adds to the list of tables for which this table's PK is the foreign key sink -void DataTable::RegisterForeignKeySource(catalog::ForeignKey *key) { - { - std::lock_guard lock(data_table_mutex_); - foreign_key_sources_.push_back(key); - } -} - -size_t DataTable::GetForeignKeySrcCount() const { - return foreign_key_sources_.size(); -} - -catalog::ForeignKey *DataTable::GetForeignKeySrc(const size_t offset) const { - return foreign_key_sources_[offset]; -} - // Get the schema for the new transformed tile group std::vector TransformTileGroupSchema( storage::TileGroup *tile_group, const Layout &layout) { diff --git a/src/storage/database.cpp b/src/storage/database.cpp index 8a7506805c8..55eaccc6892 100644 --- a/src/storage/database.cpp +++ b/src/storage/database.cpp @@ -12,7 +12,6 @@ #include -#include "catalog/foreign_key.h" #include "codegen/query_cache.h" #include "common/exception.h" #include "common/logger.h" @@ -146,15 +145,11 @@ const std::string Database::GetInfo() const { } } - if (table->HasForeignKeys()) { + if (table->GetSchema()->HasForeignKeys()) { os << "foreign tables \n"; - oid_t foreign_key_count = table->GetForeignKeyCount(); - for (oid_t foreign_key_itr = 0; foreign_key_itr < foreign_key_count; - foreign_key_itr++) { - auto foreign_key = table->GetForeignKey(foreign_key_itr); - - auto sink_table_oid = foreign_key->GetSinkTableOid(); + for (auto foreign_key : table->GetSchema()->GetForeignKeyConstraints()) { + auto sink_table_oid = foreign_key->GetFKSinkTableOid(); auto sink_table = GetTableWithOid(sink_table_oid); os << "table name : " << sink_table->GetName() << std::endl; diff --git a/test/brain/query_logger_test.cpp b/test/brain/query_logger_test.cpp index c3affa7fd40..301e3408dbd 100644 --- a/test/brain/query_logger_test.cpp +++ b/test/brain/query_logger_test.cpp @@ -53,7 +53,7 @@ class QueryLoggerTests : public PelotonTest { sleep(wait_time_); TestingSQLUtil::ExecuteSQLQueryAndCheckResult(select_query_.c_str(), - expected_result, true); + expected_result); // the select query we used will also be logged for next time expected_result.push_back(select_query_ + "|" + select_query_fingerprint_); @@ -88,7 +88,7 @@ class QueryLoggerTests : public PelotonTest { temporary_expected_result.end()); temporary_expected_result.clear(); TestingSQLUtil::ExecuteSQLQueryAndCheckResult(select_query_.c_str(), - expected_result, true); + expected_result); // the select query we used will also be logged for next time expected_result.push_back(select_query_ + "|" + @@ -97,7 +97,7 @@ class QueryLoggerTests : public PelotonTest { } else { // verify that the logging does not happen before the txn commit TestingSQLUtil::ExecuteSQLQueryAndCheckResult(select_query_.c_str(), - expected_result, true); + expected_result); // the select query we used will also be logged for next time temporary_expected_result.push_back(select_query_ + "|" + select_query_fingerprint_); diff --git a/test/catalog/catalog_test.cpp b/test/catalog/catalog_test.cpp index bda84540f89..7a15fa0f177 100644 --- a/test/catalog/catalog_test.cpp +++ b/test/catalog/catalog_test.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "catalog/catalog.h" +#include "catalog/column.h" #include "catalog/column_catalog.h" #include "catalog/database_catalog.h" #include "catalog/database_metrics_catalog.h" @@ -25,6 +26,7 @@ #include "sql/testing_sql_util.h" #include "storage/storage_manager.h" #include "type/ephemeral_pool.h" +#include "type/value_factory.h" namespace peloton { namespace test { @@ -70,8 +72,6 @@ TEST_F(CatalogTests, CreatingTable) { auto id_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "id", true); - id_column.AddConstraint( - catalog::Constraint(ConstraintType::PRIMARY, "primary_key")); auto name_column = catalog::Column(type::TypeId::VARCHAR, 32, "name", true); std::unique_ptr table_schema( @@ -81,24 +81,55 @@ TEST_F(CatalogTests, CreatingTable) { std::unique_ptr table_schema_3( new catalog::Schema({id_column, name_column})); - catalog::Catalog::GetInstance()->CreateTable(txn, - "emp_db", - DEFAULT_SCHEMA_NAME, - std::move(table_schema), - "emp_table", - false); - catalog::Catalog::GetInstance()->CreateTable(txn, - "emp_db", - DEFAULT_SCHEMA_NAME, - std::move(table_schema_2), - "department_table", - false); - catalog::Catalog::GetInstance()->CreateTable(txn, - "emp_db", - DEFAULT_SCHEMA_NAME, - std::move(table_schema_3), - "salary_table", - false); + auto catalog = catalog::Catalog::GetInstance(); + catalog->CreateTable(txn, + "emp_db", + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + "emp_table", + false); + catalog->CreateTable(txn, + "emp_db", + DEFAULT_SCHEMA_NAME, + std::move(table_schema_2), + "department_table", + false); + catalog->CreateTable(txn, + "emp_db", + DEFAULT_SCHEMA_NAME, + std::move(table_schema_3), + "salary_table", + false); + + auto emp = catalog->GetTableCatalogEntry(txn, + "emp_db", + DEFAULT_SCHEMA_NAME, + "emp_table"); + auto department = catalog->GetTableCatalogEntry(txn, + "emp_db", + DEFAULT_SCHEMA_NAME, + "department_table"); + auto salary = catalog->GetTableCatalogEntry(txn, + "emp_db", + DEFAULT_SCHEMA_NAME, + "salary_table"); + + catalog->AddPrimaryKeyConstraint(txn, + emp->GetDatabaseOid(), + emp->GetTableOid(), + {0}, + "con_primary"); + catalog->AddPrimaryKeyConstraint(txn, + department->GetDatabaseOid(), + department->GetTableOid(), + {0}, + "con_primary"); + catalog->AddPrimaryKeyConstraint(txn, + salary->GetDatabaseOid(), + salary->GetTableOid(), + {0}, + "con_primary"); + // insert random tuple into DATABASE_METRICS_CATALOG and check std::unique_ptr pool(new type::EphemeralPool()); catalog::DatabaseMetricsCatalog::GetInstance()->InsertDatabaseMetrics(txn, @@ -113,10 +144,8 @@ TEST_F(CatalogTests, CreatingTable) { param.len = 1; param.buf = (unsigned char *) pool->Allocate(1); *param.buf = 'a'; - auto database_object = - catalog::Catalog::GetInstance()->GetDatabaseCatalogEntry(txn, "emp_db"); - catalog::Catalog::GetInstance() - ->GetSystemCatalogs(database_object->GetDatabaseOid()) + auto database_object = catalog->GetDatabaseCatalogEntry(txn, "emp_db"); + catalog->GetSystemCatalogs(database_object->GetDatabaseOid()) ->GetQueryMetricsCatalog() ->InsertQueryMetrics(txn, "a query", @@ -133,20 +162,13 @@ TEST_F(CatalogTests, CreatingTable) { 1, 1, pool.get()); - auto param1 = catalog::Catalog::GetInstance() - ->GetSystemCatalogs(database_object->GetDatabaseOid()) - ->GetQueryMetricsCatalog() - ->GetParamTypes(txn, "a query"); + auto param1 = catalog->GetSystemCatalogs(database_object->GetDatabaseOid()) + ->GetQueryMetricsCatalog() + ->GetParamTypes(txn, "a query"); EXPECT_EQ(1, param1.len); EXPECT_EQ('a', *param1.buf); // check colum object - EXPECT_EQ("name", catalog::Catalog::GetInstance() - ->GetTableCatalogEntry(txn, - "emp_db", - DEFAULT_SCHEMA_NAME, - "department_table") - ->GetColumnCatalogEntry(1) - ->GetColumnName()); + EXPECT_EQ("name", department->GetColumnCatalogEntry(1)->GetColumnName()); txn_manager.CommitTransaction(txn); } @@ -155,8 +177,7 @@ TEST_F(CatalogTests, TestingCatalogCache) { auto txn = txn_manager.BeginTransaction(); auto catalog = catalog::Catalog::GetInstance(); - auto - catalog_db_object = + auto catalog_db_object = catalog->GetDatabaseCatalogEntry(txn, CATALOG_DATABASE_OID); auto catalog_table_objects = catalog_db_object->GetTableCatalogEntries(); EXPECT_NE(0, catalog_table_objects.size()); @@ -171,6 +192,7 @@ TEST_F(CatalogTests, TestingCatalogCache) { auto table = user_database->GetTable(table_idx); auto user_table_object = user_db_object->GetTableCatalogEntry(table->GetOid()); + EXPECT_EQ(user_db_object->GetDatabaseOid(), user_table_object->GetDatabaseOid()); } @@ -182,10 +204,11 @@ TEST_F(CatalogTests, TableObject) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - auto table_object = catalog::Catalog::GetInstance()->GetTableCatalogEntry(txn, - "emp_db", - DEFAULT_SCHEMA_NAME, - "department_table"); + auto table_object = + catalog::Catalog::GetInstance()->GetTableCatalogEntry(txn, + "emp_db", + DEFAULT_SCHEMA_NAME, + "department_table"); auto index_objects = table_object->GetIndexCatalogEntries(); auto column_objects = table_object->GetColumnCatalogEntries(); @@ -201,8 +224,8 @@ TEST_F(CatalogTests, TableObject) { EXPECT_EQ(type::Type::GetTypeSize(type::TypeId::INTEGER), column_objects[0]->GetColumnLength()); EXPECT_TRUE(column_objects[0]->IsInlined()); - EXPECT_TRUE(column_objects[0]->IsPrimary()); EXPECT_FALSE(column_objects[0]->IsNotNull()); + EXPECT_FALSE(column_objects[0]->HasDefault()); EXPECT_EQ(table_object->GetTableOid(), column_objects[1]->GetTableOid()); EXPECT_EQ("name", column_objects[1]->GetColumnName()); @@ -211,8 +234,8 @@ TEST_F(CatalogTests, TableObject) { EXPECT_EQ(type::TypeId::VARCHAR, column_objects[1]->GetColumnType()); EXPECT_EQ(32, column_objects[1]->GetColumnLength()); EXPECT_TRUE(column_objects[1]->IsInlined()); - EXPECT_FALSE(column_objects[1]->IsPrimary()); EXPECT_FALSE(column_objects[1]->IsNotNull()); + EXPECT_FALSE(column_objects[1]->HasDefault()); // update pg_table SET version_oid = 1 where table_name = department_table oid_t department_table_oid = table_object->GetTableOid(); @@ -487,6 +510,7 @@ TEST_F(CatalogTests, LayoutCatalogTest) { *(default_layout.get()), *(pg_layout->GetLayoutWithOid(txn, table_oid, default_layout_oid).get())); EXPECT_EQ(default_layout_oid, + catalog->GetTableCatalogEntry(txn, database_oid, table_oid)->GetDefaultLayoutOid()); @@ -546,6 +570,7 @@ TEST_F(CatalogTests, LayoutCatalogTest) { EXPECT_EQ(nullptr, pg_layout->GetLayoutWithOid(txn, table_oid, default_layout_oid)); EXPECT_EQ(ROW_STORE_LAYOUT_OID, + catalog->GetTableCatalogEntry(txn, database_oid, table_oid)->GetDefaultLayoutOid()); @@ -562,5 +587,307 @@ TEST_F(CatalogTests, LayoutCatalogTest) { txn_manager.CommitTransaction(txn); } +TEST_F(CatalogTests, ConstraintCatalogTest) { + auto db_name = "con_db"; + auto sink_table_name = "sink_table"; + auto con_table_name = "con_table"; + auto catalog = catalog::Catalog::GetInstance(); + // Create database. + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + EXPECT_EQ(ResultType::SUCCESS, catalog->CreateDatabase(txn, db_name)); + + // Create table for foreign key. + auto sink_val0 = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "sink_val0", true); + std::unique_ptr sink_table_schema( + new catalog::Schema({sink_val0})); + EXPECT_EQ(ResultType::SUCCESS, + catalog->CreateTable(txn, + db_name, + DEFAULT_SCHEMA_NAME, + std::move(sink_table_schema), + sink_table_name, + false)); + + // Create table for constraint catalog test, and set column constraints. + auto con_val0 = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "con_val0", true); + auto con_val1 = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "con_val1", true); + auto con_val2 = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "con_val2", true); + auto con_val3 = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "con_val3", true); + auto con_val4 = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "con_val4", true); + auto con_val5 = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "con_val5", true); + auto con_val6 = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "con_val6", true); + con_val5.SetNotNull(); + con_val6.SetDefaultValue(type::ValueFactory::GetIntegerValue(555)); + std::unique_ptr con_table_schema(new catalog::Schema( + {con_val0, con_val1, con_val2, con_val3, con_val4, con_val5, con_val6})); + EXPECT_EQ(ResultType::SUCCESS, + catalog->CreateTable(txn, + db_name, + DEFAULT_SCHEMA_NAME, + std::move(con_table_schema), + con_table_name, + false)); + + LOG_DEBUG("Success two table creations"); + + auto database_oid = + catalog->GetDatabaseCatalogEntry(txn, db_name)->GetDatabaseOid(); + auto sink_table_object = catalog->GetTableCatalogEntry(txn, + db_name, + DEFAULT_SCHEMA_NAME, + sink_table_name); + auto sink_table = catalog->GetTableWithName(txn, + db_name, + DEFAULT_SCHEMA_NAME, + sink_table_name); + auto sink_table_oid = sink_table_object->GetTableOid(); + auto con_table_object = catalog->GetTableCatalogEntry(txn, + db_name, + DEFAULT_SCHEMA_NAME, + con_table_name); + auto con_table = catalog->GetTableWithName(txn, + db_name, + DEFAULT_SCHEMA_NAME, + con_table_name); + auto con_table_oid = con_table_object->GetTableOid(); + + // Add primary key constraint to sink table + EXPECT_EQ(ResultType::SUCCESS, + catalog->AddPrimaryKeyConstraint(txn, + database_oid, + sink_table_oid, + {0}, + "con_primary")); + + // Add constraints for constraint catalog test. + EXPECT_EQ(ResultType::SUCCESS, + catalog->AddPrimaryKeyConstraint(txn, + database_oid, + con_table_oid, + {0, 1}, + "con_primary")); + EXPECT_EQ(ResultType::SUCCESS, + catalog->AddUniqueConstraint(txn, + database_oid, + con_table_oid, + {2}, + "con_unique")); + + EXPECT_EQ(ResultType::SUCCESS, + catalog->AddForeignKeyConstraint(txn, + database_oid, + con_table_oid, + {3}, + sink_table_oid, + {0}, + FKConstrActionType::NOACTION, + FKConstrActionType::NOACTION, + "con_foreign")); + auto exp = std::make_pair(ExpressionType::COMPARE_GREATERTHAN, + type::ValueFactory::GetIntegerValue(0)); + EXPECT_EQ(ResultType::SUCCESS, + catalog->AddCheckConstraint(txn, + database_oid, + con_table_oid, + {4}, + exp, + "con_check")); + + LOG_DEBUG("Success all constraint creations"); + + // Check constraint + auto sink_schema = sink_table->GetSchema(); + EXPECT_EQ(false, sink_schema->HasForeignKeys()); + EXPECT_EQ(true, sink_schema->HasPrimary()); + EXPECT_EQ(false, sink_schema->HasUniqueConstraints()); + auto constraint_objects = sink_table_object->GetConstraintCatalogEntries(); + EXPECT_EQ(1, constraint_objects.size()); + for (auto constraint_object_pair : constraint_objects) { + auto con_oid = constraint_object_pair.first; + auto con_object = constraint_object_pair.second; + auto column_ids = con_object->GetColumnIds(); + EXPECT_LE(1, column_ids.size()); + auto constraint = sink_table->GetSchema()->GetConstraint(con_oid); + EXPECT_EQ(constraint->GetName(), con_object->GetConstraintName()); + EXPECT_EQ(constraint->GetType(), con_object->GetConstraintType()); + EXPECT_EQ(constraint->GetTableOid(), con_object->GetTableOid()); + EXPECT_EQ(constraint->GetIndexOid(), con_object->GetIndexOid()); + EXPECT_EQ(constraint->GetColumnIds().size(), column_ids.size()); + } + + // Check foreign key as sink table + EXPECT_EQ(true, sink_schema->HasForeignKeySources()); + auto fk_sources = sink_schema->GetForeignKeySources(); + EXPECT_EQ(1, fk_sources.size()); + auto fk_source = fk_sources.at(0); + EXPECT_EQ(con_table_oid, fk_source->GetTableOid()); + EXPECT_EQ(sink_table_oid, fk_source->GetFKSinkTableOid()); + + LOG_DEBUG("%s", sink_schema->GetInfo().c_str()); + LOG_DEBUG("Complete check for sink table"); + + // Single column constraints + for (auto column_object_pair : con_table_object->GetColumnCatalogEntries()) { + auto column_id = column_object_pair.first; + auto column_object = column_object_pair.second; + auto column = con_table->GetSchema()->GetColumn(column_id); + + if (column_object->GetColumnName() == "con_val5") { + LOG_DEBUG("Check not null constraint in column:%s", + column_object->GetColumnName().c_str()); + EXPECT_TRUE(column_object->IsNotNull()); + EXPECT_EQ(column.IsNotNull(), column_object->IsNotNull()); + } else if (column_object->GetColumnName() == "con_val6") { + LOG_DEBUG("Check default constraint in column:%s", + column_object->GetColumnName().c_str()); + EXPECT_TRUE(column_object->HasDefault()); + EXPECT_EQ(column.HasDefault(), column_object->HasDefault()); + EXPECT_EQ(column.GetDefaultValue()->CompareEquals( + column_object->GetDefaultValue()), + CmpBool::CmpTrue); + } + } + + // Table constraints + auto con_schema = con_table->GetSchema(); + EXPECT_EQ(true, con_schema->HasForeignKeys()); + EXPECT_EQ(true, con_schema->HasPrimary()); + EXPECT_EQ(true, con_schema->HasUniqueConstraints()); + EXPECT_EQ(false, con_schema->HasForeignKeySources()); + constraint_objects = con_table_object->GetConstraintCatalogEntries(); + EXPECT_EQ(4, constraint_objects.size()); + for (auto constraint_object_pair : constraint_objects) { + auto con_oid = constraint_object_pair.first; + auto con_object = constraint_object_pair.second; + + LOG_DEBUG("Check constraint:%s (%s)", + con_object->GetConstraintName().c_str(), + ConstraintTypeToString(con_object->GetConstraintType()).c_str()); + + auto constraint = con_table->GetSchema()->GetConstraint(con_oid); + EXPECT_NE(nullptr, constraint); + EXPECT_EQ(constraint->GetName(), con_object->GetConstraintName()); + EXPECT_EQ(constraint->GetType(), con_object->GetConstraintType()); + EXPECT_EQ(con_table_oid, con_object->GetTableOid()); + EXPECT_EQ(constraint->GetIndexOid(), con_object->GetIndexOid()); + EXPECT_EQ(constraint->GetColumnIds().size(), + con_object->GetColumnIds().size()); + + switch (con_object->GetConstraintType()) { + case ConstraintType::PRIMARY: + case ConstraintType::UNIQUE: + break; + + case ConstraintType::FOREIGN: { + EXPECT_EQ(fk_source.get(), constraint.get()); + EXPECT_EQ(constraint->GetFKSinkTableOid(), + con_object->GetFKSinkTableOid()); + EXPECT_EQ(constraint->GetFKSinkColumnIds().size(), + con_object->GetFKSinkColumnIds().size()); + EXPECT_EQ(constraint->GetFKUpdateAction(), + con_object->GetFKUpdateAction()); + EXPECT_EQ(constraint->GetFKDeleteAction(), + con_object->GetFKDeleteAction()); + break; + } + + case ConstraintType::CHECK: { + EXPECT_EQ(1, con_object->GetColumnIds().size()); + auto column = + con_table->GetSchema()->GetColumn(con_object->GetColumnIds().at(0)); + EXPECT_EQ(constraint->GetCheckExpression().first, + con_object->GetCheckExp().first); + EXPECT_EQ(constraint->GetCheckExpression().second.CompareEquals( + con_object->GetCheckExp().second), + CmpBool::CmpTrue); + break; + } + default: + LOG_DEBUG( + "Unexpected constraint appeared: %s", + ConstraintTypeToString(con_object->GetConstraintType()).c_str()); + EXPECT_TRUE(false); + } + } + con_table_object->GetConstraintCatalogEntries(); + + txn_manager.CommitTransaction(txn); + + LOG_DEBUG("%s", con_schema->GetInfo().c_str()); + LOG_DEBUG("Complete check for constraint table"); + + // Drop constraint + txn = txn_manager.BeginTransaction(); + for (auto not_null_column_id : con_schema->GetNotNullColumns()) { + EXPECT_EQ(ResultType::SUCCESS, + catalog->DropNotNullConstraint(txn, + database_oid, + con_table_oid, + not_null_column_id)); + } + EXPECT_EQ(ResultType::SUCCESS, + catalog->DropDefaultConstraint(txn, + database_oid, + con_table_oid, + 6)); + for (auto constraint : con_schema->GetConstraints()) { + EXPECT_EQ( + ResultType::SUCCESS, + catalog->DropConstraint(txn, + database_oid, + con_table_oid, + constraint.second->GetConstraintOid())); + } + txn_manager.CommitTransaction(txn); + + LOG_DEBUG("%s", con_schema->GetInfo().c_str()); + LOG_DEBUG("Complete drop constraints in constraint table"); + + // Check dropping constraints + txn = txn_manager.BeginTransaction(); + con_table_object = catalog->GetTableCatalogEntry(txn, + db_name, + DEFAULT_SCHEMA_NAME, + con_table_name); + EXPECT_EQ(0, con_schema->GetNotNullColumns().size()); + for (oid_t column_id = 0; column_id < con_schema->GetColumnCount(); column_id++) { + EXPECT_EQ(true, con_schema->AllowNull(column_id)); + EXPECT_EQ(false, con_schema->AllowDefault(column_id)); + } + for (auto column_object_pair : con_table_object->GetColumnCatalogEntries()) { + auto column_object = column_object_pair.second; + EXPECT_EQ(false, column_object->IsNotNull()); + EXPECT_EQ(false, column_object->HasDefault()); + } + EXPECT_EQ(false, con_schema->HasForeignKeys()); + EXPECT_EQ(false, con_schema->HasPrimary()); + EXPECT_EQ(false, con_schema->HasUniqueConstraints()); + EXPECT_EQ(false, sink_table->GetSchema()->HasForeignKeySources()); + EXPECT_EQ(0, con_table_object->GetConstraintCatalogEntries().size()); + txn_manager.CommitTransaction(txn); + + // Drop database + txn = txn_manager.BeginTransaction(); + catalog->DropDatabaseWithName(txn, db_name); + txn_manager.CommitTransaction(txn); +} + } // namespace test } // namespace peloton diff --git a/test/catalog/constraints_test.cpp b/test/catalog/constraints_test.cpp index 8235e85e9a9..f9563f9b251 100644 --- a/test/catalog/constraints_test.cpp +++ b/test/catalog/constraints_test.cpp @@ -16,7 +16,6 @@ #include "catalog/testing_constraints_util.h" #include "catalog/catalog.h" -#include "catalog/foreign_key.h" #include "common/internal_types.h" #include "concurrency/testing_transaction_util.h" #include "executor/executors.h" @@ -29,7 +28,9 @@ #define CONSTRAINT_NOTNULL_TEST #define CONSTRAINT_DEFAULT_TEST -//#define CONSTRAINT_CHECK_TEST +// #define CONSTRAINT_CHECK_TEST +#define CONSTRAINT_UNIQUE_TEST +#define CONSTRAINT_FOREIGN_KEY_TEST namespace peloton { namespace test { @@ -52,15 +53,14 @@ TEST_F(ConstraintsTests, NOTNULLTest) { // 140 141 142 "143" // Set all of the columns to be NOT NULL - std::vector> constraints; + std::vector notnull_col_ids; for (int i = 0; i < CONSTRAINTS_NUM_COLS; i++) { - constraints.push_back( - {catalog::Constraint(ConstraintType::NOTNULL, "notnull_constraint")}); + notnull_col_ids.push_back(i); } - std::vector multi_constraints; + std::unordered_map default_values; storage::DataTable *data_table = - TestingConstraintsUtil::CreateAndPopulateTable(constraints, - multi_constraints); + TestingConstraintsUtil::CreateTable(notnull_col_ids, default_values); + TestingConstraintsUtil::PopulateTable(data_table); // Bootstrap auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); @@ -113,29 +113,43 @@ TEST_F(ConstraintsTests, NOTNULLTest) { #ifdef CONSTRAINT_DEFAULT_TEST TEST_F(ConstraintsTests, DEFAULTTEST) { - // Set all of the columns to be NOT NULL - std::vector> constraints; + // Set default value within col_B + std::vector notnull_col_ids; + std::unordered_map default_values; for (int i = 0; i < CONSTRAINTS_NUM_COLS; i++) { + // COL_B + if (i == 1) { + default_values[i] = type::ValueFactory::GetIntegerValue(DEFAULT_VALUE); + } + // COL_A + COL_C + COL_D + else { + // do nothing + } + } + storage::DataTable *data_table = + TestingConstraintsUtil::CreateTable(notnull_col_ids, default_values); + // Add primary key + auto catalog = catalog::Catalog::GetInstance(); + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + for (oid_t i = 0; i < CONSTRAINTS_NUM_COLS; i++) { // COL_A if (i == 0) { - constraints.push_back( - {catalog::Constraint(ConstraintType::PRIMARY, "pkey")}); - } - // COL_B - else if (i == 1) { - catalog::Constraint default_const(ConstraintType::DEFAULT, "default"); - default_const.addDefaultValue( - type::ValueFactory::GetIntegerValue(DEFAULT_VALUE)); - constraints.push_back({}); + catalog->AddPrimaryKeyConstraint(txn, + data_table->GetDatabaseOid(), + data_table->GetOid(), + {i}, + "con_primary"); } - // COL_C + COL_D + // COL_B + COL_C + COL_D else { - constraints.push_back({}); + // do nothing } } - std::vector multi_constraints; - TestingConstraintsUtil::CreateAndPopulateTable(constraints, - multi_constraints); + txn_manager.CommitTransaction(txn); + + // populate test data + TestingConstraintsUtil::PopulateTable(data_table); // Bootstrap std::vector result; @@ -153,13 +167,14 @@ TEST_F(ConstraintsTests, DEFAULTTEST) { rows_affected, error_message); EXPECT_EQ(ResultType::SUCCESS, status); - sql = StringUtil::Format("SELECT col_d FROM %s WHERE col_a = 9999", + sql = StringUtil::Format("SELECT col_b FROM %s WHERE col_a = 9999", CONSTRAINTS_TEST_TABLE); status = TestingSQLUtil::ExecuteSQLQuery(sql, result, tuple_descriptor, rows_affected, error_message); EXPECT_EQ(ResultType::SUCCESS, status); std::string resultStr = TestingSQLUtil::GetResultValueAsString(result, 0); - LOG_INFO("OUTPUT:\n%s", resultStr.c_str()); + EXPECT_EQ(std::to_string(DEFAULT_VALUE), resultStr); + LOG_INFO("OUTPUT:%s", resultStr.c_str()); } #endif @@ -173,32 +188,47 @@ TEST_F(ConstraintsTests, CHECKTest) { // 20 21 22 "23" // ..... // 140 141 142 "143" + auto catalog = catalog::Catalog::GetInstance(); + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); auto column1 = catalog::Column(type::TypeId::INTEGER, 25, "A", false, 0); - auto constraints = catalog::Constraint(ConstraintType::CHECK, "check1"); - type::Value tmp_value = type::ValueFactory::GetIntegerValue(0); - constraints.AddCheck(ExpressionType::COMPARE_GREATERTHAN, tmp_value); - column1.AddConstraint(constraints); - LOG_DEBUG("%s %s", peloton::DOUBLE_STAR.c_str(), - constraints.GetInfo().c_str()); - catalog::Schema *table_schema = new catalog::Schema({column1}); - std::string table_name("TEST_TABLE"); - bool own_schema = true; - bool adapt_table = false; - storage::DataTable *table = storage::TableFactory::GetDataTable( - INVALID_OID, INVALID_OID, table_schema, table_name, - TESTS_TUPLES_PER_TILEGROUP, own_schema, adapt_table); - std::unique_ptr data_table(table); + std::unique_ptr table_schema(new catalog::Schema({column1})); - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + std::string table_name("TEST_TABLE"); + auto result = + catalog->CreateTable(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + table_name, + false); + EXPECT_EQ(ResultType::SUCCESS, result); + + auto data_table = catalog->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + table_name); + EXPECT_NE(nullptr, data_table); + + // add check constraint + type::Value tmp_value = type::ValueFactory::GetIntegerValue(0); + catalog->AddCheckConstraint(txn, + data_table->GetDatabaseOid(), + data_table->GetOid(), + {0}, + std::make_pair(ExpressionType::COMPARE_GREATERTHAN, tmp_value), + "con_check"); + txn_manager.CommitTransaction(txn); // begin this transaction - auto txn = txn_manager.BeginTransaction(); + txn = txn_manager.BeginTransaction(); // Test1: insert a tuple with column meet the constraint requirment bool hasException = false; try { TestingConstraintsUtil::ExecuteOneInsert( - txn, data_table.get(), type::ValueFactory::GetIntegerValue(10)); + txn, data_table, type::ValueFactory::GetIntegerValue(10)); } catch (ConstraintException e) { hasException = true; } @@ -208,7 +238,7 @@ TEST_F(ConstraintsTests, CHECKTest) { hasException = false; try { TestingConstraintsUtil::ExecuteOneInsert( - txn, data_table.get(), type::ValueFactory::GetIntegerValue(-1)); + txn, data_table, type::ValueFactory::GetIntegerValue(-1)); } catch (ConstraintException e) { hasException = true; } @@ -216,7 +246,11 @@ TEST_F(ConstraintsTests, CHECKTest) { // commit this transaction txn_manager.CommitTransaction(txn); - delete data_table.release(); + + txn = txn_manager.BeginTransaction(); + auto result = catalog->DropDatabaseWithName(txn, DEFAULT_DB_NAME); + EXPECT_EQ(ResultType::SUCCESS, result); + txn_manager.CommitTransaction(txn); } #endif @@ -225,355 +259,350 @@ TEST_F(ConstraintsTests, UNIQUETest) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto catalog = catalog::Catalog::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog->CreateDatabase(DEFAULT_DB_NAME, nullptr); - auto column1 = catalog::Column(type::TypeId::INTEGER, 25, "A", false, 0); - auto column2 = catalog::Column(type::TypeId::INTEGER, 25, "B", false, 1); + std::string db_name = "db1"; + catalog->CreateDatabase(txn, db_name); + auto column1 = catalog::Column(type::TypeId::INTEGER, 25, "A", false); + auto column2 = catalog::Column(type::TypeId::INTEGER, 25, "B", false); - auto constraints = catalog::Constraint(ConstraintType::UNIQUE, "unique1"); - column1.AddConstraint(constraints); - LOG_DEBUG("%s %s", peloton::DOUBLE_STAR.c_str(), - constraints.GetInfo().c_str()); std::unique_ptr table_schema( new catalog::Schema({column1, column2})); std::string table_name("TEST_TABLE"); - catalog::Catalog::GetInstance()->CreateTable(txn, DEFAULT_DB_NAME, - DEFAULT_SCHEMA_NAME, - std::move(table_schema), - table_name, - false); - storage::DataTable *table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table_name, txn); + catalog->CreateTable(txn, + db_name, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + table_name, + false); + + auto table = catalog->GetTableWithName(txn, + db_name, + DEFAULT_SCHEMA_NAME, + table_name); + catalog->AddUniqueConstraint(txn, + table->GetDatabaseOid(), + table->GetOid(), + {0}, + "con_unique"); txn_manager.CommitTransaction(txn); - // table->AddUNIQUEIndex(); - txn = txn_manager.BeginTransaction(); // begin this transaction - // Test1: insert a tuple with column meet the unique requirment - bool hasException = false; - try { - // bool result = true; - // result = - TestingConstraintsUtil::ExecuteOneInsert( - txn, table, type::ValueFactory::GetIntegerValue(10)); - // if (result == false) hasException = true; - } catch (ConstraintException e) { - hasException = true; - } - EXPECT_FALSE(hasException); + // Test1: insert a tuple with column meet the unique requirement + bool result = TestingConstraintsUtil::ExecuteOneInsert( + txn, table, type::ValueFactory::GetIntegerValue(10)); + EXPECT_TRUE(result); // Test2: insert not a valid column violate the constraint - hasException = false; - try { - // bool result = true; - // result = - TestingConstraintsUtil::ExecuteOneInsert( + result = TestingConstraintsUtil::ExecuteOneInsert( txn, table, type::ValueFactory::GetIntegerValue(10)); - // if (result == false) hasException = true; - } catch (ConstraintException e) { - hasException = true; - } - EXPECT_TRUE(hasException); + EXPECT_FALSE(result); - hasException = false; - try { - TestingConstraintsUtil::ExecuteOneInsert( + result = TestingConstraintsUtil::ExecuteOneInsert( txn, table, type::ValueFactory::GetIntegerValue(20)); - } catch (ConstraintException e) { - hasException = true; - } - EXPECT_FALSE(hasException); + EXPECT_TRUE(result); // commit this transaction txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + catalog->DropDatabaseWithName(txn, db_name); + txn_manager.CommitTransaction(txn); +} + + TEST_F(ConstraintsTests, MULTIUNIQUETest) { + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto catalog = catalog::Catalog::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + std::string db_name = "db1"; + catalog->CreateDatabase(txn, db_name); + auto column1 = catalog::Column(type::TypeId::INTEGER, 25, "A", false); + auto column2 = catalog::Column(type::TypeId::INTEGER, 25, "B", false); + auto column3 = catalog::Column(type::TypeId::INTEGER, 25, "C", false); + std::vector cols; + cols.push_back(0); + cols.push_back(1); + std::vector columns; + columns.push_back(column1); + columns.push_back(column2); + columns.push_back(column3); + std::unique_ptr table_schema(new catalog::Schema(columns)); + std::string table_name("TEST_TABLE_1"); + catalog->CreateTable(txn, + db_name, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + table_name, + false); + + // Add multi-unique constraint + auto table = catalog->GetTableWithName(txn, + db_name, + DEFAULT_SCHEMA_NAME, + table_name); + catalog->AddUniqueConstraint(txn, + table->GetDatabaseOid(), + table->GetOid(), + cols, + "con_unique"); + txn_manager.CommitTransaction(txn); + + txn = txn_manager.BeginTransaction(); + // begin this transaction + // Test1: insert a tuple with column meet the unique requirment + std::vector ccs; + ccs.push_back(type::ValueFactory::GetIntegerValue(10)); + ccs.push_back(type::ValueFactory::GetIntegerValue(11)); + bool result = TestingConstraintsUtil::ExecuteMultiInsert(txn, table, ccs); + EXPECT_TRUE(result); + + // Test2: insert not a valid column violate the constraint + ccs.clear(); + ccs.push_back(type::ValueFactory::GetIntegerValue(10)); + ccs.push_back(type::ValueFactory::GetIntegerValue(11)); + result = TestingConstraintsUtil::ExecuteMultiInsert(txn, table, ccs); + EXPECT_FALSE(result); + + ccs.clear(); + ccs.push_back(type::ValueFactory::GetIntegerValue(10)); + ccs.push_back(type::ValueFactory::GetIntegerValue(12)); + result = TestingConstraintsUtil::ExecuteMultiInsert(txn, table, ccs); + EXPECT_TRUE(result); + + // commit this transaction + txn_manager.CommitTransaction(txn); + txn = txn_manager.BeginTransaction(); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, db_name); txn_manager.CommitTransaction(txn); } #endif -// TEST_F(ConstraintsTests, MULTIUNIQUETest) { -// auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); -// auto catalog = catalog::Catalog::GetInstance(); -// auto txn = txn_manager.BeginTransaction(); -// std::string db_name = "db1"; -// catalog->CreateDatabase(db_name, nullptr); -// auto column1 = catalog::Column(type::TypeId::INTEGER, 25, "A", false, 0); -// auto column2 = catalog::Column(type::TypeId::INTEGER, 25, "B", false, 1); -// auto column3 = catalog::Column(type::TypeId::INTEGER, 25, "C", false, 2); -// std::vector cols; -// cols.push_back(0); -// cols.push_back(1); -// std::vector columns; -// columns.push_back(column1); -// columns.push_back(column2); -// columns.push_back(column3); -// auto mc = catalog::MultiConstraint(ConstraintType::UNIQUE, "c1", cols); -// LOG_DEBUG("%s MULTI CONSTRAINTS %s %s", peloton::DOUBLE_STAR.c_str(), -// peloton::DOUBLE_STAR.c_str(), mc.GetInfo().c_str()); -// -// std::unique_ptr table_schema(new catalog::Schema(columns)); -// table_schema->AddMultiConstraints(mc); -// std::string table_name("TEST_TABLE_1"); -// catalog->CreateTable(db_name, table_name, std::move(table_schema), txn); -// txn_manager.CommitTransaction(txn); -// storage::Database *database = catalog->GetDatabaseWithName(db_name); -// storage::DataTable *table = database->GetTableWithName(table_name); -// -// // table->AddUNIQUEIndex(); -// -// txn = txn_manager.BeginTransaction(); -// // begin this transaction -// // Test1: insert a tuple with column meet the unique requirment -// bool hasException = false; -// try { -// std::vector ccs; -// ccs.push_back(type::ValueFactory::GetIntegerValue(10)); -// ccs.push_back(type::ValueFactory::GetIntegerValue(11)); -// // bool result = true; -// // result = -// TestingConstraintsUtil::ExecuteMultiInsert(txn, table, ccs); -// // if (result == false) hasException = true; -// } catch (ConstraintException e) { -// hasException = true; -// } -// EXPECT_FALSE(hasException); -// -// // Test2: insert not a valid column violate the constraint -// hasException = false; -// try { -// std::vector ccs; -// ccs.push_back(type::ValueFactory::GetIntegerValue(10)); -// ccs.push_back(type::ValueFactory::GetIntegerValue(11)); -// // bool result = true; -// // result = -// TestingConstraintsUtil::ExecuteMultiInsert(txn, table, ccs); -// // if (result == false) hasException = true; -// } catch (ConstraintException e) { -// hasException = true; -// } -// EXPECT_TRUE(hasException); -// -// hasException = false; -// try { -// std::vector ccs; -// ccs.push_back(type::ValueFactory::GetIntegerValue(10)); -// ccs.push_back(type::ValueFactory::GetIntegerValue(12)); -// TestingConstraintsUtil::ExecuteMultiInsert(txn, table, ccs); -// } catch (ConstraintException e) { -// hasException = true; -// } -// EXPECT_FALSE(hasException); -// -// // commit this transaction -// txn_manager.CommitTransaction(txn); -// txn = txn_manager.BeginTransaction(); -// catalog::Catalog::GetInstance()->DropDatabaseWithName(db_name, txn); -// txn_manager.CommitTransaction(txn); -//} - -// TEST_F(ConstraintsTests, ForeignKeySingleInsertTest) { -// // First, initial 2 tables like following -// // TABLE A -- src table TABLE B -- sink table -// // a int(primary, ref B) b int b int(primary) c int -// // 0 0 0 0 -// // 1 0 1 0 -// // 2 0 2 0 -// // ..... -// // 9 0 -// -// // create new db -// auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); -// auto txn = txn_manager.BeginTransaction(); -// auto catalog = catalog::Catalog::GetInstance(); -// std::string db_name = "db2"; -// std::string table_a_name = "tableA"; -// std::string table_b_name = "tableB"; -// catalog::Catalog::GetInstance()->CreateDatabase(db_name, nullptr); -// // txn_manager.CommitTransaction(txn); -// -// auto column1 = catalog::Column(type::TypeId::INTEGER, 25, "a", false, 0); -// auto column2 = catalog::Column(type::TypeId::INTEGER, 25, "b", false, 1); -// -// auto constraints = catalog::Constraint(ConstraintType::PRIMARY, "primary1"); -// column1.AddConstraint(constraints); -// LOG_DEBUG("%s %s", peloton::DOUBLE_STAR.c_str(), -// constraints.GetInfo().c_str()); std::unique_ptr -// tableA_schema( -// new catalog::Schema({column1, column2})); -// -// catalog->CreateTable(db_name, table_a_name, std::move(tableA_schema), txn); -// txn_manager.CommitTransaction(txn); -// -// txn = txn_manager.BeginTransaction(); -// auto column3 = catalog::Column(type::TypeId::INTEGER, 25, "b", false, 0); -// column3.AddConstraint(constraints); -// auto column4 = catalog::Column(type::TypeId::INTEGER, 25, "c", false, 1); -// std::unique_ptr tableB_schema( -// new catalog::Schema({column3, column4})); -// -// catalog->CreateTable(db_name, table_b_name, std::move(tableB_schema), txn); -// -// auto table_a = catalog->GetTableWithName(db_name, table_a_name); -// auto table_b = catalog->GetTableWithName(db_name, table_b_name); -// -// oid_t sink_table_id = table_b->GetOid(); -// std::vector sink_col_ids = { table_b->GetSchema()->GetColumnID("b") -// }; std::vector source_col_ids = { -// table_a->GetSchema()->GetColumnID("a") }; catalog::ForeignKey *foreign_key = -// new catalog::ForeignKey( -// sink_table_id, sink_col_ids, source_col_ids, -// FKConstrActionType::NOACTION, -// FKConstrActionType::NOACTION, -// "foreign_constraint1"); -// table_a->AddForeignKey(foreign_key); -// -// txn = txn_manager.BeginTransaction(); -// // begin this transaction -// // Test1: insert a tuple with column meet the unique requirment -// bool hasException = false; -// try { -// std::vector ccs; -// ccs.push_back(type::ValueFactory::GetIntegerValue(1)); -// ccs.push_back(type::ValueFactory::GetIntegerValue(2)); -// TestingConstraintsUtil::ExecuteMultiInsert(txn, table_b, ccs); -// ccs.clear(); -// ccs.push_back(type::ValueFactory::GetIntegerValue(1)); -// ccs.push_back(type::ValueFactory::GetIntegerValue(2)); -// TestingConstraintsUtil::ExecuteMultiInsert(txn, table_a, ccs); -// } catch (ConstraintException e) { -// hasException = true; -// } -// EXPECT_FALSE(hasException); -// -// hasException = true; -// try { -// std::vector ccs; -// ccs.push_back(type::ValueFactory::GetIntegerValue(3)); -// ccs.push_back(type::ValueFactory::GetIntegerValue(4)); -// TestingConstraintsUtil::ExecuteMultiInsert(txn, table_b, ccs); -// ccs.clear(); -// ccs.push_back(type::ValueFactory::GetIntegerValue(2)); -// ccs.push_back(type::ValueFactory::GetIntegerValue(5)); -// TestingConstraintsUtil::ExecuteMultiInsert(txn, table_a, ccs); -// } catch (ConstraintException e) { -// hasException = true; -// } -// EXPECT_TRUE(hasException); -// -// // commit this transaction -// txn_manager.CommitTransaction(txn); -// txn = txn_manager.BeginTransaction(); -// catalog::Catalog::GetInstance()->DropDatabaseWithName(db_name, txn); -// txn_manager.CommitTransaction(txn); -// delete foreign_key; -//} - -// TEST_F(ConstraintsTests, ForeignKeyMultiInsertTest) { -// // First, initial 2 tables like following -// // TABLE A -- src table TABLE B -- sink table -// // a int(primary, ref B) b int b int(primary) c int -// // 0 0 0 0 -// // 1 0 1 0 -// // 2 0 2 0 -// // ..... -// // 9 0 -// -// // create new db -// auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); -// auto txn = txn_manager.BeginTransaction(); -// auto catalog = catalog::Catalog::GetInstance(); -// std::string db_name = "db2"; -// std::string table_a_name = "tableA"; -// std::string table_b_name = "tableB"; -// catalog->CreateDatabase(db_name, txn); -// -// // TABLE A -// auto column1 = catalog::Column(type::TypeId::INTEGER, 25, "a", false, 0); -// auto column2 = catalog::Column(type::TypeId::INTEGER, 25, "b", false, 1); -// std::unique_ptr tableA_schema( -// new catalog::Schema({column1, column2})); -// catalog->CreateTable(db_name, table_a_name, std::move(tableA_schema), txn); -// txn_manager.CommitTransaction(txn); -// auto table_A = catalog->GetTableWithName(db_name, table_a_name); -// -// txn = txn_manager.BeginTransaction(); -// auto column3 = catalog::Column(type::TypeId::INTEGER, 25, "b", false, 0); -// auto column4 = catalog::Column(type::TypeId::INTEGER, 25, "c", false, 1); -// std::vector cols; -// cols.push_back(0); -// cols.push_back(1); -// auto mc = -// catalog::MultiConstraint(ConstraintType::PRIMARY, "multiprimary1", -// cols); -// LOG_DEBUG("%s MULTI CONSTRAINTS %s %s", peloton::DOUBLE_STAR.c_str(), -// peloton::DOUBLE_STAR.c_str(), mc.GetInfo().c_str()); -// -// // TABLE B -// catalog::Schema *table_schema = new catalog::Schema({column3, column4}); -// table_schema->AddMultiConstraints(mc); -// std::unique_ptr tableB_schema(table_schema); -// -// catalog->CreateTable(db_name, table_b_name, std::move(tableB_schema), txn); -// auto table_a = catalog->GetTableWithName(db_name, table_a_name); -// auto table_b = catalog->GetTableWithName(db_name, table_b_name); -// txn_manager.CommitTransaction(txn); -// -// // Create foreign key tableA.B -> tableB.B -// oid_t sink_table_id = table_b->GetOid(); -// std::vector sink_col_ids = { table_b->GetSchema()->GetColumnID("b") -// }; std::vector source_col_ids = { -// table_a->GetSchema()->GetColumnID("b") }; catalog::ForeignKey *foreign_key = -// new catalog::ForeignKey( -// sink_table_id, sink_col_ids, source_col_ids, -// FKConstrActionType::RESTRICT, -// FKConstrActionType::CASCADE, -// "foreign_constraint1"); -// table_A->AddForeignKey(foreign_key); -// -// // Test1: insert a tuple with column meet the constraint requirment -// -// txn = txn_manager.BeginTransaction(); -// // begin this transaction -// // Test1: insert a tuple with column meet the unique requirment -// bool hasException = false; -// try { -// std::vector ccs; -// ccs.push_back(type::ValueFactory::GetIntegerValue(1)); -// ccs.push_back(type::ValueFactory::GetIntegerValue(2)); -// TestingConstraintsUtil::ExecuteMultiInsert(txn, table_b, ccs); -// ccs.clear(); -// ccs.push_back(type::ValueFactory::GetIntegerValue(2)); -// ccs.push_back(type::ValueFactory::GetIntegerValue(1)); -// TestingConstraintsUtil::ExecuteMultiInsert(txn, table_a, ccs); -// } catch (ConstraintException e) { -// hasException = true; -// } -// EXPECT_FALSE(hasException); -// -// hasException = true; -// try { -// std::vector ccs; -// ccs.push_back(type::ValueFactory::GetIntegerValue(3)); -// ccs.push_back(type::ValueFactory::GetIntegerValue(4)); -// TestingConstraintsUtil::ExecuteMultiInsert(txn, table_b, ccs); -// ccs.clear(); -// ccs.push_back(type::ValueFactory::GetIntegerValue(2)); -// ccs.push_back(type::ValueFactory::GetIntegerValue(5)); -// TestingConstraintsUtil::ExecuteMultiInsert(txn, table_a, ccs); -// } catch (ConstraintException e) { -// hasException = true; -// } -// EXPECT_TRUE(hasException); -// -// // commit this transaction -// txn_manager.CommitTransaction(txn); -// txn = txn_manager.BeginTransaction(); -// catalog::Catalog::GetInstance()->DropDatabaseWithName(db_name, txn); -// txn_manager.CommitTransaction(txn); -// delete foreign_key; -//} +#ifdef CONSTRAINT_FOREIGN_KEY_TEST + TEST_F(ConstraintsTests, ForeignKeySingleInsertTest) { + // First, initial 2 tables like following + // TABLE A -- src table TABLE B -- sink table + // a int(primary) b int(ref B) b int(primary) c int + // 0 0 0 0 + // 1 1 1 0 + // 2 2 2 0 + // ..... + // 9 0 + + // create new db + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + auto catalog = catalog::Catalog::GetInstance(); + std::string db_name = "db2"; + std::string table_a_name = "tableA"; + std::string table_b_name = "tableB"; + catalog::Catalog::GetInstance()->CreateDatabase(txn, db_name); + + // Table A + auto column1 = catalog::Column(type::TypeId::INTEGER, 25, "a", false); + auto column2 = catalog::Column(type::TypeId::INTEGER, 25, "b", false); + std::unique_ptr tableA_schema( + new catalog::Schema({column1, column2})); + catalog->CreateTable(txn, + db_name, + DEFAULT_SCHEMA_NAME, + std::move(tableA_schema), + table_a_name, + false); + + auto table_a = catalog->GetTableWithName(txn, + db_name, + DEFAULT_SCHEMA_NAME, + table_a_name); + catalog->AddPrimaryKeyConstraint(txn, + table_a->GetDatabaseOid(), + table_a->GetOid(), + {0}, + "con_primary"); + txn_manager.CommitTransaction(txn); + + // Table B + txn = txn_manager.BeginTransaction(); + auto column3 = catalog::Column(type::TypeId::INTEGER, 25, "b", false); + auto column4 = catalog::Column(type::TypeId::INTEGER, 25, "c", false); + std::unique_ptr tableB_schema( + new catalog::Schema({column3, column4})); + + catalog->CreateTable(txn, + db_name, + DEFAULT_SCHEMA_NAME, + std::move(tableB_schema), + table_b_name, + false); + + auto table_b = catalog->GetTableWithName(txn, + db_name, + DEFAULT_SCHEMA_NAME, + table_b_name); + catalog->AddPrimaryKeyConstraint(txn, + table_b->GetDatabaseOid(), + table_b->GetOid(), + {0}, + "con_primary"); + + oid_t sink_table_id = table_b->GetOid(); + std::vector sink_col_ids = { table_b->GetSchema()->GetColumnID("b") }; + std::vector source_col_ids = { table_a->GetSchema()->GetColumnID("b") }; + catalog->AddForeignKeyConstraint(txn, + table_a->GetDatabaseOid(), + table_a->GetOid(), + source_col_ids, + sink_table_id, + sink_col_ids, + FKConstrActionType::NOACTION, + FKConstrActionType::NOACTION, + "con_foreign"); + txn_manager.CommitTransaction(txn); + + txn = txn_manager.BeginTransaction(); + // begin this transaction + // Test1: insert a tuple with column meet the constraint requirement + std::vector ccs; + ccs.push_back(type::ValueFactory::GetIntegerValue(1)); + ccs.push_back(type::ValueFactory::GetIntegerValue(2)); + bool result = TestingConstraintsUtil::ExecuteMultiInsert(txn, table_b, ccs); + EXPECT_TRUE(result); + ccs.clear(); + ccs.push_back(type::ValueFactory::GetIntegerValue(2)); + ccs.push_back(type::ValueFactory::GetIntegerValue(1)); + result = TestingConstraintsUtil::ExecuteMultiInsert(txn, table_a, ccs); + EXPECT_TRUE(result); + + ccs.clear(); + ccs.push_back(type::ValueFactory::GetIntegerValue(3)); + ccs.push_back(type::ValueFactory::GetIntegerValue(4)); + result = TestingConstraintsUtil::ExecuteMultiInsert(txn, table_b, ccs); + EXPECT_TRUE(result); + ccs.clear(); + ccs.push_back(type::ValueFactory::GetIntegerValue(2)); + ccs.push_back(type::ValueFactory::GetIntegerValue(5)); + result = TestingConstraintsUtil::ExecuteMultiInsert(txn, table_a, ccs); + EXPECT_FALSE(result); + + // commit this transaction + txn_manager.CommitTransaction(txn); + txn = txn_manager.BeginTransaction(); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, db_name); + txn_manager.CommitTransaction(txn); +} + + TEST_F(ConstraintsTests, ForeignKeyMultiInsertTest) { + // First, initial 2 tables like following + // TABLE A -- src table TABLE B -- sink table + // a int(ref B) b int(ref B) a int(primary) b int(primary) + // 0 0 0 0 + // 1 0 1 0 + // 2 0 2 0 + // ..... + // 9 0 + + // create new db + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + auto catalog = catalog::Catalog::GetInstance(); + std::string db_name = "db2"; + std::string table_a_name = "tableA"; + std::string table_b_name = "tableB"; + catalog->CreateDatabase(txn, db_name); + + // TABLE A + auto column1 = catalog::Column(type::TypeId::INTEGER, 25, "a", false); + auto column2 = catalog::Column(type::TypeId::INTEGER, 25, "b", false); + std::unique_ptr tableA_schema( + new catalog::Schema({column1, column2})); + catalog->CreateTable(txn, + db_name, + DEFAULT_SCHEMA_NAME, + std::move(tableA_schema), + table_a_name, + false); + + auto table_a = catalog->GetTableWithName(txn, + db_name, + DEFAULT_SCHEMA_NAME, + table_a_name); + txn_manager.CommitTransaction(txn); + + // TABLE B + txn = txn_manager.BeginTransaction(); + auto column3 = catalog::Column(type::TypeId::INTEGER, 25, "a", false); + auto column4 = catalog::Column(type::TypeId::INTEGER, 25, "b", false); + catalog::Schema *table_schema = new catalog::Schema({column3, column4}); + std::unique_ptr tableB_schema(table_schema); + + catalog->CreateTable(txn, + db_name, + DEFAULT_SCHEMA_NAME, + std::move(tableB_schema), + table_b_name, + false); + + std::vector cols; + cols.push_back(0); + cols.push_back(1); + auto table_b = catalog->GetTableWithName(txn, + db_name, + DEFAULT_SCHEMA_NAME, + table_b_name); + catalog->AddPrimaryKeyConstraint(txn, + table_b->GetDatabaseOid(), + table_b->GetOid(), + cols, + "con_primary"); + + // Create foreign key tableA.B -> tableB.B + oid_t sink_table_id = table_b->GetOid(); + std::vector sink_col_ids = { table_b->GetSchema()->GetColumnID("a"), + table_b->GetSchema()->GetColumnID("b") }; + std::vector source_col_ids = { table_a->GetSchema()->GetColumnID("a"), + table_a->GetSchema()->GetColumnID("b") }; + catalog->AddForeignKeyConstraint(txn, + table_a->GetDatabaseOid(), + table_a->GetOid(), + source_col_ids, + sink_table_id, + sink_col_ids, + FKConstrActionType::RESTRICT, + FKConstrActionType::CASCADE, + "con_foreign"); + txn_manager.CommitTransaction(txn); + + + txn = txn_manager.BeginTransaction(); + // begin this transaction + // Test1: insert a tuple with column meet the constraint requirement + std::vector ccs; + ccs.push_back(type::ValueFactory::GetIntegerValue(1)); + ccs.push_back(type::ValueFactory::GetIntegerValue(2)); + bool result = TestingConstraintsUtil::ExecuteMultiInsert(txn, table_b, ccs); + EXPECT_TRUE(result); + ccs.clear(); + ccs.push_back(type::ValueFactory::GetIntegerValue(1)); + ccs.push_back(type::ValueFactory::GetIntegerValue(2)); + result = TestingConstraintsUtil::ExecuteMultiInsert(txn, table_a, ccs); + EXPECT_TRUE(result); + + ccs.clear(); + ccs.push_back(type::ValueFactory::GetIntegerValue(3)); + ccs.push_back(type::ValueFactory::GetIntegerValue(4)); + result = TestingConstraintsUtil::ExecuteMultiInsert(txn, table_b, ccs); + EXPECT_TRUE(result); + ccs.clear(); + ccs.push_back(type::ValueFactory::GetIntegerValue(2)); + ccs.push_back(type::ValueFactory::GetIntegerValue(5)); + result = TestingConstraintsUtil::ExecuteMultiInsert(txn, table_a, ccs); + EXPECT_FALSE(result); + + // commit this transaction + txn_manager.CommitTransaction(txn); + txn = txn_manager.BeginTransaction(); + catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, db_name); + txn_manager.CommitTransaction(txn); +} +#endif // ======================================================== diff --git a/test/codegen/testing_codegen_util.cpp b/test/codegen/testing_codegen_util.cpp index 98879d6de59..afc498c53b0 100644 --- a/test/codegen/testing_codegen_util.cpp +++ b/test/codegen/testing_codegen_util.cpp @@ -86,23 +86,15 @@ catalog::Column PelotonCodeGenTest::GetTestColumn(uint32_t col_id) const { } // Create the test schema for all the tables -std::unique_ptr PelotonCodeGenTest::CreateTestSchema( - bool add_primary) const { +std::unique_ptr PelotonCodeGenTest::CreateTestSchema() const { // Create the columns std::vector cols = {GetTestColumn(0), GetTestColumn(1), GetTestColumn(2), GetTestColumn(3)}; // Add NOT NULL constraints on COL_A, COL_C, COL_D - cols[0].AddConstraint( - catalog::Constraint{ConstraintType::NOTNULL, "not_null"}); - if (add_primary) { - cols[0].AddConstraint( - catalog::Constraint{ConstraintType::PRIMARY, "con_primary"}); - } - cols[2].AddConstraint( - catalog::Constraint{ConstraintType::NOTNULL, "not_null"}); - cols[3].AddConstraint( - catalog::Constraint{ConstraintType::NOTNULL, "not_null"}); + cols[0].SetNotNull(); + cols[2].SetNotNull(); + cols[3].SetNotNull(); // Return the schema return std::unique_ptr{new catalog::Schema(cols)}; @@ -131,7 +123,7 @@ void PelotonCodeGenTest::CreateTestTables(concurrency::TransactionContext *txn, ->GetTableOid()); } for (int i = 4; i < 5; i++) { - auto table_schema = CreateTestSchema(true); + auto table_schema = CreateTestSchema(); catalog->CreateTable(txn, test_db_name, DEFAULT_SCHEMA_NAME, @@ -140,12 +132,15 @@ void PelotonCodeGenTest::CreateTestTables(concurrency::TransactionContext *txn, false, tuples_per_tilegroup, layout_type); - test_table_oids.push_back(catalog - ->GetTableCatalogEntry(txn, - test_db_name, - DEFAULT_SCHEMA_NAME, - test_table_names[i]) - ->GetTableOid()); + auto table_object = catalog->GetTableCatalogEntry(txn, test_db_name, + DEFAULT_SCHEMA_NAME, + test_table_names[i]); + catalog->AddPrimaryKeyConstraint(txn, + table_object->GetDatabaseOid(), + table_object->GetTableOid(), + {0}, + "con_primary"); + test_table_oids.push_back(table_object->GetTableOid()); } } diff --git a/test/common/internal_types_test.cpp b/test/common/internal_types_test.cpp index 7a616315e20..a8ff6b0881c 100644 --- a/test/common/internal_types_test.cpp +++ b/test/common/internal_types_test.cpp @@ -405,11 +405,9 @@ TEST_F(InternalTypesTests, ResultTypeTest) { TEST_F(InternalTypesTests, ConstraintTypeTest) { std::vector list = { - ConstraintType::INVALID, ConstraintType::NOT_NULL, - ConstraintType::NOTNULL, ConstraintType::DEFAULT, - ConstraintType::CHECK, ConstraintType::PRIMARY, - ConstraintType::UNIQUE, ConstraintType::FOREIGN, - ConstraintType::EXCLUSION}; + ConstraintType::INVALID, ConstraintType::CHECK, + ConstraintType::PRIMARY, ConstraintType::UNIQUE, + ConstraintType::FOREIGN, ConstraintType::EXCLUSION}; // Make sure that ToString and FromString work for (auto val : list) { diff --git a/test/concurrency/testing_transaction_util.cpp b/test/concurrency/testing_transaction_util.cpp index 7f61cc0b765..ca119c0b563 100644 --- a/test/concurrency/testing_transaction_util.cpp +++ b/test/concurrency/testing_transaction_util.cpp @@ -43,13 +43,11 @@ storage::DataTable *TestingTransactionUtil::CreateCombinedPrimaryKeyTable() { auto id_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "id", true); - id_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, "not_null")); + id_column.SetNotNull(); auto value_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "value", true); - value_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, "not_null")); + value_column.SetNotNull(); // Create the table catalog::Schema *table_schema = @@ -77,6 +75,12 @@ storage::DataTable *TestingTransactionUtil::CreateCombinedPrimaryKeyTable() { table->AddIndex(pkey_index); + // Create constraint on the table + std::shared_ptr constraint( + new catalog::Constraint(1000, ConstraintType::PRIMARY, + "con_primary", TEST_TABLE_OID, key_attrs, 1234)); + table->GetSchema()->AddConstraint(constraint); + // Insert tuple auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); @@ -92,8 +96,7 @@ storage::DataTable *TestingTransactionUtil::CreatePrimaryKeyUniqueKeyTable() { auto id_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "id", true); - id_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, "not_null")); + id_column.SetNotNull(); auto value_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "value", true); @@ -124,6 +127,12 @@ storage::DataTable *TestingTransactionUtil::CreatePrimaryKeyUniqueKeyTable() { table->AddIndex(pkey_index); + // Create primary key constraint on the table + std::shared_ptr constraint( + new catalog::Constraint(1000, ConstraintType::PRIMARY, + "con_primary", TEST_TABLE_OID, key_attrs, 1234)); + table->GetSchema()->AddConstraint(constraint); + // Create unique index on the value column std::vector key_attrs2 = {1}; auto tuple_schema2 = table->GetSchema(); @@ -140,6 +149,12 @@ storage::DataTable *TestingTransactionUtil::CreatePrimaryKeyUniqueKeyTable() { table->AddIndex(ukey_index); + // Create unique constraint on the table + std::shared_ptr unique_constraint( + new catalog::Constraint(1001, ConstraintType::UNIQUE, + "con_unique", TEST_TABLE_OID, key_attrs, 1235)); + table->GetSchema()->AddConstraint(unique_constraint); + // Insert tuple auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); @@ -153,7 +168,7 @@ storage::DataTable *TestingTransactionUtil::CreatePrimaryKeyUniqueKeyTable() { storage::DataTable *TestingTransactionUtil::CreateTable( int num_key, std::string table_name, oid_t database_id, oid_t relation_id, - oid_t index_oid, bool need_primary_index, size_t tuples_per_tilegroup) { + oid_t index_oid, bool need_primary_key, size_t tuples_per_tilegroup) { auto id_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "id", true); @@ -177,8 +192,8 @@ storage::DataTable *TestingTransactionUtil::CreateTable( key_schema->SetIndexedColumns(key_attrs); auto index_metadata = new index::IndexMetadata( - "primary_btree_index", index_oid, TEST_TABLE_OID, CATALOG_DATABASE_OID, - IndexType::BWTREE, need_primary_index ? IndexConstraintType::PRIMARY_KEY + "primary_btree_index", index_oid, relation_id, database_id, + IndexType::BWTREE, need_primary_key ? IndexConstraintType::PRIMARY_KEY : IndexConstraintType::DEFAULT, tuple_schema, key_schema, key_attrs, unique); @@ -187,6 +202,14 @@ storage::DataTable *TestingTransactionUtil::CreateTable( table->AddIndex(pkey_index); + // Create primary key constraint on the table + if (need_primary_key) { + std::shared_ptr constraint( + new catalog::Constraint(1000, ConstraintType::PRIMARY, + "con_primary", relation_id, key_attrs, index_oid)); + table->GetSchema()->AddConstraint(constraint); + } + // add this table to current database catalog::Catalog::GetInstance(); LOG_INFO("the database_id is %d", database_id); @@ -195,7 +218,7 @@ storage::DataTable *TestingTransactionUtil::CreateTable( db = storage::StorageManager::GetInstance()->GetDatabaseWithOid(database_id); } catch (CatalogException &e) { - LOG_TRACE("Can't find database %d! ", database_id); + LOG_ERROR("Can't find database %d! ", database_id); return nullptr; } PELOTON_ASSERT(db); diff --git a/test/executor/drop_test.cpp b/test/executor/drop_test.cpp index 2c1b9e5d1fb..d5f915f152b 100644 --- a/test/executor/drop_test.cpp +++ b/test/executor/drop_test.cpp @@ -286,7 +286,7 @@ TEST_F(DropTests, DroppingIndexByName) { TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table_01"); - oid_t col_id = source_table->GetSchema()->GetColumnID(id_column.column_name); + oid_t col_id = source_table->GetSchema()->GetColumnID(id_column.GetName()); std::vector source_col_ids; source_col_ids.push_back(col_id); std::string index_name1 = "Testing_Drop_Index_By_Name"; diff --git a/test/executor/testing_executor_util.cpp b/test/executor/testing_executor_util.cpp index eeb9c88095d..4adb7827475 100644 --- a/test/executor/testing_executor_util.cpp +++ b/test/executor/testing_executor_util.cpp @@ -81,7 +81,6 @@ void TestingExecutorUtil::DeleteDatabase(const std::string &db_name) { */ catalog::Column TestingExecutorUtil::GetColumnInfo(int index) { const bool is_inlined = true; - std::string not_null_constraint_name = "not_null"; catalog::Column dummy_column; switch (index) { @@ -90,8 +89,7 @@ catalog::Column TestingExecutorUtil::GetColumnInfo(int index) { type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "COL_A", is_inlined); - column.AddConstraint(catalog::Constraint(ConstraintType::NOTNULL, - not_null_constraint_name)); + column.SetNotNull(); return column; } break; @@ -100,8 +98,7 @@ catalog::Column TestingExecutorUtil::GetColumnInfo(int index) { type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "COL_B", is_inlined); - column.AddConstraint(catalog::Constraint(ConstraintType::NOTNULL, - not_null_constraint_name)); + column.SetNotNull(); return column; } break; @@ -110,8 +107,7 @@ catalog::Column TestingExecutorUtil::GetColumnInfo(int index) { type::TypeId::DECIMAL, type::Type::GetTypeSize(type::TypeId::DECIMAL), "COL_C", is_inlined); - column.AddConstraint(catalog::Constraint(ConstraintType::NOTNULL, - not_null_constraint_name)); + column.SetNotNull(); return column; } break; @@ -120,8 +116,7 @@ catalog::Column TestingExecutorUtil::GetColumnInfo(int index) { catalog::Column(type::TypeId::VARCHAR, 25, // Column length. "COL_D", !is_inlined); // inlined. - column.AddConstraint(catalog::Constraint(ConstraintType::NOTNULL, - not_null_constraint_name)); + column.SetNotNull(); return column; } break; @@ -390,7 +385,7 @@ storage::DataTable *TestingExecutorUtil::CreateTable( unique = true; index_metadata = new index::IndexMetadata( - "primary_btree_index", 123, INVALID_OID, INVALID_OID, IndexType::BWTREE, + "primary_btree_index", 123, table_oid, INVALID_OID, IndexType::BWTREE, IndexConstraintType::PRIMARY_KEY, tuple_schema, key_schema, key_attrs, unique); @@ -399,6 +394,12 @@ storage::DataTable *TestingExecutorUtil::CreateTable( table->AddIndex(pkey_index); + // Create constraint on the table + std::shared_ptr constraint( + new catalog::Constraint(1000, ConstraintType::PRIMARY, + "con_primary", table_oid, key_attrs, 123)); + table->GetSchema()->AddConstraint(constraint); + ///////////////////////////////////////////////////////////////// // Add index on table column 0 and 1 ///////////////////////////////////////////////////////////////// diff --git a/test/executor/update_test.cpp b/test/executor/update_test.cpp index 80cbc4bce7c..230d749ae0d 100644 --- a/test/executor/update_test.cpp +++ b/test/executor/update_test.cpp @@ -18,6 +18,7 @@ #include "binder/bind_node_visitor.h" #include "catalog/catalog.h" +#include "catalog/table_catalog.h" #include "catalog/schema.h" #include "common/internal_types.h" #include "common/logger.h" @@ -172,8 +173,6 @@ TEST_F(UpdateTests, UpdatingOld) { auto id_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "dept_id", true); - catalog::Constraint constraint(ConstraintType::PRIMARY, "con_primary"); - id_column.AddConstraint(constraint); auto manager_id_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "manager_id", true); @@ -182,14 +181,22 @@ TEST_F(UpdateTests, UpdatingOld) { std::unique_ptr table_schema( new catalog::Schema({id_column, manager_id_column, name_column})); - std::unique_ptr context( - new executor::ExecutorContext(txn)); - planner::CreatePlan node("department_table", DEFAULT_SCHEMA_NAME, - DEFAULT_DB_NAME, std::move(table_schema), - CreateType::TABLE); - executor::CreateExecutor create_executor(&node, context.get()); - create_executor.Init(); - create_executor.Execute(); + + catalog->CreateTable(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + "department_table", + false); + auto table_object = + catalog->GetTableCatalogEntry(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + "department_table"); + catalog->AddPrimaryKeyConstraint(txn, + table_object->GetDatabaseOid(), + table_object->GetTableOid(), + {0}, "con_primary"); LOG_INFO("Table created!"); diff --git a/test/gc/garbage_collection_test.cpp b/test/gc/garbage_collection_test.cpp index 886131e5fb0..bc7c6b1061b 100644 --- a/test/gc/garbage_collection_test.cpp +++ b/test/gc/garbage_collection_test.cpp @@ -129,12 +129,14 @@ TEST_F(GarbageCollectionTests, UpdateTest) { oid_t db_id = database->GetOid(); EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + auto prev_tc = gc_manager.GetTableCount(); + // create a table with only one key const int num_key = 1; std::unique_ptr table(TestingTransactionUtil::CreateTable( - num_key, "UPDATE_TABLE", db_id, INVALID_OID, 1234, true)); + num_key, "UPDATE_TABLE", db_id, 12345, 1234, true)); - EXPECT_TRUE(gc_manager.GetTableCount() == 1); + EXPECT_EQ(1, gc_manager.GetTableCount() - prev_tc); gc_manager.StartGC(gc_threads); @@ -223,12 +225,15 @@ TEST_F(GarbageCollectionTests, DeleteTest) { auto database = TestingExecutorUtil::InitializeDatabase("delete_db"); oid_t db_id = database->GetOid(); EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + + auto prev_tc = gc_manager.GetTableCount(); + // create a table with only one key const int num_key = 1; std::unique_ptr table(TestingTransactionUtil::CreateTable( - num_key, "DELETE_TABLE", db_id, INVALID_OID, 1234, true)); + num_key, "DELETE_TABLE", db_id, 12346, 1234, true)); - EXPECT_TRUE(gc_manager.GetTableCount() == 1); + EXPECT_EQ(1, gc_manager.GetTableCount() - prev_tc); gc_manager.StartGC(gc_threads); diff --git a/test/gc/transaction_level_gc_manager_test.cpp b/test/gc/transaction_level_gc_manager_test.cpp index e574f5bd4d8..b53f9084ad2 100644 --- a/test/gc/transaction_level_gc_manager_test.cpp +++ b/test/gc/transaction_level_gc_manager_test.cpp @@ -97,12 +97,14 @@ TEST_F(TransactionLevelGCManagerTests, UpdateDeleteTest) { oid_t db_id = database->GetOid(); EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + auto prev_tc = gc_manager.GetTableCount(); + // create a table with only one key const int num_key = 1; std::unique_ptr table(TestingTransactionUtil::CreateTable( - num_key, "TABLE0", db_id, INVALID_OID, 1234, true)); + num_key, "TABLE0", db_id, 12345, 1234, true)); - EXPECT_TRUE(gc_manager.GetTableCount() == 1); + EXPECT_EQ(1, gc_manager.GetTableCount() - prev_tc); //=========================== // update a version here. @@ -228,12 +230,14 @@ TEST_F(TransactionLevelGCManagerTests, ReInsertTest) { oid_t db_id = database->GetOid(); EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + auto prev_tc = gc_manager.GetTableCount(); + // create a table with only one key const int num_key = 1; std::unique_ptr table(TestingTransactionUtil::CreateTable( - num_key, "TABLE1", db_id, INVALID_OID, 1234, true)); + num_key, "TABLE1", db_id, 12346, 1234, true)); - EXPECT_TRUE(gc_manager.GetTableCount() == 1); + EXPECT_EQ(1, gc_manager.GetTableCount() - prev_tc); //=========================== // insert a tuple here. @@ -397,13 +401,15 @@ TEST_F(TransactionLevelGCManagerTests, ImmutabilityTest) { oid_t db_id = database->GetOid(); EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + auto prev_tc = gc_manager.GetTableCount(); + // create a table with only one key const int num_key = 25; const size_t tuples_per_tilegroup = 5; std::unique_ptr table(TestingTransactionUtil::CreateTable( - num_key, "TABLE1", db_id, INVALID_OID, 1234, true, tuples_per_tilegroup)); + num_key, "TABLE1", db_id, 12347, 1234, true, tuples_per_tilegroup)); - EXPECT_TRUE(gc_manager.GetTableCount() == 1); + EXPECT_EQ(1, gc_manager.GetTableCount() - prev_tc); oid_t num_tile_groups = (table.get())->GetTileGroupCount(); EXPECT_EQ(num_tile_groups, (num_key / tuples_per_tilegroup) + 1); diff --git a/test/include/catalog/testing_constraints_util.h b/test/include/catalog/testing_constraints_util.h index ab39e311bd1..8bd27540dbf 100644 --- a/test/include/catalog/testing_constraints_util.h +++ b/test/include/catalog/testing_constraints_util.h @@ -91,26 +91,10 @@ namespace test { class TestingConstraintsUtil { public: - /** @brief Creates a basic table with allocated and populated tuples */ - static storage::DataTable *CreateAndPopulateTable( - std::vector> constraints, - std::vector multi_constraints) { - const int tuple_count = TESTS_TUPLES_PER_TILEGROUP; - storage::DataTable *table = - TestingConstraintsUtil::CreateTable(constraints, multi_constraints); - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - auto txn = txn_manager.BeginTransaction(); - TestingConstraintsUtil::PopulateTable( - txn, table, tuple_count * DEFAULT_TILEGROUP_COUNT); - txn_manager.CommitTransaction(txn); - - return table; - }; - /** @brief Creates a basic table with allocated but not populated tuples */ static storage::DataTable *CreateTable( - std::vector> constraints, - UNUSED_ATTRIBUTE std::vector multi_constraints, + std::vector notnull_col_ids, + std::unordered_map default_values, UNUSED_ATTRIBUTE bool indexes = true) { // Create the database auto catalog = catalog::Catalog::GetInstance(); @@ -124,13 +108,24 @@ class TestingConstraintsUtil { std::vector columns; for (int i = 0; i < CONSTRAINTS_NUM_COLS; i++) { columns.push_back( - TestingConstraintsUtil::GetColumnInfo(i, constraints[i])); + TestingConstraintsUtil::GetColumnInfo(i)); } + + // set single column constraints + for (auto col_oid : notnull_col_ids) { + PELOTON_ASSERT(col_oid < CONSTRAINTS_NUM_COLS); + columns[col_oid].SetNotNull(); + } + for (auto dv : default_values) { + PELOTON_ASSERT(dv.first < CONSTRAINTS_NUM_COLS); + columns[dv.first].SetDefaultValue(dv.second); + } + std::unique_ptr table_schema(new catalog::Schema(columns)); - std::string table_name(CONSTRAINTS_TEST_TABLE); // Create table. txn = txn_manager.BeginTransaction(); + std::string table_name(CONSTRAINTS_TEST_TABLE); auto result = catalog->CreateTable(txn, DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, std::move(table_schema), table_name, false); @@ -328,6 +323,18 @@ class TestingConstraintsUtil { return executor.Execute(); }; + /** @brief Allocated and populated tuples */ + static storage::DataTable *PopulateTable(storage::DataTable *table) { + const int tuple_count = TESTS_TUPLES_PER_TILEGROUP; + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + TestingConstraintsUtil::PopulateTable( + txn, table, tuple_count * DEFAULT_TILEGROUP_COUNT); + txn_manager.CommitTransaction(txn); + + return table; + }; + static void PopulateTable(concurrency::TransactionContext *transaction, storage::DataTable *table, int num_rows) { // Ensure that the tile group is as expected. @@ -357,8 +364,7 @@ class TestingConstraintsUtil { } }; - static catalog::Column GetColumnInfo( - int index, std::vector constraints) { + static catalog::Column GetColumnInfo(int index) { catalog::Column column; switch (index) { // COL_A @@ -394,11 +400,6 @@ class TestingConstraintsUtil { std::to_string(index)); } } - - // Add any constraints that we have for this mofo - for (auto col_const : constraints) { - column.AddConstraint(col_const); - } return (column); }; diff --git a/test/include/codegen/testing_codegen_util.h b/test/include/codegen/testing_codegen_util.h index fa9fcf852cd..4c39f08c92b 100644 --- a/test/include/codegen/testing_codegen_util.h +++ b/test/include/codegen/testing_codegen_util.h @@ -89,8 +89,7 @@ class PelotonCodeGenTest : public PelotonTest { // Create the schema (common among all tables) catalog::Column GetTestColumn(uint32_t col_id) const; - std::unique_ptr CreateTestSchema( - bool add_primary = false) const; + std::unique_ptr CreateTestSchema() const; // Create the test tables void CreateTestTables(concurrency::TransactionContext *txn, diff --git a/test/include/concurrency/testing_transaction_util.h b/test/include/concurrency/testing_transaction_util.h index d7bb919a0f3..d2387f9ee98 100644 --- a/test/include/concurrency/testing_transaction_util.h +++ b/test/include/concurrency/testing_transaction_util.h @@ -122,7 +122,7 @@ class TestingTransactionUtil { int num_key = 10, std::string table_name = "TEST_TABLE", oid_t database_id = CATALOG_DATABASE_OID, oid_t relation_id = TEST_TABLE_OID, oid_t index_oid = 1234, - bool need_primary_index = false, size_t tuples_per_tilegroup = 100); + bool need_primary_key = false, size_t tuples_per_tilegroup = 100); // Create the same table as CreateTable with primary key constraints on id and // unique key constraints on value diff --git a/test/optimizer/stats_storage_test.cpp b/test/optimizer/stats_storage_test.cpp index edf42131355..131bfbdb6ac 100644 --- a/test/optimizer/stats_storage_test.cpp +++ b/test/optimizer/stats_storage_test.cpp @@ -253,4 +253,4 @@ TEST_F(StatsStorageTests, GetTableStatsTest) { } } // namespace test -} // namespace peloton \ No newline at end of file +} // namespace peloton diff --git a/test/planner/plan_util_test.cpp b/test/planner/plan_util_test.cpp index 6130d29d260..d58aae8a5e0 100644 --- a/test/planner/plan_util_test.cpp +++ b/test/planner/plan_util_test.cpp @@ -68,7 +68,7 @@ TEST_F(PlanUtilTests, GetAffectedIndexesTest) { txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - oid_t col_id = source_table->GetSchema()->GetColumnID(id_column.column_name); + oid_t col_id = source_table->GetSchema()->GetColumnID(id_column.GetName()); std::vector source_col_ids; source_col_ids.push_back(col_id); @@ -83,7 +83,7 @@ TEST_F(PlanUtilTests, GetAffectedIndexesTest) { IndexType::BWTREE); // create index on 'id' and 'first_name' - col_id = source_table->GetSchema()->GetColumnID(fname_column.column_name); + col_id = source_table->GetSchema()->GetColumnID(fname_column.GetName()); source_col_ids.push_back(col_id); catalog->CreateIndex(txn, @@ -232,11 +232,11 @@ TEST_F(PlanUtilTests, GetIndexableColumnsTest) { oid_t table_id = source_table->GetOid(); oid_t id_col_oid = - source_table->GetSchema()->GetColumnID(id_column.column_name); + source_table->GetSchema()->GetColumnID(id_column.GetName()); oid_t fname_col_oid = - source_table->GetSchema()->GetColumnID(fname_column.column_name); + source_table->GetSchema()->GetColumnID(fname_column.GetName()); oid_t lname_col_oid = - source_table->GetSchema()->GetColumnID(lname_column.column_name); + source_table->GetSchema()->GetColumnID(lname_column.GetName()); // Insert a 'test_table_job' with 'age', 'job' and 'pid' txn = txn_manager.BeginTransaction(); @@ -266,11 +266,11 @@ TEST_F(PlanUtilTests, GetIndexableColumnsTest) { "test_table_job"); oid_t table_job_id = source_table_job->GetOid(); oid_t age_col_oid = - source_table_job->GetSchema()->GetColumnID(age_column.column_name); + source_table_job->GetSchema()->GetColumnID(age_column.GetName()); oid_t job_col_oid = - source_table_job->GetSchema()->GetColumnID(job_column.column_name); + source_table_job->GetSchema()->GetColumnID(job_column.GetName()); oid_t pid_col_oid = - source_table_job->GetSchema()->GetColumnID(pid_column.column_name); + source_table_job->GetSchema()->GetColumnID(pid_column.GetName()); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); diff --git a/test/planner/planner_test.cpp b/test/planner/planner_test.cpp index 19304048dd9..fc865ab86a1 100644 --- a/test/planner/planner_test.cpp +++ b/test/planner/planner_test.cpp @@ -193,10 +193,10 @@ TEST_F(PlannerTest, UpdatePlanTestParameter) { ExpressionType::COMPARE_EQUAL, tuple_expr, parameter_expr); auto &schema_columns = schema->GetColumns(); - for (uint i = 0; i < schema_columns.size(); i++) { + for (oid_t i = 0; i < schema_columns.size(); i++) { bool is_in_target_list = false; for (auto col_id : column_ids) { - if (schema_columns[i].column_name == schema_columns[col_id].column_name) { + if (schema_columns[i].GetName() == schema_columns[col_id].GetName()) { is_in_target_list = true; break; } @@ -206,7 +206,7 @@ TEST_F(PlannerTest, UpdatePlanTestParameter) { } column_ids.clear(); - for (uint i = 0; i < schema_columns.size(); i++) { + for (oid_t i = 0; i < schema_columns.size(); i++) { column_ids.emplace_back(i); } diff --git a/test/sql/optimizer_sql_test.cpp b/test/sql/optimizer_sql_test.cpp index 9ebd230346c..b9248b1c5be 100644 --- a/test/sql/optimizer_sql_test.cpp +++ b/test/sql/optimizer_sql_test.cpp @@ -339,7 +339,7 @@ TEST_F(OptimizerSQLTests, DDLSqlTest) { auto cols = table->GetSchema()->GetColumns(); EXPECT_EQ(3, cols.size()); EXPECT_EQ("a", cols[0].GetName()); - EXPECT_EQ(true, cols[0].IsPrimary()); + EXPECT_EQ(true, table->GetSchema()->HasPrimary()); EXPECT_EQ(type::TypeId::INTEGER, cols[0].GetType()); EXPECT_EQ("b", cols[1].GetName()); EXPECT_EQ(type::TypeId::INTEGER, cols[1].GetType()); diff --git a/test/statistics/stats_test.cpp b/test/statistics/stats_test.cpp index b40efd823e6..792c4317b38 100644 --- a/test/statistics/stats_test.cpp +++ b/test/statistics/stats_test.cpp @@ -128,8 +128,6 @@ TEST_F(StatsTests, MultiThreadStatsTest) { auto id_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "dept_id", true); - catalog::Constraint constraint(ConstraintType::PRIMARY, "con_primary"); - id_column.AddConstraint(constraint); auto name_column = catalog::Column(type::TypeId::VARCHAR, 32, "dept_name", false); @@ -148,9 +146,16 @@ TEST_F(StatsTests, MultiThreadStatsTest) { storage::Database *database = catalog->GetDatabaseWithName(txn, "emp_db"); storage::DataTable *table = catalog->GetTableWithName(txn, - "emp_db", - DEFAULT_SCHEMA_NAME, - "department_table"); + "emp_db", + DEFAULT_SCHEMA_NAME, + "department_table"); + + catalog->AddPrimaryKeyConstraint(txn, + database->GetOid(), + table->GetOid(), + {0}, + "con_primary"); + txn_manager.CommitTransaction(txn); LaunchParallelTest(num_threads, TransactionTest, database, table); // Wait for aggregation to finish diff --git a/test/statistics/testing_stats_util.cpp b/test/statistics/testing_stats_util.cpp index 5c087e4aba4..169999efe9d 100644 --- a/test/statistics/testing_stats_util.cpp +++ b/test/statistics/testing_stats_util.cpp @@ -12,6 +12,8 @@ #include "statistics/testing_stats_util.h" +#include "catalog/catalog.h" +#include "catalog/table_catalog.h" #include "executor/delete_executor.h" #include "executor/executor_context.h" #include "executor/insert_executor.h" @@ -112,24 +114,32 @@ void TestingStatsUtil::CreateTable(bool has_primary_key) { auto id_column = catalog::Column( type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), "dept_id", true); - if (has_primary_key) { - catalog::Constraint constraint(ConstraintType::PRIMARY, "con_primary"); - id_column.AddConstraint(constraint); - } auto name_column = catalog::Column(type::TypeId::VARCHAR, 256, "dept_name", false); - std::unique_ptr table_schema( new catalog::Schema({id_column, name_column})); + + auto catalog = catalog::Catalog::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - std::unique_ptr context( - new executor::ExecutorContext(txn)); - planner::CreatePlan node("department_table", DEFAULT_SCHEMA_NAME, "emp_db", - std::move(table_schema), CreateType::TABLE); - executor::CreateExecutor create_executor(&node, context.get()); - create_executor.Init(); - create_executor.Execute(); + catalog->CreateTable(txn, + "emp_db", + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + "department_table", + false); + + if (has_primary_key) { + auto table_object = catalog->GetTableCatalogEntry(txn, + "emp_db", + DEFAULT_SCHEMA_NAME, + "department_table"); + catalog->AddPrimaryKeyConstraint(txn, + table_object->GetDatabaseOid(), + table_object->GetTableOid(), + {0}, + "con_primary"); + } txn_manager.CommitTransaction(txn); } From cd681eb256b79001b32f754b235f66c4b189b5ba Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 4 Jul 2018 10:48:54 -0400 Subject: [PATCH 17/19] Increase sleep time for occasionally failing ReadOnlyTest. (#1443) --- test/concurrency/serializable_transaction_test.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/concurrency/serializable_transaction_test.cpp b/test/concurrency/serializable_transaction_test.cpp index 2c352c0db7d..4548d47faaa 100644 --- a/test/concurrency/serializable_transaction_test.cpp +++ b/test/concurrency/serializable_transaction_test.cpp @@ -76,8 +76,8 @@ TEST_F(SerializableTransactionTests, ReadOnlyTransactionTest) { //manually update snapshot epoch number, so later snapshot read must get a larger epoch than table creating txn //or it may read nothing - //wait one epoch. so that global epoch is guaranteed to increase - std::this_thread::sleep_for(std::chrono::milliseconds(EPOCH_LENGTH)); + //wait two epochs. so that global epoch is guaranteed to increase + std::this_thread::sleep_for(std::chrono::milliseconds(2 * EPOCH_LENGTH)); concurrency::EpochManagerFactory::GetInstance().GetExpiredEpochId(); TransactionScheduler scheduler(1, table, &txn_manager, {0}); @@ -86,6 +86,8 @@ TEST_F(SerializableTransactionTests, ReadOnlyTransactionTest) { scheduler.Run(); + EXPECT_EQ(ResultType::SUCCESS, scheduler.schedules[0].txn_result); + //it should read all the 10 tuples EXPECT_EQ(10, scheduler.schedules[0].results.size()); From 6feb06893192a3a541e303a10f63f6f4d632ada5 Mon Sep 17 00:00:00 2001 From: saatviks Date: Fri, 6 Jul 2018 12:55:34 -0400 Subject: [PATCH 18/19] Py version control for Mac --- .travis.yml | 2 ++ script/installation/packages.sh | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1f191a1f43b..472624280e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,6 +64,8 @@ install: before_script: # first, run source_validator - python script/validators/source_validator.py + # enable aliases and make python3 point to python + - if [[ $TRAVIS_OS_NAME == 'osx' ]]; then alias python3=python && shopt -s expand_aliases; fi # build peloton (override this value to execute tests) script: diff --git a/script/installation/packages.sh b/script/installation/packages.sh index 1dff718b7d6..7b99dcdc3e4 100755 --- a/script/installation/packages.sh +++ b/script/installation/packages.sh @@ -225,8 +225,7 @@ elif [ "$DISTRO" = "DARWIN" ]; then brew install tbb brew install curl brew install wget - brew install python - brew upgrade python + python3 -m ensurepip brew install eigen brew install ant # Brew installs correct version of Protobuf(3.5.1 >= 3.4.0) From 88859a213b24dd5bbee33f0a19ef72c8a56477c6 Mon Sep 17 00:00:00 2001 From: saatviks Date: Fri, 6 Jul 2018 14:10:10 -0400 Subject: [PATCH 19/19] Minimal push --- src/brain/modelgen/LSTM.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/brain/modelgen/LSTM.py b/src/brain/modelgen/LSTM.py index 116eb4c91ce..aa1f213684e 100644 --- a/src/brain/modelgen/LSTM.py +++ b/src/brain/modelgen/LSTM.py @@ -125,8 +125,6 @@ def __repr__(self): def main(): parser = argparse.ArgumentParser(description='LSTM Model Generator') - - parser.add_argument('--nfeats', type=int, default=3, help='Input feature length(input to encoder/linear layer)') parser.add_argument('--nencoded', type=int, default=20, help='Encoded feature length(input to LSTM)') parser.add_argument('--nhid', type=int, default=20, help='Number of LSTM Hidden units')