From 657d33ce1702fc11f1fa64ca46c7dd6c07bd12ac Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Mon, 25 Sep 2023 11:46:04 -0700 Subject: [PATCH] Reduce the size of the index (#3666) This change seeks to reduce the size of the index in two ways: 1. Better schema design for the 1:N mapping tables 2. Dropping some of the mapping data that is not particularly interesting per manifest (aka per version) ### Better schema design This is achieved by the map table having no `rowid` and using a primary key with the value first. This makes the table already sorted by the value, thus the reverse lookups are fast. It also drops a fair amount of the data in the table itself to remove the `rowid`, given that it was ~1/3 of the rows. ### Dropping map data We don't actually use the fact that we know that different versions have different tags (or any other data). Thus, we can simply have one manifest entry per package identifier have all of the values and maintain the same functionality. There is a slight loss of fidelity if one is reading through the values via API, but this is deemed acceptable given the large data savings. I explicitly left the product codes alone, as this does have value to keep per version (even if we are not using it currently). --- .github/actions/spelling/expect.txt | 2 + src/AppInstallerCLITests/SQLiteIndex.cpp | 210 ++++++++++++++++-- .../AppInstallerRepositoryCore.vcxproj | 2 + ...AppInstallerRepositoryCore.vcxproj.filters | 9 + .../Microsoft/SQLiteIndex.cpp | 60 +++-- .../Microsoft/SQLiteIndex.h | 7 +- .../Microsoft/SQLiteStorageBase.cpp | 2 +- .../Microsoft/SQLiteStorageBase.h | 2 +- .../Microsoft/Schema/1_0/Interface.h | 4 + .../Microsoft/Schema/1_0/Interface_1_0.cpp | 13 +- .../Microsoft/Schema/1_0/OneToManyTable.cpp | 91 +++++++- .../Microsoft/Schema/1_0/OneToManyTable.h | 52 +++-- .../Microsoft/Schema/1_0/OneToOneTable.h | 1 + .../Microsoft/Schema/1_1/Interface.h | 2 + .../Microsoft/Schema/1_1/Interface_1_1.cpp | 21 +- .../Microsoft/Schema/1_2/Interface_1_2.cpp | 8 +- .../Microsoft/Schema/1_6/Interface_1_6.cpp | 4 +- .../Microsoft/Schema/1_7/Interface.h | 26 +++ .../Microsoft/Schema/1_7/Interface_1_7.cpp | 114 ++++++++++ .../Microsoft/Schema/MetadataTable.cpp | 23 +- .../Microsoft/Schema/MetadataTable.h | 31 ++- .../SQLiteStatementBuilder.cpp | 61 ++++- .../SQLiteStatementBuilder.h | 23 +- src/AppInstallerRepositoryCore/pch.h | 1 + 24 files changed, 636 insertions(+), 133 deletions(-) create mode 100644 src/AppInstallerRepositoryCore/Microsoft/Schema/1_7/Interface.h create mode 100644 src/AppInstallerRepositoryCore/Microsoft/Schema/1_7/Interface_1_7.cpp diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 748e0345d1..7136c77278 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -103,6 +103,7 @@ DACL datetimeoffset Dbg debian +dedupe deigh deleteifnotneeded DENYWR @@ -264,6 +265,7 @@ luffy Luffytaro maclachlan malware +mapdatafolding mapview Maxed maxvalue diff --git a/src/AppInstallerCLITests/SQLiteIndex.cpp b/src/AppInstallerCLITests/SQLiteIndex.cpp index 9884fbe33f..ff0c0e0b78 100644 --- a/src/AppInstallerCLITests/SQLiteIndex.cpp +++ b/src/AppInstallerCLITests/SQLiteIndex.cpp @@ -35,7 +35,17 @@ SQLiteIndex CreateTestIndex(const std::string& filePath, std::optional= 2 + Schema::Version versionMinus1 = Schema::Version{ 1, latestVersion.MinorVersion - 1 }; + Schema::Version versionMinus2 = Schema::Version{ 1, latestVersion.MinorVersion - 2 }; + + version = GENERATE_COPY(Schema::Version{ versionMinus2 }, Schema::Version{ versionMinus1 }, Schema::Version{ latestVersion }); } return SQLiteIndex::CreateNew(filePath, version.value()); @@ -43,41 +53,35 @@ SQLiteIndex CreateTestIndex(const std::string& filePath, std::optional= 2 + Schema::Version versionMinus1 = Schema::Version{ 1, latestVersion.MinorVersion - 1 }; + Schema::Version versionMinus2 = Schema::Version{ 1, latestVersion.MinorVersion - 2 }; + + if (index.GetVersion() == versionMinus2) + { + // Degenerate case where we don't need to do anything } - else if (index.GetVersion() == Schema::Version{ 1, 4 }) + else if (index.GetVersion() == versionMinus1) { - Schema::Version version = GENERATE(Schema::Version{ 1, 2 }, Schema::Version{ 1, 3 }, Schema::Version{ 1, 4 }); + Schema::Version version = GENERATE_COPY(Schema::Version{ versionMinus2 }, Schema::Version{ versionMinus1 }); - if (version != Schema::Version{ 1, 4 }) + if (version != versionMinus1) { index.ForceVersion(version); return version; } } - else if (index.GetVersion() == Schema::Version{ 1, 5 }) + else if (index.GetVersion() == latestVersion) { - Schema::Version version = GENERATE(Schema::Version{ 1, 2 }, Schema::Version{ 1, 3 }, Schema::Version{ 1, 4 }, Schema::Version{ 1, 5 }); + Schema::Version version = GENERATE_COPY(Schema::Version{ versionMinus2 }, Schema::Version{ versionMinus1 }, Schema::Version{ latestVersion }); - if (version != Schema::Version{ 1, 5 }) + if (version != latestVersion) { index.ForceVersion(version); return version; @@ -321,6 +325,17 @@ bool AreArpVersionsSupported(const SQLiteIndex& index, const Schema::Version& te return (index.GetVersion() >= Schema::Version{ 1, 5 } && testVersion >= Schema::Version{ 1, 5 }); } +bool IsMapDataFoldingSupported(const SQLiteIndex& index, const Schema::Version& testVersion) +{ + UNSCOPED_INFO("Index " << index.GetVersion() << " | Test " << testVersion); + return (index.GetVersion() >= Schema::Version{ 1, 7 } && testVersion >= Schema::Version{ 1, 7 }); +} + +bool IsMapDataFolded(const SQLiteIndex& index) +{ + return (index.GetVersion() >= Schema::Version{ 1, 7 }); +} + std::string GetPropertyStringByKey(const SQLiteIndex& index, SQLite::rowid_t id, PackageVersionProperty property, std::string_view version, std::string_view channel) { auto manifestId = index.GetManifestIdByKey(id, version, channel); @@ -3187,3 +3202,152 @@ TEST_CASE("SQLiteIndex_CheckConsistency_FindEmbeddedNull", "[sqliteindex]") REQUIRE(!index.CheckConsistency(true)); } + +TEST_CASE("SQLiteIndex_MapDataFolding_Tags", "[sqliteindex][mapdatafolding]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + std::string tag1 = "Tag1"; + std::string tag2 = "Tag2"; + + SQLiteIndex index = SearchTestSetup(tempFile, { + { "Id", "Name", "Publisher", "Moniker", "Version1", "", { tag1 }, { "Command" }, "Path1", {}, { "PC1" } }, + { "Id", "Name", "Publisher", "Moniker", "Version2", "", { tag2 }, { "Command" }, "Path2", {}, { "PC2" } }, + }); + + // Apply the map data folding if it is present in the created test index. + index.PrepareForPackaging(); + + Schema::Version testVersion = TestPrepareForRead(index); + + SearchRequest request1; + request1.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::Tag, MatchType::Exact, tag1)); + auto results1 = index.Search(request1); + + SearchRequest request2; + request2.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::Tag, MatchType::Exact, tag2)); + auto results2 = index.Search(request1); + + REQUIRE(results1.Matches.size() == 1); + REQUIRE(results2.Matches.size() == 1); + REQUIRE(results1.Matches[0].first == results2.Matches[0].first); +} + +TEST_CASE("SQLiteIndex_MapDataFolding_PFNs", "[sqliteindex][mapdatafolding]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + std::string pfn1 = "PFN1"; + std::string pfn2 = "PFN2"; + + SQLiteIndex index = SearchTestSetup(tempFile, { + { "Id", "Name", "Publisher", "Moniker", "Version1", "", { }, { "Command" }, "Path1", { pfn1 }, { } }, + { "Id", "Name", "Publisher", "Moniker", "Version2", "", { }, { "Command" }, "Path2", { pfn2 }, { } }, + }); + + // Apply the map data folding if it is present in the created test index. + index.PrepareForPackaging(); + + Schema::Version testVersion = TestPrepareForRead(index); + + SearchRequest request1; + request1.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::PackageFamilyName, MatchType::Exact, pfn1)); + auto results1 = index.Search(request1); + + SearchRequest request2; + request2.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::PackageFamilyName, MatchType::Exact, pfn2)); + auto results2 = index.Search(request1); + + REQUIRE(results1.Matches.size() == 1); + REQUIRE(results2.Matches.size() == 1); + REQUIRE(results1.Matches[0].first == results2.Matches[0].first); + + auto versionKeys = index.GetVersionKeysById(results1.Matches[0].first); + REQUIRE(versionKeys.size() == 2); + + auto manifestId1 = index.GetManifestIdByKey(results1.Matches[0].first, versionKeys[0].GetVersion().ToString(), versionKeys[0].GetChannel().ToString()); + auto manifestId2 = index.GetManifestIdByKey(results1.Matches[0].first, versionKeys[1].GetVersion().ToString(), versionKeys[1].GetChannel().ToString()); + + REQUIRE(manifestId1.has_value()); + REQUIRE(manifestId2.has_value()); + + auto pfnValues1 = index.GetMultiPropertyByManifestId(manifestId1.value(), PackageVersionMultiProperty::PackageFamilyName); + auto pfnValues2 = index.GetMultiPropertyByManifestId(manifestId2.value(), PackageVersionMultiProperty::PackageFamilyName); + + if (IsMapDataFoldingSupported(index, testVersion)) + { + REQUIRE(pfnValues1.size() == 2); + REQUIRE(pfnValues2.size() == 2); + REQUIRE(pfnValues1[0] != pfnValues1[1]); + } + else if (IsMapDataFolded(index)) + { + if (manifestId1 > manifestId2) + { + REQUIRE(pfnValues1.size() == 2); + REQUIRE(pfnValues2.size() == 0); + REQUIRE(pfnValues1[0] != pfnValues1[1]); + } + else + { + REQUIRE(pfnValues1.size() == 0); + REQUIRE(pfnValues2.size() == 2); + REQUIRE(pfnValues2[0] != pfnValues2[1]); + } + } + else + { + REQUIRE(pfnValues1.size() == 1); + REQUIRE(pfnValues2.size() == 1); + REQUIRE(pfnValues1[0] != pfnValues2[0]); + } +} + +TEST_CASE("SQLiteIndex_MapDataFolding_ProductCodes", "[sqliteindex][mapdatafolding]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + std::string pc1 = "PC1"; + std::string pc2 = "PC2"; + + SQLiteIndex index = SearchTestSetup(tempFile, { + { "Id", "Name", "Publisher", "Moniker", "Version1", "", { }, { "Command" }, "Path1", { }, { pc1 } }, + { "Id", "Name", "Publisher", "Moniker", "Version2", "", { }, { "Command" }, "Path2", { }, { pc2 } }, + }); + + // Apply the map data folding if it is present in the created test index. + index.PrepareForPackaging(); + + Schema::Version testVersion = TestPrepareForRead(index); + + SearchRequest request1; + request1.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::ProductCode, MatchType::Exact, pc1)); + auto results1 = index.Search(request1); + + SearchRequest request2; + request2.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::ProductCode, MatchType::Exact, pc2)); + auto results2 = index.Search(request1); + + REQUIRE(results1.Matches.size() == 1); + REQUIRE(results2.Matches.size() == 1); + REQUIRE(results1.Matches[0].first == results2.Matches[0].first); + + auto versionKeys = index.GetVersionKeysById(results1.Matches[0].first); + REQUIRE(versionKeys.size() == 2); + + auto manifestId1 = index.GetManifestIdByKey(results1.Matches[0].first, versionKeys[0].GetVersion().ToString(), versionKeys[0].GetChannel().ToString()); + auto manifestId2 = index.GetManifestIdByKey(results1.Matches[0].first, versionKeys[1].GetVersion().ToString(), versionKeys[1].GetChannel().ToString()); + + REQUIRE(manifestId1.has_value()); + REQUIRE(manifestId2.has_value()); + + auto pcValues1 = index.GetMultiPropertyByManifestId(manifestId1.value(), PackageVersionMultiProperty::ProductCode); + auto pcValues2 = index.GetMultiPropertyByManifestId(manifestId2.value(), PackageVersionMultiProperty::ProductCode); + + REQUIRE(pcValues1.size() == 1); + REQUIRE(pcValues2.size() == 1); + REQUIRE(pcValues1[0] != pcValues2[0]); +} diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj index d3563dfa38..33d79302b6 100644 --- a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj +++ b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj @@ -385,6 +385,7 @@ + @@ -488,6 +489,7 @@ + diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters index 411658d477..950f82d2de 100644 --- a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters +++ b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters @@ -91,6 +91,9 @@ {b2e78f3d-931e-432c-8485-255b1dbc9db7} + + {f610927a-6f1d-42c5-9ad9-b59790091944} + @@ -387,6 +390,9 @@ Rest\Schema\1_6 + + Microsoft\Schema\1_7 + @@ -614,6 +620,9 @@ Rest\Schema\1_6 + + Microsoft\Schema\1_7 + diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp index 1a43480f4d..af9a26a37c 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp @@ -13,6 +13,7 @@ #include "Schema/1_4/Interface.h" #include "Schema/1_5/Interface.h" #include "Schema/1_6/Interface.h" +#include "Schema/1_7/Interface.h" namespace AppInstaller::Repository::Microsoft { @@ -35,49 +36,36 @@ namespace AppInstaller::Repository::Microsoft return result; } - std::unique_ptr SQLiteIndex::CreateISQLiteIndex() const + std::unique_ptr SQLiteIndex::CreateISQLiteIndex(const Schema::Version& version) { using namespace Schema; - if (m_version == Version{ 1, 0 }) + if (version.MajorVersion == 1 || + version.IsLatest()) { - return std::make_unique(); - } - else if (m_version == Version{ 1, 1 }) - { - return std::make_unique(); - } - else if (m_version == Version{ 1, 2 }) - { - return std::make_unique(); - } - else if (m_version == Version{ 1, 3 }) - { - return std::make_unique(); - } - else if (m_version == Version{ 1, 4 }) - { - return std::make_unique(); - } - else if (m_version == Version{ 1, 5 }) - { - return std::make_unique(); - } - else if (m_version == Version{ 1, 6 } || - m_version.MajorVersion == 1 || - m_version.IsLatest()) - { - return std::make_unique(); + constexpr std::array(*)(), 8> versionCreatorMap = + { + []() { return std::unique_ptr(std::make_unique()); }, + []() { return std::unique_ptr(std::make_unique()); }, + []() { return std::unique_ptr(std::make_unique()); }, + []() { return std::unique_ptr(std::make_unique()); }, + []() { return std::unique_ptr(std::make_unique()); }, + []() { return std::unique_ptr(std::make_unique()); }, + []() { return std::unique_ptr(std::make_unique()); }, + []() { return std::unique_ptr(std::make_unique()); }, + }; + + return versionCreatorMap[std::min(static_cast(version.MinorVersion), versionCreatorMap.size() - 1)](); } // We do not have the capacity to operate on this schema version THROW_HR(HRESULT_FROM_WIN32(ERROR_NOT_SUPPORTED)); } - SQLiteIndex::SQLiteIndex(const std::string& target, Schema::Version version) : SQLiteStorageBase(target, version) + SQLiteIndex::SQLiteIndex(const std::string& target, const Schema::Version& version) : SQLiteStorageBase(target, version) { m_dbconn.EnableICU(); - m_interface = CreateISQLiteIndex(); + m_interface = CreateISQLiteIndex(version); m_version = m_interface->GetVersion(); } @@ -86,15 +74,19 @@ namespace AppInstaller::Repository::Microsoft { m_dbconn.EnableICU(); AICLI_LOG(Repo, Info, << "Opened SQLite Index with version [" << m_version << "], last write [" << GetLastWriteTime() << "]"); - m_interface = CreateISQLiteIndex(); + m_interface = CreateISQLiteIndex(m_version); THROW_HR_IF(APPINSTALLER_CLI_ERROR_CANNOT_WRITE_TO_UPLEVEL_INDEX, disposition == SQLiteStorageBase::OpenDisposition::ReadWrite && m_version != m_interface->GetVersion()); } #ifndef AICLI_DISABLE_TEST_HOOKS void SQLiteIndex::ForceVersion(const Schema::Version& version) { - m_version = version; - m_interface = CreateISQLiteIndex(); + m_interface = CreateISQLiteIndex(version); + } + + Schema::Version SQLiteIndex::GetLatestVersion() + { + return CreateISQLiteIndex(Schema::Version::Latest())->GetVersion(); } #endif diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h index f8201adc7e..679a626b07 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h @@ -56,6 +56,9 @@ namespace AppInstaller::Repository::Microsoft // Changes the version of the interface being used to operate on the database. // Should only be used for testing. void ForceVersion(const Schema::Version& version); + + // Gets the latest version of the index schema (the actual numbers, not just the latest sentinel values). + static Schema::Version GetLatestVersion(); #endif // Adds the manifest at the repository relative path to the index. @@ -138,7 +141,7 @@ namespace AppInstaller::Repository::Microsoft std::vector> GetDependentsById(AppInstaller::Manifest::string_t packageId) const; private: // Constructor used to create a new index. - SQLiteIndex(const std::string& target, Schema::Version version); + SQLiteIndex(const std::string& target, const Schema::Version& version); // Constructor used to open an existing index. SQLiteIndex(const std::string& target, SQLiteStorageBase::OpenDisposition disposition, Utility::ManagedFile&& indexFile); @@ -148,7 +151,7 @@ namespace AppInstaller::Repository::Microsoft bool UpdateManifestInternal(const Manifest::Manifest& manifest, const std::optional& relativePath); // Creates the ISQLiteIndex interface object for this version. - std::unique_ptr CreateISQLiteIndex() const; + static std::unique_ptr CreateISQLiteIndex(const Schema::Version& version); std::unique_ptr m_interface; }; diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteStorageBase.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteStorageBase.cpp index e2e3fc7804..86b056be4f 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteStorageBase.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteStorageBase.cpp @@ -105,7 +105,7 @@ namespace AppInstaller::Repository::Microsoft m_version = Schema::Version::GetSchemaVersion(m_dbconn); } - SQLiteStorageBase::SQLiteStorageBase(const std::string& target, Schema::Version version) : + SQLiteStorageBase::SQLiteStorageBase(const std::string& target, const Schema::Version& version) : m_dbconn(SQLite::Connection::Create(target, SQLite::Connection::OpenDisposition::Create)) { m_version = version; diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteStorageBase.h b/src/AppInstallerRepositoryCore/Microsoft/SQLiteStorageBase.h index 26d86c5f40..c8e3bf5257 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteStorageBase.h +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteStorageBase.h @@ -30,7 +30,7 @@ namespace AppInstaller::Repository::Microsoft Schema::Version GetVersion() const { return m_version; } protected: - SQLiteStorageBase(const std::string& target, Schema::Version version); + SQLiteStorageBase(const std::string& target, const Schema::Version& version); SQLiteStorageBase(const std::string& filePath, SQLiteStorageBase::OpenDisposition disposition, Utility::ManagedFile&& indexFile); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.h index 39e57e8e12..e91512e497 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.h @@ -3,6 +3,7 @@ #pragma once #include "Microsoft/Schema/ISQLiteIndex.h" #include "Microsoft/Schema/1_0/SearchResultsTable.h" +#include "Microsoft/Schema/1_0/OneToManyTable.h" #include #include @@ -55,6 +56,9 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 // Gets a property already knowing that the manifest id is valid. virtual std::optional GetPropertyByManifestIdInternal(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionProperty property) const; + // Gets the one to many table schema to use. + virtual OneToManyTableSchema GetOneToManyTableSchema() const; + // Force the database to shrink the file size. // This *must* be done outside of an active transaction. void Vacuum(const SQLite::Connection& connection); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface_1_0.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface_1_0.cpp index 45f1e941a8..beb47d2c4c 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface_1_0.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface_1_0.cpp @@ -176,8 +176,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 { PathPartTable::ValueName(), false, WI_IsFlagClear(options, CreateOptions::SupportPathless) } }); - TagsTable::Create_deprecated(connection); - CommandsTable::Create_deprecated(connection); + TagsTable::Create(connection, GetOneToManyTableSchema()); + CommandsTable::Create(connection, GetOneToManyTableSchema()); savepoint.Commit(); } @@ -377,8 +377,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 PathPartTable::ValueName(), }); - TagsTable::PrepareForPackaging_deprecated(connection); - CommandsTable::PrepareForPackaging_deprecated(connection); + TagsTable::PrepareForPackaging(connection, GetOneToManyTableSchema(), false, false); + CommandsTable::PrepareForPackaging(connection, GetOneToManyTableSchema(), false, false); savepoint.Commit(); @@ -641,6 +641,11 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 } } + OneToManyTableSchema Interface::GetOneToManyTableSchema() const + { + return OneToManyTableSchema::Version_1_0; + } + void Interface::Vacuum(const SQLite::Connection& connection) { SQLite::Builder::StatementBuilder builder; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToManyTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToManyTable.cpp index c172a2343d..287bde3dc6 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToManyTable.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToManyTable.cpp @@ -4,6 +4,7 @@ #include "Microsoft/Schema/1_0/OneToManyTable.h" #include "Microsoft/Schema/1_0/OneToOneTable.h" #include "Microsoft/Schema/1_0/ManifestTable.h" +#include "Microsoft/Schema/1_0/IdTable.h" #include "SQLiteStatementBuilder.h" @@ -84,6 +85,11 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 // Bind valid rowid to 1. SQLite::Statement DeleteValueById; }; + + bool SchemaVersionUsesNamedIndices(OneToManyTableSchema schemaVersion) + { + return schemaVersion != OneToManyTableSchema::Version_1_0; + } } std::string OneToManyTableGetMapTableName(std::string_view tableName) @@ -98,16 +104,31 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 return s_OneToManyTable_MapTable_ManifestName; } - void CreateOneToManyTable(SQLite::Connection& connection, bool useNamedIndices, std::string_view tableName, std::string_view valueName) + void CreateOneToManyTable(SQLite::Connection& connection, OneToManyTableSchema schemaVersion, std::string_view tableName, std::string_view valueName) { using namespace SQLite::Builder; SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, std::string{ tableName } + "_create_v1_0"); // Create the data table as a 1:1 - CreateOneToOneTable(connection, tableName, valueName, useNamedIndices); + CreateOneToOneTable(connection, tableName, valueName, SchemaVersionUsesNamedIndices(schemaVersion)); - if (useNamedIndices) + switch (schemaVersion) + { + case OneToManyTableSchema::Version_1_0: + { + // Create the mapping table + StatementBuilder createMapTableBuilder; + createMapTableBuilder.CreateTable({ tableName, s_OneToManyTable_MapTable_Suffix }).Columns({ + ColumnBuilder(s_OneToManyTable_MapTable_ManifestName, Type::Int64).NotNull(), + ColumnBuilder(valueName, Type::Int64).NotNull(), + PrimaryKeyBuilder({ valueName, s_OneToManyTable_MapTable_ManifestName }) + }); + + createMapTableBuilder.Execute(connection); + } + break; + case OneToManyTableSchema::Version_1_1: { // Create the mapping table StatementBuilder createMapTableBuilder; @@ -123,7 +144,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 On({ tableName, s_OneToManyTable_MapTable_Suffix }).Columns({ valueName, s_OneToManyTable_MapTable_ManifestName }); pkIndexBuilder.Execute(connection); } - else + break; + case OneToManyTableSchema::Version_1_7: { // Create the mapping table StatementBuilder createMapTableBuilder; @@ -131,14 +153,18 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 ColumnBuilder(s_OneToManyTable_MapTable_ManifestName, Type::Int64).NotNull(), ColumnBuilder(valueName, Type::Int64).NotNull(), PrimaryKeyBuilder({ valueName, s_OneToManyTable_MapTable_ManifestName }) - }); + }).WithoutRowID(); createMapTableBuilder.Execute(connection); } + break; + default: + THROW_HR(E_UNEXPECTED); + } StatementBuilder createMapTableIndexBuilder; createMapTableIndexBuilder.CreateIndex({ tableName, s_OneToManyTable_MapTable_Suffix, s_OneToManyTable_MapTable_IndexSuffix }). - On({ tableName, s_OneToManyTable_MapTable_Suffix }).Columns(s_OneToManyTable_MapTable_ManifestName); + On({ tableName, s_OneToManyTable_MapTable_Suffix }).Columns({ s_OneToManyTable_MapTable_ManifestName, valueName }); createMapTableIndexBuilder.Execute(connection); @@ -272,7 +298,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 savepoint.Commit(); } - void OneToManyTablePrepareForPackaging(SQLite::Connection& connection, std::string_view tableName, bool useNamedIndices, bool preserveManifestIndex, bool preserveValuesIndex) + void OneToManyTablePrepareForPackaging(SQLite::Connection& connection, std::string_view tableName, OneToManyTableSchema schemaVersion, bool preserveManifestIndex, bool preserveValuesIndex) { if (!preserveManifestIndex) { @@ -282,7 +308,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 dropMapTableIndexBuilder.Execute(connection); } - OneToOneTablePrepareForPackaging(connection, tableName, useNamedIndices, preserveValuesIndex); + OneToOneTablePrepareForPackaging(connection, tableName, SchemaVersionUsesNamedIndices(schemaVersion), preserveValuesIndex); } bool OneToManyTableCheckConsistency(const SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, bool log) @@ -299,7 +325,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 SQLite::Builder::StatementBuilder builder; builder. - Select({ QCol(s_map, SQLite::RowIDName), QCol(s_map, s_OneToManyTable_MapTable_ManifestName) }). + Select({ QCol(s_map, s_OneToManyTable_MapTable_ManifestName), QCol(s_map, valueName) }). From({ tableName, s_OneToManyTable_MapTable_Suffix }).As(s_map). LeftOuterJoin(ManifestTable::TableName()).On(QCol(s_map, s_OneToManyTable_MapTable_ManifestName), QCol(ManifestTable::TableName(), SQLite::RowIDName)). Where(QCol(ManifestTable::TableName(), SQLite::RowIDName)).IsNull(); @@ -316,7 +342,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 } AICLI_LOG(Repo, Info, << " [INVALID] " << tableName << s_OneToManyTable_MapTable_Suffix << " [" << select.GetColumn(0) << - "] refers to " << ManifestTable::TableName() << " [" << select.GetColumn(1) << "]"); + ", " << select.GetColumn(1) << "] refers to invalid " << ManifestTable::TableName()); } } @@ -331,7 +357,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 // Select map.rowid, map.tag from tags_map as map left outer join tags on map.tag = tags.rowid where tags.tag is null SQLite::Builder::StatementBuilder builder; builder. - Select({ QCol(s_map, SQLite::RowIDName), QCol(s_map, valueName) }). + Select({ QCol(s_map, s_OneToManyTable_MapTable_ManifestName), QCol(s_map, valueName) }). From({ tableName, s_OneToManyTable_MapTable_Suffix }).As(s_map). LeftOuterJoin(tableName).On(QCol(s_map, valueName), QCol(tableName, SQLite::RowIDName)). Where(QCol(tableName, valueName)).IsNull(); @@ -349,7 +375,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 } AICLI_LOG(Repo, Info, << " [INVALID] " << tableName << s_OneToManyTable_MapTable_Suffix << " [" << select.GetColumn(0) << - "] refers to " << tableName << " [" << select.GetColumn(1) << "]"); + ", " << select.GetColumn(1) << "] refers to invalid " << tableName); } result = result && secondaryResult; @@ -383,5 +409,46 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 return ((countStatement.GetColumn(0) == 0) && (countMapStatement.GetColumn(0) == 0)); } + + SQLite::Statement OneToManyTablePrepareMapDataFoldingStatement(const SQLite::Connection& connection, std::string_view tableName) + { + using namespace SQLite::Builder; + StatementBuilder builder; + + // Create a statement that will collapse (and dedupe) all rows in the map to the latest (max) manifest for a given id, like: + // UPDATE OR REPLACE map SET manifest = (SELECT MAX(rowid) FROM manifest_table WHERE id = ?1) WHERE manifest IN (SELECT rowid FROM manifest_table WHERE id = ?1) + builder.UpdateOrReplace({ tableName, s_OneToManyTable_MapTable_Suffix }).Set().Column(s_OneToManyTable_MapTable_ManifestName).Equals() + .BeginParenthetical() + .Select().Column(Aggregate::Max, SQLite::RowIDName).From(ManifestTable::TableName()).Where(IdTable::ValueName()).Equals(Unbound, 1) + .EndParenthetical() + .Where(s_OneToManyTable_MapTable_ManifestName).In() + .BeginParenthetical() + .Select(SQLite::RowIDName).From(ManifestTable::TableName()).Where(IdTable::ValueName()).Equals(Unbound, 1) + .EndParenthetical(); + + return builder.Prepare(connection); + } + } + + std::optional OneToManyTableGetMapDataFoldingManifestTargetId(const SQLite::Connection& connection, SQLite::rowid_t manifestId) + { + using namespace SQLite::Builder; + StatementBuilder builder; + + // Select the maximum manifest rowid from the manifests whose id is the same as the row with the given manifest rowid, like: + // SELECT MAX(rowid) FROM manifest_table WHERE id = (SELECT id FROM manifest_table WHERE rowid = ?) + builder.Select().Column(Aggregate::Max, SQLite::RowIDName).From(ManifestTable::TableName()).Where(IdTable::ValueName()).Equals() + .BeginParenthetical() + .Select(IdTable::ValueName()).From(ManifestTable::TableName()).Where(SQLite::RowIDName).Equals(manifestId) + .EndParenthetical(); + + SQLite::Statement statement = builder.Prepare(connection); + + if (statement.Step() && !statement.GetColumnIsNull(0)) + { + return statement.GetColumn(0); + } + + return std::nullopt; } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToManyTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToManyTable.h index dd93e14713..19dc52da3f 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToManyTable.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToManyTable.h @@ -9,6 +9,20 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 { + // Allow the different schema version to indicate which they are. + enum class OneToManyTableSchema + { + // Does not used a named unique index for either table. + // Map table has primary key and rowid. + Version_1_0, + // Uses a named unique index for both tables. + // Map table has rowid. + Version_1_1, + // Uses a named unique index for data table. (No change from 1.1) + // Map table has primary key and no rowid. + Version_1_7, + }; + namespace details { // Returns the map table name for a given table. @@ -18,7 +32,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 std::string_view OneToManyTableGetManifestColumnName(); // Create the tables. - void CreateOneToManyTable(SQLite::Connection& connection, bool useNamedIndices, std::string_view tableName, std::string_view valueName); + void CreateOneToManyTable(SQLite::Connection& connection, OneToManyTableSchema schemaVersion, std::string_view tableName, std::string_view valueName); // Gets all values associated with the given manifest id. std::vector OneToManyTableGetValuesByManifestId( @@ -41,7 +55,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 void OneToManyTableDeleteIfNotNeededByManifestId(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, SQLite::rowid_t manifestId); // Removes data that is no longer needed for an index that is to be published. - void OneToManyTablePrepareForPackaging(SQLite::Connection& connection, std::string_view tableName, bool useNamedIndices, bool preserveManifestIndex, bool preserveValuesIndex); + void OneToManyTablePrepareForPackaging(SQLite::Connection& connection, std::string_view tableName, OneToManyTableSchema schemaVersion, bool preserveManifestIndex, bool preserveValuesIndex); // Checks the consistency of the index to ensure that every referenced row exists. // Returns true if index is consistent; false if it is not. @@ -49,8 +63,14 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 // Determines if the table is empty. bool OneToManyTableIsEmpty(SQLite::Connection& connection, std::string_view tableName); + + // Prepares a statement for replacing all of the map data to point to a single manifest for a given id. + SQLite::Statement OneToManyTablePrepareMapDataFoldingStatement(const SQLite::Connection& connection, std::string_view tableName); } + // Gets the manifest id that the PrepareMapDataFoldingStatement will fold to for the given manifest id. + std::optional OneToManyTableGetMapDataFoldingManifestTargetId(const SQLite::Connection& connection, SQLite::rowid_t manifestId); + // A table that represents a value that is 1:N with a primary entry. template struct OneToManyTable @@ -74,15 +94,9 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 } // Creates the table with named indices. - static void Create(SQLite::Connection& connection) - { - details::CreateOneToManyTable(connection, true, TableInfo::TableName(), TableInfo::ValueName()); - } - - // Creates the table with standard primary keys. - static void Create_deprecated(SQLite::Connection& connection) + static void Create(SQLite::Connection& connection, OneToManyTableSchema schemaVersion) { - details::CreateOneToManyTable(connection, false, TableInfo::TableName(), TableInfo::ValueName()); + details::CreateOneToManyTable(connection, schemaVersion, TableInfo::TableName(), TableInfo::ValueName()); } // Gets all values associated with the given manifest id. @@ -110,15 +124,11 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 } // Removes data that is no longer needed for an index that is to be published. - static void PrepareForPackaging(SQLite::Connection& connection, bool preserveManifestIndex, bool preserveValuesIndex = false) + // Preserving the manifest index will improve the efficiency of finding the values associated with a manifest. + // Preserving the values index will improve searching when it is primarily done by equality. + static void PrepareForPackaging(SQLite::Connection& connection, OneToManyTableSchema schemaVersion, bool preserveManifestIndex, bool preserveValuesIndex) { - details::OneToManyTablePrepareForPackaging(connection, TableInfo::TableName(), true, preserveManifestIndex, preserveValuesIndex); - } - - // Removes data that is no longer needed for an index that is to be published. - static void PrepareForPackaging_deprecated(SQLite::Connection& connection) - { - details::OneToManyTablePrepareForPackaging(connection, TableInfo::TableName(), false, false, false); + details::OneToManyTablePrepareForPackaging(connection, TableInfo::TableName(), schemaVersion, preserveManifestIndex, preserveValuesIndex); } // Checks the consistency of the index to ensure that every referenced row exists. @@ -133,5 +143,11 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 { return details::OneToManyTableIsEmpty(connection, TableInfo::TableName()); } + + // Prepares a statement for replacing all of the map data to point to a single manifest for a given id. + static SQLite::Statement PrepareMapDataFoldingStatement(const SQLite::Connection& connection) + { + return details::OneToManyTablePrepareMapDataFoldingStatement(connection, TableInfo::TableName()); + } }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToOneTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToOneTable.h index 02594c758b..1e4989f98e 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToOneTable.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToOneTable.h @@ -113,6 +113,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 } // Removes data that is no longer needed for an index that is to be published. + // Preserving the values index will improve searching when it is primarily done by equality. static void PrepareForPackaging(SQLite::Connection& connection, bool preserveValuesIndex = false) { details::OneToOneTablePrepareForPackaging(connection, TableInfo::TableName(), true, preserveValuesIndex); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_1/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_1/Interface.h index 2e8bc227fc..3178ccc7e6 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_1/Interface.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_1/Interface.h @@ -28,6 +28,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_1 protected: std::unique_ptr CreateSearchResultsTable(const SQLite::Connection& connection) const override; void PerformQuerySearch(V1_0::SearchResultsTable& resultsTable, const RequestMatch& query) const override; + V1_0::OneToManyTableSchema GetOneToManyTableSchema() const override; + virtual SearchResult SearchInternal(const SQLite::Connection& connection, SearchRequest& request) const; virtual void PrepareForPackaging(SQLite::Connection& connection, bool vacuum); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_1/Interface_1_1.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_1/Interface_1_1.cpp index 0e2a20324f..62be2ba538 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_1/Interface_1_1.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_1/Interface_1_1.cpp @@ -51,10 +51,10 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_1 { V1_0::PathPartTable::ValueName(), false, WI_IsFlagClear(options, CreateOptions::SupportPathless) } }); - V1_0::TagsTable::Create(connection); - V1_0::CommandsTable::Create(connection); - PackageFamilyNameTable::Create(connection); - ProductCodeTable::Create(connection); + V1_0::TagsTable::Create(connection, GetOneToManyTableSchema()); + V1_0::CommandsTable::Create(connection, GetOneToManyTableSchema()); + PackageFamilyNameTable::Create(connection, GetOneToManyTableSchema()); + ProductCodeTable::Create(connection, GetOneToManyTableSchema()); savepoint.Commit(); } @@ -196,6 +196,11 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_1 V1_0::Interface::PerformQuerySearch(resultsTable, query); } + V1_0::OneToManyTableSchema Interface::GetOneToManyTableSchema() const + { + return V1_0::OneToManyTableSchema::Version_1_1; + } + ISQLiteIndex::SearchResult Interface::SearchInternal(const SQLite::Connection& connection, SearchRequest& request) const { // Update any system reference strings to be folded @@ -239,10 +244,10 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_1 V1_0::PathPartTable::ValueName(), }); - V1_0::TagsTable::PrepareForPackaging(connection, false); - V1_0::CommandsTable::PrepareForPackaging(connection, false); - PackageFamilyNameTable::PrepareForPackaging(connection, true, true); - ProductCodeTable::PrepareForPackaging(connection, true, true); + V1_0::TagsTable::PrepareForPackaging(connection, GetOneToManyTableSchema(), false, false); + V1_0::CommandsTable::PrepareForPackaging(connection, GetOneToManyTableSchema(), false, false); + PackageFamilyNameTable::PrepareForPackaging(connection, GetOneToManyTableSchema(), true, true); + ProductCodeTable::PrepareForPackaging(connection, GetOneToManyTableSchema(), true, true); savepoint.Commit(); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_2/Interface_1_2.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_2/Interface_1_2.cpp index cbd1a2ba7f..218971afd1 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_2/Interface_1_2.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_2/Interface_1_2.cpp @@ -165,8 +165,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_2 // This will mean that one can match cross locale name and publisher, but the chance that this // leads to a confusion between packages is very small. More likely would be intentional attempts // to confuse the correlation, which could be fairly easily carried out even with linked values. - NormalizedPackageNameTable::Create(connection); - NormalizedPackagePublisherTable::Create(connection); + NormalizedPackageNameTable::Create(connection, GetOneToManyTableSchema()); + NormalizedPackagePublisherTable::Create(connection, GetOneToManyTableSchema()); savepoint.Commit(); } @@ -320,8 +320,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_2 V1_1::Interface::PrepareForPackaging(connection, false); - NormalizedPackageNameTable::PrepareForPackaging(connection, true, true); - NormalizedPackagePublisherTable::PrepareForPackaging(connection, true, true); + NormalizedPackageNameTable::PrepareForPackaging(connection, GetOneToManyTableSchema(), true, true); + NormalizedPackagePublisherTable::PrepareForPackaging(connection, GetOneToManyTableSchema(), true, true); savepoint.Commit(); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_6/Interface_1_6.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_6/Interface_1_6.cpp index 0fe4e5f0ed..ac85543934 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_6/Interface_1_6.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_6/Interface_1_6.cpp @@ -24,7 +24,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_6 V1_5::Interface::CreateTables(connection, options); - UpgradeCodeTable::Create(connection); + UpgradeCodeTable::Create(connection, GetOneToManyTableSchema()); savepoint.Commit(); } @@ -142,7 +142,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_6 V1_5::Interface::PrepareForPackaging(connection, false); - UpgradeCodeTable::PrepareForPackaging(connection, true, true); + UpgradeCodeTable::PrepareForPackaging(connection, GetOneToManyTableSchema(), true, true); savepoint.Commit(); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_7/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_7/Interface.h new file mode 100644 index 0000000000..726b1e7512 --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_7/Interface.h @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include "Microsoft/Schema/ISQLiteIndex.h" +#include "Microsoft/Schema/1_6/Interface.h" + +namespace AppInstaller::Repository::Microsoft::Schema::V1_7 +{ + using namespace std::string_view_literals; + + // Interface to this schema version exposed through ISQLiteIndex. + struct Interface : public V1_6::Interface + { + static constexpr std::string_view MapDataFolded_VersionSpecifier = "1.7"sv; + + Interface(Utility::NormalizationVersion normVersion = Utility::NormalizationVersion::Initial); + + // Version 1.0 + Schema::Version GetVersion() const override; + std::vector GetMultiPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionMultiProperty property) const override; + + protected: + void PrepareForPackaging(SQLite::Connection& connection, bool vacuum) override; + V1_0::OneToManyTableSchema GetOneToManyTableSchema() const override; + }; +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_7/Interface_1_7.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_7/Interface_1_7.cpp new file mode 100644 index 0000000000..b0274bf215 --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_7/Interface_1_7.cpp @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Microsoft/Schema/1_7/Interface.h" +#include "Microsoft/Schema/MetadataTable.h" +#include "Microsoft/Schema/1_0/CommandsTable.h" +#include "Microsoft/Schema/1_0/IdTable.h" +#include "Microsoft/Schema/1_0/TagsTable.h" +#include "Microsoft/Schema/1_1/PackageFamilyNameTable.h" +#include "Microsoft/Schema/1_2/NormalizedPackageNameTable.h" +#include "Microsoft/Schema/1_2/NormalizedPackagePublisherTable.h" +#include "Microsoft/Schema/1_6/UpgradeCodeTable.h" + +namespace AppInstaller::Repository::Microsoft::Schema::V1_7 +{ + namespace + { + bool ShouldFoldPropertyLookup(const SQLite::Connection& connection) + { + // Get the metadata indicator that we folded these multi properties. + // If it contains the value for folding these properties in the 1.7 manner, also fold the incoming manifest + // to the same value that it would have been folded to so that all manifest entries will have all of these properties. + std::optional mapDataFolded = MetadataTable::TryGetNamedValue(connection, s_MetadataValueName_MapDataFolded); + + if (mapDataFolded) + { + std::vector foldedSplit = Utility::Split(mapDataFolded.value(), s_MetadataValue_MapDataFolded_Separator); + + for (const std::string& splitValue : foldedSplit) + { + if (splitValue == Interface::MapDataFolded_VersionSpecifier) + { + return true; + } + } + } + + return false; + } + } + + Interface::Interface(Utility::NormalizationVersion normVersion) : V1_6::Interface(normVersion) + { + } + + Schema::Version Interface::GetVersion() const + { + return { 1, 7 }; + } + + std::vector Interface::GetMultiPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionMultiProperty property) const + { + if (property == PackageVersionMultiProperty::PackageFamilyName || + property == PackageVersionMultiProperty::Name || + property == PackageVersionMultiProperty::Publisher || + property == PackageVersionMultiProperty::UpgradeCode) + { + if (ShouldFoldPropertyLookup(connection)) + { + std::optional maximumManifestId = V1_0::OneToManyTableGetMapDataFoldingManifestTargetId(connection, manifestId); + if (maximumManifestId) + { + manifestId = maximumManifestId.value(); + } + } + } + + return V1_6::Interface::GetMultiPropertyByManifestId(connection, manifestId, property); + } + + void Interface::PrepareForPackaging(SQLite::Connection& connection, bool vacuum) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "prepareforpackaging_v1_7"); + + // Remove data that is not particularly interesting per-manifest + std::array dataRemovalStatements{ + V1_0::CommandsTable::PrepareMapDataFoldingStatement(connection), + V1_0::TagsTable::PrepareMapDataFoldingStatement(connection), + V1_1::PackageFamilyNameTable::PrepareMapDataFoldingStatement(connection), + V1_2::NormalizedPackageNameTable::PrepareMapDataFoldingStatement(connection), + V1_2::NormalizedPackagePublisherTable::PrepareMapDataFoldingStatement(connection), + V1_6::UpgradeCodeTable::PrepareMapDataFoldingStatement(connection), + }; + + std::vector allIdentifiers = V1_0::IdTable::GetAllRowIds(connection); + + for (SQLite::rowid_t id : allIdentifiers) + { + for (SQLite::Statement& statement : dataRemovalStatements) + { + statement.Reset(); + statement.Bind(1, id); + + statement.Execute(); + } + } + + MetadataTable::SetNamedValue(connection, s_MetadataValueName_MapDataFolded, MapDataFolded_VersionSpecifier); + + V1_6::Interface::PrepareForPackaging(connection, false); + + savepoint.Commit(); + + if (vacuum) + { + Vacuum(connection); + } + } + + V1_0::OneToManyTableSchema Interface::GetOneToManyTableSchema() const + { + return V1_0::OneToManyTableSchema::Version_1_7; + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.cpp index 6244086154..ef6c66a846 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.cpp @@ -27,16 +27,31 @@ CREATE TABLE [metadata]( create.Execute(); } - SQLite::Statement MetadataTable::GetNamedValueStatement(SQLite::Connection& connection, std::string_view name) + SQLite::Statement MetadataTable::GetNamedValueStatement(const SQLite::Connection& connection, std::string_view name) + { + std::optional result = TryGetNamedValueStatement(connection, name); + THROW_HR_IF(E_NOT_SET, !result); + return std::move(result).value(); + } + + std::optional MetadataTable::TryGetNamedValueStatement(const SQLite::Connection& connection, std::string_view name) { THROW_HR_IF(E_INVALIDARG, name.empty()); + SQLite::Statement result = SQLite::Statement::Create(connection, s_MetadataTableStmt_GetNamedValue); result.Bind(1, name); - THROW_HR_IF(E_NOT_SET, !result.Step()); - return result; + + if (result.Step()) + { + return result; + } + else + { + return std::nullopt; + } } - SQLite::Statement MetadataTable::SetNamedValueStatement(SQLite::Connection& connection, std::string_view name) + SQLite::Statement MetadataTable::SetNamedValueStatement(const SQLite::Connection& connection, std::string_view name) { THROW_HR_IF(E_INVALIDARG, name.empty()); SQLite::Statement result = SQLite::Statement::Create(connection, s_MetadataTableStmt_SetNamedValue); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.h index 5205a270d2..6456932a5a 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.h @@ -16,6 +16,10 @@ namespace AppInstaller::Repository::Microsoft::Schema static constexpr std::string_view s_MetadataValueName_MinorVersion = "minorVersion"sv; static constexpr std::string_view s_MetadataValueName_LastWriteTime = "lastwritetime"sv; + // Version 1.7 + static constexpr std::string_view s_MetadataValueName_MapDataFolded = "mapDataFolded"sv; + static constexpr char s_MetadataValue_MapDataFolded_Separator = ';'; + // The metadata table for the index. // Contains a fixed-schema set of named values that can be used to determine how to read the rest of the index. struct MetadataTable @@ -24,15 +28,31 @@ namespace AppInstaller::Repository::Microsoft::Schema // Gets the named value from the metadata table, interpreting it as the given type. template - static Value GetNamedValue(SQLite::Connection& connection, std::string_view name) + static Value GetNamedValue(const SQLite::Connection& connection, std::string_view name) { SQLite::Statement statement = GetNamedValueStatement(connection, name); return statement.GetColumn(0); } + // Gets the named value from the metadata table, interpreting it as the given type. + // Returns nullopt if the value is not present. + template + static std::optional TryGetNamedValue(const SQLite::Connection& connection, std::string_view name) + { + std::optional statement = TryGetNamedValueStatement(connection, name); + if (statement) + { + return statement->GetColumn(0); + } + else + { + return std::nullopt; + } + } + // Sets the named value into the metadata table. template - static void SetNamedValue(SQLite::Connection& connection, std::string_view name, Value&& v) + static void SetNamedValue(const SQLite::Connection& connection, std::string_view name, Value&& v) { SQLite::Statement statement = SetNamedValueStatement(connection, name); statement.Bind(2, std::forward(v)); @@ -41,9 +61,12 @@ namespace AppInstaller::Repository::Microsoft::Schema private: // Internal function that gets the named value. - static SQLite::Statement GetNamedValueStatement(SQLite::Connection& connection, std::string_view name); + static SQLite::Statement GetNamedValueStatement(const SQLite::Connection& connection, std::string_view name); + + // Internal function that gets the named value, or nullopt if it is not present. + static std::optional TryGetNamedValueStatement(const SQLite::Connection& connection, std::string_view name); // Internal function that sets the named value. - static SQLite::Statement SetNamedValueStatement(SQLite::Connection& connection, std::string_view name); + static SQLite::Statement SetNamedValueStatement(const SQLite::Connection& connection, std::string_view name); }; } diff --git a/src/AppInstallerRepositoryCore/SQLiteStatementBuilder.cpp b/src/AppInstallerRepositoryCore/SQLiteStatementBuilder.cpp index 216acaac33..21eb2d25fe 100644 --- a/src/AppInstallerRepositoryCore/SQLiteStatementBuilder.cpp +++ b/src/AppInstallerRepositoryCore/SQLiteStatementBuilder.cpp @@ -84,6 +84,9 @@ namespace AppInstaller::Repository::SQLite::Builder case Aggregate::Min: out << "MIN"; break; + case Aggregate::Max: + out << "MAX"; + break; default: THROW_HR(E_UNEXPECTED); } @@ -330,9 +333,23 @@ namespace AppInstaller::Repository::SQLite::Builder return *this; } - StatementBuilder& StatementBuilder::Equals(details::unbound_t) + StatementBuilder& StatementBuilder::Equals(details::unbound_t, std::optional index) { - AppendOpAndBinder(Op::Equals); + AppendOpAndBinder(Op::Equals, index); + return *this; + } + + StatementBuilder& StatementBuilder::Equals(std::nullptr_t) + { + // This is almost certainly not what you want. + // In SQL, value = NULL is always false. + // Use StatementBuilder::IsNull instead. + THROW_HR(E_NOTIMPL); + } + + StatementBuilder& StatementBuilder::Equals() + { + m_stream << " ="; return *this; } @@ -366,14 +383,6 @@ namespace AppInstaller::Repository::SQLite::Builder return *this; } - StatementBuilder& StatementBuilder::Equals(std::nullptr_t) - { - // This is almost certainly not what you want. - // In SQL, value = NULL is always false. - // Use StatementBuilder::IsNull instead. - THROW_HR(E_NOTIMPL); - } - StatementBuilder& StatementBuilder::Not() { m_stream << " NOT"; @@ -794,6 +803,24 @@ namespace AppInstaller::Repository::SQLite::Builder return *this; } + StatementBuilder& StatementBuilder::UpdateOrReplace(std::string_view table) + { + OutputOperationAndTable(m_stream, "UPDATE OR REPLACE", table); + return *this; + } + + StatementBuilder& StatementBuilder::UpdateOrReplace(QualifiedTable table) + { + OutputOperationAndTable(m_stream, "UPDATE OR REPLACE", table); + return *this; + } + + StatementBuilder& StatementBuilder::UpdateOrReplace(std::initializer_list table) + { + OutputOperationAndTable(m_stream, "UPDATE OR REPLACE", table); + return *this; + } + StatementBuilder& StatementBuilder::Set() { m_stream << " SET "; @@ -819,6 +846,13 @@ namespace AppInstaller::Repository::SQLite::Builder return *this; } + StatementBuilder& StatementBuilder::WithoutRowID() + { + m_stream << " WITHOUT ROWID"; + return *this; + } + + StatementBuilder& StatementBuilder::As(std::string_view alias) { OutputOperationAndTable(m_stream, " AS", alias); @@ -840,7 +874,7 @@ namespace AppInstaller::Repository::SQLite::Builder Prepare(connection).Execute(); } - int StatementBuilder::AppendOpAndBinder(Op op) + int StatementBuilder::AppendOpAndBinder(Op op, std::optional index) { switch (op) { @@ -860,6 +894,11 @@ namespace AppInstaller::Repository::SQLite::Builder THROW_HR(E_UNEXPECTED); } + if (index) + { + m_stream << index.value(); + } + return m_bindIndex++; } diff --git a/src/AppInstallerRepositoryCore/SQLiteStatementBuilder.h b/src/AppInstallerRepositoryCore/SQLiteStatementBuilder.h index 193728b636..6324e1d5b5 100644 --- a/src/AppInstallerRepositoryCore/SQLiteStatementBuilder.h +++ b/src/AppInstallerRepositoryCore/SQLiteStatementBuilder.h @@ -116,7 +116,8 @@ namespace AppInstaller::Repository::SQLite::Builder // Aggregate functions. enum class Aggregate { - Min + Min, + Max, }; // Helper to mark create an integer primary key for rowid, making it stable across vacuum. @@ -246,8 +247,10 @@ namespace AppInstaller::Repository::SQLite::Builder return IsNull(); } } - StatementBuilder& Equals(details::unbound_t); + // The optional index value can be used to specify the parameter index. + StatementBuilder& Equals(details::unbound_t, std::optional index = {}); StatementBuilder& Equals(std::nullptr_t); + StatementBuilder& Equals(); StatementBuilder& LikeWithEscape(std::string_view value); StatementBuilder& Like(details::unbound_t); @@ -258,8 +261,8 @@ namespace AppInstaller::Repository::SQLite::Builder StatementBuilder& Not(); StatementBuilder& In(); - - //Appends a set of value binders for the In clause. + + // Appends a set of value binders for the In clause. StatementBuilder& In(size_t count); // IsNull(true) means the value is null; IsNull(false) means the value is not null. @@ -394,6 +397,12 @@ namespace AppInstaller::Repository::SQLite::Builder StatementBuilder& Update(QualifiedTable table); StatementBuilder& Update(std::initializer_list table); + // Begin an `update or replace` statement. + // The initializer_list form enables the table name to be constructed from multiple parts. + StatementBuilder& UpdateOrReplace(std::string_view table); + StatementBuilder& UpdateOrReplace(QualifiedTable table); + StatementBuilder& UpdateOrReplace(std::initializer_list table); + // Output the set portion of an update statement. StatementBuilder& Set(); @@ -404,6 +413,9 @@ namespace AppInstaller::Repository::SQLite::Builder StatementBuilder& BeginParenthetical(); StatementBuilder& EndParenthetical(); + // Adds the `without rowid` clause. + StatementBuilder& WithoutRowID(); + // Assign an alias to the previous item. StatementBuilder& As(std::string_view alias); @@ -427,7 +439,8 @@ namespace AppInstaller::Repository::SQLite::Builder }; // Appends given the operation. - int AppendOpAndBinder(Op op); + // The optional index value can be used to specify the parameter index. + int AppendOpAndBinder(Op op, std::optional index = {}); // Appends a set of binders for the values clause of an insert. int AppendValuesAndBinders(size_t count); diff --git a/src/AppInstallerRepositoryCore/pch.h b/src/AppInstallerRepositoryCore/pch.h index df1d971c19..e6ed263750 100644 --- a/src/AppInstallerRepositoryCore/pch.h +++ b/src/AppInstallerRepositoryCore/pch.h @@ -46,6 +46,7 @@ #include #include +#include #include #include #include