Merge from master

microsoft · Sep 26, 2023 · da93070 · da93070
2 parents 22e0ec5 + 804c125
commit da93070
Show file tree

Hide file tree

Showing 36 changed files with 815 additions and 183 deletions.
diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt
@@ -103,6 +103,7 @@ DACL
 datetimeoffset
 Dbg
 debian
+dedupe
 deigh
 deleteifnotneeded
 DENYWR
@@ -264,6 +265,7 @@ luffy
 Luffytaro
 maclachlan
 malware
+mapdatafolding
 mapview
 Maxed
 maxvalue
@@ -452,6 +454,7 @@ Syncy
 sysrefcomp
 systemnotsupported
 Tagit
+taskhostw
 TCpp
 tcs
 TEMPDIRECTORY

diff --git a/src/AppInstallerCLITests/SQLiteIndex.cpp b/src/AppInstallerCLITests/SQLiteIndex.cpp
@@ -35,49 +35,53 @@ SQLiteIndex CreateTestIndex(const std::string& filePath, std::optional<Schema::V
     // If no specific version requested, then use generator to run against the last 3 versions.
     if (!version)
     {
-        version = GENERATE(Schema::Version{ 1, 2 }, Schema::Version{ 1, 3 }, Schema::Version{ 1, 4 }, Schema::Version::Latest());
+        Schema::Version latestVersion = SQLiteIndex::GetLatestVersion();
+        if (latestVersion.MajorVersion != 1)
+        {
+            throw std::exception("You added major version 2, figure out how to deal with these tests that do back compat coverage!");
+        }
+
+        // Relies on the fact that min version is already >= 2
+        Schema::Version versionMinus1 = Schema::Version{ 1, latestVersion.MinorVersion - 1 };
+        Schema::Version versionMinus2 = Schema::Version{ 1, latestVersion.MinorVersion - 2 };
+
+        version = GENERATE_COPY(Schema::Version{ versionMinus2 }, Schema::Version{ versionMinus1 }, Schema::Version{ latestVersion });
     }
 
     return SQLiteIndex::CreateNew(filePath, version.value());
 }
 
 Schema::Version TestPrepareForRead(SQLiteIndex& index)
 {
-    if (index.GetVersion() == Schema::Version{ 1, 2 })
+    Schema::Version latestVersion = SQLiteIndex::GetLatestVersion();
+    if (latestVersion.MajorVersion != 1)
     {
-        Schema::Version version = GENERATE(Schema::Version{ 1, 2 });
-
-        if (version != Schema::Version{ 1, 2 })
-        {
-            index.ForceVersion(version);
-            return version;
-        }
+        throw std::exception("You added major version 2, figure out how to deal with these tests that do back compat coverage!");
     }
-    else if (index.GetVersion() == Schema::Version{ 1, 3 })
-    {
-        Schema::Version version = GENERATE(Schema::Version{ 1, 2 }, Schema::Version{ 1, 3 });
 
-        if (version != Schema::Version{ 1, 3 })
-        {
-            index.ForceVersion(version);
-            return version;
-        }
+    // Relies on the fact that min version is already >= 2
+    Schema::Version versionMinus1 = Schema::Version{ 1, latestVersion.MinorVersion - 1 };
+    Schema::Version versionMinus2 = Schema::Version{ 1, latestVersion.MinorVersion - 2 };
+
+    if (index.GetVersion() == versionMinus2)
+    {
+        // Degenerate case where we don't need to do anything
     }
-    else if (index.GetVersion() == Schema::Version{ 1, 4 })
+    else if (index.GetVersion() == versionMinus1)
     {
-        Schema::Version version = GENERATE(Schema::Version{ 1, 2 }, Schema::Version{ 1, 3 }, Schema::Version{ 1, 4 });
+        Schema::Version version = GENERATE_COPY(Schema::Version{ versionMinus2 }, Schema::Version{ versionMinus1 });
 
-        if (version != Schema::Version{ 1, 4 })
+        if (version != versionMinus1)
         {
             index.ForceVersion(version);
             return version;
         }
     }
-    else if (index.GetVersion() == Schema::Version{ 1, 5 })
+    else if (index.GetVersion() == latestVersion)
     {
-        Schema::Version version = GENERATE(Schema::Version{ 1, 2 }, Schema::Version{ 1, 3 }, Schema::Version{ 1, 4 }, Schema::Version{ 1, 5 });
+        Schema::Version version = GENERATE_COPY(Schema::Version{ versionMinus2 }, Schema::Version{ versionMinus1 }, Schema::Version{ latestVersion });
 
-        if (version != Schema::Version{ 1, 5 })
+        if (version != latestVersion)
         {
             index.ForceVersion(version);
             return version;
@@ -321,6 +325,17 @@ bool AreArpVersionsSupported(const SQLiteIndex& index, const Schema::Version& te
     return (index.GetVersion() >= Schema::Version{ 1, 5 } && testVersion >= Schema::Version{ 1, 5 });
 }
 
+bool IsMapDataFoldingSupported(const SQLiteIndex& index, const Schema::Version& testVersion)
+{
+    UNSCOPED_INFO("Index " << index.GetVersion() << " | Test " << testVersion);
+    return (index.GetVersion() >= Schema::Version{ 1, 7 } && testVersion >= Schema::Version{ 1, 7 });
+}
+
+bool IsMapDataFolded(const SQLiteIndex& index)
+{
+    return (index.GetVersion() >= Schema::Version{ 1, 7 });
+}
+
 std::string GetPropertyStringByKey(const SQLiteIndex& index, SQLite::rowid_t id, PackageVersionProperty property, std::string_view version, std::string_view channel)
 {
     auto manifestId = index.GetManifestIdByKey(id, version, channel);
@@ -3187,3 +3202,152 @@ TEST_CASE("SQLiteIndex_CheckConsistency_FindEmbeddedNull", "[sqliteindex]")
 
     REQUIRE(!index.CheckConsistency(true));
 }
+
+TEST_CASE("SQLiteIndex_MapDataFolding_Tags", "[sqliteindex][mapdatafolding]")
+{
+    TempFile tempFile{ "repolibtest_tempdb"s, ".db"s };
+    INFO("Using temporary file named: " << tempFile.GetPath());
+
+    std::string tag1 = "Tag1";
+    std::string tag2 = "Tag2";
+
+    SQLiteIndex index = SearchTestSetup(tempFile, {
+        { "Id", "Name", "Publisher", "Moniker", "Version1", "", { tag1 }, { "Command" }, "Path1", {}, { "PC1" } },
+        { "Id", "Name", "Publisher", "Moniker", "Version2", "", { tag2 }, { "Command" }, "Path2", {}, { "PC2" } },
+        });
+
+    // Apply the map data folding if it is present in the created test index.
+    index.PrepareForPackaging();
+
+    Schema::Version testVersion = TestPrepareForRead(index);
+
+    SearchRequest request1;
+    request1.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::Tag, MatchType::Exact, tag1));
+    auto results1 = index.Search(request1);
+
+    SearchRequest request2;
+    request2.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::Tag, MatchType::Exact, tag2));
+    auto results2 = index.Search(request1);
+
+    REQUIRE(results1.Matches.size() == 1);
+    REQUIRE(results2.Matches.size() == 1);
+    REQUIRE(results1.Matches[0].first == results2.Matches[0].first);
+}
+
+TEST_CASE("SQLiteIndex_MapDataFolding_PFNs", "[sqliteindex][mapdatafolding]")
+{
+    TempFile tempFile{ "repolibtest_tempdb"s, ".db"s };
+    INFO("Using temporary file named: " << tempFile.GetPath());
+
+    std::string pfn1 = "PFN1";
+    std::string pfn2 = "PFN2";
+
+    SQLiteIndex index = SearchTestSetup(tempFile, {
+        { "Id", "Name", "Publisher", "Moniker", "Version1", "", { }, { "Command" }, "Path1", { pfn1 }, { } },
+        { "Id", "Name", "Publisher", "Moniker", "Version2", "", { }, { "Command" }, "Path2", { pfn2 }, { } },
+        });
+
+    // Apply the map data folding if it is present in the created test index.
+    index.PrepareForPackaging();
+
+    Schema::Version testVersion = TestPrepareForRead(index);
+
+    SearchRequest request1;
+    request1.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::PackageFamilyName, MatchType::Exact, pfn1));
+    auto results1 = index.Search(request1);
+
+    SearchRequest request2;
+    request2.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::PackageFamilyName, MatchType::Exact, pfn2));
+    auto results2 = index.Search(request1);
+
+    REQUIRE(results1.Matches.size() == 1);
+    REQUIRE(results2.Matches.size() == 1);
+    REQUIRE(results1.Matches[0].first == results2.Matches[0].first);
+
+    auto versionKeys = index.GetVersionKeysById(results1.Matches[0].first);
+    REQUIRE(versionKeys.size() == 2);
+
+    auto manifestId1 = index.GetManifestIdByKey(results1.Matches[0].first, versionKeys[0].GetVersion().ToString(), versionKeys[0].GetChannel().ToString());
+    auto manifestId2 = index.GetManifestIdByKey(results1.Matches[0].first, versionKeys[1].GetVersion().ToString(), versionKeys[1].GetChannel().ToString());
+
+    REQUIRE(manifestId1.has_value());
+    REQUIRE(manifestId2.has_value());
+
+    auto pfnValues1 = index.GetMultiPropertyByManifestId(manifestId1.value(), PackageVersionMultiProperty::PackageFamilyName);
+    auto pfnValues2 = index.GetMultiPropertyByManifestId(manifestId2.value(), PackageVersionMultiProperty::PackageFamilyName);
+
+    if (IsMapDataFoldingSupported(index, testVersion))
+    {
+        REQUIRE(pfnValues1.size() == 2);
+        REQUIRE(pfnValues2.size() == 2);
+        REQUIRE(pfnValues1[0] != pfnValues1[1]);
+    }
+    else if (IsMapDataFolded(index))
+    {
+        if (manifestId1 > manifestId2)
+        {
+            REQUIRE(pfnValues1.size() == 2);
+            REQUIRE(pfnValues2.size() == 0);
+            REQUIRE(pfnValues1[0] != pfnValues1[1]);
+        }
+        else
+        {
+            REQUIRE(pfnValues1.size() == 0);
+            REQUIRE(pfnValues2.size() == 2);
+            REQUIRE(pfnValues2[0] != pfnValues2[1]);
+        }
+    }
+    else
+    {
+        REQUIRE(pfnValues1.size() == 1);
+        REQUIRE(pfnValues2.size() == 1);
+        REQUIRE(pfnValues1[0] != pfnValues2[0]);
+    }
+}
+
+TEST_CASE("SQLiteIndex_MapDataFolding_ProductCodes", "[sqliteindex][mapdatafolding]")
+{
+    TempFile tempFile{ "repolibtest_tempdb"s, ".db"s };
+    INFO("Using temporary file named: " << tempFile.GetPath());
+
+    std::string pc1 = "PC1";
+    std::string pc2 = "PC2";
+
+    SQLiteIndex index = SearchTestSetup(tempFile, {
+        { "Id", "Name", "Publisher", "Moniker", "Version1", "", { }, { "Command" }, "Path1", { }, { pc1 } },
+        { "Id", "Name", "Publisher", "Moniker", "Version2", "", { }, { "Command" }, "Path2", { }, { pc2 } },
+        });
+
+    // Apply the map data folding if it is present in the created test index.
+    index.PrepareForPackaging();
+
+    Schema::Version testVersion = TestPrepareForRead(index);
+
+    SearchRequest request1;
+    request1.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::ProductCode, MatchType::Exact, pc1));
+    auto results1 = index.Search(request1);
+
+    SearchRequest request2;
+    request2.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::ProductCode, MatchType::Exact, pc2));
+    auto results2 = index.Search(request1);
+
+    REQUIRE(results1.Matches.size() == 1);
+    REQUIRE(results2.Matches.size() == 1);
+    REQUIRE(results1.Matches[0].first == results2.Matches[0].first);
+
+    auto versionKeys = index.GetVersionKeysById(results1.Matches[0].first);
+    REQUIRE(versionKeys.size() == 2);
+
+    auto manifestId1 = index.GetManifestIdByKey(results1.Matches[0].first, versionKeys[0].GetVersion().ToString(), versionKeys[0].GetChannel().ToString());
+    auto manifestId2 = index.GetManifestIdByKey(results1.Matches[0].first, versionKeys[1].GetVersion().ToString(), versionKeys[1].GetChannel().ToString());
+
+    REQUIRE(manifestId1.has_value());
+    REQUIRE(manifestId2.has_value());
+
+    auto pcValues1 = index.GetMultiPropertyByManifestId(manifestId1.value(), PackageVersionMultiProperty::ProductCode);
+    auto pcValues2 = index.GetMultiPropertyByManifestId(manifestId2.value(), PackageVersionMultiProperty::ProductCode);
+
+    REQUIRE(pcValues1.size() == 1);
+    REQUIRE(pcValues2.size() == 1);
+    REQUIRE(pcValues1[0] != pcValues2[0]);
+}
diff --git a/src/AppInstallerCommonCore/AppInstallerTelemetry.cpp b/src/AppInstallerCommonCore/AppInstallerTelemetry.cpp
@@ -173,6 +173,11 @@ namespace AppInstaller::Logging
         m_executionStage = stage;
     }
 
+    void TelemetryTraceLogger::SetUseSummary(bool useSummary) noexcept
+    {
+        m_useSummary = useSummary;
+    }
+
     std::unique_ptr<TelemetryTraceLogger> TelemetryTraceLogger::CreateSubTraceLogger() const
     {
         THROW_HR_IF(HRESULT_FROM_WIN32(ERROR_INVALID_STATE), !this->m_isInitialized);

diff --git a/src/AppInstallerCommonCore/Public/AppInstallerTelemetry.h b/src/AppInstallerCommonCore/Public/AppInstallerTelemetry.h
@@ -168,6 +168,8 @@ namespace AppInstaller::Logging
 
         void SetExecutionStage(uint32_t stage) noexcept;
 
+        void SetUseSummary(bool useSummary) noexcept;
+
         std::unique_ptr<TelemetryTraceLogger> CreateSubTraceLogger() const;
 
         // Logs the failure info.

diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj
@@ -385,6 +385,7 @@
     <ClInclude Include="Microsoft\Schema\1_6\Interface.h" />
     <ClInclude Include="Microsoft\Schema\1_6\SearchResultsTable.h" />
     <ClInclude Include="Microsoft\Schema\1_6\UpgradeCodeTable.h" />
+    <ClInclude Include="Microsoft\Schema\1_7\Interface.h" />
     <ClInclude Include="Microsoft\Schema\IPinningIndex.h" />
     <ClInclude Include="Microsoft\Schema\IPortableIndex.h" />
     <ClInclude Include="Microsoft\Schema\ISQLiteIndex.h" />
@@ -488,6 +489,7 @@
     <ClCompile Include="Microsoft\Schema\1_5\Interface_1_5.cpp" />
     <ClCompile Include="Microsoft\Schema\1_6\Interface_1_6.cpp" />
     <ClCompile Include="Microsoft\Schema\1_6\SearchResultsTable_1_6.cpp" />
+    <ClCompile Include="Microsoft\Schema\1_7\Interface_1_7.cpp" />
     <ClCompile Include="Microsoft\Schema\MetadataTable.cpp" />
     <ClCompile Include="Microsoft\Schema\Pinning_1_0\PinningIndexInterface_1_0.cpp" />
     <ClCompile Include="Microsoft\Schema\Pinning_1_0\PinTable.cpp" />

diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters
@@ -91,6 +91,9 @@
     <Filter Include="Rest\Schema\1_6\Json">
       <UniqueIdentifier>{b2e78f3d-931e-432c-8485-255b1dbc9db7}</UniqueIdentifier>
     </Filter>
+    <Filter Include="Microsoft\Schema\1_7">
+      <UniqueIdentifier>{f610927a-6f1d-42c5-9ad9-b59790091944}</UniqueIdentifier>
+    </Filter>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="pch.h">
@@ -387,6 +390,9 @@
     <ClInclude Include="Rest\Schema\1_6\Interface.h">
       <Filter>Rest\Schema\1_6</Filter>
     </ClInclude>
+    <ClInclude Include="Microsoft\Schema\1_7\Interface.h">
+      <Filter>Microsoft\Schema\1_7</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="pch.cpp">
@@ -614,6 +620,9 @@
     <ClCompile Include="Rest\Schema\1_6\RestInterface_1_6.cpp">
       <Filter>Rest\Schema\1_6</Filter>
     </ClCompile>
+    <ClCompile Include="Microsoft\Schema\1_7\Interface_1_7.cpp">
+      <Filter>Microsoft\Schema\1_7</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <None Include="PropertySheet.props" />