diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93b3e6d0d29..23a9bb698f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,16 @@ on: push: branches: [ main, feature/*, hotfix/* ] workflow_dispatch: + inputs: + seed: + description: 'Enter an integer value between 0 and 2147483647.' + type: number + required: true + default: 0 + +env: + seed: ${{ inputs.seed || github.run_number }} + NO_COLOR: true jobs: test: @@ -31,17 +41,24 @@ jobs: shell: bash working-directory: templates - - run: dotnet test -c Release -f net8.0 --no-build --collect:"XPlat Code Coverage" --consoleLoggerParameters:"Summary;Verbosity=Minimal" + - name: Install Chromium headless shell + shell: pwsh + working-directory: src/docfx/bin/Release/net8.0 + run: | + $env:PLAYWRIGHT_NODEJS_PATH = (Get-Command node).Path + ./playwright.ps1 install chromium --only-shell + + - run: dotnet test -c Release -f net8.0 --no-build --collect:"XPlat Code Coverage" --consoleLoggerParameters:"Summary;Verbosity=Minimal" -- --seed ${{ env.seed }} id: test-net80 - - run: dotnet test -c Release -f net9.0 --no-build --collect:"XPlat Code Coverage" --consoleLoggerParameters:"Summary;Verbosity=Minimal" + - run: dotnet test -c Release -f net9.0 --no-build --collect:"XPlat Code Coverage" --consoleLoggerParameters:"Summary;Verbosity=Minimal" -- --seed ${{ env.seed }} if: matrix.os == 'ubuntu-latest' id: test-net90 - run: npm i -g @percy/cli if: matrix.os == 'ubuntu-latest' - - run: percy exec -- dotnet test -c Release -f net8.0 --filter Stage=Percy --no-build --collect:"XPlat Code Coverage" + - run: percy exec -- dotnet test -c Release -f net8.0 --no-build --collect:"XPlat Code Coverage" -- --filter-trait "Stage=Percy" if: matrix.os == 'ubuntu-latest' env: PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }} @@ -49,7 +66,7 @@ jobs: - uses: codecov/codecov-action@v5 if: matrix.os == 'ubuntu-latest' with: - fail_ci_if_error: true + fail_ci_if_error: false - run: echo "DOTNET_DbgEnableMiniDump=1" >> $GITHUB_ENV if: matrix.os == 'ubuntu-latest' @@ -69,8 +86,10 @@ jobs: name: logs-${{ matrix.os }} path: | msbuild.binlog + test/**/TestResults/*.log test/**/TestResults/*.trx test/**/TestResults/*.html + test/**/TestResults/*.ctrf - uses: actions/upload-artifact@v4 if: ${{ failure() && matrix.os == 'ubuntu-latest' }} diff --git a/docfx.sln b/docfx.sln index a5974afe119..2b9f3eb4b6c 100644 --- a/docfx.sln +++ b/docfx.sln @@ -16,6 +16,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{926A0726-B ProjectSection(SolutionItems) = preProject test\Directory.Build.props = test\Directory.Build.props test\Directory.Packages.props = test\Directory.Packages.props + test\xunit.runner.json = test\xunit.runner.json EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "docfx", "src\docfx\docfx.csproj", "{EF53214F-BA98-4026-BEED-CF771865C312}" diff --git a/samples/Directory.Packages.props b/samples/Directory.Packages.props new file mode 100644 index 00000000000..8c119d5413b --- /dev/null +++ b/samples/Directory.Packages.props @@ -0,0 +1,2 @@ + + diff --git a/samples/seed/dotnet/assembly/BuildFromAssembly.csproj b/samples/seed/dotnet/assembly/BuildFromAssembly.csproj index 1707423aa11..e0ca9544ec5 100644 --- a/samples/seed/dotnet/assembly/BuildFromAssembly.csproj +++ b/samples/seed/dotnet/assembly/BuildFromAssembly.csproj @@ -10,7 +10,7 @@ - + all runtime; build; native; contentfiles; analyzers diff --git a/test/Directory.Build.props b/test/Directory.Build.props index 58979e35535..8f776276bea 100644 --- a/test/Directory.Build.props +++ b/test/Directory.Build.props @@ -1,33 +1,80 @@ + + false en + - false - + + + + true - - - + + true + + false + + + true + + + $(TestingPlatformCommandLineArguments) --xunit-info + + + $(TestingPlatformCommandLineArguments) --results-directory "$(MSBuildThisFileDirectory)TestResults" + + + $(TestingPlatformCommandLineArguments) --ignore-exit-code 8 + + + $(TestingPlatformCommandLineArguments) --output Detailed + + + + + + $(TestingPlatformCommandLineArguments) --no-progress + + + $(TestingPlatformCommandLineArguments) --report-xunit-trx --report-xunit-trx-filename TestResults-$(MSBuildProjectName)-$(TargetFramework)-$(RUNNER_OS).trx + $(TestingPlatformCommandLineArguments) --report-xunit-html --report-xunit-html-filename TestResults-$(MSBuildProjectName)-$(TargetFramework)-$(RUNNER_OS).html + $(TestingPlatformCommandLineArguments) --report-ctrf --report-ctrf-filename TestResults-$(MSBuildProjectName)-$(TargetFramework)-$(RUNNER_OS).ctrf + + + + $(MSBuildThisFileDirectory)TestResults $(VSTestLogger);trx%3BLogFileName=TestResults-$(MSBuildProjectName)-$(TargetFramework)-$(RUNNER_OS).trx $(VSTestLogger);html%3BLogFileName=TestResults-$(MSBuildProjectName)-$(TargetFramework)-$(RUNNER_OS).html + + + + + + + + + + - - + + all + runtime; build; native; contentfiles; analyzers + + diff --git a/test/Directory.Packages.props b/test/Directory.Packages.props index 3fca3350206..918209a8dc5 100644 --- a/test/Directory.Packages.props +++ b/test/Directory.Packages.props @@ -1,4 +1,3 @@ - @@ -6,9 +5,9 @@ - - - + + + diff --git a/test/Docfx.Build.Common.Tests/Docfx.Build.Common.Tests.csproj b/test/Docfx.Build.Common.Tests/Docfx.Build.Common.Tests.csproj index 474324b4f13..cf3b83368f6 100644 --- a/test/Docfx.Build.Common.Tests/Docfx.Build.Common.Tests.csproj +++ b/test/Docfx.Build.Common.Tests/Docfx.Build.Common.Tests.csproj @@ -1,4 +1,8 @@ + + Exe + + diff --git a/test/Docfx.Build.ManagedReference.Tests/Docfx.Build.ManagedReference.Tests.csproj b/test/Docfx.Build.ManagedReference.Tests/Docfx.Build.ManagedReference.Tests.csproj index 943a81c22c2..1dca5ab1ccd 100644 --- a/test/Docfx.Build.ManagedReference.Tests/Docfx.Build.ManagedReference.Tests.csproj +++ b/test/Docfx.Build.ManagedReference.Tests/Docfx.Build.ManagedReference.Tests.csproj @@ -1,4 +1,8 @@ + + Exe + + diff --git a/test/Docfx.Build.OverwriteDocuments.Tests/Docfx.Build.OverwriteDocuments.Tests.csproj b/test/Docfx.Build.OverwriteDocuments.Tests/Docfx.Build.OverwriteDocuments.Tests.csproj index ada9e2ab801..822b2ca494c 100644 --- a/test/Docfx.Build.OverwriteDocuments.Tests/Docfx.Build.OverwriteDocuments.Tests.csproj +++ b/test/Docfx.Build.OverwriteDocuments.Tests/Docfx.Build.OverwriteDocuments.Tests.csproj @@ -1,7 +1,12 @@ + + Exe + + + diff --git a/test/Docfx.Build.RestApi.Tests/Docfx.Build.RestApi.Tests.csproj b/test/Docfx.Build.RestApi.Tests/Docfx.Build.RestApi.Tests.csproj index 6632e67a6a9..4d288bd43de 100644 --- a/test/Docfx.Build.RestApi.Tests/Docfx.Build.RestApi.Tests.csproj +++ b/test/Docfx.Build.RestApi.Tests/Docfx.Build.RestApi.Tests.csproj @@ -1,4 +1,8 @@ + + Exe + + diff --git a/test/Docfx.Build.RestApi.WithPlugins.Tests/Docfx.Build.RestApi.WithPlugins.Tests.csproj b/test/Docfx.Build.RestApi.WithPlugins.Tests/Docfx.Build.RestApi.WithPlugins.Tests.csproj index ac834570ef9..5557482545f 100644 --- a/test/Docfx.Build.RestApi.WithPlugins.Tests/Docfx.Build.RestApi.WithPlugins.Tests.csproj +++ b/test/Docfx.Build.RestApi.WithPlugins.Tests/Docfx.Build.RestApi.WithPlugins.Tests.csproj @@ -1,4 +1,8 @@ + + Exe + + diff --git a/test/Docfx.Build.SchemaDriven.Tests/Docfx.Build.SchemaDriven.Tests.csproj b/test/Docfx.Build.SchemaDriven.Tests/Docfx.Build.SchemaDriven.Tests.csproj index 4ed255dbd06..6307460d93d 100644 --- a/test/Docfx.Build.SchemaDriven.Tests/Docfx.Build.SchemaDriven.Tests.csproj +++ b/test/Docfx.Build.SchemaDriven.Tests/Docfx.Build.SchemaDriven.Tests.csproj @@ -1,4 +1,8 @@ + + Exe + + diff --git a/test/Docfx.Build.Tests/Docfx.Build.Tests.csproj b/test/Docfx.Build.Tests/Docfx.Build.Tests.csproj index 61c730cef26..92b71aea685 100644 --- a/test/Docfx.Build.Tests/Docfx.Build.Tests.csproj +++ b/test/Docfx.Build.Tests/Docfx.Build.Tests.csproj @@ -1,4 +1,8 @@ + + Exe + + diff --git a/test/Docfx.Build.Tests/ExtractSearchIndexFromHtmlTest.cs b/test/Docfx.Build.Tests/ExtractSearchIndexFromHtmlTest.cs index 41b97201b79..704db079818 100644 --- a/test/Docfx.Build.Tests/ExtractSearchIndexFromHtmlTest.cs +++ b/test/Docfx.Build.Tests/ExtractSearchIndexFromHtmlTest.cs @@ -219,7 +219,7 @@ This is article title manifest.Files.Add(manifestItem); // process the fake manifest, using tempTestFolder as the output folder - _extractor.Process(manifest, tempTestFolder); + _extractor.Process(manifest, tempTestFolder, TestContext.Current.CancellationToken); var expectedIndexJSON = @"{ ""index.html"": { diff --git a/test/Docfx.Build.Tests/PostProcessors/SitemapGeneratorTests.cs b/test/Docfx.Build.Tests/PostProcessors/SitemapGeneratorTests.cs index 4e8658a653b..e337534f446 100644 --- a/test/Docfx.Build.Tests/PostProcessors/SitemapGeneratorTests.cs +++ b/test/Docfx.Build.Tests/PostProcessors/SitemapGeneratorTests.cs @@ -5,7 +5,6 @@ using Docfx.Plugins; using Docfx.Tests.Common; using Xunit; -using Xunit.Abstractions; using DocumentType = Docfx.DataContracts.Common.Constants.DocumentType; namespace Docfx.Build.Engine.Tests; @@ -13,13 +12,6 @@ namespace Docfx.Build.Engine.Tests; [Collection("docfx STA")] public class SitemapGeneratorTests : TestBase { - private readonly ITestOutputHelper _output; - - public SitemapGeneratorTests(ITestOutputHelper output) - { - _output = output; - } - public override void Dispose() { base.Dispose(); @@ -58,7 +50,7 @@ public void TestSitemapGenerator() var sitemapPath = Path.Combine(outputFolder, "sitemap.xml"); // Act - manifest = sitemapGenerator.Process(manifest, outputFolder); + manifest = sitemapGenerator.Process(manifest, outputFolder, TestContext.Current.CancellationToken); // Assert Assert.Equal("https://example.com/", manifest.Sitemap.BaseUrl); diff --git a/test/Docfx.Build.Tests/RemoveDebugInfoTest.cs b/test/Docfx.Build.Tests/RemoveDebugInfoTest.cs index 588155d564b..e86739f3176 100644 --- a/test/Docfx.Build.Tests/RemoveDebugInfoTest.cs +++ b/test/Docfx.Build.Tests/RemoveDebugInfoTest.cs @@ -43,7 +43,7 @@ public void TestBasicFeature() new HtmlPostProcessor { Handlers = { new RemoveDebugInfo() } - }.Process(manifest, _outputFolder); + }.Process(manifest, _outputFolder, TestContext.Current.CancellationToken); var actual = File.ReadAllText(Path.Combine(_outputFolder, "a.html")); Assert.Equal("

sectionMicrosoft Bing

", actual); diff --git a/test/Docfx.Build.Tests/ValidateBookmarkTest.cs b/test/Docfx.Build.Tests/ValidateBookmarkTest.cs index 24cc7edf5de..72d155b67df 100644 --- a/test/Docfx.Build.Tests/ValidateBookmarkTest.cs +++ b/test/Docfx.Build.Tests/ValidateBookmarkTest.cs @@ -63,7 +63,7 @@ public void TestBasicFeature() new HtmlPostProcessor { Handlers = { new ValidateBookmark() } - }.Process(manifest, _outputFolder); + }.Process(manifest, _outputFolder, TestContext.Current.CancellationToken); } finally { @@ -107,7 +107,7 @@ public void TestNoCheck() new HtmlPostProcessor { Handlers = { new ValidateBookmark() } - }.Process(manifest, _outputFolder); + }.Process(manifest, _outputFolder, TestContext.Current.CancellationToken); } finally { @@ -150,7 +150,7 @@ public void TestLinkThatContainsNonAsciiChars() new HtmlPostProcessor { Handlers = { new ValidateBookmark() } - }.Process(manifest, _outputFolder); + }.Process(manifest, _outputFolder, TestContext.Current.CancellationToken); } finally { diff --git a/test/Docfx.Build.Tests/XRefArchiveBuilderTest.cs b/test/Docfx.Build.Tests/XRefArchiveBuilderTest.cs index 65314446a06..6a721e57e75 100644 --- a/test/Docfx.Build.Tests/XRefArchiveBuilderTest.cs +++ b/test/Docfx.Build.Tests/XRefArchiveBuilderTest.cs @@ -20,7 +20,7 @@ public async Task TestDownload() // sorted: true // references: [] // ``` - Assert.True(await builder.DownloadAsync(new Uri("http://dotnet.github.io/docfx/xrefmap.yml"), ZipFile)); + Assert.True(await builder.DownloadAsync(new Uri("http://dotnet.github.io/docfx/xrefmap.yml"), ZipFile, TestContext.Current.CancellationToken)); using (var xar = XRefArchive.Open(ZipFile, XRefArchiveMode.Read)) { diff --git a/test/Docfx.Build.Tests/XRefMapDownloaderTest.cs b/test/Docfx.Build.Tests/XRefMapDownloaderTest.cs index 04f21141a85..977eb64fb13 100644 --- a/test/Docfx.Build.Tests/XRefMapDownloaderTest.cs +++ b/test/Docfx.Build.Tests/XRefMapDownloaderTest.cs @@ -12,7 +12,7 @@ public class XRefMapDownloadTest public async Task BaseUrlIsSet() { var downloader = new XRefMapDownloader(); - var xrefs = await downloader.DownloadAsync(new Uri("https://dotnet.github.io/docfx/xrefmap.yml")) as XRefMap; + var xrefs = await downloader.DownloadAsync(new Uri("https://dotnet.github.io/docfx/xrefmap.yml"), TestContext.Current.CancellationToken) as XRefMap; Assert.NotNull(xrefs); Assert.Equal("https://dotnet.github.io/docfx/", xrefs.BaseUrl); } @@ -26,7 +26,7 @@ public async Task ReadLocalXRefMapWithFallback() // Get fallback TestData/xrefmap.yml which contains uid: 'str' var reader = await new XRefCollection(from u in xrefmaps - select new Uri(u, UriKind.RelativeOrAbsolute)).GetReaderAsync(basePath, fallbackFolders); + select new Uri(u, UriKind.RelativeOrAbsolute)).GetReaderAsync(basePath, fallbackFolders, TestContext.Current.CancellationToken); var xrefSpec = reader.Find("str"); Assert.NotNull(xrefSpec); @@ -40,7 +40,7 @@ public async Task ReadLocalXRefMapJsonFileTest() var path = Path.Combine(Directory.GetCurrentDirectory(), "TestData", "xrefmap.json"); var downloader = new XRefMapDownloader(); - var xrefMap = await downloader.DownloadAsync(new Uri(path)) as XRefMap; + var xrefMap = await downloader.DownloadAsync(new Uri(path), TestContext.Current.CancellationToken) as XRefMap; // Assert xrefMap.Should().NotBeNull(); @@ -54,7 +54,7 @@ public async Task ReadLocalXRefMapGZippedJsonFileTest() var path = Path.Combine(Directory.GetCurrentDirectory(), "TestData", "xrefmap.json.gz"); var downloader = new XRefMapDownloader(); - var xrefMap = await downloader.DownloadAsync(new Uri(path)) as XRefMap; + var xrefMap = await downloader.DownloadAsync(new Uri(path), TestContext.Current.CancellationToken) as XRefMap; // Assert xrefMap.Should().NotBeNull(); @@ -68,7 +68,7 @@ public async Task ReadLocalXRefMapGZippedYamlFileTest() var path = Path.Combine(Directory.GetCurrentDirectory(), "TestData", "xrefmap.yml.gz"); var downloader = new XRefMapDownloader(); - var xrefMap = await downloader.DownloadAsync(new Uri(path)) as XRefMap; + var xrefMap = await downloader.DownloadAsync(new Uri(path), TestContext.Current.CancellationToken) as XRefMap; // Assert xrefMap.Should().NotBeNull(); @@ -85,7 +85,7 @@ public async Task ReadRemoteXRefMapYamlFileTest1() var path = "https://horizongir.github.io/ZedGraph/xrefmap.yml"; var downloader = new XRefMapDownloader(); - var xrefMap = await downloader.DownloadAsync(new Uri(path)) as XRefMap; + var xrefMap = await downloader.DownloadAsync(new Uri(path), TestContext.Current.CancellationToken) as XRefMap; // Assert xrefMap.Sorted.Should().BeTrue(); @@ -114,7 +114,7 @@ public async Task ReadRemoteXRefMapJsonFileTest2() var path = "https://normanderwan.github.io/UnityXrefMaps/xrefmap.yml"; var downloader = new XRefMapDownloader(); - var xrefMap = await downloader.DownloadAsync(new Uri(path)) as XRefMap; + var xrefMap = await downloader.DownloadAsync(new Uri(path), TestContext.Current.CancellationToken) as XRefMap; // Assert xrefMap.Sorted.Should().BeTrue(); diff --git a/test/Docfx.Build.UniversalReference.Tests/Docfx.Build.UniversalReference.Tests.csproj b/test/Docfx.Build.UniversalReference.Tests/Docfx.Build.UniversalReference.Tests.csproj index 3aeb509bbd8..237d34c76c3 100644 --- a/test/Docfx.Build.UniversalReference.Tests/Docfx.Build.UniversalReference.Tests.csproj +++ b/test/Docfx.Build.UniversalReference.Tests/Docfx.Build.UniversalReference.Tests.csproj @@ -1,7 +1,7 @@ - - - + + Exe + diff --git a/test/Docfx.Common.Tests/Docfx.Common.Tests.csproj b/test/Docfx.Common.Tests/Docfx.Common.Tests.csproj index 241d5099754..3099e43dd67 100644 --- a/test/Docfx.Common.Tests/Docfx.Common.Tests.csproj +++ b/test/Docfx.Common.Tests/Docfx.Common.Tests.csproj @@ -1,4 +1,8 @@ + + Exe + + diff --git a/test/Docfx.Dotnet.Tests/Docfx.Dotnet.Tests.csproj b/test/Docfx.Dotnet.Tests/Docfx.Dotnet.Tests.csproj index b5042bdb201..4e48bc27346 100644 --- a/test/Docfx.Dotnet.Tests/Docfx.Dotnet.Tests.csproj +++ b/test/Docfx.Dotnet.Tests/Docfx.Dotnet.Tests.csproj @@ -1,4 +1,8 @@ + + Exe + + diff --git a/test/Docfx.Dotnet.Tests/GenerateMetadataFromAssemblyTest.cs b/test/Docfx.Dotnet.Tests/GenerateMetadataFromAssemblyTest.cs index 9ef41b066c9..eceb01ef205 100644 --- a/test/Docfx.Dotnet.Tests/GenerateMetadataFromAssemblyTest.cs +++ b/test/Docfx.Dotnet.Tests/GenerateMetadataFromAssemblyTest.cs @@ -14,7 +14,7 @@ public void TestGenerateMetadataFromAssembly() { { var (compilation, assembly) = CompilationHelper.CreateCompilationFromAssembly("TestData/CatLibrary.dll"); - Assert.Empty(compilation.GetDeclarationDiagnostics()); + Assert.Empty(compilation.GetDeclarationDiagnostics(TestContext.Current.CancellationToken)); var output = assembly.GenerateMetadataItem(compilation); var @class = output.Items[0].Items[2]; @@ -26,7 +26,7 @@ public void TestGenerateMetadataFromAssembly() { var (compilation, assembly) = CompilationHelper.CreateCompilationFromAssembly("TestData/CatLibrary2.dll"); - Assert.Empty(compilation.GetDeclarationDiagnostics()); + Assert.Empty(compilation.GetDeclarationDiagnostics(TestContext.Current.CancellationToken)); var output = assembly.GenerateMetadataItem(compilation); var @class = output.Items[0].Items[0]; @@ -40,7 +40,7 @@ public void TestGenerateMetadataFromAssembly() public void TestGenerateMetadataFromAssemblyWithReferences() { var (compilation, assembly) = CompilationHelper.CreateCompilationFromAssembly("TestData/TupleLibrary.dll"); - Assert.Empty(compilation.GetDeclarationDiagnostics()); + Assert.Empty(compilation.GetDeclarationDiagnostics(TestContext.Current.CancellationToken)); var output = assembly.GenerateMetadataItem(compilation); var @class = output.Items[0].Items[0]; diff --git a/test/Docfx.Glob.Tests/Docfx.Glob.Tests.csproj b/test/Docfx.Glob.Tests/Docfx.Glob.Tests.csproj index 38cbb3cdba2..4723025afbb 100644 --- a/test/Docfx.Glob.Tests/Docfx.Glob.Tests.csproj +++ b/test/Docfx.Glob.Tests/Docfx.Glob.Tests.csproj @@ -1,4 +1,8 @@ + + Exe + + diff --git a/test/Docfx.MarkdigEngine.Extensions.Tests/Docfx.MarkdigEngine.Extensions.Tests.csproj b/test/Docfx.MarkdigEngine.Extensions.Tests/Docfx.MarkdigEngine.Extensions.Tests.csproj index 6bb49f20177..3948569b174 100644 --- a/test/Docfx.MarkdigEngine.Extensions.Tests/Docfx.MarkdigEngine.Extensions.Tests.csproj +++ b/test/Docfx.MarkdigEngine.Extensions.Tests/Docfx.MarkdigEngine.Extensions.Tests.csproj @@ -1,4 +1,8 @@ + + Exe + + diff --git a/test/Docfx.MarkdigEngine.Tests/Docfx.MarkdigEngine.Tests.csproj b/test/Docfx.MarkdigEngine.Tests/Docfx.MarkdigEngine.Tests.csproj index 8c61517ccfb..4135f1d78c8 100644 --- a/test/Docfx.MarkdigEngine.Tests/Docfx.MarkdigEngine.Tests.csproj +++ b/test/Docfx.MarkdigEngine.Tests/Docfx.MarkdigEngine.Tests.csproj @@ -1,4 +1,8 @@ + + Exe + + diff --git a/test/Docfx.Tests.Common/Docfx.Tests.Common.csproj b/test/Docfx.Tests.Common/Docfx.Tests.Common.csproj index 230ab4eeb47..e2e7e0e7ff2 100644 --- a/test/Docfx.Tests.Common/Docfx.Tests.Common.csproj +++ b/test/Docfx.Tests.Common/Docfx.Tests.Common.csproj @@ -1,6 +1,10 @@ + Library false + false + false + false diff --git a/test/docfx.Snapshot.Tests/Attributes/SetBranchNameAttribute.cs b/test/docfx.Snapshot.Tests/Attributes/SetBranchNameAttribute.cs new file mode 100644 index 00000000000..43aef3a6172 --- /dev/null +++ b/test/docfx.Snapshot.Tests/Attributes/SetBranchNameAttribute.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Reflection; +using Xunit.v3; + +namespace Docfx.Tests; + +internal class UseCustomBranchNameAttribute(string branchName) : BeforeAfterTestAttribute +{ + public override void Before(MethodInfo methodUnderTest, IXunitTest test) + { + if (test.TestCase.TestCollection.TestCollectionDisplayName != "docfx STA") + throw new InvalidOperationException(@"UseCustomBranchNameAttribute change global context. Use `[Collection(""docfx STA"")]` to avoid parallel test executions."); + + Environment.SetEnvironmentVariable("DOCFX_SOURCE_BRANCH_NAME", branchName); + } + + public override void After(MethodInfo methodUnderTest, IXunitTest test) + { + Environment.SetEnvironmentVariable("DOCFX_SOURCE_BRANCH_NAME", null); + } +} diff --git a/test/docfx.Snapshot.Tests/PercyTest.cs b/test/docfx.Snapshot.Tests/PercyTest.cs index 9528ebe1ed6..222a7d9a4ba 100644 --- a/test/docfx.Snapshot.Tests/PercyTest.cs +++ b/test/docfx.Snapshot.Tests/PercyTest.cs @@ -43,13 +43,13 @@ static PercyTest() } [PercyFact] + [UseCustomBranchName("main")] public async Task SeedHtml() { var samplePath = $"{s_samplesDir}/seed"; Clean(samplePath); - using var process = Process.Start("dotnet", $"build \"{s_samplesDir}/seed/dotnet/assembly/BuildFromAssembly.csproj\""); - await process.WaitForExitAsync(); + Exec("dotnet", $"build \"{s_samplesDir}/seed/dotnet/assembly/BuildFromAssembly.csproj\""); var docfxPath = Path.GetFullPath(OperatingSystem.IsWindows() ? "docfx.exe" : "docfx"); Assert.Equal(0, Exec(docfxPath, $"metadata {samplePath}/docfx.json")); @@ -115,13 +115,14 @@ private static async Task PercySnapshot(IPage page, string name) private static int Exec(string filename, string args, string workingDirectory = null) { - var psi = new ProcessStartInfo(filename, args); - psi.EnvironmentVariables.Add("DOCFX_SOURCE_BRANCH_NAME", "main"); - if (workingDirectory != null) - psi.WorkingDirectory = Path.GetFullPath(workingDirectory); - using var process = Process.Start(psi); - process.WaitForExit(); - return process.ExitCode; + var execTask = ProcessHelper.ExecAsync( + filename, + args, + workingDirectory, + environmentVariables: [], + TestContext.Current.CancellationToken); + + return execTask.GetAwaiter().GetResult(); } private static void Clean(string samplePath) diff --git a/test/docfx.Snapshot.Tests/SamplesTest.cs b/test/docfx.Snapshot.Tests/SamplesTest.cs index 1e3d32aefc3..4245a1af873 100644 --- a/test/docfx.Snapshot.Tests/SamplesTest.cs +++ b/test/docfx.Snapshot.Tests/SamplesTest.cs @@ -48,17 +48,16 @@ public SamplesFactAttribute() } [SamplesFact] + [UseCustomBranchName("main")] public async Task Seed() { var samplePath = $"{s_samplesDir}/seed"; Clean(samplePath); - using var process = Process.Start("dotnet", $"build \"{s_samplesDir}/seed/dotnet/assembly/BuildFromAssembly.csproj\""); - await process.WaitForExitAsync(); + Exec("dotnet", $"build \"{s_samplesDir}/seed/dotnet/assembly/BuildFromAssembly.csproj\""); if (Debugger.IsAttached) { - Environment.SetEnvironmentVariable("DOCFX_SOURCE_BRANCH_NAME", "main"); Assert.Equal(0, Program.Main([$"{samplePath}/docfx.json"])); } else @@ -114,6 +113,7 @@ object ToBookmarks(IEnumerable nodes) } [SamplesFact] + [UseCustomBranchName("main")] public async Task SeedMarkdown() { var samplePath = $"{s_samplesDir}/seed"; @@ -126,39 +126,33 @@ public async Task SeedMarkdown() } [SamplesFact] + [UseCustomBranchName("main")] public async Task CSharp() { var samplePath = $"{s_samplesDir}/csharp"; Clean(samplePath); - Environment.SetEnvironmentVariable("DOCFX_SOURCE_BRANCH_NAME", "main"); - - try - { - await DotnetApiCatalog.GenerateManagedReferenceYamlFiles($"{samplePath}/docfx.json"); - await Docset.Build($"{samplePath}/docfx.json"); - } - finally - { - Environment.SetEnvironmentVariable("DOCFX_SOURCE_BRANCH_NAME", null); - } + await DotnetApiCatalog.GenerateManagedReferenceYamlFiles($"{samplePath}/docfx.json"); + await Docset.Build($"{samplePath}/docfx.json"); await VerifyDirectory($"{samplePath}/_site", IncludeFile).AutoVerify(includeBuildServer: false); } [SamplesFact] + [UseCustomBranchName("main")] public Task Extensions() { var samplePath = $"{s_samplesDir}/extensions"; Clean(samplePath); + // On some specific environment. `dotnet build` command takes about 15 minutes. + // So adding `-nodeReuse:false` parameter to resolve issue (https://github.com/dotnet/sdk/issues/9452) + // Additionaly use `--no-dependencies` parameter to suppress dependent projects build. #if DEBUG - using var process = Process.Start("dotnet", $"build \"{samplePath}/build\""); - process.WaitForExit(); + Exec("dotnet", $"build \"{samplePath}/build\" -nodereuse:false --no-dependencies"); Assert.Equal(0, Exec("dotnet", "run --no-build --project build", workingDirectory: samplePath)); #else - using var process = Process.Start("dotnet", $"build -c Release \"{samplePath}/build\""); - process.WaitForExit(); + Exec("dotnet", $"build -c Release \"{samplePath}/build\" -nodereuse:false --no-dependencies"); Assert.Equal(0, Exec("dotnet", "run --no-build -c Release --project build", workingDirectory: samplePath)); #endif @@ -167,13 +161,14 @@ public Task Extensions() private static int Exec(string filename, string args, string workingDirectory = null) { - var psi = new ProcessStartInfo(filename, args); - psi.EnvironmentVariables.Add("DOCFX_SOURCE_BRANCH_NAME", "main"); - if (workingDirectory != null) - psi.WorkingDirectory = Path.GetFullPath(workingDirectory); - using var process = Process.Start(psi); - process.WaitForExit(); - return process.ExitCode; + var execTask = ProcessHelper.ExecAsync( + filename, + args, + workingDirectory, + environmentVariables: [], + TestContext.Current.CancellationToken); + + return execTask.GetAwaiter().GetResult(); } private static void Clean(string samplePath) diff --git a/test/docfx.Snapshot.Tests/Utilities/ProcessHelper.cs b/test/docfx.Snapshot.Tests/Utilities/ProcessHelper.cs new file mode 100644 index 00000000000..8c468524067 --- /dev/null +++ b/test/docfx.Snapshot.Tests/Utilities/ProcessHelper.cs @@ -0,0 +1,115 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; + +namespace Docfx.Tests; + +internal static class ProcessHelper +{ + public static async ValueTask ExecAsync( + string filename, + string args, + string workingDirectory = null, + KeyValuePair[] environmentVariables = null, + CancellationToken cancellationToken = default) + { + var psi = new ProcessStartInfo(filename, args) + { + UseShellExecute = false, + CreateNoWindow = true, + ErrorDialog = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + RedirectStandardInput = false, + }; + + if (workingDirectory != null) + psi.WorkingDirectory = Path.GetFullPath(workingDirectory); + + if (environmentVariables != null) + { + foreach (var v in environmentVariables) + psi.EnvironmentVariables.Add(v.Key, v.Value); + } + + using var process = new Process + { + StartInfo = psi, + EnableRaisingEvents = true, + }; + process.OutputDataReceived += OnOutputDataReceived; + process.ErrorDataReceived += OnErrorDataReceived; + + try + { + process.Start(); + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + // On .NET 6 or later. Process.WaitForExitAsync wait for redirected output reads + await process.WaitForExitAsync(cancellationToken); + } + catch (OperationCanceledException) + { + TerminateProcess(process); + throw; + } + catch + { + TerminateProcess(process); + return -1; + } + + CleanupProcess(process); + return process.ExitCode; + } + + private static void CleanupProcess(Process process) + { + // Stop async event processing. (To avoid callback invoked after disposed) + process.CancelOutputRead(); + process.CancelErrorRead(); + + // Remove event handler + process.OutputDataReceived -= OnOutputDataReceived; + process.ErrorDataReceived -= OnErrorDataReceived; + } + + private static void TerminateProcess(Process process) + { + CleanupProcess(process); + + if (process.HasExited) + return; + + try + { + process.Kill(entireProcessTree: true); + } + catch + { + // Ignore exception + } + } + + private static void OnOutputDataReceived(object sender, DataReceivedEventArgs e) + { + var message = e.Data; + if (string.IsNullOrEmpty(message)) + return; + + // Ignore output message. When need to output logs. uncomment following line. + // TestContext.Current.TestOutputHelper.WriteLine(message); + } + + private static void OnErrorDataReceived(object sender, DataReceivedEventArgs e) + { + var message = e.Data; + if (string.IsNullOrEmpty(message)) + return; + + // Ignore output message. When need to output logs. uncomment following line. + // TestContext.Current.TestOutputHelper.WriteLine($"Error: {message}"); + } +} diff --git a/test/docfx.Snapshot.Tests/docfx.Snapshot.Tests.csproj b/test/docfx.Snapshot.Tests/docfx.Snapshot.Tests.csproj index 56c718ed26b..9fc45aa7574 100644 --- a/test/docfx.Snapshot.Tests/docfx.Snapshot.Tests.csproj +++ b/test/docfx.Snapshot.Tests/docfx.Snapshot.Tests.csproj @@ -1,12 +1,13 @@ + Exe net8.0 - + - + diff --git a/test/docfx.Tests/Attributes/UseNullAnsiConsoleAttribute.cs b/test/docfx.Tests/Attributes/UseNullAnsiConsoleAttribute.cs new file mode 100644 index 00000000000..78ee3f97ffe --- /dev/null +++ b/test/docfx.Tests/Attributes/UseNullAnsiConsoleAttribute.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Reflection; +using Spectre.Console; +using Xunit.v3; + +namespace Docfx.Tests; + +/// +/// Custom to temporary suppress output. +/// This attribute is required to run unit tests that calling `Program.Main` with `--help` arguments. +/// +/// This issued is confirmed by using `Visual Studio Version 17.13.0 Preview 2.0` +/// It's expected to be resolved in future releases. +/// +internal class UseNullAnsiConsoleAttribute : BeforeAfterTestAttribute +{ + private static readonly IAnsiConsole NullConsole = AnsiConsole.Create(new AnsiConsoleSettings + { + Out = new AnsiConsoleOutput(StringWriter.Null), + }); + + private IAnsiConsole SavedConsole; + + public override void Before(MethodInfo methodUnderTest, IXunitTest test) + { + if (test.TestCase.TestCollection.TestCollectionDisplayName != "docfx STA") + throw new InvalidOperationException(@"UseNullAnsiConsoleAttribute change global context. Use `[Collection(""docfx STA"")]` to avoid parallel test executions."); + + SavedConsole = AnsiConsole.Console; + AnsiConsole.Console = NullConsole; + } + + public override void After(MethodInfo methodUnderTest, IXunitTest test) + { + AnsiConsole.Console = SavedConsole; + } +} diff --git a/test/docfx.Tests/CommandLineTest.cs b/test/docfx.Tests/CommandLineTest.cs index ad509286aff..1dd4bf22354 100644 --- a/test/docfx.Tests/CommandLineTest.cs +++ b/test/docfx.Tests/CommandLineTest.cs @@ -4,6 +4,7 @@ namespace Docfx.Tests; [Collection("docfx STA")] +[UseNullAnsiConsole] public static class CommandLineTest { [Fact] diff --git a/test/docfx.Tests/JsonSchemaTest.cs b/test/docfx.Tests/JsonSchemaTest.cs index fc4e24847a5..43bec4c851f 100644 --- a/test/docfx.Tests/JsonSchemaTest.cs +++ b/test/docfx.Tests/JsonSchemaTest.cs @@ -6,7 +6,6 @@ using Docfx.DataContracts.Common; using Docfx.Tests.Common; using FluentAssertions; -using Xunit.Abstractions; using YamlDotNet.Serialization; namespace Docfx.Tests; @@ -14,13 +13,6 @@ namespace Docfx.Tests; [Collection("docfx STA")] public class JsonSchemaTest : TestBase { - private readonly ITestOutputHelper output; - - public JsonSchemaTest(ITestOutputHelper output) - { - this.output = output; - } - [Theory] [InlineData("docs/docfx.json")] [InlineData("samples/csharp/docfx.json")] diff --git a/test/docfx.Tests/MetadataCommandTest.cs b/test/docfx.Tests/MetadataCommandTest.cs index e01761fb414..7885b7fcc36 100644 --- a/test/docfx.Tests/MetadataCommandTest.cs +++ b/test/docfx.Tests/MetadataCommandTest.cs @@ -22,6 +22,10 @@ public MetadataCommandTest() { _outputFolder = GetRandomFolder(); _projectFolder = GetRandomFolder(); + + // Create empty `Directory.Build.props`. + var propsFilePath = Path.Combine(_projectFolder, "Directory.Build.props"); + File.WriteAllText(propsFilePath, ""); } [Fact] diff --git a/test/docfx.Tests/SerializationTests/TestData/TestDataAttribute.cs b/test/docfx.Tests/SerializationTests/TestData/TestDataAttribute.cs index 1e53e2bc5ec..f2f7e21345f 100644 --- a/test/docfx.Tests/SerializationTests/TestData/TestDataAttribute.cs +++ b/test/docfx.Tests/SerializationTests/TestData/TestDataAttribute.cs @@ -3,12 +3,15 @@ using System.Reflection; using Xunit.Sdk; +using Xunit.v3; namespace docfx.Tests; public class TestDataAttribute : DataAttribute { - public override IEnumerable GetData(MethodInfo testMethod) + public override bool SupportsDiscoveryEnumeration() => true; + + public override ValueTask> GetData(MethodInfo testMethod, DisposalTracker disposalTracker) { var key = GetTestDataKey(); var paths = TestData.GetTestDataFilePaths(key); @@ -28,7 +31,9 @@ public override IEnumerable GetData(MethodInfo testMethod) throw new NotSupportedException($"{className} is not supported."); } - return new TheoryData(paths); + var results = paths.Select(x => new TheoryDataRow(x)).ToArray(); + + return ValueTask.FromResult>(results); } private static string GetTestDataKey() diff --git a/test/docfx.Tests/docfx.Tests.csproj b/test/docfx.Tests/docfx.Tests.csproj index 4dd4811307a..583dade9b41 100644 --- a/test/docfx.Tests/docfx.Tests.csproj +++ b/test/docfx.Tests/docfx.Tests.csproj @@ -1,11 +1,14 @@ - + + Exe + + - + diff --git a/test/xunit.runner.json b/test/xunit.runner.json new file mode 100644 index 00000000000..d22a744e068 --- /dev/null +++ b/test/xunit.runner.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", + "diagnosticMessages": true, + "showLiveOutput": true +}