From c6d03d0efc93ec30da520539a9faa0f5bc5812b5 Mon Sep 17 00:00:00 2001 From: Karl-Johan Sjogren Date: Thu, 21 Dec 2017 13:12:24 +0100 Subject: [PATCH] Restructured everything --- .vscode/launch.json | 28 +++++++++ .vscode/tasks.json | 15 +++++ README.md | 6 +- RobotsTxtMiddleware.sln | 56 ++++++++++++++++++ .../RobotsTxtMiddleware.cs | 28 +++++++-- .../RobotsTxtMiddleware.csproj | 0 .../MiddlewareTests.cs | 59 +++++++++++++++++++ .../RobotsTxtMiddleware.Tests.csproj | 20 +++++++ 8 files changed, 205 insertions(+), 7 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 .vscode/tasks.json create mode 100644 RobotsTxtMiddleware.sln rename RobotsTxtMiddleware.cs => src/RobotsTxtMiddleware/RobotsTxtMiddleware.cs (84%) rename robots-txt-middleware.csproj => src/RobotsTxtMiddleware/RobotsTxtMiddleware.csproj (100%) create mode 100644 tests/RobotsTxtMiddleware.Tests/MiddlewareTests.cs create mode 100644 tests/RobotsTxtMiddleware.Tests/RobotsTxtMiddleware.Tests.csproj diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..86f55dd --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,28 @@ +{ + // Use IntelliSense to find out which attributes exist for C# debugging + // Use hover for the description of the existing attributes + // For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md + "version": "0.2.0", + "configurations": [ + { + "name": ".NET Core Launch (console)", + "type": "coreclr", + "request": "launch", + "preLaunchTask": "build", + // If you have changed target frameworks, make sure to update the program path. + "program": "${workspaceFolder}/tests/RobotsTxtMiddleware.Tests/bin/Debug/netcoreapp2.0/RobotsTxtMiddleware.Tests.dll", + "args": [], + "cwd": "${workspaceFolder}/tests/RobotsTxtMiddleware.Tests", + // For more information about the 'console' field, see https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md#console-terminal-window + "console": "internalConsole", + "stopAtEntry": false, + "internalConsoleOptions": "openOnSessionStart" + }, + { + "name": ".NET Core Attach", + "type": "coreclr", + "request": "attach", + "processId": "${command:pickProcess}" + } + ] +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..1becabc --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,15 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "taskName": "build", + "command": "dotnet", + "type": "process", + "args": [ + "build", + "${workspaceFolder}/tests/RobotsTxtMiddleware.Tests/RobotsTxtMiddleware.Tests.csproj" + ], + "problemMatcher": "$msCompile" + } + ] +} \ No newline at end of file diff --git a/README.md b/README.md index a88b5e0..e131a95 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ A Robots.txt middleware for ASP.NET Core ## Usage ``` - app.UseRobotsTxtMiddleware(builder => { + app.UseRobotsTxtMiddleware(builder => builder .AddSection(section => section @@ -17,6 +17,6 @@ A Robots.txt middleware for ASP.NET Core .AddUserAgent("*") .Disallow("/") ) - .AddSitemap("sitemap.xml"); - }); + .AddSitemap("sitemap.xml") + ); ``` \ No newline at end of file diff --git a/RobotsTxtMiddleware.sln b/RobotsTxtMiddleware.sln new file mode 100644 index 0000000..9089d01 --- /dev/null +++ b/RobotsTxtMiddleware.sln @@ -0,0 +1,56 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.26124.0 +MinimumVisualStudioVersion = 15.0.26124.0 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{F0FB1E81-3C38-4B4F-819E-27D258FF2899}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RobotsTxtMiddleware", "src\RobotsTxtMiddleware\RobotsTxtMiddleware.csproj", "{610B25CA-70BD-4433-89D3-FEF56E583477}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{962DBC53-02D4-46CD-842D-AFC95CEBA59B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RobotsTxtMiddleware.Tests", "tests\RobotsTxtMiddleware.Tests\RobotsTxtMiddleware.Tests.csproj", "{8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {610B25CA-70BD-4433-89D3-FEF56E583477}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {610B25CA-70BD-4433-89D3-FEF56E583477}.Debug|Any CPU.Build.0 = Debug|Any CPU + {610B25CA-70BD-4433-89D3-FEF56E583477}.Debug|x64.ActiveCfg = Debug|x64 + {610B25CA-70BD-4433-89D3-FEF56E583477}.Debug|x64.Build.0 = Debug|x64 + {610B25CA-70BD-4433-89D3-FEF56E583477}.Debug|x86.ActiveCfg = Debug|x86 + {610B25CA-70BD-4433-89D3-FEF56E583477}.Debug|x86.Build.0 = Debug|x86 + {610B25CA-70BD-4433-89D3-FEF56E583477}.Release|Any CPU.ActiveCfg = Release|Any CPU + {610B25CA-70BD-4433-89D3-FEF56E583477}.Release|Any CPU.Build.0 = Release|Any CPU + {610B25CA-70BD-4433-89D3-FEF56E583477}.Release|x64.ActiveCfg = Release|x64 + {610B25CA-70BD-4433-89D3-FEF56E583477}.Release|x64.Build.0 = Release|x64 + {610B25CA-70BD-4433-89D3-FEF56E583477}.Release|x86.ActiveCfg = Release|x86 + {610B25CA-70BD-4433-89D3-FEF56E583477}.Release|x86.Build.0 = Release|x86 + {8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Debug|x64.ActiveCfg = Debug|x64 + {8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Debug|x64.Build.0 = Debug|x64 + {8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Debug|x86.ActiveCfg = Debug|x86 + {8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Debug|x86.Build.0 = Debug|x86 + {8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Release|Any CPU.Build.0 = Release|Any CPU + {8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Release|x64.ActiveCfg = Release|x64 + {8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Release|x64.Build.0 = Release|x64 + {8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Release|x86.ActiveCfg = Release|x86 + {8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Release|x86.Build.0 = Release|x86 + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {610B25CA-70BD-4433-89D3-FEF56E583477} = {F0FB1E81-3C38-4B4F-819E-27D258FF2899} + {8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE} = {962DBC53-02D4-46CD-842D-AFC95CEBA59B} + EndGlobalSection +EndGlobal diff --git a/RobotsTxtMiddleware.cs b/src/RobotsTxtMiddleware/RobotsTxtMiddleware.cs similarity index 84% rename from RobotsTxtMiddleware.cs rename to src/RobotsTxtMiddleware/RobotsTxtMiddleware.cs index 435186c..3b43d37 100644 --- a/RobotsTxtMiddleware.cs +++ b/src/RobotsTxtMiddleware/RobotsTxtMiddleware.cs @@ -17,7 +17,7 @@ public static void UseRobotsTxtMiddleware(this IApplicationBuilder app, Func(); + app.UseMiddleware(options); } } @@ -43,7 +43,7 @@ public async Task Invoke(HttpContext context) { private async Task BuildRobotsTxt(HttpContext context) { var sb = _options.Build(); - using(var sw = new StreamWriter(context.Request.Body)) + using(var sw = new StreamWriter(context.Response.Body)) await sw.WriteAsync(sb.ToString()); } } @@ -86,7 +86,7 @@ public SectionBuilder AddUserAgent(string userAgent) { } public SectionBuilder AddComment(string comment) { - _section.UserAgents.Add(comment); + _section.Comments.Add(comment); return this; } @@ -108,23 +108,43 @@ public SectionBuilder Disallow(string path) { } public class RobotsTxtOptions { + public RobotsTxtOptions() { + Sections = new List(); + SitemapUrls = new List(); + } + public List Sections { get; } public List SitemapUrls { get; } internal StringBuilder Build() { var builder = new StringBuilder(); + foreach(var section in Sections) { + section.Build(builder); + builder.AppendLine(); + } + + foreach(var url in SitemapUrls) { + builder.AppendLine("Sitemap: " + url); + } + return builder; } } public class RobotsTxtSection { + public RobotsTxtSection() { + Comments = new List(); + UserAgents = new List(); + Rules = new List(); + } + public List Comments; public List UserAgents; public List Rules; public void Build(StringBuilder builder) { - if(UserAgents.Any()) + if(!UserAgents.Any()) return; foreach(var comment in Comments) { diff --git a/robots-txt-middleware.csproj b/src/RobotsTxtMiddleware/RobotsTxtMiddleware.csproj similarity index 100% rename from robots-txt-middleware.csproj rename to src/RobotsTxtMiddleware/RobotsTxtMiddleware.csproj diff --git a/tests/RobotsTxtMiddleware.Tests/MiddlewareTests.cs b/tests/RobotsTxtMiddleware.Tests/MiddlewareTests.cs new file mode 100644 index 0000000..1b9e6ca --- /dev/null +++ b/tests/RobotsTxtMiddleware.Tests/MiddlewareTests.cs @@ -0,0 +1,59 @@ +using System; +using System.Diagnostics; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.TestHost; +using Xunit; + +namespace RobotsTxtMiddleware.Tests { + public class MiddlewareTests { + private readonly TestServer _server; + private readonly HttpClient _client; + + public MiddlewareTests() { + _server = new TestServer(new WebHostBuilder().UseStartup()); + _client = _server.CreateClient(); + } + + [Fact] + public async Task RobotsTxtShouldRenderOnCorrectPath() { + var response = await _client.GetAsync("/robots.txt"); + response.EnsureSuccessStatusCode(); + + var result = await response.Content.ReadAsStringAsync(); + + Assert.Equal("# Allow Googlebot\nUser-agent: Googlebot\nAllow: /\n\n# Disallow the rest\nUser-agent: *\nDisallow: /\n\nSitemap: sitemap.xml\n", result); + } + + [Fact] + public async Task RobotsTxtShouldFallThroughOnInvalidPath() { + var response = await _client.GetAsync("/not-robots.txt"); + + Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); + } + } + + public class Startup { + public void Configure(IApplicationBuilder app) { + app.UseRobotsTxtMiddleware(builder => + builder + .AddSection(section => + section + .AddComment("Allow Googlebot") + .AddUserAgent("Googlebot") + .Allow("/") + ) + .AddSection(section => + section + .AddComment("Disallow the rest") + .AddUserAgent("*") + .Disallow("/") + ) + .AddSitemap("sitemap.xml") + ); + } + } +} diff --git a/tests/RobotsTxtMiddleware.Tests/RobotsTxtMiddleware.Tests.csproj b/tests/RobotsTxtMiddleware.Tests/RobotsTxtMiddleware.Tests.csproj new file mode 100644 index 0000000..e1c7a61 --- /dev/null +++ b/tests/RobotsTxtMiddleware.Tests/RobotsTxtMiddleware.Tests.csproj @@ -0,0 +1,20 @@ + + + + netcoreapp2.0 + + false + + + + + + + + + + + + + +