Restructured everything

.vscode/launch.json

@@ -0,0 +1,28 @@
+{
+   // Use IntelliSense to find out which attributes exist for C# debugging
+   // Use hover for the description of the existing attributes
+   // For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md
+   "version": "0.2.0",
+   "configurations": [
+        {
+            "name": ".NET Core Launch (console)",
+            "type": "coreclr",
+            "request": "launch",
+            "preLaunchTask": "build",
+            // If you have changed target frameworks, make sure to update the program path.
+            "program": "${workspaceFolder}/tests/RobotsTxtMiddleware.Tests/bin/Debug/netcoreapp2.0/RobotsTxtMiddleware.Tests.dll",
+            "args": [],
+            "cwd": "${workspaceFolder}/tests/RobotsTxtMiddleware.Tests",
+            // For more information about the 'console' field, see https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md#console-terminal-window
+            "console": "internalConsole",
+            "stopAtEntry": false,
+            "internalConsoleOptions": "openOnSessionStart"
+        },
+        {
+            "name": ".NET Core Attach",
+            "type": "coreclr",
+            "request": "attach",
+            "processId": "${command:pickProcess}"
+        }
+    ]
+}

.vscode/tasks.json

@@ -0,0 +1,15 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "taskName": "build",
+            "command": "dotnet",
+            "type": "process",
+            "args": [
+                "build",
+                "${workspaceFolder}/tests/RobotsTxtMiddleware.Tests/RobotsTxtMiddleware.Tests.csproj"
+            ],
+            "problemMatcher": "$msCompile"
+        }
+    ]
+}

README.md

@@ -3,7 +3,7 @@ A Robots.txt middleware for ASP.NET Core
 
 ## Usage
 ```
-    app.UseRobotsTxtMiddleware(builder => {
+    app.UseRobotsTxtMiddleware(builder =>
         builder
             .AddSection(section => 
                 section
@@ -17,6 +17,6 @@ A Robots.txt middleware for ASP.NET Core
                     .AddUserAgent("*")
                     .Disallow("/")
                 )
-            .AddSitemap("sitemap.xml");
-    });
+            .AddSitemap("sitemap.xml")
+    );
 ```

RobotsTxtMiddleware.sln

@@ -0,0 +1,56 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 15
+VisualStudioVersion = 15.0.26124.0
+MinimumVisualStudioVersion = 15.0.26124.0
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{F0FB1E81-3C38-4B4F-819E-27D258FF2899}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RobotsTxtMiddleware", "src\RobotsTxtMiddleware\RobotsTxtMiddleware.csproj", "{610B25CA-70BD-4433-89D3-FEF56E583477}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{962DBC53-02D4-46CD-842D-AFC95CEBA59B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RobotsTxtMiddleware.Tests", "tests\RobotsTxtMiddleware.Tests\RobotsTxtMiddleware.Tests.csproj", "{8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Debug|x64 = Debug|x64
+		Debug|x86 = Debug|x86
+		Release|Any CPU = Release|Any CPU
+		Release|x64 = Release|x64
+		Release|x86 = Release|x86
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{610B25CA-70BD-4433-89D3-FEF56E583477}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{610B25CA-70BD-4433-89D3-FEF56E583477}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{610B25CA-70BD-4433-89D3-FEF56E583477}.Debug|x64.ActiveCfg = Debug|x64
+		{610B25CA-70BD-4433-89D3-FEF56E583477}.Debug|x64.Build.0 = Debug|x64
+		{610B25CA-70BD-4433-89D3-FEF56E583477}.Debug|x86.ActiveCfg = Debug|x86
+		{610B25CA-70BD-4433-89D3-FEF56E583477}.Debug|x86.Build.0 = Debug|x86
+		{610B25CA-70BD-4433-89D3-FEF56E583477}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{610B25CA-70BD-4433-89D3-FEF56E583477}.Release|Any CPU.Build.0 = Release|Any CPU
+		{610B25CA-70BD-4433-89D3-FEF56E583477}.Release|x64.ActiveCfg = Release|x64
+		{610B25CA-70BD-4433-89D3-FEF56E583477}.Release|x64.Build.0 = Release|x64
+		{610B25CA-70BD-4433-89D3-FEF56E583477}.Release|x86.ActiveCfg = Release|x86
+		{610B25CA-70BD-4433-89D3-FEF56E583477}.Release|x86.Build.0 = Release|x86
+		{8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Debug|x64.ActiveCfg = Debug|x64
+		{8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Debug|x64.Build.0 = Debug|x64
+		{8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Debug|x86.ActiveCfg = Debug|x86
+		{8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Debug|x86.Build.0 = Debug|x86
+		{8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Release|x64.ActiveCfg = Release|x64
+		{8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Release|x64.Build.0 = Release|x64
+		{8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Release|x86.ActiveCfg = Release|x86
+		{8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE}.Release|x86.Build.0 = Release|x86
+	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{610B25CA-70BD-4433-89D3-FEF56E583477} = {F0FB1E81-3C38-4B4F-819E-27D258FF2899}
+		{8F7993B2-2ACB-46D7-BDE7-3845BFC31BCE} = {962DBC53-02D4-46CD-842D-AFC95CEBA59B}
+	EndGlobalSection
+EndGlobal

RobotsTxtMiddleware.cs → src/RobotsTxtMiddleware/RobotsTxtMiddleware.cs

@@ -17,7 +17,7 @@ public static void UseRobotsTxtMiddleware(this IApplicationBuilder app, Func<Rob
         }
 
         public static void UseRobotsTxtMiddleware(this IApplicationBuilder app, RobotsTxtOptions options) {
-            app.UseMiddleware<RobotsTxtMiddleware>();
+            app.UseMiddleware<RobotsTxtMiddleware>(options);
         }
     }
 
@@ -43,7 +43,7 @@ public async Task Invoke(HttpContext context) {
         private async Task BuildRobotsTxt(HttpContext context) {
             var sb = _options.Build();
 
-            using(var sw = new StreamWriter(context.Request.Body))
+            using(var sw = new StreamWriter(context.Response.Body))
                 await sw.WriteAsync(sb.ToString());
         }
     }
@@ -86,7 +86,7 @@ public SectionBuilder AddUserAgent(string userAgent) {
             }
 
             public SectionBuilder AddComment(string comment) {
-                _section.UserAgents.Add(comment);
+                _section.Comments.Add(comment);
                 return this;
             }
 
@@ -108,23 +108,43 @@ public SectionBuilder Disallow(string path) {
     }
 
     public class RobotsTxtOptions {
+        public RobotsTxtOptions() {
+            Sections = new List<RobotsTxtSection>();
+            SitemapUrls = new List<string>();
+        }
+
         public List<RobotsTxtSection> Sections { get; }
         public List<string> SitemapUrls { get; }
 
         internal StringBuilder Build() {
             var builder = new StringBuilder();
 
+            foreach(var section in Sections) {
+                section.Build(builder);
+                builder.AppendLine();
+            }
+
+            foreach(var url in SitemapUrls) {
+                builder.AppendLine("Sitemap: " + url);
+            }
+
             return builder;
         }
     }
 
     public class RobotsTxtSection {
+        public RobotsTxtSection() {
+            Comments = new List<string>();
+            UserAgents = new List<string>();
+            Rules = new List<RobotsTxtRule>();
+        }
+
         public List<string> Comments;
         public List<string> UserAgents;
         public List<RobotsTxtRule> Rules;
 
         public void Build(StringBuilder builder) {
-            if(UserAgents.Any())
+            if(!UserAgents.Any())
                 return;
 
             foreach(var comment in Comments) {

tests/RobotsTxtMiddleware.Tests/MiddlewareTests.cs

@@ -0,0 +1,59 @@
+using System;
+using System.Diagnostics;
+using System.Net;
+using System.Net.Http;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.TestHost;
+using Xunit;
+
+namespace RobotsTxtMiddleware.Tests {
+    public class MiddlewareTests {
+        private readonly TestServer _server;
+        private readonly HttpClient _client;
+
+        public MiddlewareTests() {
+            _server = new TestServer(new WebHostBuilder().UseStartup<Startup>());
+            _client = _server.CreateClient();
+        }
+
+        [Fact]
+        public async Task RobotsTxtShouldRenderOnCorrectPath() {
+            var response = await _client.GetAsync("/robots.txt");
+            response.EnsureSuccessStatusCode();
+
+            var result = await response.Content.ReadAsStringAsync();
+
+            Assert.Equal("# Allow Googlebot\nUser-agent: Googlebot\nAllow: /\n\n# Disallow the rest\nUser-agent: *\nDisallow: /\n\nSitemap: sitemap.xml\n", result);
+        }
+
+        [Fact]
+        public async Task RobotsTxtShouldFallThroughOnInvalidPath() {
+            var response = await _client.GetAsync("/not-robots.txt");
+
+            Assert.Equal(HttpStatusCode.NotFound, response.StatusCode);
+        }
+    }
+
+    public class Startup {
+        public void Configure(IApplicationBuilder app) {
+            app.UseRobotsTxtMiddleware(builder => 
+                builder
+                    .AddSection(section => 
+                        section
+                            .AddComment("Allow Googlebot")
+                            .AddUserAgent("Googlebot")
+                            .Allow("/")
+                        )
+                    .AddSection(section => 
+                        section
+                            .AddComment("Disallow the rest")
+                            .AddUserAgent("*")
+                            .Disallow("/")
+                        )
+                    .AddSitemap("sitemap.xml")
+            );
+        }
+    }
+}

tests/RobotsTxtMiddleware.Tests/RobotsTxtMiddleware.Tests.csproj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp2.0</TargetFramework>
+
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.3.0-preview-20170628-02" />
+    <PackageReference Include="xunit" Version="2.2.0" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.2.0" />
+    <PackageReference Include="Microsoft.AspNetCore.TestHost" Version="2.0.1" />    
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\RobotsTxtMiddleware\RobotsTxtMiddleware.csproj" />
+  </ItemGroup>
+
+</Project>