Merge pull request #1 from Aaronontheweb/initial-commit

Initial commit
petabridge · Feb 4, 2019 · d712ae4 · d712ae4
2 parents 9c1be12 + d59728b
commit d712ae4
Show file tree

Hide file tree

Showing 118 changed files with 34,812 additions and 0 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,12 @@
+.dockerignore
+.env
+.git
+.gitignore
+.vs
+.vscode
+docker-compose.yml
+docker-compose.*.yml
+*/bin
+*/obj
+!obj/Docker/publish/*
+!obj/Docker/empty/
diff --git a/WebCrawler.sln b/WebCrawler.sln
@@ -0,0 +1,55 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 15
+VisualStudioVersion = 15.0.27130.2027
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebCrawler.Shared", "src\WebCrawler.Shared\WebCrawler.Shared.csproj", "{236B9550-1E56-4893-809E-E2E4E93A8AD5}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebCrawler.Shared.IO", "src\WebCrawler.Shared.IO\WebCrawler.Shared.IO.csproj", "{D3BC07C3-B4E1-47F0-A444-255948A3A0A5}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebCrawler.TrackerService", "src\WebCrawler.TrackerService\WebCrawler.TrackerService.csproj", "{95B49511-2420-4E7A-A52F-15B41146A8BB}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebCrawler.CrawlService", "src\WebCrawler.CrawlService\WebCrawler.CrawlService.csproj", "{E81C11C6-C365-42F0-8430-6DBB53B66C39}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebCrawler.Web", "src\WebCrawler.Web\WebCrawler.Web.csproj", "{297D37F3-5CAA-4B4F-A6EA-0A76AFDDE80C}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Lighthouse", "src\Lighthouse\Lighthouse.csproj", "{7A2C40A8-F6D1-4A28-8FD4-886818D7CB35}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{236B9550-1E56-4893-809E-E2E4E93A8AD5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{236B9550-1E56-4893-809E-E2E4E93A8AD5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{236B9550-1E56-4893-809E-E2E4E93A8AD5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{236B9550-1E56-4893-809E-E2E4E93A8AD5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D3BC07C3-B4E1-47F0-A444-255948A3A0A5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D3BC07C3-B4E1-47F0-A444-255948A3A0A5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D3BC07C3-B4E1-47F0-A444-255948A3A0A5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D3BC07C3-B4E1-47F0-A444-255948A3A0A5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{95B49511-2420-4E7A-A52F-15B41146A8BB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{95B49511-2420-4E7A-A52F-15B41146A8BB}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{95B49511-2420-4E7A-A52F-15B41146A8BB}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{95B49511-2420-4E7A-A52F-15B41146A8BB}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E81C11C6-C365-42F0-8430-6DBB53B66C39}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E81C11C6-C365-42F0-8430-6DBB53B66C39}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E81C11C6-C365-42F0-8430-6DBB53B66C39}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E81C11C6-C365-42F0-8430-6DBB53B66C39}.Release|Any CPU.Build.0 = Release|Any CPU
+		{297D37F3-5CAA-4B4F-A6EA-0A76AFDDE80C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{297D37F3-5CAA-4B4F-A6EA-0A76AFDDE80C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{297D37F3-5CAA-4B4F-A6EA-0A76AFDDE80C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{297D37F3-5CAA-4B4F-A6EA-0A76AFDDE80C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7A2C40A8-F6D1-4A28-8FD4-886818D7CB35}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7A2C40A8-F6D1-4A28-8FD4-886818D7CB35}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7A2C40A8-F6D1-4A28-8FD4-886818D7CB35}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7A2C40A8-F6D1-4A28-8FD4-886818D7CB35}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {31B5BCF5-43CA-472A-A27B-2E79702D69E8}
+	EndGlobalSection
+EndGlobal
diff --git a/docker-compose.ci.build.yml b/docker-compose.ci.build.yml
@@ -0,0 +1,9 @@
+version: '3.4'
+
+services:
+  ci-build:
+    image: microsoft/aspnetcore-build:1.0-2.0
+    volumes:
+      - .:/src
+    working_dir: /src
+    command: /bin/bash -c "dotnet restore ./WebCrawler.sln && dotnet publish ./WebCrawler.sln -c Release -o ./obj/Docker/publish"
diff --git a/docker-compose.dcproj b/docker-compose.dcproj
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="15.0" Sdk="Microsoft.Docker.Sdk">
+  <PropertyGroup Label="Globals">
+    <ProjectVersion>2.1</ProjectVersion>
+    <DockerTargetOS>Linux</DockerTargetOS>
+    <ProjectGuid>188b87e9-7ff0-4488-a340-fe8a62688b27</ProjectGuid>
+    <DockerLaunchBrowser>True</DockerLaunchBrowser>
+    <DockerServiceUrl>http://localhost:{ServicePort}</DockerServiceUrl>
+    <DockerServiceName>webcrawler.web</DockerServiceName>
+  </PropertyGroup>
+  <ItemGroup>
+    <None Include="docker-compose.override.yml">
+      <DependentUpon>docker-compose.yml</DependentUpon>
+    </None>
+    <None Include="docker-compose.yml" />
+    <None Include=".dockerignore" />
+    <None Include="docker-compose.ci.build.yml" />
+  </ItemGroup>
+</Project>
diff --git a/docker-compose.override.yml b/docker-compose.override.yml
@@ -0,0 +1,8 @@
+version: '3.4'
+
+services:
+  webcrawler.web:
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Development
+    ports:
+      - "80"
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -0,0 +1,52 @@
+version: '3.4'
+
+services:
+  lighthouse:
+    image: petabridge/lighthouse:v0.9.2
+    ports:
+      - '9110:9110'
+      - '4053:4053'
+    environment:
+      ACTORSYSTEM: "webcrawler"
+      CLUSTER_IP: lighthouse
+      CLUSTER_PORT: 4053
+      CLUSTER_SEEDS: "akka.tcp://webcrawler@lighthouse:4053"
+
+
+  webcrawler.web:
+    image: webcrawler.web
+    build:
+      context: .
+      dockerfile: src/WebCrawler.Web/Dockerfile
+    ports:
+      - '8080:80'
+    environment:
+      CLUSTER_SEEDS: "akka.tcp://webcrawler@lighthouse:4053"
+      CLUSTER_IP: webcrawler.web
+      CLUSTER_PORT: 0
+    depends_on:
+      - "lighthouse"
+
+  webcrawler.crawlservice:
+    image: webcrawler.crawlservice
+    build:
+      context: .
+      dockerfile: src/WebCrawler.CrawlService/Dockerfile
+    environment:
+      CLUSTER_SEEDS: "akka.tcp://webcrawler@lighthouse:4053"
+      CLUSTER_IP: webcrawler.crawlservice
+      CLUSTER_PORT: 0
+    depends_on:
+      - "lighthouse"
+
+  webcrawler.trackerservice:
+    image: webcrawler.trackerservice
+    build:
+      context: .
+      dockerfile: src/WebCrawler.TrackerService/Dockerfile
+    environment:
+      CLUSTER_SEEDS: "akka.tcp://webcrawler@lighthouse:4053"
+      CLUSTER_IP: webcrawler.trackerservice
+      CLUSTER_PORT: 0
+    depends_on:
+      - "lighthouse"
diff --git a/get-dockerip.sh b/get-dockerip.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+if [ -z "$CLUSTER_IP"]; then
+	host=$(hostname -i)
+	echo "Docker container bound on $host"
+	export CLUSTER_IP="$host"
+else
+	echo "Docker container bound on $CLUSTER_IP"
+fi
+
+exec "$@"
diff --git a/src/Lighthouse/Dockerfile b/src/Lighthouse/Dockerfile
@@ -0,0 +1,25 @@
+FROM microsoft/dotnet:1.1-sdk AS build-env
+WORKDIR /app
+
+ENV ACTORSYSTEM "lighthouse"
+
+# should be a comma-delimited list
+ENV SEEDS "[]"
+
+COPY src/Lighthouse/*.csproj ./
+RUN dotnet restore
+
+COPY src/Lighthouse ./
+RUN dotnet publish -c Release --framework netcoreapp1.1 -o out
+
+FROM microsoft/dotnet:1.1-runtime AS runtime
+WORKDIR /app
+COPY --from=build-env /app/out ./
+COPY --from=build-env /app/get-dockerip.sh ./get-dockerip.sh
+ENTRYPOINT ["/bin/bash","get-dockerip.sh"]
+
+# 9110 - Petabridge.Cmd
+# 4053 - Akka.Cluster
+EXPOSE 9110 4053
+
+CMD ["dotnet", "Lighthouse.dll"]
diff --git a/src/Lighthouse/Lighthouse.csproj b/src/Lighthouse/Lighthouse.csproj
@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <Import Project="..\common.props" />
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFrameworks>$(NetCoreVersion)</TargetFrameworks>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Akka.Cluster" Version="$(AkkaVersion)" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="1.1.2" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Xml" Version="1.1.2" />
+    <PackageReference Include="Petabridge.Cmd.Cluster" Version="$(PetabridgeCmdVersion)" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Update="akka.hocon">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+</Project>
diff --git a/src/Lighthouse/LighthouseHostFactory.cs b/src/Lighthouse/LighthouseHostFactory.cs
@@ -0,0 +1,88 @@
+using System;
+using System.IO;
+using System.Linq;
+using Akka.Actor;
+using Akka.Configuration;
+using ConfigurationException = Akka.Configuration.ConfigurationException;
+
+namespace Lighthouse
+{
+    /// <summary>
+    /// Launcher for the Lighthouse <see cref="ActorSystem"/>
+    /// </summary>
+    public static class LighthouseHostFactory
+    {
+        public static ActorSystem LaunchLighthouse(string ipAddress = null, int? specifiedPort = null, string systemName = null)
+        {
+            systemName = systemName ?? Environment.GetEnvironmentVariable("ACTORSYSTEM")?.Trim();
+            ipAddress = ipAddress ?? Environment.GetEnvironmentVariable("CLUSTER_IP")?.Trim();
+            if (specifiedPort == null)
+            {
+                var envPort = Environment.GetEnvironmentVariable("CLUSTER_PORT")?.Trim();
+                if (!string.IsNullOrEmpty(envPort) && int.TryParse(envPort, out var actualPort))
+                {
+                    specifiedPort = actualPort;
+                }
+            }
+
+            var clusterConfig = ConfigurationFactory.ParseString(File.ReadAllText("akka.hocon"));
+
+            var lighthouseConfig = clusterConfig.GetConfig("lighthouse");
+            if (lighthouseConfig != null && string.IsNullOrEmpty(systemName))
+            {
+                systemName = lighthouseConfig.GetString("actorsystem", systemName);
+            }
+
+            var remoteConfig = clusterConfig.GetConfig("akka.remote");
+
+            if (string.IsNullOrEmpty(ipAddress))
+            {
+                ipAddress = remoteConfig.GetString("dot-netty.tcp.public-hostname") ??
+                            "127.0.0.1"; //localhost as a final default
+            }
+
+            int port = specifiedPort ?? remoteConfig.GetInt("dot-netty.tcp.port");
+
+            if (port == 0) throw new ConfigurationException("Need to specify an explicit port for Lighthouse. Found an undefined port or a port value of 0 in App.config.");
+
+            var selfAddress = $"akka.tcp://{systemName}@{ipAddress}:{port}";
+
+            /*
+             * Sanity check
+             */
+            Console.WriteLine($"[Lighthouse] ActorSystem: {systemName}; IP: {ipAddress}; PORT: {port}");
+            Console.WriteLine("[Lighthouse] Performing pre-boot sanity check. Should be able to parse address [{0}]", selfAddress);
+            selfAddress = new Address("akka.tcp", systemName, ipAddress.Trim(), port).ToString();
+            Console.WriteLine("[Lighthouse] Parse successful.");
+
+            var clusterSeeds = Environment.GetEnvironmentVariable("CLUSTER_SEEDS")?.Trim();
+
+            var seeds = clusterConfig.GetStringList("akka.cluster.seed-nodes");
+            if (!string.IsNullOrEmpty(clusterSeeds))
+            {
+                var tempSeeds = clusterSeeds.Trim('[', ']').Split(',');
+                if (tempSeeds.Any())
+                {
+                    seeds = tempSeeds;
+                }
+            }
+
+
+            if (!seeds.Contains(selfAddress))
+            {
+                seeds.Add(selfAddress);
+            }
+
+            var injectedClusterConfigString = seeds.Aggregate("akka.cluster.seed-nodes = [", (current, seed) => current + (@"""" + seed + @""", "));
+            injectedClusterConfigString += "]";
+
+            var finalConfig = ConfigurationFactory.ParseString(
+                string.Format(@"akka.remote.dot-netty.tcp.public-hostname = {0} 
+akka.remote.dot-netty.tcp.port = {1}", ipAddress, port))
+                .WithFallback(ConfigurationFactory.ParseString(injectedClusterConfigString))
+                .WithFallback(clusterConfig);
+
+            return ActorSystem.Create(systemName, finalConfig);
+        }
+    }
+}
diff --git a/src/Lighthouse/LighthouseService.cs b/src/Lighthouse/LighthouseService.cs
@@ -0,0 +1,61 @@
+// Copyright 2014-2015 Aaron Stannard, Petabridge LLC
+//  
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use 
+// this file except in compliance with the License. You may obtain a copy of the 
+// License at 
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0 
+// 
+// Unless required by applicable law or agreed to in writing, software distributed 
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the 
+// specific language governing permissions and limitations under the License.
+
+using System;
+using System.Threading.Tasks;
+using Akka.Actor;
+using Akka.Cluster;
+using Petabridge.Cmd.Cluster;
+using Petabridge.Cmd.Host;
+
+namespace Lighthouse
+{
+    public class LighthouseService
+    {
+        private readonly string _ipAddress;
+        private readonly int? _port;
+        private readonly string _actorSystemName;
+
+        private ActorSystem _lighthouseSystem;
+
+        public LighthouseService() : this(null, null, null) { }
+
+        public LighthouseService(string ipAddress, int? port, string actorSystemName)
+        {
+            _ipAddress = ipAddress;
+            _port = port;
+            _actorSystemName = actorSystemName;
+        }
+
+        public void Start()
+        {
+            _lighthouseSystem = LighthouseHostFactory.LaunchLighthouse(_ipAddress, _port, _actorSystemName);
+            var pbm = PetabridgeCmd.Get(_lighthouseSystem);
+            pbm.RegisterCommandPalette(ClusterCommands.Instance); // enable cluster management commands
+            pbm.Start();
+        }
+
+        /// <summary>
+        /// Task completes once the Lighthouse <see cref="ActorSystem"/> has terminated.
+        /// </summary>
+        /// <remarks>
+        /// Doesn't actually invoke termination. Need to call <see cref="StopAsync"/> for that.
+        /// </remarks>
+        public Task TerminationHandle => _lighthouseSystem.WhenTerminated;
+
+        public async Task StopAsync()
+        {
+            await CoordinatedShutdown.Get(_lighthouseSystem).Run();
+        }
+    }
+}
diff --git a/src/Lighthouse/Program.NetCore.cs b/src/Lighthouse/Program.NetCore.cs
@@ -0,0 +1,21 @@
+using System;
+
+namespace Lighthouse
+{
+    public partial class Program
+    {
+#if NETCOREAPP1_1
+        public static void Main(string[] args)
+        {
+            var lighthouseService = new LighthouseService();
+            lighthouseService.Start();
+            Console.WriteLine("Press Control + C to terminate.");
+            Console.CancelKeyPress += async (sender, eventArgs) =>
+            {
+                await lighthouseService.StopAsync();
+            };
+            lighthouseService.TerminationHandle.Wait(); 
+        }
+#endif
+    }
+}