diff --git a/README.md b/README.md index 0a75e3ec..4a6e61c3 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ - [x] Deployment - [x] Durable cluster topology (via ZooKeeper) +- [x] Web UI on scheduler port 8080 - [ ] Support deploying multiple Elasticsearch clusters to single Mesos cluster - [ ] High availability (master, indexer, replica) - [ ] Fault tolerance @@ -91,6 +92,11 @@ $ docker-compose up Now open the browser at http://localhost:5050 to view the Mesos GUI. +NOTE: If you run docker from a VM (boot2docker on OSX), use the ip address assigned to the VM instead of localhost: +``` +docker-machine inspect dev -f "{{.Driver.IPAddress}}" +``` + The Elasticsearch task can be accessed via the slave on port 9200. Find the IP address of the slave: ``` diff --git a/config/findbugs/excludeFilter.xml b/config/findbugs/excludeFilter.xml index c50cf5ca..e000005e 100644 --- a/config/findbugs/excludeFilter.xml +++ b/config/findbugs/excludeFilter.xml @@ -21,5 +21,11 @@ + + + + + + diff --git a/scheduler/build.gradle b/scheduler/build.gradle index 2f3d46ee..8582ccd2 100644 --- a/scheduler/build.gradle +++ b/scheduler/build.gradle @@ -1,5 +1,6 @@ apply plugin: 'application' apply plugin: 'com.bmuschko.docker-remote-api' +apply plugin: 'spring-boot' import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage @@ -11,6 +12,7 @@ buildscript { } dependencies { classpath 'com.bmuschko:gradle-docker-plugin:2.2' + classpath("org.springframework.boot:spring-boot-gradle-plugin:1.2.4.RELEASE") } } repositories { @@ -18,13 +20,28 @@ repositories { url "https://jitpack.io" } } +configurations { + all*.exclude group:"org.springframework.boot", module: "spring-boot-starter-logging" +} dependencies { compile project(':commons') + compile("org.springframework.boot:spring-boot-starter-web:1.2.4.RELEASE") + compile("org.springframework.boot:spring-boot-starter-log4j:1.2.4.RELEASE") compile "commons-cli:commons-cli:1.0" compile "commons-io:commons-io:2.4" - compile "log4j:log4j:1.2.16" + compile "log4j:log4j:1.2.17" + + compile 'org.webjars:angularjs:1.4.1' + compile 'org.webjars.bower:angular-route:1.4.1' + compile 'org.webjars.bower:angular-resource:1.4.1' + compile 'org.webjars:bootstrap:3.3.5' + compile 'org.webjars:font-awesome:4.3.0' + compile 'org.webjars.bower:rdash-ui:1.0.1' + compile 'org.webjars:momentjs:2.10.3' + compile 'org.webjars:angular-moment:0.10.1' + compile 'org.webjars.bower:json-formatter:0.2.7' testCompile 'org.hamcrest:hamcrest-all:1.3' testCompile 'joda-time:joda-time:2.3' diff --git a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Clock.java b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Clock.java index 93edff72..df5592bc 100644 --- a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Clock.java +++ b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Clock.java @@ -1,5 +1,6 @@ package org.apache.mesos.elasticsearch.scheduler; +import java.time.ZonedDateTime; import java.util.Date; /** @@ -11,4 +12,7 @@ public Date now() { return new Date(); } + public ZonedDateTime zonedNow() { + return ZonedDateTime.now(); + } } diff --git a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Configuration.java b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Configuration.java index ed60dc9c..40a02d7c 100644 --- a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Configuration.java +++ b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Configuration.java @@ -12,11 +12,11 @@ */ public class Configuration { - public static final double CPUS = 0.2; + private static final double CPUS = 0.2; - public static final double MEM = 512; + private static final double MEM = 512; - public static final double DISK = 250; + private static final double DISK = 250; private int numberOfHwNodes; @@ -28,6 +28,24 @@ public class Configuration { private String zookeeperUrl; + private int managementApiPort; + + public static double getCpus() { + return CPUS; + } + + public static double getMem() { + return MEM; + } + + public static double getDisk() { + return DISK; + } + + public State getState() { + return state; + } + public void setState(State state) { this.state = state; } @@ -87,4 +105,12 @@ public String getZookeeperServers() { Iterator hostPorts = zookeeperAddresses.stream().map(zk -> zk.getAddress() + ":" + zk.getPort()).iterator(); return StringUtils.join(hostPorts, ","); } + + public void setManagementApiPort(int managementApiPort) { + this.managementApiPort = managementApiPort; + } + + public int getManagementApiPort() { + return managementApiPort; + } } diff --git a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/ElasticsearchScheduler.java b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/ElasticsearchScheduler.java index 91a359bc..5d7d54c5 100644 --- a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/ElasticsearchScheduler.java +++ b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/ElasticsearchScheduler.java @@ -5,11 +5,10 @@ import org.apache.mesos.Protos; import org.apache.mesos.Scheduler; import org.apache.mesos.SchedulerDriver; +import org.apache.mesos.elasticsearch.common.Discovery; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.net.InetSocketAddress; +import java.util.*; import java.util.function.Predicate; /** @@ -24,6 +23,8 @@ public class ElasticsearchScheduler implements Scheduler { private final TaskInfoFactory taskInfoFactory; + Clock clock = new Clock(); + Set tasks = new HashSet<>(); public ElasticsearchScheduler(Configuration configuration, TaskInfoFactory taskInfoFactory) { @@ -31,6 +32,10 @@ public ElasticsearchScheduler(Configuration configuration, TaskInfoFactory taskI this.taskInfoFactory = taskInfoFactory; } + public Set getTasks() { + return tasks; + } + public void run() { LOGGER.info("Starting ElasticSearch on Mesos - [numHwNodes: " + configuration.getNumberOfHwNodes() + ", zk: " + configuration.getZookeeperUrl() + "]"); @@ -88,7 +93,12 @@ public void resourceOffers(SchedulerDriver driver, List offers) { LOGGER.info("Accepted offer: " + offer.getHostname()); Protos.TaskInfo taskInfo = taskInfoFactory.createTask(configuration, offer); driver.launchTasks(Collections.singleton(offer.getId()), Collections.singleton(taskInfo)); - tasks.add(new Task(offer.getHostname(), taskInfo.getTaskId().getValue())); + tasks.add(new Task( + offer.getHostname(), + taskInfo.getTaskId().getValue(), + clock.zonedNow(), + new InetSocketAddress(offer.getHostname(), taskInfo.getDiscovery().getPorts().getPorts(Discovery.CLIENT_PORT_INDEX).getNumber()), + new InetSocketAddress(offer.getHostname(), taskInfo.getDiscovery().getPorts().getPorts(Discovery.TRANSPORT_PORT_INDEX).getNumber()))); } } } diff --git a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Main.java b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Main.java index 3c877046..38a6ae38 100644 --- a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Main.java +++ b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Main.java @@ -3,6 +3,7 @@ import org.apache.commons.cli.*; import org.apache.mesos.elasticsearch.common.zookeeper.model.ZKAddress; import org.apache.mesos.elasticsearch.common.zookeeper.parser.ZKAddressParser; +import org.springframework.boot.builder.SpringApplicationBuilder; import java.util.List; @@ -15,6 +16,8 @@ public class Main { public static final String ZK_URL = "zk"; + public static final String MANAGEMENT_API_PORT = "m"; + private Options options; private Configuration configuration; @@ -23,6 +26,7 @@ public Main() { this.options = new Options(); this.options.addOption(NUMBER_OF_HARDWARE_NODES, "numHardwareNodes", true, "number of hardware nodes"); this.options.addOption(ZK_URL, "Zookeeper URL", true, "Zookeeper urls zk://IP:PORT,IP:PORT,IP:PORT/mesos)"); + this.options.addOption(MANAGEMENT_API_PORT, "StatusPort", true, "TCP port for status interface. Default is 8080"); } public static void main(String[] args) { @@ -34,11 +38,18 @@ public void run(String[] args) { try { parseCommandlineOptions(args); } catch (ParseException | IllegalArgumentException e) { - printUsage(); + printUsageAndExit(); return; } final ElasticsearchScheduler scheduler = new ElasticsearchScheduler(configuration, new TaskInfoFactory()); + + new SpringApplicationBuilder(WebApplication.class) + .initializers(applicationContext -> applicationContext.getBeanFactory().registerSingleton("scheduler", scheduler)) + .initializers(applicationContext -> applicationContext.getBeanFactory().registerSingleton("configuration", configuration)) + .showBanner(false) + .run(args); + scheduler.run(); } @@ -52,7 +63,7 @@ private void parseCommandlineOptions(String[] args) throws ParseException, Illeg String zkUrl = cmd.getOptionValue(ZK_URL); if (numberOfHwNodesString == null || zkUrl == null) { - printUsage(); + printUsageAndExit(); return; } @@ -66,9 +77,10 @@ private void parseCommandlineOptions(String[] args) throws ParseException, Illeg configuration.setState(new State(new ZooKeeperStateInterfaceImpl(configuration.getZookeeperServers()))); } - private void printUsage() { + private void printUsageAndExit() { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(configuration.getFrameworkName(), options); + System.exit(2); } } diff --git a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Resources.java b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Resources.java index 8360481d..5fd12545 100644 --- a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Resources.java +++ b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Resources.java @@ -66,9 +66,9 @@ public static List selectTwoPortsFromRange(List offere } public static List buildFrameworkResources() { - Protos.Resource cpus = Resources.cpus(Configuration.CPUS); - Protos.Resource mem = Resources.mem(Configuration.MEM); - Protos.Resource disk = Resources.disk(Configuration.DISK); + Protos.Resource cpus = Resources.cpus(Configuration.getCpus()); + Protos.Resource mem = Resources.mem(Configuration.getMem()); + Protos.Resource disk = Resources.disk(Configuration.getDisk()); return asList(cpus, mem, disk); } } diff --git a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Task.java b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Task.java index a5b8dc47..e9940009 100644 --- a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Task.java +++ b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Task.java @@ -1,5 +1,8 @@ package org.apache.mesos.elasticsearch.scheduler; +import java.net.InetSocketAddress; +import java.time.ZonedDateTime; + /** * Task on a host. */ @@ -8,10 +11,16 @@ public class Task { private String taskId; private String hostname; + private ZonedDateTime startedAt; + private InetSocketAddress clientAddress; + private InetSocketAddress transportAddress; - public Task(String hostname, String taskId) { + public Task(String hostname, String taskId, ZonedDateTime startedAt, InetSocketAddress clientInterface, InetSocketAddress transportAddress) { this.hostname = hostname; this.taskId = taskId; + this.startedAt = startedAt; + this.clientAddress = clientInterface; + this.transportAddress = transportAddress; } public String getHostname() { @@ -21,4 +30,16 @@ public String getHostname() { public String getTaskId() { return taskId; } + + public ZonedDateTime getStartedAt() { + return startedAt; + } + + public InetSocketAddress getClientAddress() { + return clientAddress; + } + + public InetSocketAddress getTransportAddress() { + return transportAddress; + } } diff --git a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/WebApplication.java b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/WebApplication.java new file mode 100644 index 00000000..9e86594d --- /dev/null +++ b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/WebApplication.java @@ -0,0 +1,12 @@ +package org.apache.mesos.elasticsearch.scheduler; + +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.context.annotation.ComponentScan; + +/** + * + */ +@EnableAutoConfiguration +@ComponentScan +public class WebApplication { +} diff --git a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/controllers/ClusterController.java b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/controllers/ClusterController.java new file mode 100644 index 00000000..5dc0a224 --- /dev/null +++ b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/controllers/ClusterController.java @@ -0,0 +1,108 @@ +package org.apache.mesos.elasticsearch.scheduler.controllers; + +import org.apache.log4j.Logger; +import org.apache.mesos.Protos; +import org.apache.mesos.elasticsearch.scheduler.Configuration; +import org.apache.mesos.elasticsearch.scheduler.ElasticsearchScheduler; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * + */ +@RestController +@RequestMapping("/cluster") +public class ClusterController { + public static final Logger LOGGER = Logger.getLogger(ClusterController.class); + + @Autowired + ElasticsearchScheduler scheduler; + + @Autowired + Configuration configuration; + + @RequestMapping(method = RequestMethod.GET) + public ClusterInfoResponse clusterInfo() { + ClusterInfoResponse response = new ClusterInfoResponse(); + response.name = configuration.getTaskName(); + response.configuration = toMap(configuration); + return response; + } + + private Map toMap(Configuration configuration) { + return Arrays.stream(configuration.getClass().getDeclaredMethods()).filter(this::isGetter).collect(Collectors.toMap(method -> method.getName().substring(3), this::invokeConfigurationGetter)); + } + + private boolean isGetter(Method method) { + return method.getName().startsWith("get") || method.getName().startsWith("is"); + } + + private Object invokeConfigurationGetter(Method method) { + try { + final Object result = method.invoke(configuration); + if (method.getName().toLowerCase().contains("password")) { + return "************"; + } else if (result instanceof Number || result instanceof Boolean || result instanceof String) { + return result; + } else if (result instanceof Protos.FrameworkID) { + return ((Protos.FrameworkID) result).getValue(); + } + return result.toString(); + } catch (Exception e) { + LOGGER.warn("Failed to invoce method", e); + return "--ERROR--"; + } + } + + /** + * HTTP response entity class + */ + public static class ClusterInfoResponse { + public String name; + public Map configuration; + } + + @RequestMapping(value = "/scheduler", method = RequestMethod.GET) + public ClusterSchedulerInfoResponse schedulerInfo() { + final ClusterSchedulerInfoResponse response = new ClusterSchedulerInfoResponse(); + response.docker = dockerMap("image", "tag", "id"); + return response; + } + + /** + * HTTP response entity class + */ + public static class ClusterSchedulerInfoResponse { + public Map docker = new HashMap<>(); + } + + @RequestMapping(value = "/executors", method = RequestMethod.GET) + public ClusterExecutorsInfoResponse executorsInfo() { + final ClusterExecutorsInfoResponse response = new ClusterExecutorsInfoResponse(); + response.docker = dockerMap("image", "tag", "id"); + return response; + } + + /** + * HTTP response entity class + */ + public static class ClusterExecutorsInfoResponse { + public Map docker; + } + + private Map dockerMap(String image, String tag, String id) { + final HashMap map = new HashMap<>(); + map.put("image", image); + map.put("tag", tag); + map.put("id", id); + return map; + } +} diff --git a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/controllers/TasksController.java b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/controllers/TasksController.java new file mode 100644 index 00000000..ed2c9b3d --- /dev/null +++ b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/controllers/TasksController.java @@ -0,0 +1,67 @@ +package org.apache.mesos.elasticsearch.scheduler.controllers; + +import org.apache.mesos.elasticsearch.scheduler.Configuration; +import org.apache.mesos.elasticsearch.scheduler.ElasticsearchScheduler; +import org.apache.mesos.elasticsearch.scheduler.Task; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.net.InetSocketAddress; +import java.time.format.DateTimeFormatter; +import java.util.List; + +import static java.util.stream.Collectors.toList; + +/** + * + */ +@RestController +@RequestMapping("/tasks") +public class TasksController { + + @Autowired + ElasticsearchScheduler scheduler; + + @Autowired + Configuration configuration; + + @RequestMapping + public List getTasks() { + return scheduler.getTasks().stream().map(this::from).collect(toList()); + + } + + private GetTasksResponse from(Task task) { + return new GetTasksResponse( + task.getTaskId(), + configuration.getTaskName(), + configuration.getVersion(), + task.getStartedAt().format(DateTimeFormatter.ISO_OFFSET_DATE_TIME), + toFormattedAddress(task.getClientAddress()), + toFormattedAddress(task.getTransportAddress()), + task.getHostname() + ); + } + + private String toFormattedAddress(InetSocketAddress clientAddress) { + return String.format("%s:%s", clientAddress.getAddress().getHostAddress(), clientAddress.getPort()); + } + + /** + * + */ + public static class GetTasksResponse { + public String id, name, version, startedAt, httpAddress, transportAddress, hostname; + + public GetTasksResponse(String id, String name, String version, String startedAt, String httpAddress, String transportAddress, String hostname) { + this.id = id; + this.name = name; + this.version = version; + this.startedAt = startedAt; + this.httpAddress = httpAddress; + this.transportAddress = transportAddress; + this.hostname = hostname; + } + } +} diff --git a/scheduler/src/main/resources/application.properties b/scheduler/src/main/resources/application.properties new file mode 100644 index 00000000..83f05f70 --- /dev/null +++ b/scheduler/src/main/resources/application.properties @@ -0,0 +1 @@ +spring.jackson.property-naming-strategy=CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES \ No newline at end of file diff --git a/scheduler/src/main/resources/log4j.xml b/scheduler/src/main/resources/log4j.xml index 5108c406..8d7ce89b 100644 --- a/scheduler/src/main/resources/log4j.xml +++ b/scheduler/src/main/resources/log4j.xml @@ -9,7 +9,16 @@ + + + + + + + + + diff --git a/scheduler/src/main/resources/public/app/app.js b/scheduler/src/main/resources/public/app/app.js new file mode 100644 index 00000000..3febec65 --- /dev/null +++ b/scheduler/src/main/resources/public/app/app.js @@ -0,0 +1,22 @@ +var app = angular.module('mesos-es-ui', [ + 'ngRoute', + 'ngResource', + 'angularMoment', + 'jsonFormatter', + 'mesos-es-ui.config', + 'mesos-es-ui.controllers', + 'mesos-es-ui.directives', + 'mesos-es-ui.services' +]); + +app.config(['$routeProvider', function ($routeProvider) { + $routeProvider.when('/', { + templateUrl: 'partials/cluster.html', + controller: 'ClusterController' + }).when('/tasks', { + templateUrl: 'partials/tasks.html', + controller: 'TasksController' + }).otherwise({ + redirectTo: '/' + }); +}]); diff --git a/scheduler/src/main/resources/public/app/config.js b/scheduler/src/main/resources/public/app/config.js new file mode 100644 index 00000000..21e84838 --- /dev/null +++ b/scheduler/src/main/resources/public/app/config.js @@ -0,0 +1,6 @@ +angular.module('mesos-es-ui.config', []). + value('config', { + api: { + port: "8080" + } + }); \ No newline at end of file diff --git a/scheduler/src/main/resources/public/app/controllers.js b/scheduler/src/main/resources/public/app/controllers.js new file mode 100644 index 00000000..928a7480 --- /dev/null +++ b/scheduler/src/main/resources/public/app/controllers.js @@ -0,0 +1,47 @@ +var controllers = angular.module('mesos-es-ui.controllers', []); + +controllers.controller('MainController', function($scope, $interval, config, Tasks) { + $scope.header = ""; + + var mobileView = 992; + $scope.getWidth = function() { + return window.innerWidth; + }; + $scope.$watch($scope.getWidth, function(newValue, oldValue) { + if (newValue >= mobileView) { + $scope.toggle = true; + } else { + $scope.toggle = false; + } + }); + $scope.toggleSidebar = function() { + $scope.toggle = !$scope.toggle; + }; + + window.onresize = function() { + $scope.$apply(); + }; +}); + +controllers.controller('ClusterController', function($scope, config, Cluster) { + $scope.$parent.header = 'Cluster'; + var fetchClusterConfiguration = function() { + Cluster.get(function (data) { + $scope.name = data.name; + $scope.configuration = data.configuration; + }); + }; + fetchClusterConfiguration(); +}); + +controllers.controller('TasksController', function ($scope, $interval, config, Tasks) { + $scope.$parent.header = 'Elasticsearch Tasks'; + var fetchInterval = 5000; + var fetchTasks = function() { + Tasks.query(function (data) { + $scope.tasks = data; + }); + }; + fetchTasks(); + $interval(fetchTasks, fetchInterval); +}); diff --git a/scheduler/src/main/resources/public/app/directives.js b/scheduler/src/main/resources/public/app/directives.js new file mode 100644 index 00000000..059821b4 --- /dev/null +++ b/scheduler/src/main/resources/public/app/directives.js @@ -0,0 +1,51 @@ +var directives = angular.module('mesos-es-ui.directives', []); + +directives.directive('rdLoading', function rdLoading() { + return { + restrict: 'AE', + template: '
' + }; +}); + +directives.directive('rdWidget', function rdLoading() { + return { + transclude: true, + template: '
', + restrict: 'EA' + }; +}); + +directives.directive('rdWidgetHeader', function rdLoading() { + return { + requires: '^rdWidget', + scope: { + title: '@', + icon: '@' + }, + transclude: true, + template: '
{{title}}
', + restrict: 'E' + }; +}); + +directives.directive('rdWidgetBody', function rdLoading() { + return { + requires: '^rdWidget', + scope: { + loading: '@?', + classes: '@?' + }, + transclude: true, + template: '
', + restrict: 'E' + }; +}); + +directives.directive('rdWidgetFooter', function rdLoading() { + return { + requires: '^rdWidget', + transclude: true, + template: '
', + restrict: 'E' + }; +}); diff --git a/scheduler/src/main/resources/public/app/services.js b/scheduler/src/main/resources/public/app/services.js new file mode 100644 index 00000000..08f3717b --- /dev/null +++ b/scheduler/src/main/resources/public/app/services.js @@ -0,0 +1,11 @@ +var services = angular.module('mesos-es-ui.services', []); + +services.factory('Cluster', function($resource, $location, config) { + var URL = $location.protocol() + '://' + $location.host() + ':' + config.api.port + '/cluster'; + return $resource(URL); +}); + +services.factory('Tasks', function($resource, config, $location, config) { + var URL = $location.protocol() + '://' + $location.host() + ':' + config.api.port + '/tasks'; + return $resource(URL); +}); diff --git a/scheduler/src/main/resources/public/images/bg.jpg b/scheduler/src/main/resources/public/images/bg.jpg new file mode 100644 index 00000000..538dbf5c Binary files /dev/null and b/scheduler/src/main/resources/public/images/bg.jpg differ diff --git a/scheduler/src/main/resources/public/images/elasticsearch.png b/scheduler/src/main/resources/public/images/elasticsearch.png new file mode 100644 index 00000000..46d373b5 Binary files /dev/null and b/scheduler/src/main/resources/public/images/elasticsearch.png differ diff --git a/scheduler/src/main/resources/public/images/mesos.jpg b/scheduler/src/main/resources/public/images/mesos.jpg new file mode 100644 index 00000000..12ba843f Binary files /dev/null and b/scheduler/src/main/resources/public/images/mesos.jpg differ diff --git a/scheduler/src/main/resources/public/images/mesos.png b/scheduler/src/main/resources/public/images/mesos.png new file mode 100644 index 00000000..723a3108 Binary files /dev/null and b/scheduler/src/main/resources/public/images/mesos.png differ diff --git a/scheduler/src/main/resources/public/index.html b/scheduler/src/main/resources/public/index.html new file mode 100644 index 00000000..950207cc --- /dev/null +++ b/scheduler/src/main/resources/public/index.html @@ -0,0 +1,66 @@ + + + + Elasticsearch Mesos Framework + + + + + + + + + + + + + + + + + + + + + +
+ +
+
+
+
+ +

{{header}}

+
+
+
+
+
+
+
+
+ + \ No newline at end of file diff --git a/scheduler/src/main/resources/public/partials/cluster.html b/scheduler/src/main/resources/public/partials/cluster.html new file mode 100644 index 00000000..ae134bf7 --- /dev/null +++ b/scheduler/src/main/resources/public/partials/cluster.html @@ -0,0 +1,43 @@ +
+
+ + +
+ +
+
{{configuration.NumberOfHwNodes}}
+
Nodes
+ + +
+
+ + +
+ +
+
{{configuration.Mem}}
+
Memory
+ + +
+
+ + +
+ +
+
{{configuration.Disk}}
+
Disk
+ + +
+
+ +
+
+

Configuration Overview

+ +
+
+ diff --git a/scheduler/src/main/resources/public/partials/tasks.html b/scheduler/src/main/resources/public/partials/tasks.html new file mode 100644 index 00000000..6aa50d6b --- /dev/null +++ b/scheduler/src/main/resources/public/partials/tasks.html @@ -0,0 +1,31 @@ +
+
+
+

No tasks running.

+
+
+ + + + + + + + + + + + + + + + + + + + + +
Task IDNameHostnameIP addressTransport IPStarted
{{task.id}}{{task.name}}{{task.hostname}}{{task.http_address}}{{task.transport_address}}
+
+
+
\ No newline at end of file diff --git a/scheduler/src/main/resources/public/stylesheets/style.css b/scheduler/src/main/resources/public/stylesheets/style.css new file mode 100644 index 00000000..9622d134 --- /dev/null +++ b/scheduler/src/main/resources/public/stylesheets/style.css @@ -0,0 +1,39 @@ +body { + background-image: url("../images/bg.jpg") !important; +} +h1 { + margin: 10px; +} +.content-wrapper { + margin-top: 40px; +} +.mesos-logo img { + max-height: 59px; +} +.task-list-item td { + font-family: monospace; +} +#sidebar-wrapper { + background-color: #000000; +} +ul.sidebar .sidebar-main a { + background-color: #74BD43; +} +ul.sidebar .sidebar-list a { + background: #000000; + color: #ffffff; +} +ul.sidebar .sidebar-list a:hover { + background: #222222; + color: #74BD43; + border-left: 3px solid #74BD43; +} +.sidebar-footer { + background-color: #000000; +} +.sidebar-footer div a { + color: #ffffff; +} +.sidebar-footer div a:hover { + color: #74BD43; +} diff --git a/scheduler/src/main/resources/public/stylesheets/style.less b/scheduler/src/main/resources/public/stylesheets/style.less new file mode 100644 index 00000000..d0c974eb --- /dev/null +++ b/scheduler/src/main/resources/public/stylesheets/style.less @@ -0,0 +1,64 @@ +body { + background-image: url("../images/bg.jpg") !important; +} + +.row { + &.header { +// border-bottom: 1px solid #222222; + } +} + +h1 { + margin: 10px; +} + +.content-wrapper { + margin-top: 40px; +} + +.mesos-logo { + img { + max-height: 59px; + } +} + +.task-list-item { + td { + font-family: monospace; + } +} + +#sidebar-wrapper { + background-color: #000000; +} + +ul.sidebar { + .sidebar-main { + a { + background-color: #74BD43; + } + } + .sidebar-list { + a { + background: #000000; + color: #ffffff; + &:hover { + background: #222222; + color: #74BD43; + border-left: 3px solid #74BD43; + } + } + } +} + +.sidebar-footer { + background-color: #000000; + div { + a { + color: #ffffff; + &:hover { + color: #74BD43; + } + } + } +} diff --git a/scheduler/src/test/java/org/apache/mesos/elasticsearch/scheduler/ElasticsearchSchedulerTest.java b/scheduler/src/test/java/org/apache/mesos/elasticsearch/scheduler/ElasticsearchSchedulerTest.java index de6b3600..fe8afeb0 100644 --- a/scheduler/src/test/java/org/apache/mesos/elasticsearch/scheduler/ElasticsearchSchedulerTest.java +++ b/scheduler/src/test/java/org/apache/mesos/elasticsearch/scheduler/ElasticsearchSchedulerTest.java @@ -2,12 +2,15 @@ import org.apache.mesos.Protos; import org.apache.mesos.SchedulerDriver; +import org.apache.mesos.elasticsearch.common.Discovery; import org.apache.mesos.elasticsearch.scheduler.matcher.RequestMatcher; import org.joda.time.DateTime; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; +import java.net.InetSocketAddress; +import java.time.ZonedDateTime; import java.util.*; import static java.util.Collections.*; @@ -56,6 +59,10 @@ public class ElasticsearchSchedulerTest { private TaskInfoFactory taskInfoFactory; private org.apache.mesos.elasticsearch.scheduler.Configuration configuration; + private ZonedDateTime now = ZonedDateTime.now(); + private InetSocketAddress transportAddress = new InetSocketAddress("localhost", 9300); + private InetSocketAddress clientAddress = new InetSocketAddress("localhost", 9200); + @Before public void before() { @@ -83,12 +90,12 @@ public void before() { public void testRegistered() { scheduler.registered(driver, frameworkID, masterInfo); - Mockito.verify(driver).requestResources(Mockito.argThat(new RequestMatcher().cpus(Configuration.CPUS).mem(Configuration.MEM).disk(Configuration.DISK))); + Mockito.verify(driver).requestResources(Mockito.argThat(new RequestMatcher().cpus(Configuration.getCpus()).mem(Configuration.getMem()).disk(Configuration.getDisk()))); } @Test public void testResourceOffers_isSlaveAlreadyRunningTask() { - scheduler.tasks = asSet(new Task[]{new Task("host1", "1"), new Task("host2", "2")}); + scheduler.tasks = asSet(new Task[]{new Task("host1", "1", now, clientAddress, transportAddress), new Task("host2", "2", now, clientAddress, transportAddress)}); Protos.Offer.Builder offer = newOffer("host1"); @@ -99,7 +106,7 @@ public void testResourceOffers_isSlaveAlreadyRunningTask() { @Test public void testResourceOffers_enoughNodes() { - scheduler.tasks = asSet(new Task[]{new Task("host1", "1"), new Task("host2", "2"), new Task("host3", "3")}); + scheduler.tasks = asSet(new Task[]{new Task("host1", "1", now, clientAddress, transportAddress), new Task("host2", "2", now, clientAddress, transportAddress), new Task("host3", "3", now, clientAddress, transportAddress)}); Protos.Offer.Builder offer = newOffer("host4"); @@ -110,7 +117,7 @@ public void testResourceOffers_enoughNodes() { @Test public void testResourceOffers_noPorts() { - scheduler.tasks = asSet(new Task[]{new Task("host1", "1"), new Task("host2", "2")}); + scheduler.tasks = asSet(new Task[]{new Task("host1", "1", now, clientAddress, transportAddress), new Task("host2", "2", now, clientAddress, transportAddress)}); Protos.Offer.Builder offer = newOffer("host3"); @@ -122,7 +129,7 @@ public void testResourceOffers_noPorts() { @SuppressWarnings("unchecked") @Test public void testResourceOffers_singlePort() { - scheduler.tasks = asSet(new Task[]{new Task("host1", "task1")}); + scheduler.tasks = asSet(new Task[]{new Task("host1", "task1", now, clientAddress, transportAddress)}); Protos.Offer.Builder offerBuilder = newOffer("host3"); offerBuilder.addResources(portRange(9200, 9200)); @@ -144,6 +151,14 @@ public void testResourceOffers_launchTasks() { .setName(configuration.getTaskName()) .setTaskId(Protos.TaskID.newBuilder().setValue(UUID.randomUUID().toString())) .setSlaveId(Protos.SlaveID.newBuilder().setValue(UUID.randomUUID().toString()).build()) + .setDiscovery( + Protos.DiscoveryInfo.newBuilder() + .setVisibility(Protos.DiscoveryInfo.Visibility.EXTERNAL) + .setPorts(Protos.Ports.newBuilder() + .addPorts(Discovery.CLIENT_PORT_INDEX, Protos.Port.newBuilder().setNumber(9200)) + .addPorts(Discovery.TRANSPORT_PORT_INDEX, Protos.Port.newBuilder().setNumber(9300)) + ) + ) .build(); when(taskInfoFactory.createTask(configuration, offerBuilder.build())).thenReturn(taskInfo); @@ -155,9 +170,9 @@ public void testResourceOffers_launchTasks() { private Protos.Offer.Builder newOffer(String hostname) { Protos.Offer.Builder builder = newOfferBuilder(UUID.randomUUID().toString(), hostname, UUID.randomUUID().toString(), frameworkID); - builder.addResources(cpus(Configuration.CPUS)); - builder.addResources(mem(Configuration.MEM)); - builder.addResources(disk(Configuration.DISK)); + builder.addResources(cpus(Configuration.getCpus())); + builder.addResources(mem(Configuration.getMem())); + builder.addResources(disk(Configuration.getDisk())); return builder; } diff --git a/settings.gradle b/settings.gradle index ee93a736..ddea6fa5 100644 --- a/settings.gradle +++ b/settings.gradle @@ -4,4 +4,4 @@ rootProject.name = 'elastic-search' include "commons" include "scheduler" include "executor" -include "system-test" \ No newline at end of file +include "system-test" diff --git a/system-test/src/test/resources/mesos-es/docker-compose.yml b/system-test/src/test/resources/mesos-es/docker-compose.yml index 779ad29c..c4176549 100644 --- a/system-test/src/test/resources/mesos-es/docker-compose.yml +++ b/system-test/src/test/resources/mesos-es/docker-compose.yml @@ -1,3 +1,20 @@ +zookeeper: + image: jplock/zookeeper:3.4.5 + hostname: zookeeper +mesosmaster: + build: master + hostname: mesosmaster + environment: + - MESOS_QUORUM=1 + - MESOS_WORK_DIR=/var/lib/mesos + - MESOS_LOG_DIR=/var/log + - MESOS_ZK=zk://zookeeper:2181/mesos + volumes: + - /var/log/mesos:/var/log/mesos + ports: + - "5050:5050" + links: + - zookeeper slave1: extends: file: slave.yml @@ -57,6 +74,7 @@ scheduler: command: "-zk zk://zookeeper:2181/mesos -n 3" ports: - "8000:8000" + - "8080:8080" links: - slave1 - zookeeper