Merge branch 'master' of https://github.com/mesos/elasticsearch

README.md

@@ -7,6 +7,7 @@
 
 - [x] Deployment
 - [x] Durable cluster topology (via ZooKeeper)
+- [x] Web UI on scheduler port 8080
 - [ ] Support deploying multiple Elasticsearch clusters to single Mesos cluster
 - [ ] High availability (master, indexer, replica)
 - [ ] Fault tolerance
@@ -91,6 +92,11 @@ $ docker-compose up
 
 Now open the browser at http://localhost:5050 to view the Mesos GUI.
 
+NOTE: If you run docker from a VM (boot2docker on OSX), use the ip address assigned to the VM instead of localhost:
+```
+docker-machine inspect dev -f "{{.Driver.IPAddress}}"
+```
+
 The Elasticsearch task can be accessed via the slave on port 9200. Find the IP address of the slave: 
 
 ```

config/findbugs/excludeFilter.xml

@@ -21,5 +21,11 @@
     <Match>
         <Bug code="SnVI"/>
     </Match>
+    <Match>
+        <!-- Ignore all unread public fields on responses classes in the controllers-->
+        <Bug code="UrF, UuF"/>
+        <Class name="~org\.apache\.mesos\.elasticsearch\.scheduler\.controllers\.[A-Za-z0-9]+Controller\$[A-Za-z0-9]+Response"/>
+        <!--<Class name="org.apache.mesos.elasticsearch.scheduler.controllers.ClusterController"/>-->
+    </Match>
 
 </FindBugsFilter>

scheduler/build.gradle

@@ -1,5 +1,6 @@
 apply plugin: 'application'
 apply plugin: 'com.bmuschko.docker-remote-api'
+apply plugin: 'spring-boot'
 
 import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
 
@@ -11,20 +12,36 @@ buildscript {
     }
     dependencies {
         classpath 'com.bmuschko:gradle-docker-plugin:2.2'
+        classpath("org.springframework.boot:spring-boot-gradle-plugin:1.2.4.RELEASE")
     }
 }
 repositories {
     maven {
         url "https://jitpack.io"
     }
 }
+configurations {
+    all*.exclude group:"org.springframework.boot", module: "spring-boot-starter-logging"
+}
 dependencies {
 
     compile project(':commons')
 
+    compile("org.springframework.boot:spring-boot-starter-web:1.2.4.RELEASE")
+    compile("org.springframework.boot:spring-boot-starter-log4j:1.2.4.RELEASE")
     compile "commons-cli:commons-cli:1.0"
     compile "commons-io:commons-io:2.4"
-    compile "log4j:log4j:1.2.16"
+    compile "log4j:log4j:1.2.17"
+
+    compile 'org.webjars:angularjs:1.4.1'
+    compile 'org.webjars.bower:angular-route:1.4.1'
+    compile 'org.webjars.bower:angular-resource:1.4.1'
+    compile 'org.webjars:bootstrap:3.3.5'
+    compile 'org.webjars:font-awesome:4.3.0'
+    compile 'org.webjars.bower:rdash-ui:1.0.1'
+    compile 'org.webjars:momentjs:2.10.3'
+    compile 'org.webjars:angular-moment:0.10.1'
+    compile 'org.webjars.bower:json-formatter:0.2.7'
 
     testCompile 'org.hamcrest:hamcrest-all:1.3'
     testCompile 'joda-time:joda-time:2.3'

scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Clock.java

@@ -1,5 +1,6 @@
 package org.apache.mesos.elasticsearch.scheduler;
 
+import java.time.ZonedDateTime;
 import java.util.Date;
 
 /**
@@ -11,4 +12,7 @@ public Date now() {
         return new Date();
     }
 
+    public ZonedDateTime zonedNow() {
+        return ZonedDateTime.now();
+    }
 }

scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Configuration.java

@@ -12,11 +12,11 @@
  */
 public class Configuration {
 
-    public static final double CPUS = 0.2;
+    private static final double CPUS = 0.2;
 
-    public static final double MEM = 512;
+    private static final double MEM = 512;
 
-    public static final double DISK = 250;
+    private static final double DISK = 250;
 
     private int numberOfHwNodes;
 
@@ -28,6 +28,24 @@ public class Configuration {
 
     private String zookeeperUrl;
 
+    private int managementApiPort;
+
+    public static double getCpus() {
+        return CPUS;
+    }
+
+    public static double getMem() {
+        return MEM;
+    }
+
+    public static double getDisk() {
+        return DISK;
+    }
+
+    public State getState() {
+        return state;
+    }
+
     public void setState(State state) {
         this.state = state;
     }
@@ -87,4 +105,12 @@ public String getZookeeperServers() {
         Iterator<String> hostPorts = zookeeperAddresses.stream().map(zk -> zk.getAddress() + ":" + zk.getPort()).iterator();
         return StringUtils.join(hostPorts, ",");
     }
+
+    public void setManagementApiPort(int managementApiPort) {
+        this.managementApiPort = managementApiPort;
+    }
+
+    public int getManagementApiPort() {
+        return managementApiPort;
+    }
 }

scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/ElasticsearchScheduler.java

@@ -5,11 +5,10 @@
 import org.apache.mesos.Protos;
 import org.apache.mesos.Scheduler;
 import org.apache.mesos.SchedulerDriver;
+import org.apache.mesos.elasticsearch.common.Discovery;
 
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.net.InetSocketAddress;
+import java.util.*;
 import java.util.function.Predicate;
 
 /**
@@ -24,13 +23,19 @@ public class ElasticsearchScheduler implements Scheduler {
 
     private final TaskInfoFactory taskInfoFactory;
 
+    Clock clock = new Clock();
+
     Set<Task> tasks = new HashSet<>();
 
     public ElasticsearchScheduler(Configuration configuration, TaskInfoFactory taskInfoFactory) {
         this.configuration = configuration;
         this.taskInfoFactory = taskInfoFactory;
     }
 
+    public Set<Task> getTasks() {
+        return tasks;
+    }
+
     public void run() {
         LOGGER.info("Starting ElasticSearch on Mesos - [numHwNodes: " + configuration.getNumberOfHwNodes() + ", zk: " + configuration.getZookeeperUrl() + "]");
 
@@ -88,7 +93,12 @@ public void resourceOffers(SchedulerDriver driver, List<Protos.Offer> offers) {
                 LOGGER.info("Accepted offer: " + offer.getHostname());
                 Protos.TaskInfo taskInfo = taskInfoFactory.createTask(configuration, offer);
                 driver.launchTasks(Collections.singleton(offer.getId()), Collections.singleton(taskInfo));
-                tasks.add(new Task(offer.getHostname(), taskInfo.getTaskId().getValue()));
+                tasks.add(new Task(
+                        offer.getHostname(),
+                        taskInfo.getTaskId().getValue(),
+                        clock.zonedNow(),
+                        new InetSocketAddress(offer.getHostname(), taskInfo.getDiscovery().getPorts().getPorts(Discovery.CLIENT_PORT_INDEX).getNumber()),
+                        new InetSocketAddress(offer.getHostname(), taskInfo.getDiscovery().getPorts().getPorts(Discovery.TRANSPORT_PORT_INDEX).getNumber())));
             }
         }
     }

scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Main.java

@@ -3,6 +3,7 @@
 import org.apache.commons.cli.*;
 import org.apache.mesos.elasticsearch.common.zookeeper.model.ZKAddress;
 import org.apache.mesos.elasticsearch.common.zookeeper.parser.ZKAddressParser;
+import org.springframework.boot.builder.SpringApplicationBuilder;
 
 import java.util.List;
 
@@ -15,6 +16,8 @@ public class Main {
 
     public static final String ZK_URL = "zk";
 
+    public static final String MANAGEMENT_API_PORT = "m";
+
     private Options options;
 
     private Configuration configuration;
@@ -23,6 +26,7 @@ public Main() {
         this.options = new Options();
         this.options.addOption(NUMBER_OF_HARDWARE_NODES, "numHardwareNodes", true, "number of hardware nodes");
         this.options.addOption(ZK_URL, "Zookeeper URL", true, "Zookeeper urls zk://IP:PORT,IP:PORT,IP:PORT/mesos)");
+        this.options.addOption(MANAGEMENT_API_PORT, "StatusPort", true, "TCP port for status interface. Default is 8080");
     }
 
     public static void main(String[] args) {
@@ -34,11 +38,18 @@ public void run(String[] args) {
         try {
             parseCommandlineOptions(args);
         } catch (ParseException | IllegalArgumentException e) {
-            printUsage();
+            printUsageAndExit();
             return;
         }
 
         final ElasticsearchScheduler scheduler = new ElasticsearchScheduler(configuration, new TaskInfoFactory());
+
+        new SpringApplicationBuilder(WebApplication.class)
+                .initializers(applicationContext -> applicationContext.getBeanFactory().registerSingleton("scheduler", scheduler))
+                .initializers(applicationContext -> applicationContext.getBeanFactory().registerSingleton("configuration", configuration))
+                .showBanner(false)
+                .run(args);
+
         scheduler.run();
     }
 
@@ -52,7 +63,7 @@ private void parseCommandlineOptions(String[] args) throws ParseException, Illeg
         String zkUrl = cmd.getOptionValue(ZK_URL);
 
         if (numberOfHwNodesString == null || zkUrl == null) {
-            printUsage();
+            printUsageAndExit();
             return;
         }
 
@@ -66,9 +77,10 @@ private void parseCommandlineOptions(String[] args) throws ParseException, Illeg
         configuration.setState(new State(new ZooKeeperStateInterfaceImpl(configuration.getZookeeperServers())));
     }
 
-    private void printUsage() {
+    private void printUsageAndExit() {
         HelpFormatter formatter = new HelpFormatter();
         formatter.printHelp(configuration.getFrameworkName(), options);
+        System.exit(2);
     }
 
 }

scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Resources.java

@@ -66,9 +66,9 @@ public static List<Integer> selectTwoPortsFromRange(List<Protos.Resource> offere
     }
 
     public static List<Protos.Resource> buildFrameworkResources() {
-        Protos.Resource cpus = Resources.cpus(Configuration.CPUS);
-        Protos.Resource mem = Resources.mem(Configuration.MEM);
-        Protos.Resource disk = Resources.disk(Configuration.DISK);
+        Protos.Resource cpus = Resources.cpus(Configuration.getCpus());
+        Protos.Resource mem = Resources.mem(Configuration.getMem());
+        Protos.Resource disk = Resources.disk(Configuration.getDisk());
         return asList(cpus, mem, disk);
     }
 }

scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Task.java

@@ -1,5 +1,8 @@
 package org.apache.mesos.elasticsearch.scheduler;
 
+import java.net.InetSocketAddress;
+import java.time.ZonedDateTime;
+
 /**
  * Task on a host.
  */
@@ -8,10 +11,16 @@ public class Task {
     private String taskId;
 
     private String hostname;
+    private ZonedDateTime startedAt;
+    private InetSocketAddress clientAddress;
+    private InetSocketAddress transportAddress;
 
-    public Task(String hostname, String taskId) {
+    public Task(String hostname, String taskId, ZonedDateTime startedAt, InetSocketAddress clientInterface, InetSocketAddress transportAddress) {
         this.hostname = hostname;
         this.taskId = taskId;
+        this.startedAt = startedAt;
+        this.clientAddress = clientInterface;
+        this.transportAddress = transportAddress;
     }
 
     public String getHostname() {
@@ -21,4 +30,16 @@ public String getHostname() {
     public String getTaskId() {
         return taskId;
     }
+
+    public ZonedDateTime getStartedAt() {
+        return startedAt;
+    }
+
+    public InetSocketAddress getClientAddress() {
+        return clientAddress;
+    }
+
+    public InetSocketAddress getTransportAddress() {
+        return transportAddress;
+    }
 }

scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/WebApplication.java

@@ -0,0 +1,12 @@
+package org.apache.mesos.elasticsearch.scheduler;
+
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.context.annotation.ComponentScan;
+
+/**
+ *
+ */
+@EnableAutoConfiguration
+@ComponentScan
+public class WebApplication {
+}