Merge branch 'master' of https://github.com/DSC-SPIDAL/twister2 into …

…tset-new

dashboard/server/src/main/java/edu/iu/dsc/tws/dashboard/services/JobService.java

@@ -61,7 +61,7 @@ public Job createJob(Job job) {
       computeResource.setJob(job);
     });
 
-    if (scalableResourcesCount.get() != 1) {
+    if (scalableResourcesCount.get() > 1) {
       throw new RuntimeException("Job should have exactly one "
           + "scalable resource. Found : " + scalableResourcesCount);
     }

docs/docs/compiling/linux.md

@@ -46,7 +46,7 @@ sudo pip install wheel
 4. Installing maven and configure it as follows :
 
 ```text
-  wget http://mirrors.ibiblio.org/apache/maven/maven-3/3.5.2/binaries/apache-maven-3.5.2-bin.tar.gz
+  wget http://apache.mirrors.pair.com/maven/maven-3/3.6.2/binaries/apache-maven-3.6.2-bin.tar.gz
 ```
 
 Extract this to a directory called maven configure the environmental variables
@@ -111,4 +111,4 @@ After you've build the Twister2 distribution, you can extract it and use it to s
 ```bash
 cd bazel-bin/scripts/package/
 tar -xvf twister2-0.3.0.tar.gz
-```
+```

twister2/examples/src/java/edu/iu/dsc/tws/examples/ml/svm/data/IterativeSVMDataObjectCompute.java

@@ -72,7 +72,7 @@ public IterativeSVMDataObjectCompute(String edgeName, int parallelism, int datas
     this.features = features;
     this.delemiter = del;
     int size = this.datasize % this.parallelism == 0
-        ? (this.datasize / parallelism) + 1 : (this.datasize / parallelism);
+        ? (this.datasize / parallelism) : (this.datasize / parallelism);
     this.dataPointsLocal = new double[size][this.features + 1];
   }
 
@@ -82,7 +82,7 @@ public IterativeSVMDataObjectCompute(String edgeName, int datasize, int features
     this.features = features;
     this.delemiter = del;
     int size = this.datasize % this.parallelism == 0
-        ? (this.datasize / parallelism) + 1 : (this.datasize / parallelism);
+        ? (this.datasize / parallelism)  : (this.datasize / parallelism);
     this.dataPointsLocal = new double[size][this.features + 1];
   }
 

twister2/examples/src/java/edu/iu/dsc/tws/examples/ml/svm/job/SvmSgdIterativeRunner.java

@@ -13,7 +13,6 @@
 
 import java.util.Arrays;
 import java.util.HashMap;
-import java.util.Random;
 import java.util.logging.Logger;
 
 import edu.iu.dsc.tws.api.comms.messaging.types.MessageTypes;
@@ -37,6 +36,7 @@
 import edu.iu.dsc.tws.examples.ml.svm.data.IterativeSVMWeightVectorObjectCompute;
 import edu.iu.dsc.tws.examples.ml.svm.data.IterativeSVMWeightVectorObjectDirectSink;
 import edu.iu.dsc.tws.examples.ml.svm.data.SVMDataObjectSource;
+import edu.iu.dsc.tws.examples.ml.svm.math.Matrix;
 import edu.iu.dsc.tws.examples.ml.svm.streamer.IterativeDataStream;
 import edu.iu.dsc.tws.examples.ml.svm.streamer.IterativePredictionDataStreamer;
 import edu.iu.dsc.tws.examples.ml.svm.util.BinaryBatchModel;
@@ -214,20 +214,24 @@ private void loadTrainingData() {
         .getOutput(trainingDatapointsTaskGraph, datapointsExecutionPlan,
             Constants.SimpleGraphConfig.DATA_OBJECT_SINK);
     double[][] datapoints = null;
+
     for (int i = 0; i < trainingDoubleDataPointObject.getPartitions().length; i++) {
       datapoints = trainingDoubleDataPointObject.getPartitions()[i].getConsumer()
           .next();
       LOG.info(String.format("Training Datapoints : %d,%d", datapoints.length, datapoints[0]
           .length));
-      int randomIndex = new Random()
-          .nextInt(this.svmJobParameters.getSamples() / dataStreamerParallelism - 1);
-      LOG.info(String.format("Random DataPoint[%d] : %s", randomIndex, Arrays
-          .toString(datapoints[randomIndex])));
-      System.out.println("---------Training Data-------------");
     }
+    System.out.println("---------Training Data-------------");
+    double sum = 0;
+    int count = 0;
     for (int j = 0; j < datapoints.length; j++) {
-      System.out.println(Arrays.toString(datapoints[j]));
+      sum = Matrix.sum(datapoints[j]);
+      if (sum == 0) {
+        count++;
+      }
     }
+    System.out.println(String.format("Training: %d,%d, Zero Sum: %d", datapoints.length,
+        datapoints[0].length, count));
   }
 
   private void loadTestingData() {
@@ -243,16 +247,20 @@ private void loadTestingData() {
       datapoints = testingDoubleDataPointObject.getPartitions()[i].getConsumer().next();
       LOG.info(String.format("Partition[%d] Testing Datapoints : %d,%d", i, datapoints.length,
           datapoints[0].length));
-      int randomIndex = new Random()
-          .nextInt(this.svmJobParameters.getTestingSamples() / dataStreamerParallelism - 1);
-      LOG.info(String.format("Random DataPoint[%d] : %s", randomIndex, Arrays
-          .toString(datapoints[randomIndex])));
     }
 
     System.out.println("---------Testing Data-------------");
+    double sum = 0;
+    int count = 0;
     for (int j = 0; j < datapoints.length; j++) {
-      System.out.println(Arrays.toString(datapoints[j]));
+      sum = Matrix.sum(datapoints[j]);
+      if (sum == 0) {
+        count++;
+      }
     }
+    System.out.println(String.format("%d,%d, Zero Sum: %d", datapoints.length,
+        datapoints[0].length, count));
+
   }
 
   private ComputeGraph buildTrainingDataPointsTG() {

twister2/examples/src/java/edu/iu/dsc/tws/examples/ml/svm/math/Matrix.java

@@ -33,6 +33,14 @@ public static double[] scalarMultiply(double[] x, double y) {
     return result;
   }
 
+  public static double sum(double[] arr) {
+    double sum = -1;
+    for (int i = 0; i < arr.length; i++) {
+      sum += arr[i];
+    }
+    return sum;
+  }
+
   public static double[] scalarMultiplyR(double[] x, double y, double[] result) {
     for (int i = 0; i < x.length; i++) {
       result[i] = x[i] * y;

twister2/master/src/java/edu/iu/dsc/tws/master/dashclient/DashboardClient.java

@@ -55,6 +55,7 @@ public DashboardClient(String dashHost, String jobID) {
   public boolean registerJob(JobAPI.Job job, JobMasterAPI.NodeInfo jobMasterNodeInfo) {
 
     RegisterJob registerJob = new RegisterJob(jobID, job, jobMasterNodeInfo);
+    LOG.fine("Registering job to dashboard: " + job + "jobMasterNodeInfo: " + jobMasterNodeInfo);
     String path = "jobs/";
 
     try {

twister2/proto/job.proto

@@ -42,8 +42,10 @@ message ComputeResource {
     double disk_giga_bytes = 3;
     int32 instances = 4;
     int32 workers_per_pod = 5;
-    bool scalable = 6;
 
+    oneof scalableRequired {
+        bool scalable = 6;
+    }
     oneof indexRequired {
         int32 index = 7;
     }