From c7dd4c61b6d954504b6e03a58a4cdd47a5077883 Mon Sep 17 00:00:00 2001 From: Gio Borje Date: Mon, 23 May 2016 13:45:52 -0700 Subject: [PATCH 1/2] Log warnings for numIterations * miniBatchFraction < 1.0 during gradient descent --- .../apache/spark/mllib/optimization/GradientDescent.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala index a67ea836e5681..407db1e78c6f1 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala @@ -197,6 +197,11 @@ object GradientDescent extends Logging { "< 1.0 can be unstable because of the stochasticity in sampling.") } + if (numIterations * miniBatchFraction < 1.0) { + logWarning("Not all examples will be used if numIterations * miniBatchFraction " + + "< 1.0") + } + val stochasticLossHistory = new ArrayBuffer[Double](numIterations) // Record previous weight and current one to calculate solution vector difference From 0754ce08148adb9debd70fdfc65a9c0d40532af5 Mon Sep 17 00:00:00 2001 From: Gio Borje Date: Tue, 24 May 2016 19:20:45 -0700 Subject: [PATCH 2/2] Add current numIterations and miniBatchFraction values to warning log --- .../org/apache/spark/mllib/optimization/GradientDescent.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala index 407db1e78c6f1..735e780909ef2 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala @@ -198,8 +198,8 @@ object GradientDescent extends Logging { } if (numIterations * miniBatchFraction < 1.0) { - logWarning("Not all examples will be used if numIterations * miniBatchFraction " + - "< 1.0") + logWarning("Not all examples will be used if numIterations * miniBatchFraction < 1.0: " + + s"numIterations=$numIterations and miniBatchFraction=$miniBatchFraction") } val stochasticLossHistory = new ArrayBuffer[Double](numIterations)