Use local installations for benchmarking

Makefile

@@ -1,3 +1,13 @@
+# These are the directories that contain everything we need after 'make setup'
+# is completed.  We assume that 'matlab' is available on the PATH.
+export INCLUDEPATH := $(shell pwd)/libraries/include/
+export LIBPATH := $(shell pwd)/libraries/lib/
+export BINPATH := $(shell pwd)/libraries/bin/
+export JAVAPATH := $(shell pwd)/libraries/share/
+export DEBUGBINPATH := $(shell pwd)/libraries/debug/bin/
+export DEBUGINCLUDEPATH := $(shell pwd)/libraries/debug/include/
+export DEBUGLIBPATH := $(shell pwd)/libraries/debug/lib/
+
 # Locate the python bin.
 PYTHON_BIN := $(shell which python3.3)
 ifndef PYTHON_BIN
@@ -38,52 +48,10 @@ COPY := False
 USER := ""
 PASSWORD := ""
 
-################################################################################################
-# How to use:                                                                                  #
-# Specify the environment variables in this file or set these variables from the command line. #
-# All environment path variables should end with a slash.                                      #
-# Note that four settings need to be changed for this to work.                                 #
-# 1) SHOGUN_PATH                                                                               #
-# 2) PYTHONPATH                                                                                #
-# 3) WEKA_CLASSPATH                                                                            #
-# 4) MATLAB_BIN                                                                                #
-################################################################################################
-
-# Set the environment variable for the mlpack executables.
-
-ifdef $(shell which mlpack_knn)
-	export MLPACK_BIN=$(shell dirname $(firstword $(shell which mlpack_knn)))/
-else
-	export MLPACK_BIN=""
-endif
-
-# Set the environment variable for the mlpack executables.
-export MLPACK_BIN_DEBUG=$(MLPACK_BIN)
-
-# Export the MLPACK_PATH environment variable.
-export MLPACK_PATH=$(shell dirname $(MLPACK_BIN))/
+# Set the environment variable for the compiled mlpack executables.
 export MLPACK_BIN_SRC=methods/mlpack/src/build/
 export MLPACK_BIN_DEBUG_SRC=methods/mlpack/src/build/
 
-# Set the environment variable for the matlab executable.
-# You can use the following command to search the 'matlab' file everytime:
-# export MATLAB_BIN=$(shell find / -name matlab  -print 2>/dev/null -quit)
-export MATLAB_BIN=""
-
-# Export the MATLABPATH environment variable.
-export MATLABPATH=$(shell pwd)/methods/matlab/
-
-# Export the WEKA_CLASSPATH environment variable.
-# You can use the following command to search the 'weka.jar' file everytime:
-# export WEKA_CLASSPATH=".:$(shell find / -name weka.jar  -print 2>/dev/null -quit)"
-export WEKA_CLASSPATH=".:/Users/marcus/Downloads/weka-3-6-11/weka.jar"
-
-# Export the SHOGUN_PATH environment variable.
-export SHOGUN_PATH=""
-
-# Export the PYTHONPATH environment variable.
-export PYTHONPATH=""
-
 # Set the environment variable for the the ms_print executable.
 export MS_PRINT_BIN=$(shell which ms_print)
 
@@ -92,12 +60,14 @@ export VALGRIND_BIN=$(shell which valgrind)
 
 # Export the path to the FLANN library.
 export FLANN_PATH=methods/flann/
-
 # Export the path to the ANN library.
 export ANN_PATH=methods/ann/
 
-# Export the path to the HLearn library.
-export HLEARN_PATH=""
+# Set LD_LIBRARY_PATH correctly.
+export LD_LIBRARY_PATH=$(shell echo $(LIBPATH))
+# Set PYTHONPATH correctly.
+PYVERSION=$(shell python3 -c 'import sys; print("python" + sys.version[0:3])')
+export PYTHONPATH=$(shell pwd)/libraries/lib/$(shell echo $(PYVERSION))/dist-packages:$(shell pwd)/libraries/lib/$(shell echo $(PYVERSION))/site-packages
 
 # Color settings.
 NO_COLOR=\033[0m
@@ -200,11 +170,14 @@ endif
 
 .scripts:
 	# Compile the java files for the weka methods.
-	javac -cp $(shell echo $(WEKA_CLASSPATH)) -d methods/weka methods/weka/src/*.java
+	javac -cp $(shell echo $(JAVAPATH)/weka.jar) -d methods/weka methods/weka/src/*.java
 	# Compile the ann scripts.
-	g++ -O0 -std=c++11 methods/ann/src/allknn.cpp -o methods/ann/allknn -I$(MLPACK_PATH)/include -I$(ANN_PATH)/include -L$(MLPACK_PATH)/lib -L$(ANN_PATH)/lib -lANN -lmlpack -lboost_program_options
+	g++ -O0 -std=c++11 methods/ann/src/allknn.cpp -o methods/ann/allknn -I$(INCLUDEPATH) -L$(LIBPATH) -lANN -lmlpack -lboost_program_options
 	# Compile the FLANN scripts.
-	g++ -O0 -std=c++11 methods/flann/src/allknn.cpp -o methods/flann/allknn -I$(MLPACK_PATH)/include -I$(FLANN_PATH)/include -L$(MLPACK_PATH)/lib -L$(FLANN_PATH)/lib -lmlpack -lboost_program_options
+	g++ -O0 -std=c++11 methods/flann/src/allknn.cpp -o methods/flann/allknn -I$(INCLUDEPATH) -L$(LIBPATH) -lmlpack -lboost_program_options -llz4
+	# Compile the mlpack scripts.  (Can't do this until ANN is released or a
+	# git version of mlpack is used.)
+	#cd methods/mlpack/src/ && ./build_scripts.sh
 
 .setup:
 	cd libraries/ && ./download_packages.sh && ./install_all.sh

README.md

@@ -35,6 +35,7 @@ Benchmarks are run with the `make` command.
 * `make run`        -- Perform the benchmark.
 * `make memory`     -- Get memory profiling information.
 * `make test`       -- Test the configuration file. Check for correct syntax and then try to open files referred in the configuration file.
+* `make setup`      -- Download and set up all of the libraries to compare against.
 * `make scripts`    -- Make additional scripts.
 
 

libraries/annoy_install.sh

@@ -26,4 +26,5 @@ tar -xzpf annoy*.tar.gz --strip-components=1 -C annoy/
 
 cd annoy/
 python3 setup.py build
-python3 setup.py install --prefix=../../ -O2
+PYVER=`python3 -c 'import sys; print("python" + sys.version[0:3])'`;
+PYTHONPATH=../lib/$PYVER/site-packages/ python3 setup.py install --prefix=../ -O2

libraries/matlab_install.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 #
 # This does not install MATLAB but instead simply checks for its presence.
-if [ ! -f /opt/matlab/bin/matlab ];
+if [ ! -f `which matlab` ];
 then
   echo "MATLAB not found!"
   exit 1

libraries/mlpack_install.sh

@@ -30,3 +30,10 @@ mkdir build/
 cd build/
 cmake -DCMAKE_INSTALL_PREFIX=../../ -DBUILD_TESTS=OFF ../
 make install
+
+# Also install debug version.
+cd ..
+mkdir build-debug/
+cd build-debug/
+cmake -DDEBUG=ON -DCMAKE_INSTALL_PREFIX=../../debug/ -DBUILD_TEST=OFF ../
+make install

libraries/mlpy_install.sh

@@ -26,4 +26,4 @@ tar -xzpf mlpy*.tar.gz --strip-components=1 -C mlpy/
 
 cd mlpy/
 python3 setup.py build
-python3 setup.py install --prefix=../../ -O2
+python3 setup.py install --prefix=../ -O2

libraries/scikit_install.sh

@@ -26,4 +26,4 @@ tar -xzpf scikit*.tar.gz --strip-components=1 -C scikit/
 
 cd scikit/
 python3 setup.py build
-python3 setup.py install --prefix=../../ -O2
+python3 setup.py install --prefix=../ -O2

methods/mlpack/allkfn.py

@@ -43,8 +43,8 @@ class ALLKFN(object):
   @param path - Path to the mlpack executable.
   @param verbose - Display informational messages.
   '''
-  def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN"],
-      verbose=True, debug=os.environ["MLPACK_BIN_DEBUG"]):
+  def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
+      verbose=True, debug=os.environ["DEBUGBINPATH"]):
     self.verbose = verbose
     self.dataset = dataset
     self.path = path

methods/mlpack/allknn.py

@@ -43,8 +43,8 @@ class ALLKNN(object):
   @param path - Path to the mlpack executable.
   @param verbose - Display informational messages.
   '''
-  def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN"],
-    verbose=True, debug=os.environ["MLPACK_BIN_DEBUG"]):
+  def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
+    verbose=True, debug=os.environ["DEBUGBINPATH"]):
     self.verbose = verbose
     self.dataset = dataset
     self.path = path

methods/mlpack/allkrann.py

@@ -43,8 +43,8 @@ class ALLKRANN(object):
   @param path - Path to the mlpack executable.
   @param verbose - Display informational messages.
   '''
-  def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN"],
-      verbose=True, debug=os.environ["MLPACK_BIN_DEBUG"]):
+  def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
+      verbose=True, debug=os.environ["DEBUGBINPATH"]):
     self.verbose = verbose
     self.dataset = dataset
     self.path = path

methods/mlpack/decision_stump.py

@@ -51,8 +51,8 @@ class DecisionStump(object):
   @param path - Path to the mlpack executable.
   @param verbose - Display informational messages.
   '''
-  def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN"],
-      verbose=True, debug=os.environ["MLPACK_BIN_DEBUG"]):
+  def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
+      verbose=True, debug=os.environ["DEBUGBINPATH"]):
     self.verbose = verbose
     self.dataset = dataset
     self.path = path

methods/mlpack/det.py

@@ -45,8 +45,8 @@ class DET(object):
   @param path - Path to the mlpack executable.
   @param verbose - Display informational messages.
   '''
-  def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN"],
-      verbose=True, debug=os.environ["MLPACK_BIN_DEBUG"]):
+  def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
+      verbose=True, debug=os.environ["DEBUGBINPATH"]):
     self.verbose = verbose
     self.dataset = dataset
     self.path = path

methods/mlpack/emst.py

@@ -43,8 +43,8 @@ class EMST(object):
   @param path - Path to the mlpack executable.
   @param verbose - Display informational messages.
   '''
-  def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN"],
-      verbose=True, debug=os.environ["MLPACK_BIN_DEBUG"]):
+  def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
+      verbose=True, debug=os.environ["DEBUGBINPATH"]):
     self.verbose = verbose
     self.dataset = dataset
     self.path = path

methods/mlpack/fastmks.py

@@ -43,8 +43,8 @@ class FastMKS(object):
   @param path - Path to the mlpack executable.
   @param verbose - Display informational messages.
   '''
-  def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN"],
-      verbose=True, debug=os.environ["MLPACK_BIN_DEBUG"]):
+  def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
+      verbose=True, debug=os.environ["DEBUGBINPATH"]):
     self.verbose = verbose
     self.dataset = dataset
     self.path = path

methods/mlpack/hmm_generate.py

@@ -43,8 +43,8 @@ class HMMGENERATE(object):
   @param path - Path to the mlpack executable.
   @param verbose - Display informational messages.
   '''
-  def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN"],
-      verbose=True, debug=os.environ["MLPACK_BIN_DEBUG"]):
+  def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
+      verbose=True, debug=os.environ["DEBUGBINPATH"]):
     self.verbose = verbose
     self.dataset = dataset
     self.path = path

methods/mlpack/hmm_loglik.py

@@ -44,8 +44,8 @@ class HMMLOGLIK(object):
   @param path - Path to the mlpack executable.
   @param verbose - Display informational messages.
   '''
-  def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN"],
-      verbose=True, debug=os.environ["MLPACK_BIN_DEBUG"]):
+  def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
+      verbose=True, debug=os.environ["DEBUGBINPATH"]):
     self.verbose = verbose
     self.dataset = dataset
     self.path = path

methods/mlpack/hmm_train.py

@@ -43,8 +43,8 @@ class HMMTRAIN(object):
   @param path - Path to the mlpack executable.
   @param verbose - Display informational messages.
   '''
-  def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN"],
-      verbose=True, debug=os.environ["MLPACK_BIN_DEBUG"]):
+  def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
+      verbose=True, debug=os.environ["DEBUGBINPATH"]):
     self.verbose = verbose
     self.dataset = dataset
     self.path = path

methods/mlpack/hmm_viterbi.py

@@ -45,8 +45,8 @@ class HMMVITERBI(object):
   @param path - Path to the mlpack executable.
   @param verbose - Display informational messages.
   '''
-  def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN"],
-      verbose=True, debug=os.environ["MLPACK_BIN_DEBUG"]):
+  def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
+      verbose=True, debug=os.environ["DEBUGBINPATH"]):
     self.verbose = verbose
     self.dataset = dataset
     self.path = path

methods/mlpack/ica.py

@@ -43,8 +43,8 @@ class ICA(object):
   @param path - Path to the mlpack executable.
   @param verbose - Display informational messages.
   '''
-  def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN"],
-      verbose=True, debug=os.environ["MLPACK_BIN_DEBUG"]):
+  def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
+      verbose=True, debug=os.environ["DEBUGBINPATH"]):
     self.verbose = verbose
     self.dataset = dataset
     self.path = path

methods/mlpack/kernel_pca.py

@@ -43,8 +43,8 @@ class KPCA(object):
   @param path - Path to the mlpack executable.
   @param verbose - Display informational messages.
   '''
-  def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN"],
-      verbose=True, debug=os.environ["MLPACK_BIN_DEBUG"]):
+  def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
+      verbose=True, debug=os.environ["DEBUGBINPATH"]):
     self.verbose = verbose
     self.dataset = dataset
     self.path = path

methods/mlpack/kmeans.py

@@ -43,8 +43,8 @@ class KMEANS(object):
   @param path - Path to the mlpack executable.
   @param verbose - Display informational messages.
   '''
-  def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN"],
-      verbose=True, debug=os.environ["MLPACK_BIN_DEBUG"]):
+  def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
+      verbose=True, debug=os.environ["DEBUGBINPATH"]):
     self.verbose = verbose
     self.dataset = dataset
     self.path = path

methods/mlpack/lars.py

@@ -43,8 +43,8 @@ class LARS(object):
   @param path - Path to the mlpack executable.
   @param verbose - Display informational messages.
   '''
-  def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN"],
-    verbose=True, debug=os.environ["MLPACK_BIN_DEBUG"]):
+  def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
+    verbose=True, debug=os.environ["DEBUGBINPATH"]):
     self.verbose = verbose
     self.dataset = dataset
     self.path = path

methods/mlpack/linear_regression.py

@@ -50,8 +50,8 @@ class LinearRegression(object):
   @param path - Path to the mlpack executable.
   @param verbose - Display informational messages.
   '''
-  def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN"],
-      verbose=True, debug=os.environ["MLPACK_BIN_DEBUG"]):
+  def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
+      verbose=True, debug=os.environ["DEBUGBINPATH"]):
     self.verbose = verbose
     self.dataset = dataset
     self.path = path

methods/mlpack/local_coordinate_coding.py

@@ -43,8 +43,8 @@ class LocalCoordinateCoding(object):
   @param path - Path to the mlpack executable.
   @param verbose - Display informational messages.
   '''
-  def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN"],
-      verbose=True, debug=os.environ["MLPACK_BIN_DEBUG"]):
+  def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
+      verbose=True, debug=os.environ["DEBUGBINPATH"]):
     self.verbose = verbose
     self.dataset = dataset
     self.path = path

methods/mlpack/logistic_regression.py

@@ -51,8 +51,8 @@ class LogisticRegression(object):
   @param path - Path to the mlpack executable.
   @param verbose - Display informational messages.
   '''
-  def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN"],
-      verbose=True, debug=os.environ["MLPACK_BIN_DEBUG"]):
+  def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
+      verbose=True, debug=os.environ["DEBUGBINPATH"]):
     self.verbose = verbose
     self.dataset = dataset
     self.path = path

methods/mlpack/lsh.py

@@ -44,8 +44,8 @@ class LSH(object):
   @param path - Path to the mlpack executable.
   @param verbose - Display informational messages.
   '''
-  def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN"],
-      verbose=True, debug=os.environ["MLPACK_BIN_DEBUG"]):
+  def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
+      verbose=True, debug=os.environ["DEBUGBINPATH"]):
     self.verbose = verbose
     self.dataset = dataset
     self.path = path

methods/mlpack/mlp_backward.py

@@ -80,7 +80,7 @@ def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN_SRC"],
   successful.
   '''
   def RunMetrics(self, options):
-    Log.Info("Perform MLP Backard.", self.verbose)
+    Log.Info("Perform MLP Backward.", self.verbose)
 
     # Split the command using shell-like syntax.
     cmd = shlex.split(self.path + "mlp_backward -v " + options)

methods/mlpack/nbc.py

@@ -51,8 +51,8 @@ class NBC(object):
   @param path - Path to the mlpack executable.
   @param verbose - Display informational messages.
   '''
-  def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN"],
-      verbose=True, debug=os.environ["MLPACK_BIN_DEBUG"]):
+  def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
+      verbose=True, debug=os.environ["DEBUGBINPATH"]):
     self.verbose = verbose
     self.dataset = dataset
     self.path = path

methods/mlpack/nca.py

@@ -43,8 +43,8 @@ class NCA(object):
   @param path - Path to the mlpack executable.
   @param verbose - Display informational messages.
   '''
-  def __init__(self, dataset, timeout=0, path=os.environ["MLPACK_BIN"],
-      verbose=True, debug=os.environ["MLPACK_BIN_DEBUG"]):
+  def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
+      verbose=True, debug=os.environ["DEBUGBINPATH"]):
     self.verbose = verbose
     self.dataset = dataset
     self.path = path