Skip to content

Commit

Permalink
Preparing for 2.1.0 release.
Browse files Browse the repository at this point in the history
  • Loading branch information
@MatthewGerber
MatthewGerber committed May 27, 2015
2 parents eb9cf0d + 3b013c8 commit 647fc07
Show file tree
Hide file tree
Showing 77 changed files with 5,036 additions and 12,650 deletions.
5 changes: 2 additions & 3 deletions ATT/ATT.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,8 @@
<SpecificVersion>False</SpecificVersion>
<HintPath>..\Libraries\LAIR.MachineLearning.dll</HintPath>
</Reference>
<Reference Include="LAIR.Misc, Version=1.0.0.0, Culture=neutral, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\Libraries\LAIR.Misc.dll</HintPath>
<Reference Include="LAIR.Math">
<HintPath>..\Libraries\LAIR.Math.dll</HintPath>
</Reference>
<Reference Include="LAIR.ResourceAPIs.PostGIS, Version=1.0.0.0, Culture=neutral, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
Expand Down
42 changes: 28 additions & 14 deletions ATT/Classifiers/AdaBoost.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,7 @@ public class AdaBoost : Classifier
{
static AdaBoost()
{
if (Configuration.RCranMirror != null)
R.InstallPackages(R.CheckForMissingPackages(new string[] { "ada" }), Configuration.RCranMirror, Configuration.RPackageInstallDirectory);
R.InstallPackages(R.CheckForMissingPackages(new string[] { "ada" }), Configuration.RCranMirror, Configuration.RPackageInstallDirectory);
}

private int _iterations;
Expand Down Expand Up @@ -107,7 +106,7 @@ public override void Consume(FeatureVectorList featureVectors)
}

protected override void BuildModel()
{
{
StringBuilder rCmd = new StringBuilder(@"
trainRaw=read.csv(""" + RawTrainPath.Replace(@"\", "/") + @""", header = TRUE, sep = ',')" + @"
trainNorm=trainRaw
Expand Down Expand Up @@ -223,9 +222,11 @@ public override void Classify(FeatureVectorList featureVectors)
}
write.table(mult, file=""" + PredictionsPath.Replace("\\", "/") + @""", row.names=FALSE, sep=',')" + @"
");
R.Execute(rCmd.ToString(), false);
string output, error;
R.Execute(rCmd.ToString(), false, out output, out error);

if (File.Exists(PredictionsPath))
try
{
using (StreamReader predictionsFile = new StreamReader(PredictionsPath))
{
string[] colnames = predictionsFile.ReadLine().Split(',');
Expand All @@ -238,7 +239,7 @@ public override void Classify(FeatureVectorList featureVectors)

for (int i = 0; i < colnames.Length; i++)
{
string label = colnames[i].Replace("\"", @""); ;
string label = colnames[i].Replace("\"", "");
float prob = float.Parse(lines[i]);
featureVectors[row].DerivedFrom.PredictionConfidenceScores.Add(label, prob);
}
Expand All @@ -250,14 +251,27 @@ public override void Classify(FeatureVectorList featureVectors)
if (row != featureVectors.Count)
throw new Exception("Number of predictions doesn't match number of input vectors");
}
else
Console.Out.WriteLine("WARNING: AdaBoost failed to classify points. See previous messages for hints.");

File.Delete(ColumnMaxMinPath);
File.Delete(ClassPath);
File.Delete(AdaModelPath);
File.Delete(RawPredictionInstancesPath);
File.Delete(PredictionsPath);
}
catch (Exception ex)
{
throw new Exception("ERROR: AdaBoost failed to classify points. Output and error messages follow:" + Environment.NewLine +
"\tException message: " + ex.Message + Environment.NewLine +
"\tR output: " + output + Environment.NewLine +
"\tR orror: " + error);
}
finally
{
try { File.Delete(ColumnMaxMinPath); }
catch { }
try { File.Delete(ClassPath); }
catch { }
try { File.Delete(AdaModelPath); }
catch { }
try { File.Delete(RawPredictionInstancesPath); }
catch { }
try { File.Delete(PredictionsPath); }
catch { }
}
}
}

Expand Down
58 changes: 44 additions & 14 deletions ATT/Classifiers/RandomForest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,7 @@ public class RandomForest : Classifier
{
static RandomForest()
{
if (Configuration.RCranMirror != null)
R.InstallPackages(R.CheckForMissingPackages(new string[] { "randomForest" }), Configuration.RCranMirror, Configuration.RPackageInstallDirectory);
R.InstallPackages(R.CheckForMissingPackages(new string[] { "randomForest" }), Configuration.RCranMirror, Configuration.RPackageInstallDirectory);
}

private int _numTrees;
Expand Down Expand Up @@ -104,7 +103,7 @@ public override void Consume(FeatureVectorList featureVectors)
}
}
}

protected override void BuildModel()
{
StringBuilder rCmd = new StringBuilder(@"
Expand All @@ -126,9 +125,26 @@ protected override void BuildModel()
rf=randomForest(Class ~., data=trainNorm, ntree=" + _numTrees + ", importance=TRUE, seed=99)" + @"
save(rf, file=""" + RandomForestModelPath.Replace("\\", "/") + @""")" + @"
");
R.Execute(rCmd.ToString(), false);
string output, error;
R.Execute(rCmd.ToString(), false, out output, out error);

File.Delete(RawTrainPath);
try
{
if (!File.Exists(RandomForestModelPath))
throw new Exception("RandomForest model was not created at \"" + RandomForestModelPath + "\".");
}
catch (Exception ex)
{
throw new Exception("ERROR: RandomForest failed to build model. Output and error messages follow:" + Environment.NewLine +
"\tException message: " + ex.Message + Environment.NewLine +
"\tR output: " + output + Environment.NewLine +
"\tR orror: " + error);
}
finally
{
try { File.Delete(RawTrainPath); }
catch { }
}
}

public override IEnumerable<string> SelectFeatures(Prediction prediction)
Expand Down Expand Up @@ -182,9 +198,11 @@ public override void Classify(FeatureVectorList featureVectors)
names(dfp)[names(dfp)=='NULL.'] <- 'NULL'
write.table(dfp, file=""" + PredictionsPath.Replace("\\", "/") + @""", row.names=FALSE, sep=',')" + @"
");
R.Execute(rCmd.ToString(), false);
string output, error;
R.Execute(rCmd.ToString(), false, out output, out error);

if (File.Exists(PredictionsPath))
try
{
using (StreamReader predictionsFile = new StreamReader(PredictionsPath))
{
string[] colnames = predictionsFile.ReadLine().Split(',');
Expand All @@ -210,13 +228,25 @@ public override void Classify(FeatureVectorList featureVectors)
if (row != featureVectors.Count)
throw new Exception("Number of predictions doesn't match number of input vectors");
}
else
Console.Out.WriteLine("WARNING: RandomForest failed to classify points. See previous messages for hints.");

File.Delete(ColumnMaxMinPath);
File.Delete(RandomForestModelPath);
File.Delete(RawPredictionInstancesPath);
File.Delete(PredictionsPath);
}
catch (Exception ex)
{
throw new Exception("ERROR: RandomForest failed to classify points. Output and error messages follow:" + Environment.NewLine +
"\tException message: " + ex.Message + Environment.NewLine +
"\tR output: " + output + Environment.NewLine +
"\tR orror: " + error);
}
finally
{
try { File.Delete(ColumnMaxMinPath); }
catch { }
try { File.Delete(RandomForestModelPath); }
catch { }
try { File.Delete(RawPredictionInstancesPath); }
catch { }
try { File.Delete(PredictionsPath); }
catch { }
}
}
}

Expand Down
77 changes: 76 additions & 1 deletion ATT/Configuration.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
using LAIR.ResourceAPIs.R;
using PTL.ATT.Models;
using System.Reflection;
using LAIR.Extensions;

namespace PTL.ATT
{
Expand Down Expand Up @@ -108,6 +109,7 @@ public static int PostgresCommandTimeout
#region postgis
private static string _shp2pgsqlPath;
private static string _pgsql2shpPath;
private static string _postgisShapefileDirectory;

public static string Shp2PgsqlPath
{
Expand All @@ -120,6 +122,12 @@ public static string Pgsql2ShpPath
get { return _pgsql2shpPath; }
set { _pgsql2shpPath = value; }
}

public static string PostGisShapefileDirectory
{
get { return Configuration._postgisShapefileDirectory; }
set { Configuration._postgisShapefileDirectory = value; }
}
#endregion

#region r
Expand Down Expand Up @@ -149,6 +157,36 @@ public static Dictionary<Type, Dictionary<string, string>> ClassifierTypeOptions
}
#endregion

#region incidents
private static string _incidentsImportDirectory;

public static string IncidentsImportDirectory
{
get { return Configuration._incidentsImportDirectory; }
set { Configuration._incidentsImportDirectory = value; }
}
#endregion

#region events
private static string _eventsImportDirectory;

public static string EventsImportDirectory
{
get { return Configuration._eventsImportDirectory; }
set { Configuration._eventsImportDirectory = value; }
}
#endregion

#region importers
private static string _importersLoadDirectory;

public static string ImportersLoadDirectory
{
get { return Configuration._importersLoadDirectory; }
set { Configuration._importersLoadDirectory = value; }
}
#endregion

#region models
private static string _modelsDirectory;
private static Dictionary<Type, Type> _modelTypeFeatureExtractorType;
Expand Down Expand Up @@ -195,6 +233,11 @@ public static int ProcessorCount
private static string _path = null;
private static bool _initialized = false;

public static bool Initialized
{
get { return _initialized; }
}

public static string LicenseText
{
get
Expand Down Expand Up @@ -238,6 +281,7 @@ public static void Initialize(string path, bool initializeDB)
XmlParser postgisP = new XmlParser(p.OuterXML("postgis"));
_shp2pgsqlPath = postgisP.ElementText("shp2pgsql");
_pgsql2shpPath = postgisP.ElementText("pgsql2shp");
_postgisShapefileDirectory = postgisP.ElementText("shapefile_directory");

if (string.IsNullOrWhiteSpace(_shp2pgsqlPath) || !File.Exists(_shp2pgsqlPath))
throw new FileNotFoundException("Failed to locate shp2pgsql executable. Check configuration.");
Expand Down Expand Up @@ -288,6 +332,15 @@ public static void Initialize(string path, bool initializeDB)
_classifierTypeOptions.Add(type, optionValue);
}

XmlParser incidentsP = new XmlParser(p.OuterXML("incidents"));
_incidentsImportDirectory = incidentsP.ElementText("import_directory");

XmlParser eventsP = new XmlParser(p.OuterXML("events"));
_eventsImportDirectory = eventsP.ElementText("import_directory");

XmlParser importersP = new XmlParser(p.OuterXML("importers"));
_importersLoadDirectory = importersP.ElementText("load_directory");

XmlParser modelingP = new XmlParser(p.OuterXML("modeling"));
_modelsDirectory = modelingP.ElementText("model_directory");
if (string.IsNullOrWhiteSpace(_modelsDirectory))
Expand Down Expand Up @@ -338,11 +391,33 @@ public static void Initialize(string path, bool initializeDB)
throw new Exception("Invalid processor count (must be >= 1): " + _processorCount);

if (initializeDB)
{
DB.Initialize();
_initialized = true;
}
}

_initialized = true;
/// <summary>
/// Resets the entire ATT system, deleting and recreating tables. ATT system must be initialized before calling this.
/// </summary>
/// <param name="tablesToKeep">Tables to keep</param>
public static void Reset(IEnumerable<string> tablesToKeep)
{
if (!_initialized)
throw new InvalidOperationException("ATT system has not been initialized. Cannot reset.");

string tablesToDrop = DB.Tables.Where(t => tablesToKeep == null || !tablesToKeep.Contains(t)).Concatenate(",");
if (!string.IsNullOrWhiteSpace(tablesToDrop))
{
DB.Connection.ExecuteNonQuery("DROP TABLE " + tablesToDrop + " CASCADE");
PTL.ATT.Configuration.Reload(true);
}
}

/// <summary>
/// Reloads the configuration, keeping all existing data.
/// </summary>
/// <param name="reinitializeDB">Whether or not to reinitialize the database.</param>
public static void Reload(bool reinitializeDB)
{
_initialized = false;
Expand Down
5 changes: 5 additions & 0 deletions ATT/DB.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@ public static class DB
{
public static ConnectionPool Connection;

public static string[] Tables
{
get { return Connection.GetTables().Where(t => t != "spatial_ref_sys").ToArray(); }
}

public static void Initialize()
{
Connection = new ConnectionPool(Configuration.PostgresHost, Configuration.PostgresPort, Configuration.PostgresSSL, Configuration.PostgresUser, Configuration.PostgresPassword, Configuration.PostgresDatabase, Configuration.PostgresConnectionTimeout, Configuration.PostgresRetryLimit, Configuration.PostgresCommandTimeout, Configuration.PostgresMaxPoolSize);
Expand Down
11 changes: 11 additions & 0 deletions ATT/Evaluation/Plot.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,17 @@ protected Plot(string title, long slice, Dictionary<string, List<PointF>> series
_imageFormat = format;
}

/// <summary>
/// Renders this surveillance plot
/// </summary>
/// <param name="height">Height in pixels</param>
/// <param name="width">Width in pixels</param>
/// <param name="includeTitle">Whether or not to include the title</param>
/// <param name="plotSeriesDifference">Pass non-null to plot a series difference. If both elements are null, the series difference with largest difference is plotted. Or you can pass specific series names to plot a specific difference. If only one series name is provided, the maximum difference between that series and another will be computed.</param>
/// <param name="blackAndWhite">Whether or not to use black and white only</param>
/// <param name="retainImageOnDisk">Whether or not to retain the image on disk</param>
/// <param name="args">Additional arguments: (0) plot margins in 0,0,0,0 format (default is 5,4,4,2), (1) additional arguments to plot and lines commands (e.g., cex), (2) additional arguments to legend command (e.g., cex)</param>
/// <returns>Path to rendered image file</returns>
public void Render(int height, int width, bool includeTitle, Tuple<string, string> plotSeriesDifference, bool blackAndWhite, bool retainImageOnDisk, params string[] args)
{
_imagePath = CreateImageOnDisk(height, width, includeTitle, plotSeriesDifference, blackAndWhite, args);
Expand Down
Loading

0 comments on commit 647fc07

Please sign in to comment.