CreatePlotsFromCSV.py

###############################################################################
# Copyright (C) 2013-2018 Jacob Barhak
# Copyright (C) 2009-2012 The Regents of the University of Michigan
# 
# This file is part of the MIcroSimulation Tool (MIST).
# The MIcroSimulation Tool (MIST) is free software: you
# can redistribute it and/or modify it under the terms of the GNU General
# Public License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
# 
# The MIcroSimulation Tool (MIST) is distributed in the
# hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
###############################################################################
# 
# ADDITIONAL CLARIFICATION
# 
# The MIcroSimulation Tool (MIST) is distributed in the 
# hope that it will be useful, but "as is" and WITHOUT ANY WARRANTY of any 
# kind, including any warranty that it will not infringe on any property 
# rights of another party or the IMPLIED WARRANTIES OF MERCHANTABILITY or 
# FITNESS FOR A PARTICULAR PURPOSE. THE AUTHORS assume no responsibilities 
# with respect to the use of the MIcroSimulation Tool (MIST).  
# 
# The MIcroSimulation Tool (MIST) was derived from the Indirect Estimation  
# and Simulation Tool (IEST) and uses code distributed under the IEST name.
# The change of the name signifies a split from the original design that 
# focuses on microsimulation. For the sake of completeness, the copyright 
# statement from the original tool developed by the University of Michigan
# is provided below and is also mentioned above.
# 
###############################################################################
############################ Original Copyright ###############################
###############################################################################
# This file was originally missing a copyright statement yet was part of the
# IEST system with a GPL license.

from __future__ import division
import DataDef as DB
#import csv
import sys
import os

# Some constants to control the look
PLOT_MAX_LEGEND_ROW_BEFORE_SCALE = 4
PLOT_MAX_LEGEND_CHARACTER_COUNT_BEFORE_SCALE = 20
PLOT_LEGEND_TRANSPERANCY = 0.5
PlotGenerationFileNamePrefix = 'PlotGeneration'

def CreatePlots(InputFile, OutputFileName, PlotSequence):
    """ Create a plot report from the AssemblySequence specified """

    def LocateParam (ParamName, ParamCalcMethod, Data):
        " locate the parameter in the parameter list in the data"
        ParamIndex = None
        for (RowNum,Row) in enumerate(Data):
            if ParamCalcMethod == '' or Row[1]==None or Row[1]=='':
                # if no calculation method is specified of if it is blank
                # in the data then ignore it - this means that the first 
                # parameter name match will be returned - regardless of
                # calculation method
                RowParamCalc = ''
            else:
                # calculation method may contain an asterisk symbol - ignore
                # it in the data
                if Row[1][-1] == '*':
                    RowParamCalc = Row[1][:-1]
                else:
                    RowParamCalc = Row[1]
            # Handle the case where an empty string is converted to None
            FirstColumnToCompare = DB.Iif(Row[0] == None, '', Row[0])
            # Check if this is correct
            if ParamName == FirstColumnToCompare and ParamCalcMethod == RowParamCalc:
                # row located
                ParamIndex = RowNum
                break
        return ParamIndex


    def CreateLegendTitleMatchList (LegendList, Data):
        "Create a list of indices of matching legends"
        # Initialize the Legend Title match list 
        LegendTitleMatchList = []
        # Check if this is a special case of no titles that can be
        # generated by using empty string that will be loaded as None
        if not (all(map(DB.IsNone,Data[0])) and LegendList==['']):
            # Handle the regular case where legends should match
            # the first line in the data
            TitleRow=Data[0]
            # Loop through all legends
            for (LegendEnum, Legend) in enumerate(LegendList):
                TitleMatches = []
                for (ColumnEnum,Title) in enumerate(TitleRow):
                    if TitleRow[ColumnEnum] == Legend:
                        # If found a title match, count this as part of the
                        # legend
                        TitleMatches = TitleMatches + [ColumnEnum]
                # Assemble the Legend Title Match List 
                LegendTitleMatchList = LegendTitleMatchList + [TitleMatches]
        else:
            # Handle the special case where there is no title since
            # only one scenario was executed and the title was an
            # empty string. In this case, look for all the columns
            # after the second column that are not the same as the
            # first column.
            TransposeData = map(None,*Data)
            TitleMatches = []
            for ColumnEnum in range(2,len(TransposeData)):
                if TransposeData[0] != TransposeData[ColumnEnum]:
                    TitleMatches = TitleMatches + [ColumnEnum]
            # Assemble the Legend Title Match List 
            LegendTitleMatchList = LegendTitleMatchList + [TitleMatches]
        return LegendTitleMatchList


    def LoopThroughSequence (ParamList, LegendList, StyleList, LegendTitleMatchList, Data, IsNested = False):
        "Loop throgh the plot sequence and call the generator code"
        # reset the style counter and the X axis
        StyleCounter = 0
        LongestLegendTextSize = 1
        AxisX = None
        PlotText = ""
        if not IsNested:
            # figure out the default font size for legends
            PlotText = PlotText + "DefaultLegendFontSize = matplotlib.font_manager.FontProperties(size=matplotlib.rcParams['legend.fontsize']).get_size_in_points()\n"
        PlotText = PlotText + "# New plot sequence\n"
        for (ParamEnum,ParamEntry) in enumerate(ParamList):
            DB.MessageToUser('Processing plot for param: ' +repr(ParamEntry))
            if DB.IsList(ParamEntry):
                # detect nesting of 2 or more levels
                if IsNested:
                    raise ValueError, "Create Plots Error: Double nested list was detected - make sure there is only one nesting level"
                # recurse to resolve the nested level
                PlotText = PlotText + LoopThroughSequence (ParamEntry, LegendList, StyleList, LegendTitleMatchList, Data, True)
            else:
                # For nested, clear only once, otherwise clear for each plot
                if ParamEnum == 0 or not IsNested:
                    PlotText = PlotText + "HandleAxes.clear()\n"
                (ParamName, ParamCalcMethod, AxisTitle) = ParamEntry
                # try to locate this parameter
                ParamIndex = LocateParam (ParamName, ParamCalcMethod, Data)
                # Check to see if the parameter was found
                if ParamIndex == None:
                    # If not found print warning and save the warning as a plot
                    TheErrorText = "Create Plots Warning: No parameter entry : " + str(ParamEntry)
                    PlotText = PlotText + 'HandleAxes.set_title(\"\"\"' + TheErrorText + '\"\"\")\n'
                    PlotText = PlotText + "HandlePDF.savefig(HandleFigure)\n"
                    DB.MessageToUser(TheErrorText)
                    continue
                else:
                    # If found, resolve the title
                    if AxisTitle == '':
                        TitleToUse = ParamName + ' - ' + ParamCalcMethod
                    else:
                        TitleToUse = AxisTitle
                    # Check if the x axis was already defined
                    if AxisX == None:
                        # Record the axis and continue looping
                        AxisX = (ParamIndex, TitleToUse, ParamEntry)
                        continue
                    else:
                        # X axis is known - so this is a series for the plot.
                        if not IsNested:
                            # if not nested param list, the styles will repeat
                            # in each graph as if taken from the beginning of 
                            # the styles list, therefore reset the style counter
                            StyleCounter = 0
                            LongestLegendTextSize = 1
                            PlotText = PlotText + "HandleAxes.set_title(" + repr(TitleToUse) + ")\n"
                        if ParamEnum == 0 or not IsNested:
                            PlotText = PlotText + "HandleAxes.set_xlabel('" + AxisX[1] + "')\n"
                        # Start looping through legends, 
                        for (LegendEnum,Legend) in enumerate(LegendList):
                            # Collect the series values for the parameter
                            # and legend
                            MatchingColumnIndexList = LegendTitleMatchList[LegendEnum]
                            XSerieValues = [ Data[AxisX[0]][ColumnIndex] for ColumnIndex in MatchingColumnIndexList]
                            YSerieValues = [ Data[ParamIndex][ColumnIndex] for ColumnIndex in MatchingColumnIndexList]
                            if not IsNested:
                                LegendTextToUse = Legend
                            else:
                                LegendTextToUse = Legend + ' - ' + TitleToUse
                            if StyleCounter < len(StyleList):
                                StyleString = ", '" + StyleList[StyleCounter] + "'"
                            else:
                                StyleString = ''
                            PlotText = PlotText + "HandleAxes.plot( " + DB.SmartStr(XSerieValues) + " ," + DB.SmartStr(YSerieValues) + StyleString + " , label = '" + LegendTextToUse + "')\n"
                            LongestLegendTextSize = max(LongestLegendTextSize, len(LegendTextToUse))
                            StyleCounter = StyleCounter + 1
                if ParamEnum == len(ParamList)-1 or not IsNested:
                    # calculate the scale for the legend font - which also
                    # effects the line size. The scale is designed to not
                    # exceed a certain numbers of rows/characters
                    ScaleFontByRows = min(1.0, PLOT_MAX_LEGEND_ROW_BEFORE_SCALE / StyleCounter)
                    ScaleFontByColumns = min (1.0, PLOT_MAX_LEGEND_CHARACTER_COUNT_BEFORE_SCALE / LongestLegendTextSize)
                    ScaleFactor = min (ScaleFontByRows,ScaleFontByColumns)
                    # After each complete plot entry draw the legends
                    # A nested plot will draw legends only once at the
                    # end of the plot
                    PlotText = PlotText + "HandleSmallerFont = matplotlib.font_manager.FontProperties(size= DefaultLegendFontSize*" + str(ScaleFactor) + " )\n"
                    PlotText = PlotText + "LineHandles, LabelHandles = HandleAxes.get_legend_handles_labels()\n"
                    PlotText = PlotText + "TheLegend = HandleAxes.legend(LineHandles, LabelHandles, loc = 0, prop = HandleSmallerFont, handlelength = 2.0/" + str(ScaleFactor) + ")\n"
                    PlotText = PlotText + "TheLegend.get_frame().set_alpha("+str(PLOT_LEGEND_TRANSPERANCY)+")\n"
                    PlotText = PlotText + "HandlePDF.savefig(HandleFigure)\n"
        return PlotText


    # Parse the plot sequence
    if len(PlotSequence) != 3:
        raise ValueError, "Create Plots Error: Plot sequence does not contain the 3 components: ParamList, LegendList, StyleList"
    ParamList, LegendList, StyleList = PlotSequence
    # Load the data
    (DataColumns,Data) = DB.ImportDataFromCSV(FileName = InputFile, ImportColumnNames = False, ConvertTextToProperDataType = True, TextCellsAllowed = True)
    NumberOfRows = len(Data)
    if NumberOfRows == 0:
        raise ValueError, "Create Plots Error: Input file contains no rows - check that the file format is ok"
    PlotText = '################################################################################\n'
    PlotText = PlotText + '# This plot script was automatically Generated on: '+ DB.datetime.datetime.now().isoformat(' ')[:19] + '         #\n'
    PlotText = PlotText + '# by the MIcro Simulation Tool (MIST).                                         #\n'
    PlotText = PlotText + '################################################################################\n'
    PlotText = PlotText + 'from __future__ import division\n'
    PlotText = PlotText + "import matplotlib\n"
    PlotText = PlotText + "matplotlib.use('PDF')\n"
    PlotText = PlotText + "import matplotlib.pyplot as plt\n"
    PlotText = PlotText + "import matplotlib.font_manager\n"
    PlotText = PlotText + "import matplotlib.backends.backend_pdf\n"
    PlotText = PlotText + "import DataDef as DB\n"
    PlotText = PlotText + "Inf = DB.Inf\n"
    PlotText = PlotText + "NaN = DB.NaN\n"
    PlotText = PlotText + "inf = DB.inf\n"
    PlotText = PlotText + "nan = DB.nan\n"
    PlotText = PlotText + "HandlePDF = matplotlib.backends.backend_pdf.PdfPages('"+OutputFileName+"')\n"
    PlotText = PlotText + "HandleFigure = plt.figure()\n"
    PlotText = PlotText + "HandleAxes = HandleFigure.add_subplot(111)\n"
    LegendTitleMatchList = CreateLegendTitleMatchList (LegendList, Data)
    TheActualPlotsText = LoopThroughSequence (ParamList, LegendList, StyleList, LegendTitleMatchList, Data, False)
    PlotText = PlotText + TheActualPlotsText
    PlotText = PlotText +"HandlePDF.close()\n"

    (ScriptFileDescriptor, ScriptFileName) = DB.tempfile.mkstemp ( DB.PythonSuffix, PlotGenerationFileNamePrefix , DB.SessionTempDirecory , True)
    (ScriptPathOnly , ScriptFileNameOnly, ScriptFileNameFullPath) = DB.DetermineFileNameAndPath(ScriptFileName)
    # Now actually create the file
    try:
        GenFile = os.fdopen(ScriptFileDescriptor,'w')
        GenFile.write(PlotText)
        GenFile.close()
    except:
        (ExceptType, ExceptValue, ExceptTraceback) = sys.exc_info()
        raise ValueError, 'Create Plots Error: The script file ' + ScriptFileNameFullPath  +' cannot be created. Please make sure you specified a valid path for the file and that that file name is not in use by the system. Here are details about the error: ' + str(ExceptValue)
    (ScriptFileNameNoExtension , ScriptFileNameOnlyExtension ) = os.path.splitext(ScriptFileNameOnly)
    if ScriptFileNameOnlyExtension.lower() not in ['.py', 'py']:
        raise ValueError, 'Create Plots Error: The plot script generation file name ' + ScriptFileNameFullPath + ' does not have a python extension of "py"'
    # Make sure the module is in the system path. First save the current
    # system path and then change it
    OldSysPath = sys.path
    # Insert the new path at the beginning of the search list so that
    # the correct file will be run in case of duplicate filenames in
    # different directories.
    sys.path.insert(0,ScriptPathOnly)
    # Now try running the generation - enclose this in a try catch clause
    try:
        # Run the Generation
        __import__(ScriptFileNameNoExtension)
        # remove the module from sys.modules to allow reload later on
        del(sys.modules[ScriptFileNameNoExtension])
    except:
        (ExceptType, ExceptValue, ExceptTraceback) = sys.exc_info()
        if ScriptFileNameNoExtension in sys.modules:
            del(sys.modules[ScriptFileNameNoExtension])
        sys.path = OldSysPath
        ErrorText = 'Create Plots Error: Generation Execution Error: An error was encountered while running the plot generation script file ' + ScriptFileNameFullPath + ' . Here are details about the error: ' + str(ExceptValue)
        # report the error
        raise ValueError, ErrorText
    sys.path = OldSysPath
    # Return the plot generation file name
    return (ScriptPathOnly , ScriptFileNameOnly, ScriptFileNameFullPath)


if __name__ == "__main__":
    # Redirect stdout to File if needed
    (sys.stdout, BackupOfOriginal) = DB.RedirectOutputToValidFile(sys.stdout)
    if len(sys.argv) == 4:
        InputFileName = sys.argv[1]
        OutputFileName = sys.argv[2]
        PlotSequenceCandidate = sys.argv[3]
        InputIsSequence = True
        if PlotSequenceCandidate.strip()[0]=='[' and PlotSequenceCandidate.strip()[-1]== ']':
            # This seems to be a valid python list this means the user provided
            # the sequence itself
            PlotSequence = eval(PlotSequenceCandidate, DB.EmptyEvalDict)
        else:
            # if this is not a list, then this means the user provided
            # a file
            TheSequenceFile = open(PlotSequenceCandidate,'r')
            PlotSequenceText = TheSequenceFile.read()
            TheSequenceFile.close()
            PlotSequence = eval(PlotSequenceText, DB.EmptyEvalDict)
        CreatePlots(InputFileName,OutputFileName,PlotSequence)
    else:
        print 'Info: this script can be invoked from command line using the following syntax:'
        print ' CreatePlotsFromCSV.py InputFileName OutputFileName PlotSequence'
        print ' InputFileName = File name with the assembled CSV report after simulation'
        print '                 where rows represent parameters and columns represent'
        print '                 different scenarios of running the simulation.'
        print '                 Upper most row contains scenario titles and the two'
        print '                 left most columns contain parameter name and calculation'
        print '                 method.'
        print ' OutputFileName = The name of the pdf file that will contain all the plots' 
        print '                  in different pages.' 
        print ' PlotSequence = a file name or a string representing the graphs to be made '
        print '                of the form [ParamList, LegendList, StyleList] where'
        print '     ParamList = ([ ParamDataX, ParamDataY1, ParamDataY2...] where'
        print '         ParamData# = ( ParamName, ParamCalcMethod, AxisTitle ) where'
        print '             ParamName = The parameter name as used in the simulation'
        print '             ParamCalcMethod = The statistics calculation method used to'
        print '                               calculate the report - in short form.'
        print '                               ignoring the * at the end. If left blank'
        print '                               then first parameter occurrence is used'
        print '             AxisTitle = The title to use for the axis. if an empty string '
        print '                         is specified, then the system will use the combined'
        print '                         ParamName and ParamCalcMethod.'
        print '         ParamData# can also be a ParamList in this case the system will'
        print '                    generate a single graph for showing all the parameters'
        print '                    in a single graph rather than creating a graph for'
        print '                    each parameter alone. Note that Further nesting of'
        print '                    ParamList in ParamData is not possible beyond 1 level'
        print '                    Note that such a list causes redefinition of the X axis'
        print '         There must be at least 2 non list ParamData items in a list. The'
        print '         first defines the X axis data. The rest define multiple Y axis data.'
        print '         unless specified as a nested list, each Y data set will be specified'
        print '         in a separate graph using the same X data set. '
        print '     LegendList = The list of title strings to be used as series in the'
        print '                  plot. These must correspond to row titles in InputFileName'
        print '     StyleList = A list of strings representing the style of the series. If'
        print '                 not provided for all series, the system will decide on style'
    # Redirect stdout back if needed
    sys.stdout = DB.RedirectOutputBackwards(sys.stdout, BackupOfOriginal)