From 95bd18013b96412d404100c80ce4c7e2b57f5536 Mon Sep 17 00:00:00 2001 From: hwang381 Date: Mon, 23 Nov 2015 18:43:55 -0600 Subject: [PATCH] Add support for parsing arbitrary tags 1. LogParserResults.java - Add Maps for totalCount and linksFile for extra tags, add its associated getters and setters. - Add Set for extra tags and its getter and setter 2. LogParserUtils.java 1. standardizeStatus(final String status) The legal status condition check is removed so that arbitrary status can come through 2. compilePatterns(final String[] parsingRulesArray, final Logger logger) The extra tags are extracted and set in CompliedPatterns result 3. LogParserParser.java 1. Add extra tags to statusCount, linkFiles, writers 2. parseLog(final Run build) Add link file paths for extra paths Close file writer for extra tags Pass in extra tags for writeReferenceHtml(...) Set totalCount, linksFile and extraTag in LogParserResult for extra tags 3. colorLine(final String line, final String status) Set color as default if the tag is extra 4. CompliedPatterns.java Add List for extra tags 5. LogParserDisplayConsts.java Add default color, icon, link list display and link list display plural for extra tags 6. LogParserWriter.java 1. writeReferenceHtml(...) Add an additional parameter for extra tags Also write links for extra tags 2. writeLinks(...) Set icon, link list display and link list display plural as default if the tag is extra 7. LogParserAction.java 1. buildDataset() Add extra tags to dataset as well --- .../plugins/logparser/CompiledPatterns.java | 11 +++ .../plugins/logparser/LogParserAction.java | 9 +++ .../plugins/logparser/LogParserConsts.java | 7 +- .../logparser/LogParserDisplayConsts.java | 16 ++++ .../plugins/logparser/LogParserParser.java | 71 +++++++++++++----- .../plugins/logparser/LogParserResult.java | 58 ++++++++++++++ .../plugins/logparser/LogParserUtils.java | 22 ++++-- .../plugins/logparser/LogParserWriter.java | 30 ++++++-- .../logparser/LogParserWorkflowTest.java | 45 +++++++---- .../plugins/logparser/maven-project1.zip | Bin 4551 -> 4626 bytes 10 files changed, 223 insertions(+), 46 deletions(-) diff --git a/src/main/java/hudson/plugins/logparser/CompiledPatterns.java b/src/main/java/hudson/plugins/logparser/CompiledPatterns.java index ad5ca8c..85ce41d 100755 --- a/src/main/java/hudson/plugins/logparser/CompiledPatterns.java +++ b/src/main/java/hudson/plugins/logparser/CompiledPatterns.java @@ -1,16 +1,20 @@ package hudson.plugins.logparser; import javax.annotation.CheckForNull; +import java.util.ArrayList; +import java.util.List; import java.util.regex.Pattern; public class CompiledPatterns { private String errorMsg; private Pattern[] compiledPatterns; + private List extraTags; public CompiledPatterns() { this.errorMsg = null; this.compiledPatterns = null; + this.extraTags = new ArrayList(); } public String getError() { @@ -34,4 +38,11 @@ public void setCompiledPatters(final Pattern[] compiledPatterns) { this.compiledPatterns = compiledPatterns; } + public List getExtraTags() { + return extraTags; + } + + public void setExtraTags(List extraTags) { + this.extraTags = extraTags; + } } diff --git a/src/main/java/hudson/plugins/logparser/LogParserAction.java b/src/main/java/hudson/plugins/logparser/LogParserAction.java index ec396ef..19d803f 100755 --- a/src/main/java/hudson/plugins/logparser/LogParserAction.java +++ b/src/main/java/hudson/plugins/logparser/LogParserAction.java @@ -142,6 +142,12 @@ private CategoryDataset buildDataSet() { new ChartUtil.NumberOnlyBuildLabel(a.getOwner())); dsb.add(a.result.getTotalInfos(), "infos", new ChartUtil.NumberOnlyBuildLabel(a.getOwner())); + dsb.add(a.result.getTotalDebugs(), "debugs", + new ChartUtil.NumberOnlyBuildLabel(a.getOwner())); + for (String extraTag : a.result.getExtraTags()) { + dsb.add(a.result.getTotalCountsByExtraTag(extraTag), extraTag, + new ChartUtil.NumberOnlyBuildLabel(a.getOwner())); + } } return dsb.build(); } @@ -211,6 +217,8 @@ public String generateToolTip(CategoryDataset dataset, int row, return "Errors: " + result.getTotalErrors(); case 1: return "Warnings: " + result.getTotalWarnings(); + case 2: + return "Debugs: " + result.getTotalDebugs(); default: return "Infos: " + result.getTotalInfos(); } @@ -220,6 +228,7 @@ public String generateToolTip(CategoryDataset dataset, int row, ar.setSeriesPaint(0, ColorPalette.RED); // error ar.setSeriesPaint(1, ColorPalette.BLUE); // info ar.setSeriesPaint(2, ColorPalette.YELLOW); // warning + ar.setSeriesPaint(3, ColorPalette.BLUE); // debug // crop extra space around the graph plot.setInsets(new RectangleInsets(0, 0, 0, 5.0)); diff --git a/src/main/java/hudson/plugins/logparser/LogParserConsts.java b/src/main/java/hudson/plugins/logparser/LogParserConsts.java index 361589b..0fef390 100755 --- a/src/main/java/hudson/plugins/logparser/LogParserConsts.java +++ b/src/main/java/hudson/plugins/logparser/LogParserConsts.java @@ -10,6 +10,7 @@ public class LogParserConsts { public static final String ERROR = "ERROR"; public static final String WARNING = "WARNING"; public static final String INFO = "INFO"; + public static final String DEBUG = "DEBUG"; public static final String NONE = "NONE"; public static final String START = "START"; // marks a beginning of a section public static final String DEFAULT = NONE; @@ -18,9 +19,9 @@ public class LogParserConsts { public static final String CANNOT_PARSE = "log-parser plugin ERROR: Cannot parse log "; public static final String NOT_INT = " is not an integer - using default"; - public static final List LEGAL_STATUS = Arrays.asList(ERROR, WARNING, INFO, NONE, START); - public static final List STATUSES_WITH_LINK_FILES = Arrays.asList(ERROR, WARNING, INFO); - public static final List STATUSES_WITH_SECTIONS_IN_LINK_FILES = Arrays.asList(ERROR, WARNING); + public static final List LEGAL_STATUS = Arrays.asList(ERROR, WARNING, INFO, DEBUG, NONE, START); + public static final List STATUSES_WITH_LINK_FILES = Arrays.asList(ERROR, WARNING, INFO, DEBUG); + public static final List STATUSES_WITH_SECTIONS_IN_LINK_FILES = Arrays.asList(ERROR, WARNING, DEBUG); public static String getHtmlOpeningTags() { final String hudsonRoot = Jenkins.getActiveInstance().getRootUrl(); diff --git a/src/main/java/hudson/plugins/logparser/LogParserDisplayConsts.java b/src/main/java/hudson/plugins/logparser/LogParserDisplayConsts.java index f8a72b1..4bd5702 100755 --- a/src/main/java/hudson/plugins/logparser/LogParserDisplayConsts.java +++ b/src/main/java/hudson/plugins/logparser/LogParserDisplayConsts.java @@ -1,6 +1,7 @@ package hudson.plugins.logparser; import java.util.HashMap; +import org.apache.commons.lang.WordUtils; public class LogParserDisplayConsts { @@ -9,26 +10,41 @@ public class LogParserDisplayConsts { final private HashMap linkListDisplay = new HashMap(); final private HashMap linkListDisplayPlural = new HashMap(); + public static final String DEFAULT_COLOR = "blue"; + public static final String DEFAULT_ICON = "blue.gif"; + + public static String getDefaultLinkListDisplay(String status) { + return WordUtils.capitalize(status); + } + + public static String getDefaultLinkListDisplayPlural(String status) { + return getDefaultLinkListDisplay(status) + "s"; + } + public LogParserDisplayConsts() { // Color of each status colorTable.put(LogParserConsts.ERROR, "red"); colorTable.put(LogParserConsts.WARNING, "orange"); colorTable.put(LogParserConsts.INFO, "blue"); colorTable.put(LogParserConsts.START, "blue"); + colorTable.put(LogParserConsts.DEBUG, "blue"); // Icon for each status in the summary iconTable.put(LogParserConsts.ERROR, "red.gif"); iconTable.put(LogParserConsts.WARNING, "yellow.gif"); iconTable.put(LogParserConsts.INFO, "blue.gif"); + iconTable.put(LogParserConsts.DEBUG, "blue.gif"); // How to display in link summary html linkListDisplay.put(LogParserConsts.ERROR, "Error"); linkListDisplay.put(LogParserConsts.WARNING, "Warning"); linkListDisplay.put(LogParserConsts.INFO, "Info"); + linkListDisplay.put(LogParserConsts.DEBUG, "Debug"); linkListDisplayPlural.put(LogParserConsts.ERROR, "Errors"); linkListDisplayPlural.put(LogParserConsts.WARNING, "Warnings"); linkListDisplayPlural.put(LogParserConsts.INFO, "Infos"); + linkListDisplayPlural.put(LogParserConsts.DEBUG, "Debugs"); } public HashMap getColorTable() { diff --git a/src/main/java/hudson/plugins/logparser/LogParserParser.java b/src/main/java/hudson/plugins/logparser/LogParserParser.java index bddc97f..aca8316 100755 --- a/src/main/java/hudson/plugins/logparser/LogParserParser.java +++ b/src/main/java/hudson/plugins/logparser/LogParserParser.java @@ -16,6 +16,8 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Pattern; @@ -29,6 +31,7 @@ public class LogParserParser { final private String[] parsingRulesArray; final private Pattern[] compiledPatterns; final private CompiledPatterns compiledPatternsPlusError; + final private List extraTags; // if key is 3-ERROR it shows how many errors are in section 3 final private HashMap statusCountPerSection = new HashMap(); @@ -47,11 +50,6 @@ public LogParserParser(final FilePath parsingRulesFile, // init logger final Logger logger = Logger.getLogger(getClass().getName()); - // Count of lines in this status - statusCount.put(LogParserConsts.ERROR, 0); - statusCount.put(LogParserConsts.WARNING, 0); - statusCount.put(LogParserConsts.INFO, 0); - this.parsingRulesArray = LogParserUtils .readParsingRules(parsingRulesFile); @@ -61,9 +59,19 @@ public LogParserParser(final FilePath parsingRulesFile, this.parsingRulesArray, logger); this.compiledPatterns = this.compiledPatternsPlusError .getCompiledPatterns(); + this.extraTags = this.compiledPatternsPlusError.getExtraTags(); this.preformattedHtml = preformattedHtml; this.channel = channel; + + // Count of lines in this status + statusCount.put(LogParserConsts.ERROR, 0); + statusCount.put(LogParserConsts.WARNING, 0); + statusCount.put(LogParserConsts.INFO, 0); + statusCount.put(LogParserConsts.DEBUG, 0); + for (String extraTag : this.extraTags) { + statusCount.put(extraTag, 0); + } } /* @@ -90,16 +98,24 @@ public LogParserResult parseLog(final Run build) throws IOException, Inter // Determine parsed log files final String parsedFilePath = logDirectory + "/log_content.html"; final String errorLinksFilePath = logDirectory + "/logerrorLinks.html"; - final String warningLinksFilePath = logDirectory - + "/logwarningLinks.html"; + final String warningLinksFilePath = logDirectory + "/logwarningLinks.html"; final String infoLinksFilePath = logDirectory + "/loginfoLinks.html"; + final String debugLinksFilePath = logDirectory + "/logdebugLinks.html"; + final Map linksFilePathByExtraTags = new HashMap(); + for (String extraTag : this.extraTags) { + linksFilePathByExtraTags.put(extraTag, logDirectory + "/log" + extraTag + "Links.html"); + } final String buildRefPath = logDirectory + "/log_ref.html"; final String buildWrapperPath = logDirectory + "/log.html"; - // Record file paths in hash + // Record file paths in HashMap linkFiles.put(LogParserConsts.ERROR, errorLinksFilePath); linkFiles.put(LogParserConsts.WARNING, warningLinksFilePath); linkFiles.put(LogParserConsts.INFO, infoLinksFilePath); + linkFiles.put(LogParserConsts.DEBUG, debugLinksFilePath); + for (String extraTag : this.extraTags) { + linkFiles.put(extraTag, linksFilePathByExtraTags.get(extraTag)); + } // Open console log for reading and all other files for writing final BufferedWriter writer = new BufferedWriter(new FileWriter( @@ -112,18 +128,23 @@ public LogParserResult parseLog(final Run build) throws IOException, Inter warningLinksFilePath))); writers.put(LogParserConsts.INFO, new BufferedWriter(new FileWriter( infoLinksFilePath))); + writers.put(LogParserConsts.DEBUG, new BufferedWriter(new FileWriter( + debugLinksFilePath))); + for (String extraTag : this.extraTags) { + writers.put(extraTag, new BufferedWriter(new FileWriter( + linksFilePathByExtraTags.get(extraTag)))); + } // Loop on the console log as long as there are input lines and parse // line by line // At the end of this loop, we will have: // - a parsed log with colored lines - // - 3 links files which will be consolidated into one referencing html + // - 4 links files which will be consolidated into one referencing html // file. // Create dummy header and section for beginning of log final String shortLink = " Beginning of log"; - LogParserWriter.writeHeaderTemplateToAllLinkFiles(writers, - sectionCounter); // This enters a line which will later be + LogParserWriter.writeHeaderTemplateToAllLinkFiles(writers, sectionCounter); // This enters a line which will later be // replaced by the actual header and count for // this header headerForSection.add(shortLink); @@ -156,6 +177,10 @@ public LogParserResult parseLog(final Run build) throws IOException, Inter ((BufferedWriter) writers.get(LogParserConsts.ERROR)).close(); ((BufferedWriter) writers.get(LogParserConsts.WARNING)).close(); ((BufferedWriter) writers.get(LogParserConsts.INFO)).close(); + ((BufferedWriter) writers.get(LogParserConsts.DEBUG)).close(); + for (String extraTag : this.extraTags) { + ((BufferedWriter) writers.get(extraTag)).close(); + } // Build the reference html from the warnings/errors/info html files // created in the loop above @@ -163,7 +188,7 @@ public LogParserResult parseLog(final Run build) throws IOException, Inter statusCountPerSection, displayConstants.getIconTable(), displayConstants.getLinkListDisplay(), displayConstants.getLinkListDisplayPlural(), statusCount, - linkFiles); + linkFiles, extraTags); // Write the wrapping html for the reference page and the parsed log page LogParserWriter.writeWrapperHtml(buildWrapperPath); @@ -175,16 +200,23 @@ public LogParserResult parseLog(final Run build) throws IOException, Inter final LogParserResult result = new LogParserResult(); result.setHtmlLogFile(parsedFilePath); result.setTotalErrors((Integer) statusCount.get(LogParserConsts.ERROR)); - result.setTotalWarnings((Integer) statusCount - .get(LogParserConsts.WARNING)); + result.setTotalWarnings((Integer) statusCount.get(LogParserConsts.WARNING)); result.setTotalInfos((Integer) statusCount.get(LogParserConsts.INFO)); + result.setTotalDebugs((Integer) statusCount.get(LogParserConsts.DEBUG)); + for (String extraTag : this.extraTags) { + result.putTotalCountsByExtraTag(extraTag, (Integer) statusCount.get(extraTag)); + } result.setErrorLinksFile(errorLinksFilePath); result.setWarningLinksFile(warningLinksFilePath); result.setInfoLinksFile(infoLinksFilePath); + result.setDebugLinksFile(debugLinksFilePath); + for (String extraTag : this.extraTags) { + result.putLinksFileByExtraTag(extraTag, linksFilePathByExtraTags.get(extraTag)); + } result.setParsedLogURL(parsedLogURL); result.setHtmlLogPath(logDirectory); - result.setBadParsingRulesError(this.compiledPatternsPlusError - .getError()); + result.setBadParsingRulesError(this.compiledPatternsPlusError.getError()); + result.setExtraTags(this.extraTags); return result; @@ -250,10 +282,13 @@ public void incrementCounterPerSection(final String status, } private String colorLine(final String line, final String status) { - final String color = (String) displayConstants.getColorTable().get(status); + String color = (String) displayConstants.getColorTable().get(status); + if (color == null) { + color = LogParserDisplayConsts.DEFAULT_COLOR; + } final StringBuffer result = new StringBuffer(""); result.append(line); diff --git a/src/main/java/hudson/plugins/logparser/LogParserResult.java b/src/main/java/hudson/plugins/logparser/LogParserResult.java index 9d469f1..e352c70 100755 --- a/src/main/java/hudson/plugins/logparser/LogParserResult.java +++ b/src/main/java/hudson/plugins/logparser/LogParserResult.java @@ -5,17 +5,27 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.io.Reader; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; public class LogParserResult { private int totalErrors = 0; private int totalWarnings = 0; private int totalInfos = 0; + private int totalDebugs = 0; + private Map totalCountsByExtraTag = new HashMap(); private String htmlLogFile; private String errorLinksFile; private String warningLinksFile; private String infoLinksFile; + private String debugLinksFile; + private Map linkedFilesByExtraTag = new HashMap(); + private Set extraTags = new HashSet(); private String parsedLogURL; private String htmlLogPath; @@ -55,6 +65,14 @@ public int getTotalInfos() { return totalInfos; } + public int getTotalDebugs() { + return totalDebugs; + } + + public int getTotalCountsByExtraTag(String tag) { + return totalCountsByExtraTag.get(tag); + } + public String getHtmlLogFile() { return htmlLogFile; } @@ -75,6 +93,14 @@ public String getInfoLinksFile() { return infoLinksFile; } + public String getDebugLinksFile() { + return debugLinksFile; + } + + public String getLinksFileByExtraTag(String tag) { + return linkedFilesByExtraTag.get(tag); + } + public String getParsedLogURL() { return parsedLogURL; } @@ -103,6 +129,14 @@ public Reader getInfoLinksReader() throws IOException { return getReader(getInfoLinksFile()); } + public Reader getDebugLinkedReader() throws IOException { + return getReader(getDebugLinksFile()); + } + + public Reader getLinkedReaderByExtraTag(String tag) throws IOException { + return getReader(getLinksFileByExtraTag(tag)); + } + public void setHtmlLogFile(final String file) { this.htmlLogFile = file; } @@ -123,6 +157,14 @@ public void setInfoLinksFile(final String file) { this.infoLinksFile = file; } + public void setDebugLinksFile(final String file) { + this.debugLinksFile = file; + } + + public void putLinksFileByExtraTag(final String tag, final String file) { + this.linkedFilesByExtraTag.put(tag, file); + } + public void setTotalErrors(final int totalErrors) { this.totalErrors = totalErrors; } @@ -135,6 +177,14 @@ public void setTotalInfos(final int totalInfos) { this.totalInfos = totalInfos; } + public void setTotalDebugs(final int totalDebugs) { + this.totalDebugs = totalDebugs; + } + + public void putTotalCountsByExtraTag(final String tag, final int totalCounts) { + this.totalCountsByExtraTag.put(tag, totalCounts); + } + public void setParsedLogURL(final String parsedLogURL) { this.parsedLogURL = parsedLogURL; } @@ -143,6 +193,14 @@ public File getHtmlLogFileToRead() { return new File(this.htmlLogFile); } + public void setExtraTags(Collection extraTags) { + this.extraTags.addAll(extraTags); + } + + public Set getExtraTags() { + return this.extraTags; + } + public String getHtmlContent() { final StringBuffer result = new StringBuffer(""); String line = ""; diff --git a/src/main/java/hudson/plugins/logparser/LogParserUtils.java b/src/main/java/hudson/plugins/logparser/LogParserUtils.java index 4ed303d..089e3a2 100755 --- a/src/main/java/hudson/plugins/logparser/LogParserUtils.java +++ b/src/main/java/hudson/plugins/logparser/LogParserUtils.java @@ -5,6 +5,8 @@ import java.io.FileReader; import java.io.IOException; import java.io.LineNumberReader; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Locale; import java.util.logging.Level; @@ -43,15 +45,17 @@ public static String standardizeStatus(final String status) { } else if (result.equalsIgnoreCase("warn") || result.equalsIgnoreCase("end")) { result = LogParserConsts.WARNING; - } else { + } else if (LogParserConsts.LEGAL_STATUS.contains(result.toUpperCase(Locale.ENGLISH))) { result = result.toUpperCase(Locale.ENGLISH); } // If some non-existent status is in the configuration - disregard it - final List legals = LogParserConsts.LEGAL_STATUS; - if (!legals.contains(result)) { - result = LogParserConsts.DEFAULT; - } + // Arbitrary tag: this condition check is commented out so that arbitrary status can come through + + //final List legals = LogParserConsts.LEGAL_STATUS; + //if (!legals.contains(result)) { + // result = LogParserConsts.DEFAULT; + //} return result; } @@ -61,6 +65,7 @@ public static CompiledPatterns compilePatterns( Pattern[] result = new Pattern[parsingRulesArray.length]; final StringBuffer badParsingRules = new StringBuffer(); + List extraTags = new ArrayList(); for (int i = 0; i < parsingRulesArray.length; i++) { final String parsingRule = parsingRulesArray[i]; @@ -69,6 +74,12 @@ public static CompiledPatterns compilePatterns( try { final String ruleParts[] = parsingRule.split("\\s"); String regexp = ruleParts[1]; + String tag = ruleParts[0]; + if (!LogParserConsts.LEGAL_STATUS.contains(tag.toUpperCase(Locale.ENGLISH))) { + if (!Arrays.asList("OK", "END", "WARN").contains(tag.toUpperCase(Locale.ENGLISH))) { + extraTags.add(tag); + } + } final int firstDash = parsingRule.indexOf('/'); final int lastDash = parsingRule.lastIndexOf('/'); @@ -92,6 +103,7 @@ public static CompiledPatterns compilePatterns( final CompiledPatterns fullResult = new CompiledPatterns(); fullResult.setCompiledPatters(result); fullResult.setError(badParsingRules.toString()); + fullResult.setExtraTags(extraTags); return fullResult; } diff --git a/src/main/java/hudson/plugins/logparser/LogParserWriter.java b/src/main/java/hudson/plugins/logparser/LogParserWriter.java index 0f9ef01..21054da 100755 --- a/src/main/java/hudson/plugins/logparser/LogParserWriter.java +++ b/src/main/java/hudson/plugins/logparser/LogParserWriter.java @@ -54,7 +54,8 @@ public static void writeReferenceHtml(final String buildRefPath, final HashMap linkListDisplay, final HashMap linkListDisplayPlural, final HashMap statusCount, - final HashMap linkFiles) throws IOException { + final HashMap linkFiles, + final List extraTags) throws IOException { final String refStart = "