Merge pull request #8 from levelops/bugfix/prop-3540-trunk

FIX: Issue removing too many old report files from a Jenkins

.vscode/settings.json

@@ -29,6 +29,6 @@
         "#"
     ],
     // "java.home": "/usr/lib/jvm/java-13-openjdk-amd64",
-    "java.jdt.ls.java.home": "/usr/lib/jvm/java-13-openjdk-amd64",
+    // "java.jdt.ls.java.home": "/usr/lib/jvm/java-13-openjdk-amd64",
     "java.import.gradle.wrapper.enabled": true
 }

pom.xml

@@ -82,7 +82,7 @@
     </profiles>
 
     <properties>
-        <revision>1.0.27</revision>
+        <revision>1.0.28</revision>
         <!-- <changelist>999999-SNAPSHOT</changelist> -->
         <changelist></changelist>
         <product.build.sourceEncoding>UTF-8</product.build.sourceEncoding>

src/main/java/io/jenkins/plugins/propelo/commons/service/JenkinsStatusService.java

@@ -95,7 +95,7 @@ private File createFileIfNotExists(File expandedLevelOpsPluginDir) throws IOExce
         try {
             FileUtils.createDirectoryRecursively(expandedLevelOpsPluginDir);
         } catch (IOException e) {
-            LOGGER.log(Level.WARNING, "JenkinsStatusService.createFileIfNotExists Error creating expandedLevelOpsPluginDir!", e);
+            LOGGER.log(Level.SEVERE, "JenkinsStatusService.createFileIfNotExists Error creating expandedLevelOpsPluginDir!", e);
             throw e;
         }
         File file = new File(expandedLevelOpsPluginDir, JENKINS_HEARTBEAT_INFO_FILE);

src/main/java/io/jenkins/plugins/propelo/job_reporter/extensions/LevelOpsRunListener.java

@@ -36,10 +36,10 @@
 import java.io.File;
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
+import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Objects;
 import java.util.UUID;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -179,6 +179,7 @@ public void onFinalized(Run run) {
             //performJobRunCompleteNotification fn logs have this.
             //LOGGER.log(Level.INFO, "Completed processing complete event jobFullName={0}, build number = {1}", new Object[]{jobRunDetail.getJobFullName(), jobRunDetail.getBuildNumber()});
         } catch (Exception e) {
+            // TODO: Delete data files after having retried a few configurable number of times
             LOGGER.log(Level.SEVERE, "Error in RunListener onCompleted!", e);
             reportError(ExceptionUtils.getFullStackTrace(e));
             if (e instanceof InterruptedException) {
@@ -187,15 +188,32 @@ public void onFinalized(Run run) {
         }
     }
 
+    /**
+     * Deletes data transmitted to propelo
+     */
     private void deleteJobRunDataCompleteDirectoryContents() {
+        LOGGER.log(Level.FINEST, "Delete Job Run Complete Data Directory (date-$date) starting");
         File dataDirectoryWithVersion = plugin.getDataDirectoryWithVersion();
-        if (dataDirectoryWithVersion != null && dataDirectoryWithVersion.exists()) {
-            LOGGER.log(Level.FINEST, "Delete Job Run Complete Data Directory (date-$date) starting");
-            for (File file : Objects.requireNonNull(dataDirectoryWithVersion.listFiles())) {
-                if (!file.getName().equalsIgnoreCase(DateUtils.getDateFormattedDirName())) {
-                    FileUtils.deleteQuietly(file);
-                }
-            }
+        if (dataDirectoryWithVersion == null || !dataDirectoryWithVersion.exists()) {
+            LOGGER.log(Level.FINE, "Skipping job run complete data directory delete... data directory = {0}", dataDirectoryWithVersion);
+            return;
+        }
+        String todayDirName = DateUtils.getDateFormattedDirName();
+        try {
+            Files.newDirectoryStream(
+                dataDirectoryWithVersion.toPath(),
+                (entry) -> {
+                    boolean use = entry.getFileName().toString().equalsIgnoreCase(todayDirName);
+                    LOGGER.log(Level.FINER, "Filtering files... accept {0}? {1}", new Object[]{todayDirName, use});
+                    return use;
+                }).forEach(item -> {
+                    LOGGER.log(Level.FINE, "Deleting old historic report: {0}", item.toString());
+                    FileUtils.deleteQuietly(item.toFile());
+                });
+        } catch (SecurityException | IOException e) {
+            LOGGER.log(Level.SEVERE, e, () -> {
+                return String.format("Unable to delete all files from the historic reports in the directory '%s' other than %s", dataDirectoryWithVersion.toPath().toString(), todayDirName);
+            });
         }
     }
 
@@ -227,7 +245,7 @@ private void reportError(final String payload) {
         GenericRequestService genericRequestService = new GenericRequestService(LevelOpsPluginConfigService.getInstance().getLevelopsConfig().getApiUrl(), mapper);
         try {
             genericRequestService.performGenericRequest(plugin.getLevelOpsApiKey().getPlainText(), "pluginErrorReport", payload, plugin.isTrustAllCertificates(), null, ProxyConfigService.generateConfigFromJenkinsProxyConfiguration(Jenkins.getInstanceOrNull()));
-        } catch (IOException e) {
+        } catch (Exception e) {
             LOGGER.log(Level.SEVERE, "Unabled to report errors back to Propelo....", e);
         }
     }

src/main/java/io/jenkins/plugins/propelo/job_reporter/plugins/PropeloPluginImpl.java

@@ -10,9 +10,9 @@
 import io.jenkins.plugins.propelo.commons.service.BlueOceanRestClient;
 import io.jenkins.plugins.propelo.commons.service.JenkinsInstanceGuidService;
 import io.jenkins.plugins.propelo.commons.service.JenkinsStatusService;
+import io.jenkins.plugins.propelo.commons.service.JenkinsStatusService.LoadFileException;
 import io.jenkins.plugins.propelo.commons.service.LevelOpsPluginConfigValidator;
 import io.jenkins.plugins.propelo.commons.service.ProxyConfigService;
-import io.jenkins.plugins.propelo.commons.service.JenkinsStatusService.LoadFileException;
 import io.jenkins.plugins.propelo.commons.utils.DateUtils;
 import io.jenkins.plugins.propelo.commons.utils.EnvironmentVariableNotDefinedException;
 import io.jenkins.plugins.propelo.commons.utils.JsonUtils;
@@ -34,13 +34,12 @@
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.net.UnknownHostException;
+import java.nio.file.Files;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import java.util.Objects;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import static io.jenkins.plugins.propelo.commons.plugins.Common.REPORTS_DIR_NAME;
@@ -80,6 +79,8 @@ public void start() throws Exception {
             LOGGER.info("No stored configuration detected");
             migrateOldPluginConfig();
         }
+        LOGGER.info("Checking work directory permissions...");
+        checkWorkDirectoryAccess();
         LOGGER.info("Deleting Older directories during plugin initialization.Started");
         deleteOlderDirectories();
         LOGGER.info("Deleting Older directories during plugin initialization.Completed");
@@ -233,16 +234,42 @@ public File getReportsDirectory() {
         return buildReportsDirectory(this.getExpandedLevelOpsPluginPath());
     }
 
+    private boolean checkWorkDirectoryAccess() {
+        File expandedLevelOpsPluginDir = getExpandedLevelOpsPluginDir();
+        File tmp = null;
+        try {
+            tmp = File.createTempFile("propelo", "access_check", expandedLevelOpsPluginDir);
+            return true;
+        }
+        catch (IOException e){
+            LOGGER.log(Level.SEVERE, "Unable to use the propelo plugin directory {0}. Either the path doesn't refer to a directory or the directory cannot be accessed.", expandedLevelOpsPluginDir.toPath());
+        }
+        finally{
+            if (tmp != null) {
+                FileUtils.deleteQuietly(tmp);
+            }
+        }
+        return false;
+    }
+
     private void deleteOlderDirectories() {
         File currentDataDirectoryWithVersion = getDataDirectoryWithVersion();
         File expandedLevelOpsPluginDir = getExpandedLevelOpsPluginDir();
-        if (expandedLevelOpsPluginDir != null && expandedLevelOpsPluginDir.exists() && currentDataDirectoryWithVersion != null) {
-            for (File file : Objects.requireNonNull(expandedLevelOpsPluginDir.listFiles(), "Unable to use the Propelo plugin directory '" + expandedLevelOpsPluginDir.getPath() + "'. Either the path doesn't refer to a firectory or the directory cannot be accessed.")) {
-                Matcher matcher = OLDER_DIRECTORIES_PATTERN.matcher(file.getName());
-                if (matcher.find() && !file.getName().equalsIgnoreCase(currentDataDirectoryWithVersion.getName())) {
-                    FileUtils.deleteQuietly(file);
-                }
-            }
+        if (expandedLevelOpsPluginDir == null || !expandedLevelOpsPluginDir.exists() || currentDataDirectoryWithVersion == null) {
+            LOGGER.log(Level.FINE, "Skipping old directories deletion: plugin_dir={0}, todays_data_directory_name={1}", new Object[]{expandedLevelOpsPluginDir, currentDataDirectoryWithVersion});
+            return;
+        }
+        try {
+            Files.newDirectoryStream(expandedLevelOpsPluginDir.toPath(), (path) -> {
+                boolean use = OLDER_DIRECTORIES_PATTERN.matcher(path.getFileName().toString()).find() && !currentDataDirectoryWithVersion.getName().equalsIgnoreCase(path.getFileName().toString());
+                LOGGER.log(Level.FINEST, "Filering files... accept {0}? {1}", new Object[]{path.toString(), use});
+                return use;
+            }).forEach(path -> {
+                LOGGER.log(Level.FINER, "Deleting file: {0}", path);
+                FileUtils.deleteQuietly(path.toFile());
+            });
+        } catch (SecurityException | IOException e) {
+            LOGGER.log(Level.SEVERE, "Unable to delete all the old directories in " + expandedLevelOpsPluginDir, e);
         }
     }