From ff56ebc876fe220535ff6130a19b72e39d2c5989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicklas=20K=C3=B6rtge?= Date: Fri, 20 Sep 2024 16:47:31 +0200 Subject: [PATCH] increase heap size in docker image, update docker compose, add config gor java dependency jar (#28) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicklas Körtge --- docker-compose.yaml | 2 +- src/main/docker/Dockerfile.jvm | 5 ++++- src/main/java/com/ibm/Init.java | 9 +++++---- src/main/java/com/ibm/Utils.java | 10 ++++++++++ .../com/ibm/configuration/Configuration.java | 17 ++++++++++++++++- .../com/ibm/configuration/IConfiguration.java | 5 +++++ src/main/java/com/ibm/scan/JavaScanner.java | 15 +++++++++------ src/main/resources/application.yml | 1 + .../java/com/ibm/DefaultTestConfiguration.java | 17 ++++++++++++++++- 9 files changed, 67 insertions(+), 14 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 1c5569c3b..84eb4fb51 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -34,7 +34,7 @@ services: deploy: resources: reservations: - memory: 8192m + memory: 16g profiles: - prod - ext-compliance diff --git a/src/main/docker/Dockerfile.jvm b/src/main/docker/Dockerfile.jvm index dd2e2dfac..568e9e596 100644 --- a/src/main/docker/Dockerfile.jvm +++ b/src/main/docker/Dockerfile.jvm @@ -7,9 +7,12 @@ COPY --chown=185 target/quarkus-app/lib/ /deployments/lib/ COPY --chown=185 target/quarkus-app/*.jar /deployments/ COPY --chown=185 target/quarkus-app/app/ /deployments/app/ COPY --chown=185 target/quarkus-app/quarkus/ /deployments/quarkus/ +# copy the crypto lib dependecies for java into the image +COPY --chown=185 src/main/resources/java/scan/*.jar /deployments/java/scan/ EXPOSE 8080 USER 185 -ENV JAVA_OPTS="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager" +ENV CBOMKIT_JAVA_JAR_DIR="/deployments/java/scan/" +ENV JAVA_OPTS="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager -Xmx8g" ENV JAVA_APP_JAR="/deployments/quarkus-run.jar" diff --git a/src/main/java/com/ibm/Init.java b/src/main/java/com/ibm/Init.java index 87cf5a3e6..2bb7bd3c7 100644 --- a/src/main/java/com/ibm/Init.java +++ b/src/main/java/com/ibm/Init.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.github.packageurl.MalformedPackageURLException; import com.github.packageurl.PackageURL; +import com.ibm.configuration.Configuration; import com.ibm.model.Identifiers; import com.ibm.model.PurlVersion; import io.quarkus.runtime.Quarkus; @@ -39,10 +40,10 @@ public class Init implements QuarkusApplication { @Override public int run(String... args) throws Exception { - try (InputStream in = - // Thread.currentThread().getContextClassLoader().getResourceAsStream("purls.json")) - // { - this.getClass().getClassLoader().getResourceAsStream("purls.json")) { + // check if jars exists + new Configuration().getJavaDependencyJARS(); + // load purls + try (InputStream in = this.getClass().getClassLoader().getResourceAsStream("purls.json")) { LOG.info("Try to load purls"); ObjectMapper mapper = new ObjectMapper(); JsonNode jsonNode = mapper.readValue(in, JsonNode.class); diff --git a/src/main/java/com/ibm/Utils.java b/src/main/java/com/ibm/Utils.java index 8ea23d3f0..b14f89d66 100644 --- a/src/main/java/com/ibm/Utils.java +++ b/src/main/java/com/ibm/Utils.java @@ -26,9 +26,11 @@ import com.ibm.model.IdentifiableScan; import com.ibm.model.api.ScanRequest; import java.io.File; +import java.io.FileFilter; import java.util.Collections; import java.util.List; import java.util.Optional; +import javax.annotation.Nonnull; import org.cyclonedx.model.Component; import org.cyclonedx.model.Evidence; import org.cyclonedx.model.component.evidence.Occurrence; @@ -107,4 +109,12 @@ public static void addProperties( properties.add(purlProp); } } + + @Nonnull + public static Optional getJarFiles(@Nonnull String directoryPath) { + final File directory = new File(directoryPath); + final FileFilter jarFilter = + file -> file.isFile() && file.getName().toLowerCase().endsWith(".jar"); + return Optional.ofNullable(directory.listFiles(jarFilter)); + } } diff --git a/src/main/java/com/ibm/configuration/Configuration.java b/src/main/java/com/ibm/configuration/Configuration.java index 69b14d5d2..a6b0cc117 100644 --- a/src/main/java/com/ibm/configuration/Configuration.java +++ b/src/main/java/com/ibm/configuration/Configuration.java @@ -19,6 +19,7 @@ */ package com.ibm.configuration; +import com.ibm.Utils; import com.ibm.compliance.BasicQuantumSafeComplianceService; import com.ibm.compliance.IComplianceService; import com.ibm.compliance.ibmregulator.IBMRegulatorClient; @@ -33,8 +34,10 @@ import com.ibm.scan.ScannerManager; import io.quarkus.rest.client.reactive.QuarkusRestClientBuilder; import jakarta.enterprise.context.ApplicationScoped; +import java.io.File; import java.net.URI; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Optional; import org.eclipse.microprofile.config.ConfigProvider; @@ -88,8 +91,20 @@ public IScannerManager getScannerManager() { .ifPresent(api -> registry.add((new IBMqsScanner(api)))); return new ScannerManager(registry); } - registry.add((new JavaScanner())); + registry.add((new JavaScanner(this))); registry.add((new PythonScanner())); return new ScannerManager(registry); } + + @Override + public @NotNull List getJavaDependencyJARS() { + return ConfigProvider.getConfig() + .getOptionalValue("service.scanning.java-jar-dir", String.class) + .flatMap(Utils::getJarFiles) + .map(files -> Arrays.stream(files).toList()) + .orElseThrow( + () -> + new IllegalStateException( + "Could not load jar dependencies for java scanning")); // Error + } } diff --git a/src/main/java/com/ibm/configuration/IConfiguration.java b/src/main/java/com/ibm/configuration/IConfiguration.java index 1e612ea2a..7850ba7f8 100644 --- a/src/main/java/com/ibm/configuration/IConfiguration.java +++ b/src/main/java/com/ibm/configuration/IConfiguration.java @@ -22,6 +22,8 @@ import com.ibm.compliance.IComplianceService; import com.ibm.repository.IScanRepository; import com.ibm.scan.IScannerManager; +import java.io.File; +import java.util.List; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -34,4 +36,7 @@ public interface IConfiguration { @Nonnull IScannerManager getScannerManager(); + + @Nonnull + List getJavaDependencyJARS(); } diff --git a/src/main/java/com/ibm/scan/JavaScanner.java b/src/main/java/com/ibm/scan/JavaScanner.java index 796f71e60..143492a7c 100644 --- a/src/main/java/com/ibm/scan/JavaScanner.java +++ b/src/main/java/com/ibm/scan/JavaScanner.java @@ -19,6 +19,7 @@ */ package com.ibm.scan; +import com.ibm.configuration.IConfiguration; import com.ibm.message.IMessageDispatcher; import com.ibm.model.Project; import com.ibm.model.api.ScanRequest; @@ -50,13 +51,11 @@ public class JavaScanner extends AbstractScanner { private List projects = null; private List visitors = null; private SonarComponents sonarComponents = null; + private final IConfiguration configuration; - private static final List JARS = - Collections.singletonList( - new File("src/main/resources/java/scan/bcprov-jdk18on-1.78.1.jar")); - - public JavaScanner() { + public JavaScanner(@Nonnull IConfiguration config) { LOG.info("Created Java scanner (*" + JAVA_FILE_EXTENSION + ")"); + this.configuration = config; } @SuppressWarnings("all") @@ -119,7 +118,11 @@ public IScanner.ScanResult scan() throws CancelScanException { new JavaAstScannerExtension(sonarComponents, iMessageDispatcher, projectStr); // add bc to classpath to resolve types VisitorsBridge visitorBridge = - new VisitorsBridge(visitors, JARS, sonarComponents, JAVA_VERSION); + new VisitorsBridge( + visitors, + configuration.getJavaDependencyJARS(), + sonarComponents, + JAVA_VERSION); jscanner.setVisitorBridge(visitorBridge); jscanner.scan(project.getInputFiles()); counter++; diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index c3530f75e..7622d497a 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -25,6 +25,7 @@ quarkus: service: clone-dir: ${CBOMKIT_CLONEDIR:/home/user/.cbomkit} # specifies the directory in which the cloned Git repositories are stored (temporary) scanning: + java-jar-dir: ${CBOMKIT_JAVA_JAR_DIR:src/main/resources/java/scan/} ibm-qs-explorer: # if the ibm qs explorer should be used as the service to scan, enable it here and provide the url enabled: false url: ${CBOMKIT_QS_EXPLORER_API_BASE:http://localhost:8000/api/v1/scan"} diff --git a/src/test/java/com/ibm/DefaultTestConfiguration.java b/src/test/java/com/ibm/DefaultTestConfiguration.java index e11300d06..56814c0e3 100644 --- a/src/test/java/com/ibm/DefaultTestConfiguration.java +++ b/src/test/java/com/ibm/DefaultTestConfiguration.java @@ -32,9 +32,12 @@ import com.ibm.scan.ScannerManager; import io.quarkus.test.Mock; import jakarta.enterprise.context.ApplicationScoped; +import java.io.File; import java.sql.Timestamp; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import org.eclipse.microprofile.config.ConfigProvider; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -96,8 +99,20 @@ public IScanRepository getCBOMRepository() { public IScannerManager getScannerManager() { // register scanners final List registry = new ArrayList<>(); - registry.add((new JavaScanner())); + registry.add((new JavaScanner(this))); registry.add((new PythonScanner())); return new ScannerManager(registry); } + + @Override + public @NotNull List getJavaDependencyJARS() { + return ConfigProvider.getConfig() + .getOptionalValue("service.scanning.java-jar-dir", String.class) + .flatMap(Utils::getJarFiles) + .map(files -> Arrays.stream(files).toList()) + .orElseThrow( + () -> + new IllegalStateException( + "Could not load jar dependencies for java scanning")); // Error + } }