diff --git a/dev/fattest.simplicity/src/componenttest/containers/ArtifactoryImageNameSubstitutor.java b/dev/fattest.simplicity/src/componenttest/containers/ArtifactoryImageNameSubstitutor.java index 5cc8be0d2cd..61e73dd141d 100644 --- a/dev/fattest.simplicity/src/componenttest/containers/ArtifactoryImageNameSubstitutor.java +++ b/dev/fattest.simplicity/src/componenttest/containers/ArtifactoryImageNameSubstitutor.java @@ -12,6 +12,9 @@ *******************************************************************************/ package componenttest.containers; +import java.util.Arrays; +import java.util.List; + import org.testcontainers.DockerClientFactory; import org.testcontainers.dockerclient.EnvironmentAndSystemPropertyClientProviderStrategy; import org.testcontainers.utility.DockerImageName; @@ -40,6 +43,27 @@ public class ArtifactoryImageNameSubstitutor extends ImageNameSubstitutor { */ private static final String mirror = "wasliberty-docker-remote"; + /** + * TODO remove this temp mirror and either build these images at runtime + * or pull from a self hosted docker repository + * + * Artifactory keeps a set of community docker images that have been removed + * from DockerHub within this organization specifically for the Liberty builds. + */ + private static final String tempMirror = "wasliberty-infrastructure-docker"; + + /** + * TODO remove this temp repository list and either build these images at runtime + * or pull from a self hosted docker repository + * + * Artifactory keeps a set of community docker images that have been removed + * from DockerHub with this set of repository names. + */ + private static final List tempRepositories = Arrays.asList("kyleaure/"); + + private static final boolean mockBehavior = System.getenv().containsKey("MOCK_ARTIFACTORY_BEHAVIOR") + && System.getenv().get("MOCK_ARTIFACTORY_BEHAVIOR").equalsIgnoreCase("true"); + @Override public DockerImageName apply(final DockerImageName original) { DockerImageName result = null; @@ -59,8 +83,8 @@ public DockerImageName apply(final DockerImageName original) { // Priority 2a: If the image is known to only exist in an Artifactory organization if (original.getRepository().contains("wasliberty-")) { result = DockerImageName.parse(original.asCanonicalNameString()) - .withRegistry(ArtifactoryRegistry.instance().getRegistry()) - .asCompatibleSubstituteFor(original); + .withRegistry(ArtifactoryRegistry.instance().getRegistry()) + .asCompatibleSubstituteFor(original); needsArtifactory = true; reason = "This image only exists in Artifactory, must use Artifactory registry."; break; @@ -80,7 +104,7 @@ public DockerImageName apply(final DockerImageName original) { // Priority 4: Always use Artifactory if using remote docker host. if (DockerClientFactory.instance().isUsing(EnvironmentAndSystemPropertyClientProviderStrategy.class)) { - result = DockerImageName.parse(mirror + '/' + original.asCanonicalNameString()) + result = DockerImageName.parse(whichMirror(original) + '/' + original.asCanonicalNameString()) .withRegistry(ArtifactoryRegistry.instance().getRegistry()) .asCompatibleSubstituteFor(original); needsArtifactory = true; @@ -98,7 +122,7 @@ public DockerImageName apply(final DockerImageName original) { // Priority 6: If Artifactory registry is available use it to avoid rate limits on other registries if (ArtifactoryRegistry.instance().isArtifactoryAvailable()) { - result = DockerImageName.parse(mirror + '/' + original.asCanonicalNameString()) + result = DockerImageName.parse(whichMirror(original) + '/' + original.asCanonicalNameString()) .withRegistry(ArtifactoryRegistry.instance().getRegistry()) .asCompatibleSubstituteFor(original); needsArtifactory = true; @@ -106,13 +130,23 @@ public DockerImageName apply(final DockerImageName original) { break; } + // Priority 7: If we need to mock this behavior for image name generation + if (mockBehavior) { + result = DockerImageName.parse(whichMirror(original) + '/' + original.asCanonicalNameString()) + .withRegistry(ArtifactoryRegistry.instance().getRegistry()) + .asCompatibleSubstituteFor(original); + needsArtifactory = true; + reason = "Mocking artifactory behavior."; + break; + } + //default - use original result = original; reason = "Default behavior: use default docker registry."; } while (false); // We determined we need Artifactory, but it is unavailable. - if (needsArtifactory && !ArtifactoryRegistry.instance().isArtifactoryAvailable()) { + if (needsArtifactory && !mockBehavior && !ArtifactoryRegistry.instance().isArtifactoryAvailable()) { throw new RuntimeException("Need to swap image " + original.asCanonicalNameString() + " --> " + result.asCanonicalNameString() + System.lineSeparator() + "Reason: " + reason + System.lineSeparator() + "Error: The Artifactory registry was not added to the docker config.", // @@ -161,4 +195,19 @@ private static boolean isSyntheticImage(DockerImageName dockerImage) { return isSynthetic || isCommittedImage; } + /** + * Determine which internal mirror a docker image should belong to + */ + private static String whichMirror(DockerImageName dockerImage) { + for (String tempRepository : tempRepositories) { + if (dockerImage.getRepository().startsWith(tempRepository)) { + Log.info(c, "whichMirror", "Using mirror " + tempMirror + " for docker image " + dockerImage.asCanonicalNameString()); + return tempMirror; + } + } + + Log.info(c, "whichMirror", "Using mirror " + mirror + " for docker image " + dockerImage.asCanonicalNameString()); + return mirror; + } + } diff --git a/dev/fattest.simplicity/test/componenttest/containers/ArtifactoryImageNameSubstitutorTest.java b/dev/fattest.simplicity/test/componenttest/containers/ArtifactoryImageNameSubstitutorTest.java index 2ec67d6de32..d6d0aeab9f8 100644 --- a/dev/fattest.simplicity/test/componenttest/containers/ArtifactoryImageNameSubstitutorTest.java +++ b/dev/fattest.simplicity/test/componenttest/containers/ArtifactoryImageNameSubstitutorTest.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2023 IBM Corporation and others. + * Copyright (c) 2023, 2024 IBM Corporation and others. * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License 2.0 * which accompanies this distribution, and is available at @@ -157,7 +157,7 @@ public void testDockerClientStrategy() throws Exception { setDockerClientStrategy(new EnvironmentAndSystemPropertyClientProviderStrategy()); input = DockerImageName.parse("kyleaure/oracle-xe:1.0.0"); - expected = DockerImageName.parse("example.com/wasliberty-docker-remote/kyleaure/oracle-xe:1.0.0"); + expected = DockerImageName.parse("example.com/wasliberty-infrastructure-docker/kyleaure/oracle-xe:1.0.0"); assertEquals(expected, new ArtifactoryImageNameSubstitutor().apply(input)); //Using local docker host @@ -205,7 +205,7 @@ public void testSystemPropertyModified() throws Exception { expected = DockerImageName.parse("openliberty:1.0.0"); assertEquals(expected, new ArtifactoryImageNameSubstitutor().apply(expected)); - //No force, with Artifactory, use artifactory + //No force, with Artifactory, use Artifactory setDockerClientStrategy(new UnixSocketClientProviderStrategy()); System.setProperty(FORCE_EXTERNAL, "false"); setArtifactoryRegistryAvailable(true); diff --git a/dev/io.openliberty.org.testcontainers/.classpath b/dev/io.openliberty.org.testcontainers/.classpath index f86804bc5cd..a86d82fe577 100644 --- a/dev/io.openliberty.org.testcontainers/.classpath +++ b/dev/io.openliberty.org.testcontainers/.classpath @@ -1,6 +1,11 @@ - + + + + + + diff --git a/dev/io.openliberty.org.testcontainers/build.gradle b/dev/io.openliberty.org.testcontainers/build.gradle index 557cf1259ce..ecbb6abdd57 100644 --- a/dev/io.openliberty.org.testcontainers/build.gradle +++ b/dev/io.openliberty.org.testcontainers/build.gradle @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2022 IBM Corporation and others. + * Copyright (c) 2022, 2024 IBM Corporation and others. * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License 2.0 * which accompanies this distribution, and is available at @@ -10,72 +10,39 @@ * Contributors: * IBM Corporation - initial API and implementation *******************************************************************************/ +configurations { + runtime +} -task assembleTestContainerData { - doLast{ - long start = System.currentTimeMillis() - - //First create a list of projects that use testcontainers - File projectList = new File(project.getProjectDir(), 'cache/projects') - projectList.getParentFile().mkdirs() - projectList.delete() - - //Investigate all gradle files - def projectArr = ['fattest.simplicity'] - rootDir.eachDir() { nextDir -> - if(nextDir == project.getProjectDir()) - return // continue: do not check this project - - File gradleFile = new File(nextDir, 'build.gradle') - if(gradleFile.exists() && gradleFile.readLines().findAll { it.contains('copyTestContainers') }.size() > 0 ) - projectArr << gradleFile.getParentFile().getName() //Cache project name - } - - //Next create a list of all images used by testcontainer projects - File imageList = new File(project.getProjectDir(), 'cache/images') - imageList.delete() - - //Investigate all bnd files - def imageArr = [] - def prefix = 'fat.test.container.images:' - projectArr.each { foundProject -> - File bndFile = new File(rootDir, foundProject + '/bnd.bnd') - - if(bndFile.exists() && bndFile.readLines().findAll { it.contains(prefix) }.size() > 0) { - - //Cache image names - def getNext = false - bndFile.eachLine { line -> - if( line.contains(prefix) || getNext ) { - line = line.replaceAll('\\s', '') - prefix - if ( line[-1] == '\\') - getNext = true - else - getNext = false - - line.split(',').each { image -> - imageArr << image - } //line.split - }//line.contains - }//bndFile.eachLine - }//bndFile.exists - }//projectArr.each - - // Write lists to file - projectList << '# NOTICE: This file was automatically updated to reflect changes made to test projects.' << '\n' - projectList << '# Please check these changes into GitHub' << '\n' - projectArr.sort().forEach() { projectList << it << '\n' } - - imageList << '# NOTICE: This file was automatically updated to reflect changes made to test projects.' << '\n' - imageList << '# Please check these changes into GitHub.' << '\n' - imageArr.unique().sort() - imageArr -= '\\' - imageArr.forEach() { imageList << it << '\n' } - - println 'Cached test container project and image data in ' + projectList + ' and ' + imageList - - // Average execution time is 500 ms - long end = System.currentTimeMillis() - println 'Execution time in ms: ' + ( end - start ) - } +dependencies { + runtime project(':io.openliberty.com.fasterxml.jackson') + runtime project(':com.ibm.ws.componenttest') + runtime project(':fattest.simplicity') +} + +task assembleTestContainerData(type: JavaExec) { + onlyIf { // Do not execute this task during our builds + !isAutomatedBuild + } + + // Setup environment to mock external state + environment "TESTCONTAINERS_IMAGE_SUBSTITUTOR", "componenttest.containers.ArtifactoryImageNameSubstitutor" + environment "MOCK_ARTIFACTORY_BEHAVIOR", "true" + + classpath = configurations.runtime.plus(sourceSets.main.runtimeClasspath) + main = "io.openliberty.org.testcontainers.generate.CacheFiles" + args project.getProjectDir() +} + +task generateCustomImages(type: JavaExec) { + onlyIf { // Prototype - should never be run + false + } + + // Ensure cache has been generated + dependsOn assembleTestContainerData + + classpath = configurations.runtime.plus(sourceSets.main.runtimeClasspath) + main = "io.openliberty.org.testcontainers.generate.CustomImages" + args project.getProjectDir() } \ No newline at end of file diff --git a/dev/io.openliberty.org.testcontainers/cache/externals b/dev/io.openliberty.org.testcontainers/cache/externals new file mode 100644 index 00000000000..1fd80480da8 --- /dev/null +++ b/dev/io.openliberty.org.testcontainers/cache/externals @@ -0,0 +1,45 @@ +# NOTICE: This file was automatically updated to reflect changes made to test projects. +# Please check these changes into GitHub +alpine:3.17 +confluentinc/cp-kafka:7.1.1 +elastic/logstash:7.16.3 +fbicodex/spnego-kdc-server:1.0 +gvenzl/oracle-free:23.3-full-faststart +gvenzl/oracle-free:23.3-slim-faststart +icr.io/db2_community/db2:11.5.9.0 +infinispan/server:10.0.1.Final +infinispan/server:13.0.10.Final +jaegertracing/all-in-one:1.54 +jboss/keycloak:16.1.1 +jiwoo/nexus:1.0 +jiwoo/simple-keyserver:1.0 +jiwoo/squid-proxy:1.0 +jonhawkes/postgresql-ssl:1.0 +kyleaure/cloudant-developer:1.0 +kyleaure/couchdb-ssl:1.0 +kyleaure/db2-krb5:2.0 +kyleaure/db2-ssl:3.0 +kyleaure/krb5-server:1.0 +kyleaure/oracle-21.3.0-faststart:1.0.full.krb5 +kyleaure/oracle-21.3.0-faststart:1.0.full.ssl +kyleaure/postgres-krb5:1.0 +kyleaure/postgres-ssl:1.0 +kyleaure/postgres-test-table:3.0 +kyleaure/sqlserver-ssl:2019-CU18-ubuntu-20.04 +letsencrypt/pebble-challtestsrv:latest +letsencrypt/pebble:latest +mariadb:10.3 +mcr.microsoft.com/mssql/server:2019-CU28-ubuntu-20.04 +mongo:6.0.6 +openzipkin/zipkin-slim:2.23 +otel/opentelemetry-collector-contrib:0.103.0 +otel/opentelemetry-collector:0.74.0 +postgres:17.0-alpine +ryanesch/acme-boulder:1.2 +seleniarm/standalone-chromium:4.8.3 +selenium/standalone-chrome:4.8.3 +testcontainers/ryuk:0.9.0 +testcontainers/sshd:1.2.0 +testcontainers/vnc-recorder:1.3.0 +zachhein/krb5-server:0.2 +zachhein/ldap-server:0.5 diff --git a/dev/io.openliberty.org.testcontainers/cache/images b/dev/io.openliberty.org.testcontainers/cache/images index ec254468f81..15bb6d35ff6 100644 --- a/dev/io.openliberty.org.testcontainers/cache/images +++ b/dev/io.openliberty.org.testcontainers/cache/images @@ -1,45 +1,45 @@ # NOTICE: This file was automatically updated to reflect changes made to test projects. -# Please check these changes into GitHub. -alpine:3.17 -confluentinc/cp-kafka:7.1.1 -elastic/logstash:7.16.3 -fbicodex/spnego-kdc-server:1.0 -gvenzl/oracle-free:23.3-full-faststart -gvenzl/oracle-free:23.3-slim-faststart +# Please check these changes into GitHub icr.io/db2_community/db2:11.5.9.0 -infinispan/server:10.0.1.Final -infinispan/server:13.0.10.Final -jaegertracing/all-in-one:1.54 -jboss/keycloak:16.1.1 -jiwoo/nexus:1.0 -jiwoo/simple-keyserver:1.0 -jiwoo/squid-proxy:1.0 -jonhawkes/postgresql-ssl:1.0 -kyleaure/cloudant-developer:1.0 -kyleaure/couchdb-ssl:1.0 -kyleaure/db2-krb5:2.0 -kyleaure/db2-ssl:3.0 -kyleaure/krb5-server:1.0 -kyleaure/oracle-21.3.0-faststart:1.0.full.krb5 -kyleaure/oracle-21.3.0-faststart:1.0.full.ssl -kyleaure/postgres-krb5:1.0 -kyleaure/postgres-ssl:1.0 -kyleaure/postgres-test-table:3.0 -kyleaure/sqlserver-ssl:2019-CU18-ubuntu-20.04 -letsencrypt/pebble-challtestsrv:latest -letsencrypt/pebble:latest -mariadb:10.3 mcr.microsoft.com/mssql/server:2019-CU28-ubuntu-20.04 -mongo:6.0.6 -openzipkin/zipkin-slim:2.23 -otel/opentelemetry-collector-contrib:0.103.0 -otel/opentelemetry-collector:0.74.0 -postgres:17.0-alpine -ryanesch/acme-boulder:1.2 -seleniarm/standalone-chromium:4.8.3 -selenium/standalone-chrome:4.8.3 -testcontainers/ryuk:0.9.0 -testcontainers/sshd:1.2.0 -testcontainers/vnc-recorder:1.3.0 -zachhein/krb5-server:0.2 -zachhein/ldap-server:0.5 +wasliberty-docker-remote/alpine:3.17 +wasliberty-docker-remote/confluentinc/cp-kafka:7.1.1 +wasliberty-docker-remote/elastic/logstash:7.16.3 +wasliberty-docker-remote/fbicodex/spnego-kdc-server:1.0 +wasliberty-docker-remote/gvenzl/oracle-free:23.3-full-faststart +wasliberty-docker-remote/gvenzl/oracle-free:23.3-slim-faststart +wasliberty-docker-remote/infinispan/server:10.0.1.Final +wasliberty-docker-remote/infinispan/server:13.0.10.Final +wasliberty-docker-remote/jaegertracing/all-in-one:1.54 +wasliberty-docker-remote/jboss/keycloak:16.1.1 +wasliberty-docker-remote/jiwoo/nexus:1.0 +wasliberty-docker-remote/jiwoo/simple-keyserver:1.0 +wasliberty-docker-remote/jiwoo/squid-proxy:1.0 +wasliberty-docker-remote/jonhawkes/postgresql-ssl:1.0 +wasliberty-docker-remote/letsencrypt/pebble-challtestsrv:latest +wasliberty-docker-remote/letsencrypt/pebble:latest +wasliberty-docker-remote/mariadb:10.3 +wasliberty-docker-remote/mongo:6.0.6 +wasliberty-docker-remote/openzipkin/zipkin-slim:2.23 +wasliberty-docker-remote/otel/opentelemetry-collector-contrib:0.103.0 +wasliberty-docker-remote/otel/opentelemetry-collector:0.74.0 +wasliberty-docker-remote/postgres:17.0-alpine +wasliberty-docker-remote/ryanesch/acme-boulder:1.2 +wasliberty-docker-remote/seleniarm/standalone-chromium:4.8.3 +wasliberty-docker-remote/selenium/standalone-chrome:4.8.3 +wasliberty-docker-remote/testcontainers/ryuk:0.9.0 +wasliberty-docker-remote/testcontainers/sshd:1.2.0 +wasliberty-docker-remote/testcontainers/vnc-recorder:1.3.0 +wasliberty-docker-remote/zachhein/krb5-server:0.2 +wasliberty-docker-remote/zachhein/ldap-server:0.5 +wasliberty-infrastructure-docker/kyleaure/cloudant-developer:1.0 +wasliberty-infrastructure-docker/kyleaure/couchdb-ssl:1.0 +wasliberty-infrastructure-docker/kyleaure/db2-krb5:2.0 +wasliberty-infrastructure-docker/kyleaure/db2-ssl:3.0 +wasliberty-infrastructure-docker/kyleaure/krb5-server:1.0 +wasliberty-infrastructure-docker/kyleaure/oracle-21.3.0-faststart:1.0.full.krb5 +wasliberty-infrastructure-docker/kyleaure/oracle-21.3.0-faststart:1.0.full.ssl +wasliberty-infrastructure-docker/kyleaure/postgres-krb5:1.0 +wasliberty-infrastructure-docker/kyleaure/postgres-ssl:1.0 +wasliberty-infrastructure-docker/kyleaure/postgres-test-table:3.0 +wasliberty-infrastructure-docker/kyleaure/sqlserver-ssl:2019-CU18-ubuntu-20.04 diff --git a/dev/io.openliberty.org.testcontainers/dockerfiles/kyleaure/cloudant-developer/1.0/Dockerfile b/dev/io.openliberty.org.testcontainers/dockerfiles/kyleaure/cloudant-developer/1.0/Dockerfile new file mode 100644 index 00000000000..9590b0c745f --- /dev/null +++ b/dev/io.openliberty.org.testcontainers/dockerfiles/kyleaure/cloudant-developer/1.0/Dockerfile @@ -0,0 +1,5 @@ +FROM ibmcom/cloudant-developer:2.0.1 + +#Contains vulnerable log4j library for features not needed for testing +RUN rm -rf /opt/cloudant/search/ +RUN rm -rf /opt/cloudant/etc/log4j.properties diff --git a/dev/io.openliberty.org.testcontainers/src/io/openliberty/org/testcontainers/generate/CacheFiles.java b/dev/io.openliberty.org.testcontainers/src/io/openliberty/org/testcontainers/generate/CacheFiles.java new file mode 100644 index 00000000000..6005a3f556d --- /dev/null +++ b/dev/io.openliberty.org.testcontainers/src/io/openliberty/org/testcontainers/generate/CacheFiles.java @@ -0,0 +1,152 @@ +/******************************************************************************* + * Copyright (c) 2024 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * IBM Corporation - initial API and implementation + *******************************************************************************/ +package io.openliberty.org.testcontainers.generate; + +import java.io.BufferedWriter; +import java.io.File; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.ImageNameSubstitutor; + +public class CacheFiles { + + public static void main(String[] args) { + long start = System.currentTimeMillis(); + + if(args == null || args[0] == null || args.length > 1) { + throw new RuntimeException("CacheFiles expects a single argument (projectPath) which is the path to the io.openliberty.org.testcontainers project."); + } + + //Get data from calling script + String projectPath = args[0]; + File thisProject = new File(projectPath); + + //Create the cache directory and files + File projectList = new File(projectPath, "cache/projects"); + projectList.mkdirs(); + projectList.delete(); + + File imageList = new File(projectPath, "cache/images"); + imageList.delete(); + + File externalsList = new File(projectPath, "cache/externals"); + externalsList.delete(); + + //Create structures to store data + HashSet projects = new HashSet<>(); + HashSet externals = new HashSet<>(); + HashSet images = new HashSet<>(); + + //Investigate all bnd.bnd files + File rootDir = thisProject.getParentFile(); + for(File nextDir : rootDir.listFiles()) { + if (!nextDir.isDirectory()) { + continue; + } + + File bndFile = new File(nextDir.getAbsolutePath(), "bnd.bnd"); + if (!bndFile.exists()) { + continue; + } + + File gradleFile = new File(nextDir.getAbsolutePath(), "build.gradle"); + if (!gradleFile.exists()) { + continue; + } + + try { + if (!nextDir.getName().equals("fattest.simplicity") && !Files.lines(gradleFile.toPath()).anyMatch(line -> line.contains("copyTestContainers"))) { + continue; + } + + //Found a project that uses TestContainers + projects.add(nextDir.getName()); + + List bndLines = Files.lines(bndFile.toPath()).collect(Collectors.toList()); + if (!bndLines.stream().anyMatch(line -> line.contains("fat.test.container.images"))) { + continue; + } + + AtomicBoolean getNext = new AtomicBoolean(false); + bndLines.stream() + .forEach(line -> { + if (line.contains("fat.test.container.images") || getNext.get()) { + String trimmedLine = line.trim() + .replaceAll("\\s", "") + .replaceAll("fat.test.container.images:", "") + .replaceAll("fat.test.container.images=", ""); + if(trimmedLine.endsWith("\\")) { + getNext.set(true); + } else { + getNext.set(false); + } + + trimmedLine = trimmedLine.replaceAll("\\\\", ""); + + externals.addAll(Arrays.asList(trimmedLine.split(","))); + } + }); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + String header = "# NOTICE: This file was automatically updated to reflect changes made to test projects." + System.lineSeparator() + + "# Please check these changes into GitHub" + System.lineSeparator(); + + externals.remove(""); //Remove any blank lines that made it into the list. + + // Duplicate the externals list with internal organizations + externals.stream() + .forEach(image -> { + DockerImageName applied = ImageNameSubstitutor.instance().apply(DockerImageName.parse(image)); + images.add(applied.asCanonicalNameString()); + }); + + + //Save projects and images to files + try (BufferedWriter projectWriter = Files.newBufferedWriter(projectList.toPath(), Charset.defaultCharset()); + BufferedWriter imageWriter = Files.newBufferedWriter(imageList.toPath(), Charset.defaultCharset()); + BufferedWriter externalsWriter = Files.newBufferedWriter(externalsList.toPath(), Charset.defaultCharset());) { + + projectWriter.append(header); + for(String project : new TreeSet(projects)) { + projectWriter.append(project + System.lineSeparator()); + } + + imageWriter.append(header); + for(String image : new TreeSet(images)) { + imageWriter.append(image + System.lineSeparator()); + } + + externalsWriter.append(header); + for(String external : new TreeSet(externals)) { + externalsWriter.append(external + System.lineSeparator()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + // Average execution time is 300 ms + long end = System.currentTimeMillis(); + System.out.println( "Execution time in ms: " + ( end - start )); + } +} diff --git a/dev/io.openliberty.org.testcontainers/src/io/openliberty/org/testcontainers/generate/CustomImages.java b/dev/io.openliberty.org.testcontainers/src/io/openliberty/org/testcontainers/generate/CustomImages.java new file mode 100644 index 00000000000..ec2048bc28e --- /dev/null +++ b/dev/io.openliberty.org.testcontainers/src/io/openliberty/org/testcontainers/generate/CustomImages.java @@ -0,0 +1,99 @@ +/******************************************************************************* + * Copyright (c) 2024 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * IBM Corporation - initial API and implementation + *******************************************************************************/ +package io.openliberty.org.testcontainers.generate; + +import java.io.File; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import org.testcontainers.images.RemoteDockerImage; +import org.testcontainers.images.builder.ImageFromDockerfile; +import org.testcontainers.utility.DockerImageName; + +/** + * Allows external contributors a convenient way to build the custom images we use in our build + */ +public class CustomImages { + + public static void main(String[] args) { + long start = System.currentTimeMillis(); + + if(args == null || args[0] == null || args.length > 1) { + throw new RuntimeException("CustomImages expects a single argument (projectPath) which is the path to the io.openliberty.org.testcontainers project."); + } + + //Get data from calling script + String projectPath = args[0]; + + // Get image list if it exists + File imageList = new File(projectPath, "cache/externals"); + if(!imageList.exists()) { + System.out.println("Could not find file: " + imageList.getAbsolutePath()); + return; + } + + List images = Arrays.asList(); + try { + images = Files.lines(imageList.toPath()) + .filter(line -> !line.startsWith("#")) + .map(line -> DockerImageName.parse(line)) + .collect(Collectors.toList()); + } catch (Exception e) { + System.out.println("Could not read file: " + imageList.getAbsolutePath()); + return; + } + + // Try to pull all images in the list and generate a list of images that need to be created + final Set unpullableImageNames = new HashSet<>(images); + images.stream() + .map(name -> new RemoteDockerImage(name)) + .forEach(image -> { + try { + unpullableImageNames.remove(DockerImageName.parse(image.get())); + } catch (Exception e) { + System.out.println("Could not pull image " + image.toString() + " because " + e.getMessage()); + } + }); + + + File dockerfiles = new File(projectPath, "dockerfiles"); + if(!dockerfiles.exists() && !dockerfiles.isDirectory()) { + System.out.println("Could not find directory: " + dockerfiles.getAbsolutePath()); + return; + } + + // Try to find a Dockerfile for each unpullable image and build the image + final Set unbuildableImageNames = new HashSet<>(unpullableImageNames); + + unpullableImageNames.stream() + .map(name -> new ImageFromDockerfile(name.asCanonicalNameString(), false) + .withDockerfile( + new File(dockerfiles.getAbsolutePath(), name.getUnversionedPart() + "/" + name.getVersionPart() + "/Dockerfile").toPath())) + .forEach(image -> { + try { + unbuildableImageNames.remove(DockerImageName.parse(image.get())); + } catch (Exception e) { + System.out.println("Could not build image " + image.getDockerImageName() + " because " + e.getMessage()); + } + }); + + System.out.println("Could not pull or build " + unbuildableImageNames.size() + " image(s)"); + + long end = System.currentTimeMillis(); + System.out.println( "Execution time in ms: " + ( end - start )); + } +}