Merge pull request #30233 from KyleAure/switch-artifactory-repository

dev/fattest.simplicity/src/componenttest/containers/ArtifactoryImageNameSubstitutor.java

@@ -12,6 +12,9 @@
  *******************************************************************************/
 package componenttest.containers;
 
+import java.util.Arrays;
+import java.util.List;
+
 import org.testcontainers.DockerClientFactory;
 import org.testcontainers.dockerclient.EnvironmentAndSystemPropertyClientProviderStrategy;
 import org.testcontainers.utility.DockerImageName;
@@ -40,6 +43,27 @@ public class ArtifactoryImageNameSubstitutor extends ImageNameSubstitutor {
      */
     private static final String mirror = "wasliberty-docker-remote";
 
+    /**
+     * TODO remove this temp mirror and either build these images at runtime
+     * or pull from a self hosted docker repository
+     *
+     * Artifactory keeps a set of community docker images that have been removed
+     * from DockerHub within this organization specifically for the Liberty builds.
+     */
+    private static final String tempMirror = "wasliberty-infrastructure-docker";
+
+    /**
+     * TODO remove this temp repository list and either build these images at runtime
+     * or pull from a self hosted docker repository
+     *
+     * Artifactory keeps a set of community docker images that have been removed
+     * from DockerHub with this set of repository names.
+     */
+    private static final List<String> tempRepositories = Arrays.asList("kyleaure/");
+
+    private static final boolean mockBehavior = System.getenv().containsKey("MOCK_ARTIFACTORY_BEHAVIOR")
+                                                && System.getenv().get("MOCK_ARTIFACTORY_BEHAVIOR").equalsIgnoreCase("true");
+
     @Override
     public DockerImageName apply(final DockerImageName original) {
         DockerImageName result = null;
@@ -59,8 +83,8 @@ public DockerImageName apply(final DockerImageName original) {
             // Priority 2a: If the image is known to only exist in an Artifactory organization
             if (original.getRepository().contains("wasliberty-")) {
                 result = DockerImageName.parse(original.asCanonicalNameString())
-                                        .withRegistry(ArtifactoryRegistry.instance().getRegistry())
-                                        .asCompatibleSubstituteFor(original);
+                                .withRegistry(ArtifactoryRegistry.instance().getRegistry())
+                                .asCompatibleSubstituteFor(original);
                 needsArtifactory = true;
                 reason = "This image only exists in Artifactory, must use Artifactory registry.";
                 break;
@@ -80,7 +104,7 @@ public DockerImageName apply(final DockerImageName original) {
 
             // Priority 4: Always use Artifactory if using remote docker host.
             if (DockerClientFactory.instance().isUsing(EnvironmentAndSystemPropertyClientProviderStrategy.class)) {
-                result = DockerImageName.parse(mirror + '/' + original.asCanonicalNameString())
+                result = DockerImageName.parse(whichMirror(original) + '/' + original.asCanonicalNameString())
                                 .withRegistry(ArtifactoryRegistry.instance().getRegistry())
                                 .asCompatibleSubstituteFor(original);
                 needsArtifactory = true;
@@ -98,21 +122,31 @@ public DockerImageName apply(final DockerImageName original) {
 
             // Priority 6: If Artifactory registry is available use it to avoid rate limits on other registries
             if (ArtifactoryRegistry.instance().isArtifactoryAvailable()) {
-                result = DockerImageName.parse(mirror + '/' + original.asCanonicalNameString())
+                result = DockerImageName.parse(whichMirror(original) + '/' + original.asCanonicalNameString())
                                 .withRegistry(ArtifactoryRegistry.instance().getRegistry())
                                 .asCompatibleSubstituteFor(original);
                 needsArtifactory = true;
                 reason = "Artifactory was available.";
                 break;
             }
 
+            // Priority 7: If we need to mock this behavior for image name generation
+            if (mockBehavior) {
+                result = DockerImageName.parse(whichMirror(original) + '/' + original.asCanonicalNameString())
+                                .withRegistry(ArtifactoryRegistry.instance().getRegistry())
+                                .asCompatibleSubstituteFor(original);
+                needsArtifactory = true;
+                reason = "Mocking artifactory behavior.";
+                break;
+            }
+
             //default - use original
             result = original;
             reason = "Default behavior: use default docker registry.";
         } while (false);
 
         // We determined we need Artifactory, but it is unavailable.
-        if (needsArtifactory && !ArtifactoryRegistry.instance().isArtifactoryAvailable()) {
+        if (needsArtifactory && !mockBehavior && !ArtifactoryRegistry.instance().isArtifactoryAvailable()) {
             throw new RuntimeException("Need to swap image " + original.asCanonicalNameString() + " --> " + result.asCanonicalNameString()
                                        + System.lineSeparator() + "Reason: " + reason
                                        + System.lineSeparator() + "Error: The Artifactory registry was not added to the docker config.", //
@@ -161,4 +195,19 @@ private static boolean isSyntheticImage(DockerImageName dockerImage) {
         return isSynthetic || isCommittedImage;
     }
 
+    /**
+     * Determine which internal mirror a docker image should belong to
+     */
+    private static String whichMirror(DockerImageName dockerImage) {
+        for (String tempRepository : tempRepositories) {
+            if (dockerImage.getRepository().startsWith(tempRepository)) {
+                Log.info(c, "whichMirror", "Using mirror " + tempMirror + " for docker image " + dockerImage.asCanonicalNameString());
+                return tempMirror;
+            }
+        }
+
+        Log.info(c, "whichMirror", "Using mirror " + mirror + " for docker image " + dockerImage.asCanonicalNameString());
+        return mirror;
+    }
+
 }

dev/fattest.simplicity/test/componenttest/containers/ArtifactoryImageNameSubstitutorTest.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2023 IBM Corporation and others.
+ * Copyright (c) 2023, 2024 IBM Corporation and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License 2.0
  * which accompanies this distribution, and is available at
@@ -157,7 +157,7 @@ public void testDockerClientStrategy() throws Exception {
         setDockerClientStrategy(new EnvironmentAndSystemPropertyClientProviderStrategy());
 
         input = DockerImageName.parse("kyleaure/oracle-xe:1.0.0");
-        expected = DockerImageName.parse("example.com/wasliberty-docker-remote/kyleaure/oracle-xe:1.0.0");
+        expected = DockerImageName.parse("example.com/wasliberty-infrastructure-docker/kyleaure/oracle-xe:1.0.0");
         assertEquals(expected, new ArtifactoryImageNameSubstitutor().apply(input));
 
         //Using local docker host
@@ -205,7 +205,7 @@ public void testSystemPropertyModified() throws Exception {
         expected = DockerImageName.parse("openliberty:1.0.0");
         assertEquals(expected, new ArtifactoryImageNameSubstitutor().apply(expected));
 
-        //No force, with Artifactory, use artifactory
+        //No force, with Artifactory, use Artifactory
         setDockerClientStrategy(new UnixSocketClientProviderStrategy());
         System.setProperty(FORCE_EXTERNAL, "false");
         setArtifactoryRegistryAvailable(true);

dev/io.openliberty.org.testcontainers/.classpath

@@ -1,6 +1,11 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <classpath>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8"/>
+	<classpathentry kind="src" path="src"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
+		<attributes>
+			<attribute name="module" value="true"/>
+		</attributes>
+	</classpathentry>
 	<classpathentry kind="con" path="aQute.bnd.classpath.container"/>
 	<classpathentry kind="output" path="bin"/>
 </classpath>

dev/io.openliberty.org.testcontainers/build.gradle

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2022 IBM Corporation and others.
+ * Copyright (c) 2022, 2024 IBM Corporation and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License 2.0
  * which accompanies this distribution, and is available at
@@ -10,72 +10,39 @@
  * Contributors:
  *     IBM Corporation - initial API and implementation
  *******************************************************************************/
+configurations {
+  runtime
+}
 
-task assembleTestContainerData {
-	doLast{
-	  	long start = System.currentTimeMillis()
-
-	  	//First create a list of projects that use testcontainers
-	  	File projectList = new File(project.getProjectDir(), 'cache/projects')
-	  	projectList.getParentFile().mkdirs()
-	  	projectList.delete()
-
-	  	//Investigate all gradle files
-	  	def projectArr = ['fattest.simplicity']
-	    rootDir.eachDir() { nextDir ->
-	      if(nextDir == project.getProjectDir())
-	        return // continue: do not check this project
-
-	      File gradleFile = new File(nextDir, 'build.gradle')
-	      if(gradleFile.exists() && gradleFile.readLines().findAll { it.contains('copyTestContainers') }.size() > 0 )
-	        projectArr << gradleFile.getParentFile().getName() //Cache project name
-	    }
-
-	    //Next create a list of all images used by testcontainer projects
-	    File imageList   = new File(project.getProjectDir(), 'cache/images')
-	    imageList.delete()
-
-	    //Investigate all bnd files
-	    def imageArr = []
-	    def prefix = 'fat.test.container.images:'
-	    projectArr.each { foundProject ->
-	      File bndFile = new File(rootDir, foundProject + '/bnd.bnd')
-
-	      if(bndFile.exists() && bndFile.readLines().findAll { it.contains(prefix) }.size() > 0) {
-
-	        //Cache image names
-	        def getNext = false
-	        bndFile.eachLine { line ->
-	          if( line.contains(prefix) || getNext ) {
-	            line = line.replaceAll('\\s', '') - prefix
-	            if ( line[-1] == '\\')
-	              getNext = true
-	            else
-	              getNext = false
-
-	            line.split(',').each { image ->
-	              imageArr << image
-	            } //line.split
-	          }//line.contains
-	        }//bndFile.eachLine
-	      }//bndFile.exists
-	    }//projectArr.each
-
-	    // Write lists to file
-	    projectList << '# NOTICE: This file was automatically updated to reflect changes made to test projects.' << '\n'
-	    projectList	<< '# Please check these changes into GitHub' << '\n'
-	    projectArr.sort().forEach() { projectList << it << '\n' }
-
-	    imageList   << '# NOTICE: This file was automatically updated to reflect changes made to test projects.' << '\n'
-	    imageList	<< '# Please check these changes into GitHub.' << '\n'
-	    imageArr.unique().sort()
-	    imageArr -= '\\'
-	    imageArr.forEach() { imageList << it << '\n' }
-
-	    println 'Cached test container project and image data in ' + projectList + ' and ' + imageList
-
-	    // Average execution time is 500 ms
-	    long end = System.currentTimeMillis()
-	    println 'Execution time in ms: ' + ( end - start )
-	}
+dependencies {
+  runtime project(':io.openliberty.com.fasterxml.jackson')
+  runtime project(':com.ibm.ws.componenttest')
+  runtime project(':fattest.simplicity')
+}
+
+task assembleTestContainerData(type: JavaExec) {
+    onlyIf { // Do not execute this task during our builds
+        !isAutomatedBuild
+    }
+
+    // Setup environment to mock external state
+    environment "TESTCONTAINERS_IMAGE_SUBSTITUTOR", "componenttest.containers.ArtifactoryImageNameSubstitutor"
+    environment "MOCK_ARTIFACTORY_BEHAVIOR", "true"
+
+    classpath = configurations.runtime.plus(sourceSets.main.runtimeClasspath)
+    main = "io.openliberty.org.testcontainers.generate.CacheFiles"
+    args project.getProjectDir()
+}
+
+task generateCustomImages(type: JavaExec) {
+    onlyIf { // Prototype - should never be run
+        false
+    }
+
+    // Ensure cache has been generated
+    dependsOn assembleTestContainerData
+
+    classpath = configurations.runtime.plus(sourceSets.main.runtimeClasspath)
+    main = "io.openliberty.org.testcontainers.generate.CustomImages"
+    args project.getProjectDir()
 }

dev/io.openliberty.org.testcontainers/cache/externals

@@ -0,0 +1,45 @@
+# NOTICE: This file was automatically updated to reflect changes made to test projects.
+# Please check these changes into GitHub
+alpine:3.17
+confluentinc/cp-kafka:7.1.1
+elastic/logstash:7.16.3
+fbicodex/spnego-kdc-server:1.0
+gvenzl/oracle-free:23.3-full-faststart
+gvenzl/oracle-free:23.3-slim-faststart
+icr.io/db2_community/db2:11.5.9.0
+infinispan/server:10.0.1.Final
+infinispan/server:13.0.10.Final
+jaegertracing/all-in-one:1.54
+jboss/keycloak:16.1.1
+jiwoo/nexus:1.0
+jiwoo/simple-keyserver:1.0
+jiwoo/squid-proxy:1.0
+jonhawkes/postgresql-ssl:1.0
+kyleaure/cloudant-developer:1.0
+kyleaure/couchdb-ssl:1.0
+kyleaure/db2-krb5:2.0
+kyleaure/db2-ssl:3.0
+kyleaure/krb5-server:1.0
+kyleaure/oracle-21.3.0-faststart:1.0.full.krb5
+kyleaure/oracle-21.3.0-faststart:1.0.full.ssl
+kyleaure/postgres-krb5:1.0
+kyleaure/postgres-ssl:1.0
+kyleaure/postgres-test-table:3.0
+kyleaure/sqlserver-ssl:2019-CU18-ubuntu-20.04
+letsencrypt/pebble-challtestsrv:latest
+letsencrypt/pebble:latest
+mariadb:10.3
+mcr.microsoft.com/mssql/server:2019-CU28-ubuntu-20.04
+mongo:6.0.6
+openzipkin/zipkin-slim:2.23
+otel/opentelemetry-collector-contrib:0.103.0
+otel/opentelemetry-collector:0.74.0
+postgres:17.0-alpine
+ryanesch/acme-boulder:1.2
+seleniarm/standalone-chromium:4.8.3
+selenium/standalone-chrome:4.8.3
+testcontainers/ryuk:0.9.0
+testcontainers/sshd:1.2.0
+testcontainers/vnc-recorder:1.3.0
+zachhein/krb5-server:0.2
+zachhein/ldap-server:0.5

dev/io.openliberty.org.testcontainers/cache/images

@@ -1,45 +1,45 @@
 # NOTICE: This file was automatically updated to reflect changes made to test projects.
-# Please check these changes into GitHub.
-alpine:3.17
-confluentinc/cp-kafka:7.1.1
-elastic/logstash:7.16.3
-fbicodex/spnego-kdc-server:1.0
-gvenzl/oracle-free:23.3-full-faststart
-gvenzl/oracle-free:23.3-slim-faststart
+# Please check these changes into GitHub
 icr.io/db2_community/db2:11.5.9.0
-infinispan/server:10.0.1.Final
-infinispan/server:13.0.10.Final
-jaegertracing/all-in-one:1.54
-jboss/keycloak:16.1.1
-jiwoo/nexus:1.0
-jiwoo/simple-keyserver:1.0
-jiwoo/squid-proxy:1.0
-jonhawkes/postgresql-ssl:1.0
-kyleaure/cloudant-developer:1.0
-kyleaure/couchdb-ssl:1.0
-kyleaure/db2-krb5:2.0
-kyleaure/db2-ssl:3.0
-kyleaure/krb5-server:1.0
-kyleaure/oracle-21.3.0-faststart:1.0.full.krb5
-kyleaure/oracle-21.3.0-faststart:1.0.full.ssl
-kyleaure/postgres-krb5:1.0
-kyleaure/postgres-ssl:1.0
-kyleaure/postgres-test-table:3.0
-kyleaure/sqlserver-ssl:2019-CU18-ubuntu-20.04
-letsencrypt/pebble-challtestsrv:latest
-letsencrypt/pebble:latest
-mariadb:10.3
 mcr.microsoft.com/mssql/server:2019-CU28-ubuntu-20.04
-mongo:6.0.6
-openzipkin/zipkin-slim:2.23
-otel/opentelemetry-collector-contrib:0.103.0
-otel/opentelemetry-collector:0.74.0
-postgres:17.0-alpine
-ryanesch/acme-boulder:1.2
-seleniarm/standalone-chromium:4.8.3
-selenium/standalone-chrome:4.8.3
-testcontainers/ryuk:0.9.0
-testcontainers/sshd:1.2.0
-testcontainers/vnc-recorder:1.3.0
-zachhein/krb5-server:0.2
-zachhein/ldap-server:0.5
+wasliberty-docker-remote/alpine:3.17
+wasliberty-docker-remote/confluentinc/cp-kafka:7.1.1
+wasliberty-docker-remote/elastic/logstash:7.16.3
+wasliberty-docker-remote/fbicodex/spnego-kdc-server:1.0
+wasliberty-docker-remote/gvenzl/oracle-free:23.3-full-faststart
+wasliberty-docker-remote/gvenzl/oracle-free:23.3-slim-faststart
+wasliberty-docker-remote/infinispan/server:10.0.1.Final
+wasliberty-docker-remote/infinispan/server:13.0.10.Final
+wasliberty-docker-remote/jaegertracing/all-in-one:1.54
+wasliberty-docker-remote/jboss/keycloak:16.1.1
+wasliberty-docker-remote/jiwoo/nexus:1.0
+wasliberty-docker-remote/jiwoo/simple-keyserver:1.0
+wasliberty-docker-remote/jiwoo/squid-proxy:1.0
+wasliberty-docker-remote/jonhawkes/postgresql-ssl:1.0
+wasliberty-docker-remote/letsencrypt/pebble-challtestsrv:latest
+wasliberty-docker-remote/letsencrypt/pebble:latest
+wasliberty-docker-remote/mariadb:10.3
+wasliberty-docker-remote/mongo:6.0.6
+wasliberty-docker-remote/openzipkin/zipkin-slim:2.23
+wasliberty-docker-remote/otel/opentelemetry-collector-contrib:0.103.0
+wasliberty-docker-remote/otel/opentelemetry-collector:0.74.0
+wasliberty-docker-remote/postgres:17.0-alpine
+wasliberty-docker-remote/ryanesch/acme-boulder:1.2
+wasliberty-docker-remote/seleniarm/standalone-chromium:4.8.3
+wasliberty-docker-remote/selenium/standalone-chrome:4.8.3
+wasliberty-docker-remote/testcontainers/ryuk:0.9.0
+wasliberty-docker-remote/testcontainers/sshd:1.2.0
+wasliberty-docker-remote/testcontainers/vnc-recorder:1.3.0
+wasliberty-docker-remote/zachhein/krb5-server:0.2
+wasliberty-docker-remote/zachhein/ldap-server:0.5
+wasliberty-infrastructure-docker/kyleaure/cloudant-developer:1.0
+wasliberty-infrastructure-docker/kyleaure/couchdb-ssl:1.0
+wasliberty-infrastructure-docker/kyleaure/db2-krb5:2.0
+wasliberty-infrastructure-docker/kyleaure/db2-ssl:3.0
+wasliberty-infrastructure-docker/kyleaure/krb5-server:1.0
+wasliberty-infrastructure-docker/kyleaure/oracle-21.3.0-faststart:1.0.full.krb5
+wasliberty-infrastructure-docker/kyleaure/oracle-21.3.0-faststart:1.0.full.ssl
+wasliberty-infrastructure-docker/kyleaure/postgres-krb5:1.0
+wasliberty-infrastructure-docker/kyleaure/postgres-ssl:1.0
+wasliberty-infrastructure-docker/kyleaure/postgres-test-table:3.0
+wasliberty-infrastructure-docker/kyleaure/sqlserver-ssl:2019-CU18-ubuntu-20.04