diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/PipInstallTask.groovy b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/PipInstallTask.groovy index 447f583f..eb1a3120 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/PipInstallTask.groovy +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/PipInstallTask.groovy @@ -119,7 +119,7 @@ class PipInstallTask extends DefaultTask implements FailureReasonProvider, Suppo void pipInstall() { def extension = ExtensionUtils.getPythonExtension(project) - ProgressLoggerFactory progressLoggerFactory = getServices().get(ProgressLoggerFactory) + ProgressLoggerFactory progressLoggerFactory = (ProgressLoggerFactory) getServices().get(ProgressLoggerFactory) ProgressLogger progressLogger = progressLoggerFactory.newOperation(PipInstallTask) progressLogger.setDescription("Installing Libraries") diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/EmptyWheelCache.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/EmptyWheelCache.java index 7946e59b..e27e8cc0 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/EmptyWheelCache.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/EmptyWheelCache.java @@ -22,15 +22,23 @@ public class EmptyWheelCache implements WheelCache { @Override - public Optional findWheel(String library, String version, PythonDetails pythonDetails) { + public Optional findWheel(String name, String version, PythonDetails pythonDetails) { return Optional.empty(); } @Override - public Optional findWheel(String library, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer) { + public Optional findWheel(String name, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer) { return Optional.empty(); } @Override - public void storeWheel(File wheelFile, WheelCacheLayer wheelCacheLayer) { } + public void storeWheel(File wheel) { } + + @Override + public void storeWheel(File wheel, WheelCacheLayer wheelCacheLayer) { } + + @Override + public Optional getTargetDirectory() { + return Optional.empty(); + } } diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/FileBackedWheelCache.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/FileBackedWheelCache.java index 742de6dd..0f49a994 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/FileBackedWheelCache.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/FileBackedWheelCache.java @@ -38,36 +38,37 @@ public FileBackedWheelCache(File cacheDir, PythonAbiContainer pythonAbiContainer this.pythonAbiContainer = pythonAbiContainer; } - /** - * Find's a wheel in the wheel cache. - * - * @param library name of the library - * @param version version of the library - * @param pythonDetails details on the python to find a wheel for - * @return A wheel that could be used in it's place. If not found, {@code Optional.empty()} - */ @Override - public Optional findWheel(String library, String version, PythonDetails pythonDetails) { - return findWheel(library, version, pythonDetails.getVirtualEnvInterpreter()); + public Optional findWheel(String name, String version, PythonDetails pythonDetails) { + return findWheel(name, version, pythonDetails.getVirtualEnvInterpreter()); } @Override - public Optional findWheel(String library, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer) { + public Optional findWheel(String name, String version, PythonDetails pythonDetails, + WheelCacheLayer wheelCacheLayer) { return Optional.empty(); } @Override - public void storeWheel(File wheelFile, WheelCacheLayer wheelCacheLayer) { } + public void storeWheel(File wheel) { } + + @Override + public void storeWheel(File wheel, WheelCacheLayer wheelCacheLayer) { } + + @Override + public Optional getTargetDirectory() { + return Optional.empty(); + } /** - * Find's a wheel in the wheel cache. + * Finds a wheel in the cache. * - * @param library name of the library - * @param version version of the library - * @param pythonExecutable Python Executable - * @return A wheel that could be used in it's place. If not found, {@code Optional.empty()} + * @param name package name + * @param version package version + * @param pythonExecutable Python interpreter executable file + * @return the wheel if found in the cache, otherwise {@code Optional.empty()} */ - public Optional findWheel(String library, String version, File pythonExecutable) { + public Optional findWheel(String name, String version, File pythonExecutable) { if (cacheDir == null) { return Optional.empty(); } @@ -79,13 +80,13 @@ public Optional findWheel(String library, String version, File pythonExecu * See PEP 427: https://www.python.org/dev/peps/pep-0427/ */ String wheelPrefix = ( - library.replace("-", "_") + name.replace("-", "_") + "-" + version.replace("-", "_") + "-" ); - logger.info("Searching for {} {} with prefix {}", library, version, wheelPrefix); - File[] files = cacheDir.listFiles((dir, name) -> name.startsWith(wheelPrefix) && name.endsWith(".whl")); + logger.info("Searching for {} {} with prefix {}", name, version, wheelPrefix); + File[] files = cacheDir.listFiles((dir, entry) -> entry.startsWith(wheelPrefix) && entry.endsWith(".whl")); if (files == null) { return Optional.empty(); @@ -96,7 +97,7 @@ public Optional findWheel(String library, String version, File pythonExecu .map(Optional::get) .collect(Collectors.toList()); - logger.info("Wheels for version of library: {}", wheelDetails); + logger.info("Wheels for version of package: {}", wheelDetails); Optional foundWheel = wheelDetails.stream() .filter(it -> wheelMatches(pythonExecutable, it)) diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/LayeredWheelCache.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/LayeredWheelCache.java index b078a9dd..9eecbff8 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/LayeredWheelCache.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/LayeredWheelCache.java @@ -42,52 +42,63 @@ public LayeredWheelCache(Map layeredCacheMap, PythonAbiCo this.pythonAbiContainer = pythonAbiContainer; } - /** - * Find a wheel from all wheel cache layers. - * - * @param library name of the library - * @param version version of the library - * @param pythonDetails details on the python to find a wheel for - * @return A wheel that could be used in it's place. If not found, {@code Optional.empty()} - */ @Override - public Optional findWheel(String library, String version, PythonDetails pythonDetails) { + public Optional findWheel(String name, String version, PythonDetails pythonDetails) { // TODO: Make sure layeredCacheMap is a LinkedHashMap when we initialize it in the plugin. for (WheelCacheLayer wheelCacheLayer : layeredCacheMap.keySet()) { - Optional layerResult = findWheelInLayer(library, version, pythonDetails.getVirtualEnvInterpreter(), wheelCacheLayer); + Optional wheel = findWheel(name, version, pythonDetails.getVirtualEnvInterpreter(), wheelCacheLayer); - if (layerResult.isPresent()) { - return layerResult; + if (wheel.isPresent()) { + return wheel; } } return Optional.empty(); } - /** - * Find wheel based on cache layer. - * - * @param library name of the library - * @param version version of the library - * @param pythonDetails details on the python to find a wheel for - * @param wheelCacheLayer which {@link WheelCacheLayer} to fetch wheel - * @return a wheel that could be used in the target layer. If not found, {@code Optional.empty()} - */ @Override - public Optional findWheel(String library, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer) { - return findWheelInLayer(library, version, pythonDetails.getVirtualEnvInterpreter(), wheelCacheLayer); + public Optional findWheel(String name, String version, PythonDetails pythonDetails, + WheelCacheLayer wheelCacheLayer) { + return findWheel(name, version, pythonDetails.getVirtualEnvInterpreter(), wheelCacheLayer); + } + + + @Override + public void storeWheel(File wheel) { + for (WheelCacheLayer wheelCacheLayer : layeredCacheMap.keySet()) { + storeWheel(wheel, wheelCacheLayer); + } + } + + @Override + public void storeWheel(File wheel, WheelCacheLayer wheelCacheLayer) { + File cacheDir = layeredCacheMap.get(wheelCacheLayer); + + if (wheel != null && cacheDir != null) { + try { + Files.copy(wheel.toPath(), new File(cacheDir, wheel.getName()).toPath()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + @Override + public Optional getTargetDirectory() { + return Optional.of(layeredCacheMap.get(WheelCacheLayer.PROJECT_LAYER)); } /** * Find a wheel from target layer. * - * @param library name of the library - * @param version version of the library - * @param pythonExecutable python executable - * @param wheelCacheLayer which {@link WheelCacheLayer} to fetch wheel - * @return A wheel that could be used in it's place. If not found, {@code Optional.empty()} + * @param name package name + * @param version package version + * @param pythonExecutable Python interpreter executable file + * @param wheelCacheLayer the {@link WheelCacheLayer} to fetch the wheel from + * @return the wheel if found in the specified layer, otherwise {@code Optional.empty()} */ - public Optional findWheelInLayer(String library, String version, File pythonExecutable, WheelCacheLayer wheelCacheLayer) { + private Optional findWheel(String name, String version, File pythonExecutable, + WheelCacheLayer wheelCacheLayer) { File cacheDir = layeredCacheMap.get(wheelCacheLayer); if (cacheDir == null) { @@ -101,13 +112,13 @@ public Optional findWheelInLayer(String library, String version, File pyth * See PEP 427: https://www.python.org/dev/peps/pep-0427/ */ String wheelPrefix = ( - library.replace("-", "_") + name.replace("-", "_") + "-" + version.replace("-", "_") + "-" ); - logger.info("Searching for {} {} with prefix {}", library, version, wheelPrefix); - File[] files = cacheDir.listFiles((dir, name) -> name.startsWith(wheelPrefix) && name.endsWith(".whl")); + logger.info("Searching for {} {} with prefix {}", name, version, wheelPrefix); + File[] files = cacheDir.listFiles((dir, entry) -> entry.startsWith(wheelPrefix) && entry.endsWith(".whl")); if (files == null) { return Optional.empty(); @@ -118,7 +129,7 @@ public Optional findWheelInLayer(String library, String version, File pyth .map(Optional::get) .collect(Collectors.toList()); - logger.info("Wheels for version of library: {}", wheelDetails); + logger.info("Wheels for version of package: {}", wheelDetails); Optional foundWheel = wheelDetails.stream() .filter(it -> wheelMatches(pythonExecutable, it)) @@ -129,25 +140,6 @@ public Optional findWheelInLayer(String library, String version, File pyth return foundWheel.map(it -> it.getFile()); } - /** - * Store given wheel file to target layer. - * - * @param wheelFile the wheel file to store - * @param wheelCacheLayer which {@link WheelCacheLayer} to store wheel - */ - @Override - public void storeWheel(File wheelFile, WheelCacheLayer wheelCacheLayer) { - File cacheDir = layeredCacheMap.get(wheelCacheLayer); - - if (wheelFile != null && cacheDir != null) { - try { - Files.copy(wheelFile.toPath(), new File(cacheDir, wheelFile.getName()).toPath()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - } - private boolean wheelMatches(File pythonExecutable, PythonWheelDetails wheelDetails) { return pythonAbiContainer.matchesSupportedVersion( pythonExecutable, diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/WheelCache.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/WheelCache.java index 13b8ada3..9c76ef2d 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/WheelCache.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/WheelCache.java @@ -22,24 +22,50 @@ import java.util.Optional; public interface WheelCache extends Serializable { - Optional findWheel(String library, String version, PythonDetails pythonDetails); + /** + * Finds a wheel in the cache. + * + *

The wheel can be stored in any cache layer.

+ * + * @param name package name + * @param version package version + * @param pythonDetails the Python interpreter and other details + * @return the wheel if found, otherwise {@code Optional.empty()} + */ + Optional findWheel(String name, String version, PythonDetails pythonDetails); /** - * Find wheel based on cache layer. + * Finds a wheel in the cache layer. * - * @param library name of the library - * @param version version of the library - * @param pythonDetails details on the python to find a wheel for - * @param wheelCacheLayer which {@link WheelCacheLayer} to fetch wheel - * @return a wheel that could be used in the target layer. If not found, {@code Optional.empty()} + * @param name package name + * @param version package version + * @param pythonDetails the Python interpreter and other details + * @param wheelCacheLayer the {@link WheelCacheLayer} to fetch the wheel from + * @return the wheel if found in the specified layer, otherwise {@code Optional.empty()} */ - Optional findWheel(String library, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer); + Optional findWheel(String name, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer); /** - * Store given wheel file to target layer. + * Stores the wheel file into all cache layers. + * + * @param wheel the wheel file to store + */ + void storeWheel(File wheel); + + /** + * Stores the wheel file into the cache layer. + * + * @param wheel the wheel file to store + * @param wheelCacheLayer the {@link WheelCacheLayer} to store the wheel in + */ + void storeWheel(File wheel, WheelCacheLayer wheelCacheLayer); + + /** + * Gets the default directory for wheel build target. + * + *

This should be project layer cache directory

* - * @param wheelFile the wheel file to store - * @param wheelCacheLayer which {@link WheelCacheLayer} to store wheel + * @return the directory used for wheel build target */ - void storeWheel(File wheelFile, WheelCacheLayer wheelCacheLayer); + Optional getTargetDirectory(); } diff --git a/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/wheel/LayeredWheelCacheTest.groovy b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/wheel/LayeredWheelCacheTest.groovy index 0e8d41da..a368ed62 100644 --- a/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/wheel/LayeredWheelCacheTest.groovy +++ b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/wheel/LayeredWheelCacheTest.groovy @@ -34,79 +34,125 @@ class LayeredWheelCacheTest extends Specification { private Map cacheMap private DefaultPythonDetails pythonDetails private LayeredWheelCache cache + private File otherCache void setup() { projectLayerCache = temporaryFolder.newFolder('project-cache') hostLayerCache = temporaryFolder.newFolder('host-cache') + otherCache = temporaryFolder.newFolder('other-cache') def virtualEnv = temporaryFolder.newFile('venv') pythonExec = new File(virtualEnv, 'bin/python') + pythonDetails = new DefaultPythonDetails(new ProjectBuilder().build(), virtualEnv) def formats = new DefaultPythonAbiContainer() formats.addSupportedAbi(new AbiDetails(pythonExec, 'py2', 'none', 'any')) - pythonDetails = new DefaultPythonDetails(new ProjectBuilder().build(), virtualEnv) - cacheMap = [(WheelCacheLayer.PROJECT_LAYER): projectLayerCache, (WheelCacheLayer.HOST_LAYER): hostLayerCache] + cacheMap = [ + (WheelCacheLayer.PROJECT_LAYER): projectLayerCache, + (WheelCacheLayer.HOST_LAYER): hostLayerCache, + ] cache = new LayeredWheelCache(cacheMap, formats) } def "can find Sphinx-1.6.3 in host layer"() { - setup: + setup: "put the wheel in host layer only" new File(hostLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl').createNewFile() - expect: + expect: "wheel is found in host layer, but not in project layer" cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.HOST_LAYER).isPresent() !cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.PROJECT_LAYER).isPresent() } def "can find Sphinx-1.6.3 in project layer"() { - setup: + setup: "put the wheel in project layer only" new File(projectLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl').createNewFile() - expect: + expect: "wheel is in project layer, but not in host layer" cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.PROJECT_LAYER).isPresent() !cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.HOST_LAYER).isPresent() } def "can find Sphinx-1.6.3 in all layers"() { - setup: + setup: "put the wheel in both layers" new File(hostLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl').createNewFile() new File(projectLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl').createNewFile() - expect: + expect: "wheel is in both layers" cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.HOST_LAYER).isPresent() cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.PROJECT_LAYER).isPresent() } - def "can find Sphinx-1.6.3 despite layers"() { - setup: + def "can find Sphinx-1.6.3 regardless of layers when in project layer"() { + setup: "put the wheel in project layer" new File(projectLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl').createNewFile() - expect: + expect: "wheel is found without the need to specify the layer" + cache.findWheel('Sphinx', '1.6.3', pythonDetails).isPresent() + } + + def "can find Sphinx-1.6.3 regardless of layers when in host layer"() { + setup: "put the wheel in host layer" + new File(hostLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl').createNewFile() + + expect: "wheel is found without the need to specify the layer" cache.findWheel('Sphinx', '1.6.3', pythonDetails).isPresent() } - def "cannot find Sphinx-1.6.3 if not put to any cache layer"() { - expect: + def "cannot find Sphinx-1.6.3 if not stored in any cache layer"() { + expect: "wheel is not found in any layer" !cache.findWheel('Sphinx', '1.6.3', pythonDetails).isPresent() } - def "can find Sphinx-1.6.3 from target folder"() { - setup: - new File(hostLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl').createNewFile() + def "can store Sphinx-1.6.3 to host layer from project layer"() { + setup: "building the wheel in the project layer is the default" + def wheelFile = new File(projectLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl') + wheelFile.createNewFile() + + when: "wheel is stored in host layer" + cache.storeWheel(wheelFile, WheelCacheLayer.HOST_LAYER) - expect: - cache.findWheelInLayer('Sphinx', '1.6.3', pythonExec, WheelCacheLayer.HOST_LAYER).isPresent() - !cache.findWheelInLayer('Sphinx', '1.6.3', pythonExec, WheelCacheLayer.PROJECT_LAYER).isPresent() + then: "wheel is found in host layer" + cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.HOST_LAYER).isPresent() } - def "can store Sphinx-1.6.3 to target layer"() { - setup: - def wheelFile = new File(projectLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl') + def "can store Sphinx-1.6.3 to project layer from other cache"() { + setup: "put the wheel in another cache" + def wheelFile = new File(otherCache, 'Sphinx-1.6.3-py2.py3-none-any.whl') wheelFile.createNewFile() + + when: "wheel is stored in project layer only" + cache.storeWheel(wheelFile, WheelCacheLayer.PROJECT_LAYER) + + then: "wheel is found in project layer, but not in host layer" + cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.PROJECT_LAYER).isPresent() + !cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.HOST_LAYER).isPresent() + } + + def "can store Sphinx-1.6.3 to host layer from other cache"() { + setup: "put the wheel in another cache" + def wheelFile = new File(otherCache, 'Sphinx-1.6.3-py2.py3-none-any.whl') + wheelFile.createNewFile() + + when: "wheel is stored in host layer only" cache.storeWheel(wheelFile, WheelCacheLayer.HOST_LAYER) - expect: + then: "wheel is found in host layer, but not in project layer" + !cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.PROJECT_LAYER).isPresent() cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.HOST_LAYER).isPresent() } + + def "can store Sphinx-1.6.3 to both layers"() { + setup: "put the wheel in another cache" + def wheelFile = new File(otherCache, 'Sphinx-1.6.3-py2.py3-none-any.whl') + wheelFile.createNewFile() + + when: "wheel is stored without specifying layer" + cache.storeWheel(wheelFile) + + then: "wheel is found in both layers" + cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.PROJECT_LAYER).isPresent() + cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.HOST_LAYER).isPresent() + } + }