From 28d7635b683e663c2ffc5c6ca63c9cfe81a05b04 Mon Sep 17 00:00:00 2001 From: Zvezdan Petkovic Date: Sat, 6 Apr 2019 22:26:48 -0700 Subject: [PATCH] Complete the wheel cache API and backing storage. These changes complete the layered cache support in the backing storage and wheel cache API. Cleanup unnecessary javadoc from classes, since interface has the docs complete now. Also, renaming and clarifications. Finally, a small correction for the missing cast and type warning from the IDE. --- .../gradle/python/tasks/PipInstallTask.groovy | 2 +- .../gradle/python/wheel/EmptyWheelCache.java | 14 ++- .../python/wheel/FileBackedWheelCache.java | 45 ++++----- .../python/wheel/LayeredWheelCache.java | 96 +++++++++---------- .../gradle/python/wheel/WheelCache.java | 50 +++++++--- .../python/wheel/LayeredWheelCacheTest.groovy | 92 +++++++++++++----- 6 files changed, 186 insertions(+), 113 deletions(-) diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/PipInstallTask.groovy b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/PipInstallTask.groovy index 447f583f..eb1a3120 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/PipInstallTask.groovy +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/PipInstallTask.groovy @@ -119,7 +119,7 @@ class PipInstallTask extends DefaultTask implements FailureReasonProvider, Suppo void pipInstall() { def extension = ExtensionUtils.getPythonExtension(project) - ProgressLoggerFactory progressLoggerFactory = getServices().get(ProgressLoggerFactory) + ProgressLoggerFactory progressLoggerFactory = (ProgressLoggerFactory) getServices().get(ProgressLoggerFactory) ProgressLogger progressLogger = progressLoggerFactory.newOperation(PipInstallTask) progressLogger.setDescription("Installing Libraries") diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/EmptyWheelCache.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/EmptyWheelCache.java index 7946e59b..e27e8cc0 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/EmptyWheelCache.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/EmptyWheelCache.java @@ -22,15 +22,23 @@ public class EmptyWheelCache implements WheelCache { @Override - public Optional findWheel(String library, String version, PythonDetails pythonDetails) { + public Optional findWheel(String name, String version, PythonDetails pythonDetails) { return Optional.empty(); } @Override - public Optional findWheel(String library, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer) { + public Optional findWheel(String name, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer) { return Optional.empty(); } @Override - public void storeWheel(File wheelFile, WheelCacheLayer wheelCacheLayer) { } + public void storeWheel(File wheel) { } + + @Override + public void storeWheel(File wheel, WheelCacheLayer wheelCacheLayer) { } + + @Override + public Optional getTargetDirectory() { + return Optional.empty(); + } } diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/FileBackedWheelCache.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/FileBackedWheelCache.java index 742de6dd..0f49a994 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/FileBackedWheelCache.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/FileBackedWheelCache.java @@ -38,36 +38,37 @@ public FileBackedWheelCache(File cacheDir, PythonAbiContainer pythonAbiContainer this.pythonAbiContainer = pythonAbiContainer; } - /** - * Find's a wheel in the wheel cache. - * - * @param library name of the library - * @param version version of the library - * @param pythonDetails details on the python to find a wheel for - * @return A wheel that could be used in it's place. If not found, {@code Optional.empty()} - */ @Override - public Optional findWheel(String library, String version, PythonDetails pythonDetails) { - return findWheel(library, version, pythonDetails.getVirtualEnvInterpreter()); + public Optional findWheel(String name, String version, PythonDetails pythonDetails) { + return findWheel(name, version, pythonDetails.getVirtualEnvInterpreter()); } @Override - public Optional findWheel(String library, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer) { + public Optional findWheel(String name, String version, PythonDetails pythonDetails, + WheelCacheLayer wheelCacheLayer) { return Optional.empty(); } @Override - public void storeWheel(File wheelFile, WheelCacheLayer wheelCacheLayer) { } + public void storeWheel(File wheel) { } + + @Override + public void storeWheel(File wheel, WheelCacheLayer wheelCacheLayer) { } + + @Override + public Optional getTargetDirectory() { + return Optional.empty(); + } /** - * Find's a wheel in the wheel cache. + * Finds a wheel in the cache. * - * @param library name of the library - * @param version version of the library - * @param pythonExecutable Python Executable - * @return A wheel that could be used in it's place. If not found, {@code Optional.empty()} + * @param name package name + * @param version package version + * @param pythonExecutable Python interpreter executable file + * @return the wheel if found in the cache, otherwise {@code Optional.empty()} */ - public Optional findWheel(String library, String version, File pythonExecutable) { + public Optional findWheel(String name, String version, File pythonExecutable) { if (cacheDir == null) { return Optional.empty(); } @@ -79,13 +80,13 @@ public Optional findWheel(String library, String version, File pythonExecu * See PEP 427: https://www.python.org/dev/peps/pep-0427/ */ String wheelPrefix = ( - library.replace("-", "_") + name.replace("-", "_") + "-" + version.replace("-", "_") + "-" ); - logger.info("Searching for {} {} with prefix {}", library, version, wheelPrefix); - File[] files = cacheDir.listFiles((dir, name) -> name.startsWith(wheelPrefix) && name.endsWith(".whl")); + logger.info("Searching for {} {} with prefix {}", name, version, wheelPrefix); + File[] files = cacheDir.listFiles((dir, entry) -> entry.startsWith(wheelPrefix) && entry.endsWith(".whl")); if (files == null) { return Optional.empty(); @@ -96,7 +97,7 @@ public Optional findWheel(String library, String version, File pythonExecu .map(Optional::get) .collect(Collectors.toList()); - logger.info("Wheels for version of library: {}", wheelDetails); + logger.info("Wheels for version of package: {}", wheelDetails); Optional foundWheel = wheelDetails.stream() .filter(it -> wheelMatches(pythonExecutable, it)) diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/LayeredWheelCache.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/LayeredWheelCache.java index b078a9dd..9eecbff8 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/LayeredWheelCache.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/LayeredWheelCache.java @@ -42,52 +42,63 @@ public LayeredWheelCache(Map layeredCacheMap, PythonAbiCo this.pythonAbiContainer = pythonAbiContainer; } - /** - * Find a wheel from all wheel cache layers. - * - * @param library name of the library - * @param version version of the library - * @param pythonDetails details on the python to find a wheel for - * @return A wheel that could be used in it's place. If not found, {@code Optional.empty()} - */ @Override - public Optional findWheel(String library, String version, PythonDetails pythonDetails) { + public Optional findWheel(String name, String version, PythonDetails pythonDetails) { // TODO: Make sure layeredCacheMap is a LinkedHashMap when we initialize it in the plugin. for (WheelCacheLayer wheelCacheLayer : layeredCacheMap.keySet()) { - Optional layerResult = findWheelInLayer(library, version, pythonDetails.getVirtualEnvInterpreter(), wheelCacheLayer); + Optional wheel = findWheel(name, version, pythonDetails.getVirtualEnvInterpreter(), wheelCacheLayer); - if (layerResult.isPresent()) { - return layerResult; + if (wheel.isPresent()) { + return wheel; } } return Optional.empty(); } - /** - * Find wheel based on cache layer. - * - * @param library name of the library - * @param version version of the library - * @param pythonDetails details on the python to find a wheel for - * @param wheelCacheLayer which {@link WheelCacheLayer} to fetch wheel - * @return a wheel that could be used in the target layer. If not found, {@code Optional.empty()} - */ @Override - public Optional findWheel(String library, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer) { - return findWheelInLayer(library, version, pythonDetails.getVirtualEnvInterpreter(), wheelCacheLayer); + public Optional findWheel(String name, String version, PythonDetails pythonDetails, + WheelCacheLayer wheelCacheLayer) { + return findWheel(name, version, pythonDetails.getVirtualEnvInterpreter(), wheelCacheLayer); + } + + + @Override + public void storeWheel(File wheel) { + for (WheelCacheLayer wheelCacheLayer : layeredCacheMap.keySet()) { + storeWheel(wheel, wheelCacheLayer); + } + } + + @Override + public void storeWheel(File wheel, WheelCacheLayer wheelCacheLayer) { + File cacheDir = layeredCacheMap.get(wheelCacheLayer); + + if (wheel != null && cacheDir != null) { + try { + Files.copy(wheel.toPath(), new File(cacheDir, wheel.getName()).toPath()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + @Override + public Optional getTargetDirectory() { + return Optional.of(layeredCacheMap.get(WheelCacheLayer.PROJECT_LAYER)); } /** * Find a wheel from target layer. * - * @param library name of the library - * @param version version of the library - * @param pythonExecutable python executable - * @param wheelCacheLayer which {@link WheelCacheLayer} to fetch wheel - * @return A wheel that could be used in it's place. If not found, {@code Optional.empty()} + * @param name package name + * @param version package version + * @param pythonExecutable Python interpreter executable file + * @param wheelCacheLayer the {@link WheelCacheLayer} to fetch the wheel from + * @return the wheel if found in the specified layer, otherwise {@code Optional.empty()} */ - public Optional findWheelInLayer(String library, String version, File pythonExecutable, WheelCacheLayer wheelCacheLayer) { + private Optional findWheel(String name, String version, File pythonExecutable, + WheelCacheLayer wheelCacheLayer) { File cacheDir = layeredCacheMap.get(wheelCacheLayer); if (cacheDir == null) { @@ -101,13 +112,13 @@ public Optional findWheelInLayer(String library, String version, File pyth * See PEP 427: https://www.python.org/dev/peps/pep-0427/ */ String wheelPrefix = ( - library.replace("-", "_") + name.replace("-", "_") + "-" + version.replace("-", "_") + "-" ); - logger.info("Searching for {} {} with prefix {}", library, version, wheelPrefix); - File[] files = cacheDir.listFiles((dir, name) -> name.startsWith(wheelPrefix) && name.endsWith(".whl")); + logger.info("Searching for {} {} with prefix {}", name, version, wheelPrefix); + File[] files = cacheDir.listFiles((dir, entry) -> entry.startsWith(wheelPrefix) && entry.endsWith(".whl")); if (files == null) { return Optional.empty(); @@ -118,7 +129,7 @@ public Optional findWheelInLayer(String library, String version, File pyth .map(Optional::get) .collect(Collectors.toList()); - logger.info("Wheels for version of library: {}", wheelDetails); + logger.info("Wheels for version of package: {}", wheelDetails); Optional foundWheel = wheelDetails.stream() .filter(it -> wheelMatches(pythonExecutable, it)) @@ -129,25 +140,6 @@ public Optional findWheelInLayer(String library, String version, File pyth return foundWheel.map(it -> it.getFile()); } - /** - * Store given wheel file to target layer. - * - * @param wheelFile the wheel file to store - * @param wheelCacheLayer which {@link WheelCacheLayer} to store wheel - */ - @Override - public void storeWheel(File wheelFile, WheelCacheLayer wheelCacheLayer) { - File cacheDir = layeredCacheMap.get(wheelCacheLayer); - - if (wheelFile != null && cacheDir != null) { - try { - Files.copy(wheelFile.toPath(), new File(cacheDir, wheelFile.getName()).toPath()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - } - private boolean wheelMatches(File pythonExecutable, PythonWheelDetails wheelDetails) { return pythonAbiContainer.matchesSupportedVersion( pythonExecutable, diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/WheelCache.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/WheelCache.java index 13b8ada3..9c76ef2d 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/WheelCache.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/WheelCache.java @@ -22,24 +22,50 @@ import java.util.Optional; public interface WheelCache extends Serializable { - Optional findWheel(String library, String version, PythonDetails pythonDetails); + /** + * Finds a wheel in the cache. + * + *

The wheel can be stored in any cache layer.

+ * + * @param name package name + * @param version package version + * @param pythonDetails the Python interpreter and other details + * @return the wheel if found, otherwise {@code Optional.empty()} + */ + Optional findWheel(String name, String version, PythonDetails pythonDetails); /** - * Find wheel based on cache layer. + * Finds a wheel in the cache layer. * - * @param library name of the library - * @param version version of the library - * @param pythonDetails details on the python to find a wheel for - * @param wheelCacheLayer which {@link WheelCacheLayer} to fetch wheel - * @return a wheel that could be used in the target layer. If not found, {@code Optional.empty()} + * @param name package name + * @param version package version + * @param pythonDetails the Python interpreter and other details + * @param wheelCacheLayer the {@link WheelCacheLayer} to fetch the wheel from + * @return the wheel if found in the specified layer, otherwise {@code Optional.empty()} */ - Optional findWheel(String library, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer); + Optional findWheel(String name, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer); /** - * Store given wheel file to target layer. + * Stores the wheel file into all cache layers. + * + * @param wheel the wheel file to store + */ + void storeWheel(File wheel); + + /** + * Stores the wheel file into the cache layer. + * + * @param wheel the wheel file to store + * @param wheelCacheLayer the {@link WheelCacheLayer} to store the wheel in + */ + void storeWheel(File wheel, WheelCacheLayer wheelCacheLayer); + + /** + * Gets the default directory for wheel build target. + * + *

This should be project layer cache directory

* - * @param wheelFile the wheel file to store - * @param wheelCacheLayer which {@link WheelCacheLayer} to store wheel + * @return the directory used for wheel build target */ - void storeWheel(File wheelFile, WheelCacheLayer wheelCacheLayer); + Optional getTargetDirectory(); } diff --git a/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/wheel/LayeredWheelCacheTest.groovy b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/wheel/LayeredWheelCacheTest.groovy index 0e8d41da..a368ed62 100644 --- a/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/wheel/LayeredWheelCacheTest.groovy +++ b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/wheel/LayeredWheelCacheTest.groovy @@ -34,79 +34,125 @@ class LayeredWheelCacheTest extends Specification { private Map cacheMap private DefaultPythonDetails pythonDetails private LayeredWheelCache cache + private File otherCache void setup() { projectLayerCache = temporaryFolder.newFolder('project-cache') hostLayerCache = temporaryFolder.newFolder('host-cache') + otherCache = temporaryFolder.newFolder('other-cache') def virtualEnv = temporaryFolder.newFile('venv') pythonExec = new File(virtualEnv, 'bin/python') + pythonDetails = new DefaultPythonDetails(new ProjectBuilder().build(), virtualEnv) def formats = new DefaultPythonAbiContainer() formats.addSupportedAbi(new AbiDetails(pythonExec, 'py2', 'none', 'any')) - pythonDetails = new DefaultPythonDetails(new ProjectBuilder().build(), virtualEnv) - cacheMap = [(WheelCacheLayer.PROJECT_LAYER): projectLayerCache, (WheelCacheLayer.HOST_LAYER): hostLayerCache] + cacheMap = [ + (WheelCacheLayer.PROJECT_LAYER): projectLayerCache, + (WheelCacheLayer.HOST_LAYER): hostLayerCache, + ] cache = new LayeredWheelCache(cacheMap, formats) } def "can find Sphinx-1.6.3 in host layer"() { - setup: + setup: "put the wheel in host layer only" new File(hostLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl').createNewFile() - expect: + expect: "wheel is found in host layer, but not in project layer" cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.HOST_LAYER).isPresent() !cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.PROJECT_LAYER).isPresent() } def "can find Sphinx-1.6.3 in project layer"() { - setup: + setup: "put the wheel in project layer only" new File(projectLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl').createNewFile() - expect: + expect: "wheel is in project layer, but not in host layer" cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.PROJECT_LAYER).isPresent() !cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.HOST_LAYER).isPresent() } def "can find Sphinx-1.6.3 in all layers"() { - setup: + setup: "put the wheel in both layers" new File(hostLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl').createNewFile() new File(projectLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl').createNewFile() - expect: + expect: "wheel is in both layers" cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.HOST_LAYER).isPresent() cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.PROJECT_LAYER).isPresent() } - def "can find Sphinx-1.6.3 despite layers"() { - setup: + def "can find Sphinx-1.6.3 regardless of layers when in project layer"() { + setup: "put the wheel in project layer" new File(projectLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl').createNewFile() - expect: + expect: "wheel is found without the need to specify the layer" + cache.findWheel('Sphinx', '1.6.3', pythonDetails).isPresent() + } + + def "can find Sphinx-1.6.3 regardless of layers when in host layer"() { + setup: "put the wheel in host layer" + new File(hostLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl').createNewFile() + + expect: "wheel is found without the need to specify the layer" cache.findWheel('Sphinx', '1.6.3', pythonDetails).isPresent() } - def "cannot find Sphinx-1.6.3 if not put to any cache layer"() { - expect: + def "cannot find Sphinx-1.6.3 if not stored in any cache layer"() { + expect: "wheel is not found in any layer" !cache.findWheel('Sphinx', '1.6.3', pythonDetails).isPresent() } - def "can find Sphinx-1.6.3 from target folder"() { - setup: - new File(hostLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl').createNewFile() + def "can store Sphinx-1.6.3 to host layer from project layer"() { + setup: "building the wheel in the project layer is the default" + def wheelFile = new File(projectLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl') + wheelFile.createNewFile() + + when: "wheel is stored in host layer" + cache.storeWheel(wheelFile, WheelCacheLayer.HOST_LAYER) - expect: - cache.findWheelInLayer('Sphinx', '1.6.3', pythonExec, WheelCacheLayer.HOST_LAYER).isPresent() - !cache.findWheelInLayer('Sphinx', '1.6.3', pythonExec, WheelCacheLayer.PROJECT_LAYER).isPresent() + then: "wheel is found in host layer" + cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.HOST_LAYER).isPresent() } - def "can store Sphinx-1.6.3 to target layer"() { - setup: - def wheelFile = new File(projectLayerCache, 'Sphinx-1.6.3-py2.py3-none-any.whl') + def "can store Sphinx-1.6.3 to project layer from other cache"() { + setup: "put the wheel in another cache" + def wheelFile = new File(otherCache, 'Sphinx-1.6.3-py2.py3-none-any.whl') wheelFile.createNewFile() + + when: "wheel is stored in project layer only" + cache.storeWheel(wheelFile, WheelCacheLayer.PROJECT_LAYER) + + then: "wheel is found in project layer, but not in host layer" + cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.PROJECT_LAYER).isPresent() + !cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.HOST_LAYER).isPresent() + } + + def "can store Sphinx-1.6.3 to host layer from other cache"() { + setup: "put the wheel in another cache" + def wheelFile = new File(otherCache, 'Sphinx-1.6.3-py2.py3-none-any.whl') + wheelFile.createNewFile() + + when: "wheel is stored in host layer only" cache.storeWheel(wheelFile, WheelCacheLayer.HOST_LAYER) - expect: + then: "wheel is found in host layer, but not in project layer" + !cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.PROJECT_LAYER).isPresent() cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.HOST_LAYER).isPresent() } + + def "can store Sphinx-1.6.3 to both layers"() { + setup: "put the wheel in another cache" + def wheelFile = new File(otherCache, 'Sphinx-1.6.3-py2.py3-none-any.whl') + wheelFile.createNewFile() + + when: "wheel is stored without specifying layer" + cache.storeWheel(wheelFile) + + then: "wheel is found in both layers" + cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.PROJECT_LAYER).isPresent() + cache.findWheel('Sphinx', '1.6.3', pythonDetails, WheelCacheLayer.HOST_LAYER).isPresent() + } + }