Skip to content

Commit

Permalink
Add digests to OutputArtifacts
Browse files Browse the repository at this point in the history
`OutputArtifact` values are constructed either form the build output events or
from a file system directory. File digests are available in the former case
only. `OutputArtifact` values constructed from the file system (see
`com.google.idea.blaze.base.sync.workspace.ArtifactLocationDecoderImpl.outputArtifactFromExecRoot`)
are used in some special cases only.

Introduce `OutputArtifactWithDigest` interface that extends `OutputArtifact`
and separate code paths to ensure digest values are available when used.

(cherry picked from commit 314feac)
  • Loading branch information
Googler authored and mai93 committed Jun 5, 2023
1 parent a0495d9 commit 9d44548
Show file tree
Hide file tree
Showing 31 changed files with 328 additions and 231 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
*/
package com.google.idea.blaze.android.filecache;

import com.google.idea.blaze.base.command.buildresult.OutputArtifact;
import com.google.idea.blaze.base.command.buildresult.OutputArtifactWithoutDigest;
import com.google.idea.blaze.base.scope.BlazeContext;
import java.nio.file.Path;
import java.util.Collection;
Expand All @@ -35,19 +35,21 @@ public interface ArtifactCache {
void clearCache();

/**
* Fetches and caches the given collection of {@link OutputArtifact} to disk.
* Fetches and caches the given collection of {@link OutputArtifactWithoutDigest} to disk.
*
* @param artifacts Collection of artifacts to add to cache
* @param removeMissingArtifacts if true, will remove any cached artifact that is not present in
* {@code artifacts}.
*/
void putAll(
Collection<OutputArtifact> artifacts, BlazeContext context, boolean removeMissingArtifacts);
Collection<? extends OutputArtifactWithoutDigest> artifacts,
BlazeContext context,
boolean removeMissingArtifacts);

/**
* Returns the {@link Path} corresponding to the given {@link OutputArtifact}, or {@code null} if
* the artifact is not tracked in cache.
* Returns the {@link Path} corresponding to the given {@link OutputArtifactWithoutDigest}, or
* {@code null} if the artifact is not tracked in cache.
*/
@Nullable
Path get(OutputArtifact artifact);
Path get(OutputArtifactWithoutDigest artifact);
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import com.google.devtools.intellij.model.ProjectData;
import com.google.devtools.intellij.model.ProjectData.LocalFile;
import com.google.devtools.intellij.model.ProjectData.LocalFileOrOutputArtifact;
import com.google.idea.blaze.base.command.buildresult.OutputArtifact;
import com.google.idea.blaze.base.command.buildresult.OutputArtifactWithoutDigest;
import com.google.idea.blaze.base.filecache.ArtifactState;
import java.util.Objects;

Expand Down Expand Up @@ -66,7 +66,7 @@ public int hashCode() {
*
* @throws ArtifactNotFoundException if the artifact is not present.
*/
public static ArtifactMetadata forArtifact(OutputArtifact artifact)
public static ArtifactMetadata forArtifact(OutputArtifactWithoutDigest artifact)
throws ArtifactNotFoundException {
ArtifactState artifactState = artifact.toArtifactState();
if (artifactState == null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,12 @@
*/
package com.google.idea.blaze.android.filecache;

import com.google.idea.blaze.base.command.buildresult.OutputArtifact;
import com.google.idea.blaze.base.command.buildresult.OutputArtifactWithoutDigest;
import java.io.IOException;

/** Signals that a particular artifact cannot be located. */
public class ArtifactNotFoundException extends IOException {
public ArtifactNotFoundException(OutputArtifact artifact) {
public ArtifactNotFoundException(OutputArtifactWithoutDigest artifact) {
super(artifact.getRelativePath() + " not found.");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
import com.google.idea.blaze.base.command.buildresult.BlazeArtifact;
import com.google.idea.blaze.base.command.buildresult.OutputArtifact;
import com.google.idea.blaze.base.command.buildresult.OutputArtifactWithoutDigest;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.util.PathUtil;
import java.util.Objects;
Expand Down Expand Up @@ -76,7 +76,7 @@ public int hashCode() {

/** Returns a {@link CacheEntry} corresponding to the given {@code OutputArtifact}. */
@VisibleForTesting
public static CacheEntry forArtifact(OutputArtifact blazeArtifact)
public static CacheEntry forArtifact(OutputArtifactWithoutDigest blazeArtifact)
throws ArtifactNotFoundException {
ArtifactMetadata artifactMetadata = ArtifactMetadata.forArtifact(blazeArtifact);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import com.google.common.util.concurrent.ListenableFuture;
import com.google.idea.blaze.base.async.FutureUtil;
import com.google.idea.blaze.base.command.buildresult.BlazeArtifact;
import com.google.idea.blaze.base.command.buildresult.OutputArtifact;
import com.google.idea.blaze.base.command.buildresult.OutputArtifactWithoutDigest;
import com.google.idea.blaze.base.io.FileOperationProvider;
import com.google.idea.blaze.base.logging.EventLoggingService;
import com.google.idea.blaze.base.prefetch.FetchExecutor;
Expand Down Expand Up @@ -58,7 +58,7 @@
import javax.annotation.Nullable;

/**
* LocalArtifactCache is an on disk cache for {@link OutputArtifact}s.
* LocalArtifactCache is an on disk cache for {@link OutputArtifactWithoutDigest}s.
*
* <p>The {@link #get} returns the path to a local copy of the cached artifact (if present).
*
Expand All @@ -75,9 +75,9 @@
* is set to true.
* </pre>
*
* Internally, every {@link OutputArtifact} is mapped to a {@link CacheEntry} object, which captures
* metadata about the artifact including data like timestamp or objfs blobId that allows it to
* determine when an artifact has been updated.
* Internally, every {@link OutputArtifactWithoutDigest} is mapped to a {@link CacheEntry} object,
* which captures metadata about the artifact including data like timestamp or objfs blobId that
* allows it to determine when an artifact has been updated.
*
* <p>An in memory map of these CacheEntries allows for quick retrieval of the cached objects. But
* this map is also persisted to the disk after calls to {@link #clearCache} and {@link #putAll} and
Expand Down Expand Up @@ -156,9 +156,11 @@ public synchronized void clearCache() {
*/
@Override
public synchronized void putAll(
Collection<OutputArtifact> artifacts, BlazeContext context, boolean removeMissingArtifacts) {
Collection<? extends OutputArtifactWithoutDigest> artifacts,
BlazeContext context,
boolean removeMissingArtifacts) {
// Utility maps for the passed artifacts
Map<String, OutputArtifact> keyToArtifact = new HashMap<>();
Map<String, OutputArtifactWithoutDigest> keyToArtifact = new HashMap<>();
Map<String, CacheEntry> keyToCacheEntry = new HashMap<>();
artifacts.forEach(
a -> {
Expand All @@ -184,7 +186,7 @@ public synchronized void putAll(
|| !cacheState.get(kv.getKey()).equals(kv.getValue()))
.map(Entry::getKey)
.collect(ImmutableList.toImmutableList());
ImmutableMap<String, OutputArtifact> updatedKeyToArtifact =
ImmutableMap<String, OutputArtifactWithoutDigest> updatedKeyToArtifact =
updatedKeys.stream().collect(ImmutableMap.toImmutableMap(k -> k, keyToArtifact::get));
ImmutableMap<String, CacheEntry> updatedKeyToCacheEntry =
updatedKeys.stream().collect(ImmutableMap.toImmutableMap(k -> k, keyToCacheEntry::get));
Expand Down Expand Up @@ -251,7 +253,7 @@ public synchronized void putAll(

@Override
@Nullable
public synchronized Path get(OutputArtifact artifact) {
public synchronized Path get(OutputArtifactWithoutDigest artifact) {
CacheEntry queriedEntry;
try {
queriedEntry = CacheEntry.forArtifact(artifact);
Expand Down Expand Up @@ -318,11 +320,11 @@ private void loadCacheData() {
}

/**
* Returns a list of futures copying the given {@link OutputArtifact}s to disk. The returned
* futures return the cache key on successful copy, or an empty string on copy failure.
* Returns a list of futures copying the given {@link OutputArtifactWithoutDigest}s to disk. The
* returned futures return the cache key on successful copy, or an empty string on copy failure.
*/
private ImmutableList<ListenableFuture<String>> copyLocally(
Map<String, OutputArtifact> updatedKeyToArtifact,
Map<String, OutputArtifactWithoutDigest> updatedKeyToArtifact,
Map<String, CacheEntry> updatedKeyToCacheEntry) {
return updatedKeyToArtifact.entrySet().stream()
.map(
Expand All @@ -348,7 +350,7 @@ private ImmutableList<ListenableFuture<String>> copyLocally(
.collect(ImmutableList.toImmutableList());
}

private static void copyLocally(OutputArtifact blazeArtifact, Path destinationPath)
private static void copyLocally(OutputArtifactWithoutDigest blazeArtifact, Path destinationPath)
throws IOException {
try (InputStream stream = blazeArtifact.getInputStream()) {
Files.copy(stream, destinationPath, StandardCopyOption.REPLACE_EXISTING);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import com.google.idea.blaze.android.sync.aspects.strategy.RenderResolveOutputGroupProvider;
import com.google.idea.blaze.android.sync.importer.BlazeImportUtil;
import com.google.idea.blaze.base.command.buildresult.BlazeArtifact;
import com.google.idea.blaze.base.command.buildresult.OutputArtifact;
import com.google.idea.blaze.base.command.buildresult.OutputArtifactWithoutDigest;
import com.google.idea.blaze.base.filecache.FileCache;
import com.google.idea.blaze.base.ideinfo.AndroidIdeInfo;
import com.google.idea.blaze.base.ideinfo.ArtifactLocation;
Expand Down Expand Up @@ -124,7 +124,7 @@ private void onSync(

boolean removeMissingFiles = syncMode == SyncMode.INCREMENTAL;

ImmutableList<OutputArtifact> artifactsToCache =
ImmutableList<OutputArtifactWithoutDigest> artifactsToCache =
getArtifactsToCache(projectViewSet, projectData);

artifactCache.putAll(artifactsToCache, context, removeMissingFiles);
Expand All @@ -139,7 +139,7 @@ private void refresh(BlazeContext context, BlazeBuildOutputs buildOutput) {
return;
}

ImmutableList<OutputArtifact> renderJars =
ImmutableList<OutputArtifactWithoutDigest> renderJars =
buildOutput
.getOutputGroupArtifacts(
RenderResolveOutputGroupProvider.RESOLVE_OUTPUT_GROUP::contains)
Expand All @@ -152,7 +152,7 @@ private void refresh(BlazeContext context, BlazeBuildOutputs buildOutput) {
artifactCache.putAll(renderJars, context, false);
}

private ImmutableList<OutputArtifact> getArtifactsToCache(
private ImmutableList<OutputArtifactWithoutDigest> getArtifactsToCache(
ProjectViewSet projectViewSet, BlazeProjectData projectData) {
List<ArtifactLocation> renderJars =
BlazeImportUtil.getSourceTargetsStream(project, projectData, projectViewSet)
Expand All @@ -163,10 +163,10 @@ private ImmutableList<OutputArtifact> getArtifactsToCache(
.collect(Collectors.toList());

ArtifactLocationDecoder decoder = projectData.getArtifactLocationDecoder();
List<OutputArtifact> blazeArtifacts =
List<OutputArtifactWithoutDigest> blazeArtifacts =
decoder.resolveOutputs(renderJars).stream()
.filter(artifact -> artifact instanceof OutputArtifact)
.map(artifact -> (OutputArtifact) artifact)
.filter(artifact -> artifact instanceof OutputArtifactWithoutDigest)
.map(artifact -> (OutputArtifactWithoutDigest) artifact)
.collect(Collectors.toList());
return ImmutableList.copyOf(blazeArtifacts);
}
Expand All @@ -191,12 +191,12 @@ public File getCachedJarForBinaryTarget(
}

BlazeArtifact jarArtifact = artifactLocationDecoder.resolveOutput(jarArtifactLocation);
if (!(jarArtifact instanceof OutputArtifact)) {
if (!(jarArtifact instanceof OutputArtifactWithoutDigest)) {
Logger.getInstance(RenderJarCache.class)
.warn("Unexpected render jar that is not an OutputArtifact: " + jarArtifactLocation);
return null;
}
Path jarPath = artifactCache.get((OutputArtifact) jarArtifact);
Path jarPath = artifactCache.get((OutputArtifactWithoutDigest) jarArtifact);
return jarPath == null ? null : jarPath.toFile();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import com.android.SdkConstants;
import com.google.idea.blaze.base.command.buildresult.BlazeArtifact;
import com.google.idea.blaze.base.command.buildresult.OutputArtifact;
import com.google.idea.blaze.base.command.buildresult.OutputArtifactWithoutDigest;
import com.google.idea.blaze.base.command.buildresult.SourceArtifact;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.util.PathUtil;
Expand All @@ -36,8 +36,8 @@ public final class UnpackedAarUtils {

/* Converts {@link BlazeArtifact} to the key which is used to create aar directory name */
public static String getArtifactKey(BlazeArtifact artifact) {
if (artifact instanceof OutputArtifact) {
return ((OutputArtifact) artifact).getKey();
if (artifact instanceof OutputArtifactWithoutDigest) {
return ((OutputArtifactWithoutDigest) artifact).getKey();
}
if (artifact instanceof SourceArtifact) {
return ((SourceArtifact) artifact).getFile().getPath();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@
import com.google.idea.blaze.android.sync.aspects.strategy.RenderResolveOutputGroupProvider;
import com.google.idea.blaze.android.sync.model.AndroidResourceModule;
import com.google.idea.blaze.android.sync.model.AndroidResourceModuleRegistry;
import com.google.idea.blaze.base.command.buildresult.LocalFileOutputArtifact;
import com.google.idea.blaze.base.command.buildresult.OutputArtifact;
import com.google.idea.blaze.base.command.buildresult.LocalFileOutputArtifactWithoutDigest;
import com.google.idea.blaze.base.command.buildresult.OutputArtifactWithoutDigest;
import com.google.idea.blaze.base.filecache.FileCache;
import com.google.idea.blaze.base.ideinfo.AndroidIdeInfo;
import com.google.idea.blaze.base.ideinfo.ArtifactLocation;
Expand Down Expand Up @@ -144,21 +144,21 @@ public void workspaceModule_canFindAllClassesInAllBinaries() throws ArtifactNotF

CacheEntry binACacheEntry =
CacheEntry.forArtifact(
(LocalFileOutputArtifact)
(LocalFileOutputArtifactWithoutDigest)
artifactLocationDecoder.resolveOutput(
getArtifactLocation("com/google/example/simple/bin_a.jar")));
String binAJar = cacheDir.getAbsoluteFile() + "/" + binACacheEntry.getFileName();

CacheEntry binBCacheEntry =
CacheEntry.forArtifact(
(LocalFileOutputArtifact)
(LocalFileOutputArtifactWithoutDigest)
artifactLocationDecoder.resolveOutput(
getArtifactLocation("com/google/example/simple/bin_b.jar")));
String binBJar = cacheDir.getAbsoluteFile() + "/" + binBCacheEntry.getFileName();

CacheEntry binCCacheEntry =
CacheEntry.forArtifact(
(LocalFileOutputArtifact)
(LocalFileOutputArtifactWithoutDigest)
artifactLocationDecoder.resolveOutput(
getArtifactLocation("com/google/example/simple/bin_c.jar")));
String binCJar = cacheDir.getAbsoluteFile() + "/" + binCCacheEntry.getFileName();
Expand Down Expand Up @@ -228,21 +228,21 @@ public void resourceModule_canFindSourceClasses() throws ArtifactNotFoundExcepti

CacheEntry binACacheEntry =
CacheEntry.forArtifact(
(LocalFileOutputArtifact)
(LocalFileOutputArtifactWithoutDigest)
artifactLocationDecoder.resolveOutput(
getArtifactLocation("com/google/example/simple/bin_a.jar")));
String binAJar = cacheDir.getAbsoluteFile() + "/" + binACacheEntry.getFileName();

CacheEntry binBCacheEntry =
CacheEntry.forArtifact(
(LocalFileOutputArtifact)
(LocalFileOutputArtifactWithoutDigest)
artifactLocationDecoder.resolveOutput(
getArtifactLocation("com/google/example/simple/bin_b.jar")));
String binBJar = cacheDir.getAbsoluteFile() + "/" + binBCacheEntry.getFileName();

CacheEntry binCCacheEntry =
CacheEntry.forArtifact(
(LocalFileOutputArtifact)
(LocalFileOutputArtifactWithoutDigest)
artifactLocationDecoder.resolveOutput(
getArtifactLocation("com/google/example/simple/bin_c.jar")));
String binCJar = cacheDir.getAbsoluteFile() + "/" + binCCacheEntry.getFileName();
Expand Down Expand Up @@ -285,21 +285,21 @@ public void resourceModule_canFindDependencyClasses() throws ArtifactNotFoundExc

CacheEntry binACacheEntry =
CacheEntry.forArtifact(
(LocalFileOutputArtifact)
(LocalFileOutputArtifactWithoutDigest)
artifactLocationDecoder.resolveOutput(
getArtifactLocation("com/google/example/simple/bin_a.jar")));
String binAJar = cacheDir.getAbsoluteFile() + "/" + binACacheEntry.getFileName();

CacheEntry binBCacheEntry =
CacheEntry.forArtifact(
(LocalFileOutputArtifact)
(LocalFileOutputArtifactWithoutDigest)
artifactLocationDecoder.resolveOutput(
getArtifactLocation("com/google/example/simple/bin_b.jar")));
String binBJar = cacheDir.getAbsoluteFile() + "/" + binBCacheEntry.getFileName();

CacheEntry binCCacheEntry =
CacheEntry.forArtifact(
(LocalFileOutputArtifact)
(LocalFileOutputArtifactWithoutDigest)
artifactLocationDecoder.resolveOutput(
getArtifactLocation("com/google/example/simple/bin_c.jar")));
String binCJar = cacheDir.getAbsoluteFile() + "/" + binCCacheEntry.getFileName();
Expand Down Expand Up @@ -447,8 +447,8 @@ private void createBinaryJars() throws ArtifactNotFoundException {
fileSystem.createDirectory(cacheDirFile.getAbsolutePath());
String cacheDir = cacheDirFile.getPath();

OutputArtifact binAArtifact =
(OutputArtifact)
OutputArtifactWithoutDigest binAArtifact =
(OutputArtifactWithoutDigest)
artifactLocationDecoder.resolveOutput(
getArtifactLocation("com/google/example/simple/bin_a.jar"));
CacheEntry binACacheEntry = CacheEntry.forArtifact(binAArtifact);
Expand All @@ -465,8 +465,8 @@ private void createBinaryJars() throws ArtifactNotFoundException {
fileSystem.createFile(binAJar + "!/com/google/example/simple/trans_dep_a/R$dimen.class");
artifactCache.addTrackedFile(binAArtifact, binAJar);

OutputArtifact binBArtifact =
(OutputArtifact)
OutputArtifactWithoutDigest binBArtifact =
(OutputArtifactWithoutDigest)
artifactLocationDecoder.resolveOutput(
getArtifactLocation("com/google/example/simple/bin_b.jar"));
CacheEntry binBCacheEntry = CacheEntry.forArtifact(binBArtifact);
Expand All @@ -483,8 +483,8 @@ private void createBinaryJars() throws ArtifactNotFoundException {
fileSystem.createFile(binBJar + "!/com/google/example/simple/trans_dep_b/R$dimen.class");
artifactCache.addTrackedFile(binBArtifact, binBJar);

OutputArtifact binCArtifact =
(OutputArtifact)
OutputArtifactWithoutDigest binCArtifact =
(OutputArtifactWithoutDigest)
artifactLocationDecoder.resolveOutput(
getArtifactLocation("com/google/example/simple/bin_c.jar"));
CacheEntry binCCacheEntry = CacheEntry.forArtifact(binCArtifact);
Expand Down
Loading

0 comments on commit 9d44548

Please sign in to comment.