Add support for multiple module interfaces per cc_library

src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java

@@ -75,7 +75,6 @@ public class ActionCacheChecker {
 
   @Nullable private final ActionCache actionCache; // Null when not enabled.
 
-
   /** Cache config parameters for ActionCacheChecker. */
   @AutoValue
   public abstract static class CacheConfig {
@@ -179,6 +178,7 @@ private static TreeArtifactValue getCachedTreeMetadata(
    * @param actionInputs the action inputs; usually action.getInputs(), but might be a previously
    *     cached set of discovered inputs for actions that discover them.
    * @param outputMetadataStore metadata provider for action outputs.
+   * @param mandatoryInputsDigest the digest of mandatory inputs, or null if not discovering inputs.
    * @param cachedOutputMetadata cached metadata that should be used instead of {@code
    *     outputMetadataStore}.
    * @param outputChecker used to check whether remote metadata should be trusted.
@@ -195,6 +195,7 @@ private static boolean isUpToDate(
       NestedSet<Artifact> actionInputs,
       InputMetadataProvider inputMetadataProvider,
       OutputMetadataStore outputMetadataStore,
+      @Nullable byte[] mandatoryInputsDigest,
       @Nullable CachedOutputMetadata cachedOutputMetadata,
       @Nullable OutputChecker outputChecker,
       ImmutableMap<String, String> effectiveEnvironment,
@@ -209,7 +210,8 @@ private static boolean isUpToDate(
             effectiveEnvironment,
             effectiveExecProperties,
             outputPermissions,
-            useArchivedTreeArtifacts);
+            useArchivedTreeArtifacts,
+            mandatoryInputsDigest);
 
     for (Artifact artifact : action.getOutputs()) {
       if (artifact.isTreeArtifact()) {
@@ -459,6 +461,7 @@ private TreeArtifactValue constructProxyTreeMetadata(SpecialArtifact parent)
   public Token getTokenIfNeedToExecute(
       Action action,
       List<Artifact> resolvedCacheArtifacts,
+      @Nullable byte[] mandatoryInputsDigest,
       Map<String, String> clientEnv,
       OutputPermissions outputPermissions,
       EventHandler handler,
@@ -515,6 +518,7 @@ public Token getTokenIfNeedToExecute(
         clientEnv,
         outputPermissions,
         remoteDefaultExecProperties,
+        mandatoryInputsDigest,
         cachedOutputMetadata,
         outputChecker,
         useArchivedTreeArtifacts)) {
@@ -548,6 +552,7 @@ private boolean mustExecute(
       Map<String, String> clientEnv,
       OutputPermissions outputPermissions,
       ImmutableMap<String, String> remoteDefaultExecProperties,
+      @Nullable byte[] mandatoryInputsDigest,
       @Nullable CachedOutputMetadata cachedOutputMetadata,
       @Nullable OutputChecker outputChecker,
       boolean useArchivedTreeArtifacts)
@@ -587,6 +592,7 @@ private boolean mustExecute(
         actionInputs,
         inputMetadataProvider,
         outputMetadataStore,
+        mandatoryInputsDigest,
         cachedOutputMetadata,
         outputChecker,
         effectiveEnvironment,
@@ -623,7 +629,7 @@ private static FileArtifactValue getOutputMetadataOrConstant(
   // to trigger a re-execution, so we should catch the IOException explicitly there. In others, we
   // should propagate the exception, because it is unexpected (e.g., bad file system state).
   @Nullable
-  private static FileArtifactValue getInputMetadataMaybe(
+  public static FileArtifactValue getInputMetadataMaybe(
       InputMetadataProvider inputMetadataProvider, Artifact artifact) {
     try {
       return getInputMetadataOrConstant(inputMetadataProvider, artifact);
@@ -665,7 +671,8 @@ public void updateActionCache(
       Map<String, String> clientEnv,
       OutputPermissions outputPermissions,
       ImmutableMap<String, String> remoteDefaultExecProperties,
-      boolean useArchivedTreeArtifacts)
+      boolean useArchivedTreeArtifacts,
+      @Nullable byte[] mandatoryInputsDigest)
       throws IOException, InterruptedException {
     checkState(cacheConfig.enabled(), "cache unexpectedly disabled, action: %s", action);
     Preconditions.checkArgument(token != null, "token unexpectedly null, action: %s", action);
@@ -694,7 +701,8 @@ public void updateActionCache(
             effectiveEnvironment,
             effectiveExecProperties,
             outputPermissions,
-            useArchivedTreeArtifacts);
+            useArchivedTreeArtifacts,
+            mandatoryInputsDigest);
 
     for (Artifact output : action.getOutputs()) {
       // Remove old records from the cache if they used different key.
@@ -737,6 +745,14 @@ public void updateActionCache(
     actionCache.put(key, builder.build());
   }
 
+  public boolean mandatoryInputsMatch(Action action, byte[] mandatoryInputsDigest) {
+    checkArgument(action.discoversInputs());
+    ActionCache.Entry entry = getCacheEntry(action);
+    return entry != null
+        && !entry.isCorrupted()
+        && Arrays.equals(entry.getMandatoryInputsDigest(), mandatoryInputsDigest);
+  }
+
   @Nullable
   public List<Artifact> getCachedInputs(Action action, PackageRootResolver resolver)
       throws PackageRootResolver.PackageRootException, InterruptedException {
@@ -810,6 +826,7 @@ public List<Artifact> getCachedInputs(Action action, PackageRootResolver resolve
   public Token getTokenUnconditionallyAfterFailureToRecordActionCacheHit(
       Action action,
       List<Artifact> resolvedCacheArtifacts,
+      @Nullable byte[] mandatoryInputsDigest,
       Map<String, String> clientEnv,
       OutputPermissions outputPermissions,
       EventHandler handler,
@@ -825,6 +842,7 @@ public Token getTokenUnconditionallyAfterFailureToRecordActionCacheHit(
     return getTokenIfNeedToExecute(
         action,
         resolvedCacheArtifacts,
+        mandatoryInputsDigest,
         clientEnv,
         outputPermissions,
         handler,
@@ -887,5 +905,4 @@ private Token(Action action) {
       this.cacheKey = action.getPrimaryOutput().getExecPathString();
     }
   }
-
 }

src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java

@@ -82,12 +82,13 @@ public interface ActionCache {
   final class Entry {
     /** Unique instance standing for a corrupted cache entry. */
     public static final ActionCache.Entry CORRUPTED =
-        new Entry(null, null, ImmutableMap.of(), ImmutableMap.of(), ImmutableList.of());
+        new Entry(null, null, null, ImmutableMap.of(), ImmutableMap.of(), ImmutableList.of());
 
     // Digest of all relevant properties of the action for cache invalidation purposes.
     // Null if the entry is corrupted.
     @Nullable private final byte[] digest;
 
+    @Nullable private final byte[] mandatoryInputsDigest;
     // List of input paths discovered by the action.
     // Null if the action does not discover inputs.
     @Nullable private final ImmutableList<String> discoveredInputPaths;
@@ -100,11 +101,13 @@ final class Entry {
 
     Entry(
         @Nullable byte[] digest,
+        @Nullable byte[] mandatoryInputsDigest,
         @Nullable ImmutableList<String> discoveredInputPaths,
         ImmutableMap<String, FileArtifactValue> outputFileMetadata,
         ImmutableMap<String, SerializableTreeArtifactValue> outputTreeMetadata,
         ImmutableList<String> proxyOutputs) {
       this.digest = digest;
+      this.mandatoryInputsDigest = mandatoryInputsDigest;
       this.discoveredInputPaths = discoveredInputPaths;
       this.outputFileMetadata = outputFileMetadata;
       this.outputTreeMetadata = outputTreeMetadata;
@@ -128,7 +131,11 @@ public byte[] getDigest() {
     /** Returns whether the action discovers inputs. */
     public boolean discoversInputs() {
       checkState(!isCorrupted());
-      return discoveredInputPaths != null;
+      if (discoveredInputPaths == null) {
+        return false;
+      }
+      checkState(mandatoryInputsDigest != null);
+      return true;
     }
 
     /**
@@ -140,6 +147,12 @@ public ImmutableList<String> getDiscoveredInputPaths() {
       return discoveredInputPaths;
     }
 
+    @Nullable
+    public byte[] getMandatoryInputsDigest() {
+      checkState(discoversInputs());
+      return mandatoryInputsDigest;
+    }
+
     /** Gets the metadata of an output file. */
     @Nullable
     public FileArtifactValue getOutputFile(Artifact output) {
@@ -187,6 +200,7 @@ public String toString() {
       return MoreObjects.toStringHelper(this)
           .add("digest", digest)
           .add("discoveredInputPaths", discoveredInputPaths)
+          .add("mandatoryInputsDigest", mandatoryInputsDigest)
           .add("outputFileMetadata", outputFileMetadata)
           .add("outputTreeMetadata", outputTreeMetadata)
           .add("proxyOutputs", proxyOutputs)
@@ -205,6 +219,9 @@ void dump(PrintStream out) {
           out.format("    %s\n", path);
         }
       }
+      if (mandatoryInputsDigest != null) {
+        out.format("  mandatoryInputsDigest = %s\n", formatDigest(mandatoryInputsDigest));
+      }
 
       if (!outputFileMetadata.isEmpty()) {
         out.println("  outputFileMetadata =");
@@ -275,6 +292,7 @@ public static final class Builder {
       // Discovered inputs.
       // Null if the action does not discover inputs.
       @Nullable private final ImmutableList.Builder<String> discoveredInputPaths;
+      @Nullable private byte[] mandatoryInputsDigest;
 
       private final ImmutableMap.Builder<String, FileArtifactValue> outputFileMetadata =
           ImmutableMap.builder();
@@ -293,20 +311,27 @@ public static final class Builder {
        * @param discoversInputs whether the action discovers inputs.
        * @param outputPermissions the requested output permissions.
        * @param useArchivedTreeArtifacts whether archived tree artifacts are enabled.
+       * @param mandatoryInputsDigest the digest of the mandatory inputs, or null if the action
+       *     doesn't discover inputs.
        */
       public Builder(
           String actionKey,
           boolean discoversInputs,
           ImmutableMap<String, String> clientEnv,
           ImmutableMap<String, String> execProperties,
           OutputPermissions outputPermissions,
-          boolean useArchivedTreeArtifacts) {
+          boolean useArchivedTreeArtifacts,
+          @Nullable byte[] mandatoryInputsDigest) {
         this.actionKey = actionKey;
         this.clientEnv = clientEnv;
         this.execProperties = execProperties;
         this.discoveredInputPaths = discoversInputs ? ImmutableList.builder() : null;
         this.outputPermissions = outputPermissions;
         this.useArchivedTreeArtifacts = useArchivedTreeArtifacts;
+        checkArgument(
+            (mandatoryInputsDigest != null) == discoversInputs,
+            "mandatoryInputsDigest must be set iff the action discovers inputs");
+        this.mandatoryInputsDigest = mandatoryInputsDigest;
       }
 
       /** Adds metadata of an input file. */
@@ -390,6 +415,7 @@ public Entry build() {
                 execProperties,
                 outputPermissions,
                 useArchivedTreeArtifacts),
+            mandatoryInputsDigest,
             discoveredInputPaths != null ? discoveredInputPaths.build() : null,
             outputFileMetadata.buildOrThrow(),
             outputTreeMetadata.buildOrThrow(),

src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java

@@ -75,7 +75,7 @@ public class CompactPersistentActionCache implements ActionCache {
   // cache records.
   private static final int VALIDATION_KEY = -10;
 
-  private static final int VERSION = 23;
+  private static final int VERSION = 24;
 
   /**
    * A timestamp, represented as the number of minutes since the Unix epoch.
@@ -848,7 +848,8 @@ private byte[] encode(ActionCache.Entry entry) throws IOException {
     int maxDiscoveredInputsSize = 1; // presence marker
     if (entry.discoversInputs()) {
       maxDiscoveredInputsSize +=
-          VarInt.MAX_VARINT_SIZE // length
+          (1 + DigestUtils.ESTIMATED_SIZE) // mandatoryInputsDigest
+              + VarInt.MAX_VARINT_SIZE // length
               + (VarInt.MAX_VARINT_SIZE // execPath
                   * entry.getDiscoveredInputPaths().size());
     }
@@ -899,6 +900,7 @@ private byte[] encode(ActionCache.Entry entry) throws IOException {
 
     VarInt.putVarInt(entry.discoversInputs() ? 1 : 0, sink);
     if (entry.discoversInputs()) {
+      MetadataDigestUtils.write(entry.getMandatoryInputsDigest(), sink);
       ImmutableList<String> discoveredInputPaths = entry.getDiscoveredInputPaths();
       VarInt.putVarInt(discoveredInputPaths.size(), sink);
       for (String discoveredInputPath : discoveredInputPaths) {
@@ -974,13 +976,18 @@ private ActionCache.Entry decodeInternal(byte[] data) throws IOException {
 
       byte[] digest = MetadataDigestUtils.read(source);
 
+      byte[] mandatoryInputsDigest = null;
       ImmutableList<String> discoveredInputPaths = null;
       int discoveredInputsPresenceMarker = VarInt.getVarInt(source);
       if (discoveredInputsPresenceMarker != 0) {
         if (discoveredInputsPresenceMarker != 1) {
           throw new IOException(
               "Invalid presence marker for discovered inputs: " + discoveredInputsPresenceMarker);
         }
+        mandatoryInputsDigest = MetadataDigestUtils.read(source);
+        if (mandatoryInputsDigest.length != digest.length) {
+          throw new IOException("Corrupted mandatory inputs digest");
+        }
         int numDiscoveredInputs = VarInt.getVarInt(source);
         if (numDiscoveredInputs < 0) {
           throw new IOException("Invalid discovered input count: " + numDiscoveredInputs);
@@ -1002,6 +1009,7 @@ private ActionCache.Entry decodeInternal(byte[] data) throws IOException {
         }
         return new ActionCache.Entry(
             digest,
+            mandatoryInputsDigest,
             discoveredInputPaths,
             /* outputFileMetadata= */ ImmutableMap.of(),
             /* outputTreeMetadata= */ ImmutableMap.of(),
@@ -1082,6 +1090,7 @@ private ActionCache.Entry decodeInternal(byte[] data) throws IOException {
       }
       return new ActionCache.Entry(
           digest,
+          mandatoryInputsDigest,
           discoveredInputPaths,
           outputFiles.buildOrThrow(),
           outputTrees.buildOrThrow(),

src/main/java/com/google/devtools/build/lib/actions/cache/MetadataDigestUtils.java

@@ -28,6 +28,8 @@
 public final class MetadataDigestUtils {
   private MetadataDigestUtils() {}
 
+  private static final byte[] EMPTY_DIGEST = new Fingerprint().digestAndReset();
+
   /**
    * @param source the byte buffer source.
    * @return the digest from the given buffer.
@@ -54,7 +56,7 @@ public static void write(byte[] digest, OutputStream sink) throws IOException {
    * @param mdMap A collection of (execPath, FileArtifactValue) pairs. Values may be null.
    */
   public static byte[] fromMetadata(Map<String, FileArtifactValue> mdMap) {
-    byte[] result = new byte[1]; // reserve the empty string
+    byte[] result = EMPTY_DIGEST.clone();
     // Profiling showed that MessageDigest engine instantiation was a hotspot, so create one
     // instance for this computation to amortize its cost.
     Fingerprint fp = new Fingerprint();

src/main/java/com/google/devtools/build/lib/analysis/actions/StarlarkAction.java

@@ -345,6 +345,15 @@ public NestedSet<Artifact> getAllowedDerivedInputs() {
       return getInputs();
     }
 
+    @Override
+    public NestedSet<Artifact> getMandatoryInputs() {
+      if (unusedInputsList.isPresent()) {
+        return NestedSetBuilder.emptySet(Order.STABLE_ORDER);
+      } else {
+        return getInputs();
+      }
+    }
+
     @Nullable
     @Override
     public NestedSet<Artifact> discoverInputs(ActionExecutionContext actionExecutionContext)
@@ -358,7 +367,11 @@ public NestedSet<Artifact> discoverInputs(ActionExecutionContext actionExecution
         NestedSet<Artifact> inputFilesForExtraAction =
             shadowedActionObj.getInputFilesForExtraAction(actionExecutionContext);
         if (inputFilesForExtraAction == null) {
-          return null;
+          if (unusedInputsList.isPresent()) {
+            return allStarlarkActionInputs;
+          } else {
+            return null;
+          }
         }
         updateInputs(
             createInputs(