bryan-minimal · Jul 29, 2026 · Jul 28, 2026 · Jul 28, 2026 · Jul 29, 2026 · twitchyliquid64
@@ -607,6 +607,16 @@ pub enum IndexSourceMode {
     Pinned,
     /// The global root index only.
     Root,
+    /// The union of the per-commit *closure* snapshots of the pinned chain.
+    /// Asymmetric by design: the upstream's closure is required (it is the
+    /// signed catalog — missing is an error), while each sideload's closure
+    /// is unioned best-effort — a sideload not covered by build
+    /// infrastructure publishes no closure, is skipped with a log line, and
+    /// its specs resolve locally. Distinct from [`Self::Pinned`] in both the
+    /// object read (`<commit>.closure.shisha`, the bounded signed catalog,
+    /// vs `<commit>.shisha`, a byte copy of the whole root index) and in
+    /// covering sideload closures at their own pins.
+    Closure,
 }
 
 impl std::str::FromStr for IndexSourceMode {
@@ -616,8 +626,9 @@ impl std::str::FromStr for IndexSourceMode {
             "auto" => Ok(Self::Auto),
             "pinned" => Ok(Self::Pinned),
             "root" => Ok(Self::Root),
+            "closure" => Ok(Self::Closure),
             other => Err(format!(
-                "unknown index source {other:?} (expected \"auto\", \"pinned\" or \"root\")"
+                "unknown index source {other:?} (expected \"auto\", \"pinned\", \"root\" or \"closure\")"
             )),
         }
     }
@@ -636,6 +647,14 @@ pub enum CacheConfig {
     /// Read the immutable per-commit snapshot `object`; if it doesn't exist,
     /// fail rather than fall back.
     CommitIndexOnly { object: String },
+    /// Read and union the immutable per-commit closure snapshots. The
+    /// upstream's closure is required (it is the signed catalog); sideload
+    /// closures are best-effort — a sideload repo not covered by build
+    /// infrastructure publishes no index, and its specs resolve locally.
+    CommitClosures {
+        upstream: String,
+        sideloads: Vec<String>,
+    },
 }
 
 /// Object key of the per-commit index snapshot for a repo + commit:
@@ -644,6 +663,12 @@ pub fn commit_index_object(repo_url: &str, commit_sha: &str) -> String {
     format!("{}/{commit_sha}.shisha", repo_slug(repo_url))
 }
 
+/// Object key of the per-commit *closure* snapshot (the bounded catalog at a
+/// commit): `<host>/<owner>/<repo>/<commit>.closure.shisha`.
+pub fn commit_closure_object(repo_url: &str, commit_sha: &str) -> String {
+    format!("{}/{commit_sha}.closure.shisha", repo_slug(repo_url))
+}
+
 /// Stable `host/owner/repo` slug for a repo URL: query/fragment, scheme,
 /// credentials, trailing `/` and any `.git` suffix stripped; scp-like syntax
 /// (`git@host:owner/repo`) normalized to `host/owner/repo`.
@@ -1099,6 +1124,44 @@ impl File {
             (IndexSourceMode::Pinned, None) => Err(
                 "index_source \"pinned\" requires a git upstream with a locked_commit".to_string(),
             ),
+            (IndexSourceMode::Closure, _) => {
+                let Some(u) = self.upstream.as_ref() else {
+                    return Err(
+                        "index_source \"closure\" requires a git upstream with a locked_commit"
+                            .to_string(),
+                    );
+                };
+                let LinkConfig::Git {
+                    repo,
+                    locked_commit: Some(commit),
+                    ..
+                } = &u.link
+                else {
+                    return Err(
+                        "index_source \"closure\" requires a git upstream with a locked_commit"
+                            .to_string(),
+                    );
+                };
+                // Sideloads in declaration order; links without a locked
+                // commit (or backed by a directory) have no remote presence
+                // and contribute nothing.
+                let sideloads = u
+                    .sideloads()
+                    .iter()
+                    .filter_map(|s| match s.link() {
+                        LinkConfig::Git {
+                            repo,
+                            locked_commit: Some(commit),
+                            ..
+                        } => Some(commit_closure_object(repo, commit)),<
8000
/span>
+                        _ => None,
+                    })
+                    .collect();
+                Ok(CacheConfig::CommitClosures {
+                    upstream: commit_closure_object(repo, commit),
+                    sideloads,
+                })
+            }
         }
     }
 }
@@ -1734,4 +1797,62 @@ mod tests {
             }
         );
     }
+
+    #[test]
+    fn cache_config_closure_unions_pinned_links_and_skips_the_rest() {
+        const UP: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+        const SIDE: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
+        let mf: File = toml::from_str(&format!(
+            r#"
+            [upstream]
+            repo = "https://github.com/gominimal/pkgs"
+            locked_commit = "{UP}"
+
+            [[upstream.sideload]]
+            repo = "https://github.com/acme/extra"
+            locked_commit = "{SIDE}"
+
+            [[upstream.sideload]]
+            dir = "/tmp/local-overlay"
+
+            [[upstream.sideload]]
+            repo = "https://github.com/acme/unpinned"
+            "#
+        ))
+        .unwrap();
+        // Required upstream, then pinned sideloads in declaration order; the
+        // dir and unpinned links have no remote presence.
+        assert_eq!(
+            mf.cache_config(Some(IndexSourceMode::Closure)).unwrap(),
+            CacheConfig::CommitClosures {
+                upstream: format!("github.com/gominimal/pkgs/{UP}.closure.shisha"),
+                sideloads: vec![format!("github.com/acme/extra/{SIDE}.closure.shisha")],
+            }
+        );
+
+        // An unpinned upstream is a loud error — even with a pinned sideload:
+        // the upstream closure is the required (signed) catalog.
+        let unpinned: File = toml::from_str(&format!(
+            r#"
+            [upstream]
+            repo = "https://github.com/gominimal/pkgs"
+
+            [[upstream.sideload]]
+            repo = "https://github.com/acme/extra"
+            locked_commit = "{SIDE}"
+            "#
+        ))
+        .unwrap();
+        assert!(
+            unpinned
+                .cache_config(Some(IndexSourceMode::Closure))
+                .is_err()
+        );
+
+        // The env-override spelling parses.
+        assert_eq!(
+            "closure".parse::<IndexSourceMode>().unwrap(),
+            IndexSourceMode::Closure
+        );
+    }
 }
@@ -138,6 +138,32 @@ impl IndexFile {
         }
         Ok(())
     }
+
+    /// Absorbs `other`'s entries into `self` (other wins on overlap).
+    /// Returns the number of entries whose
10BC8
 value *changed* — overlapping
+    /// records should be byte-identical across per-link closures, so a
+    /// non-zero count is a divergence signal the caller should surface.
+    pub fn merge(&mut self, other: IndexFile) -> usize {
+        let mut conflicts = 0;
+        for (k, v) in other.idx {
+            if let Some(prev) = self.idx.insert(k, v.clone())
+                && prev != v
+            {
+                conflicts += 1;
+            }
+        }
+        conflicts
+    }
+
+    /// The number of entries in the index.
+    pub fn len(&self) -> usize {
+        self.idx.len()
+    }
+
+    /// Whether the index has no entries.
+    pub fn is_empty(&self) -> bool {
+        self.idx.is_empty()
+    }
 }
 
 impl Extend<(SpecHash, [u8; 32])> for IndexFile {
@@ -207,4 +233,21 @@ mod tests {
             ))
         );
     }
+
+    #[test]
+    fn merge_unions_and_counts_only_value_changes() {
+        let h = |b: u8| common::SpecHash::from_bytes([b; 32]);
+        let mut a = IndexFile::default();
+        a.extend([(h(1), [0x11; 32]), (h(2), [0x22; 32])]);
+        let mut b = IndexFile::default();
+        // h(2) identical (no conflict), h(3) new, h(1) DIFFERENT (conflict).
+        b.extend([(h(2), [0x22; 32]), (h(3), [0x33; 32]), (h(1), [0xFF; 32])]);
+
+        let conflicts = a.merge(b);
+        assert_eq!(conflicts, 1);
+        assert_eq!(a.len(), 3);
+        // Other wins on overlap.
+        assert_eq!(a.sha256(&h(1)), Some([0xFF; 32]));
+        assert_eq!(a.sha256(&h(3)), Some([0x33; 32]));
+    }
 }
@@ -260,6 +260,20 @@ impl RemoteCache<AnyBackend> {
                 },
                 false,
             ),
+            mfile::CacheConfig::CommitClosures {
+                upstream,
+                sideloads,
+            } => {
+                return Self::new_any_closure_union(
+                    url,
+                    gcs_storage,
+                    index_dir,
+                    ot,
+                    upstream,
+                    sideloads,
+                )
+                .await;
+            }
         };
         if let IndexSource::Snapshot { object } = &source {
             tracing::debug!("cache index: per-commit snapshot {object}");
@@ -280,6 +294,73 @@ impl RemoteCache<AnyBackend> {
             other => other,
         }
     }
+
+    /// Reads and unions per-commit *closure* snapshots into one index. The
+    /// upstream's closure is required — it is the signed catalog, and its
+    /// absence surfaces as [`Error::SnapshotMissing`] naming the object,
+    /// never a silent fallback. Sideload closures are best-effort: a sideload
+    /// repo not covered by build infrastructure publishes no index, so a
+    /// missing one is logged and skipped (its specs resolve locally as they
+    /// always have). Every closure is fetched through the same
+    /// immutable-snapshot path as the byte-copy (whole-record check, local
+    /// cache forever).
+    async fn new_any_closure_union(
+        url: AnyUrl,
+        gcs_storage: Option<Storage>,
+        index_dir: Option<PathBuf>,
+        ot: Option<OpTracker>,
+        upstream: &str,
+        sideloads: &[String],
+    ) -> Result<Self, Error<AnyRespError>> {
+        tracing::debug!("cache index: per-commit closure {upstream}");
+        let mut rc = Self::new_any(
+            url.clone(),
+            gcs_storage.clone(),
+            index_dir.clone(),
+            ot.clone(),
+            IndexSource::Snapshot {
+                object: upstream.to_string(),
+            },
+        )
+        .await?;
+
+        for object in sideloads {
+            let side = Self::new_any(
+                url.clone(),
+                gcs_storage.clone(),
+                index_dir.clone(),
+                ot.clone(),
+                IndexSource::Snapshot {
+                    object: object.clone(),
+                },
+            )
+            .await;
+            match side {
+                Ok(s) => {
+                    let conflicts = rc.index.merge(s.index);
+                    if conflicts > 0 {
+                        // Overlapping spec hashes must agree across links;
+                        // divergence means two closures claim different bytes
+                        // for one spec — surface it, last link wins.
+                        tracing::warn!(
+                            "closure union: {conflicts} conflicting entries merging {object}"
+                        );
+                    }
+                }
+                Err(Error::SnapshotMissing { object }) => {
+                    tracing::info!(
+                        "sideload closure {object} not published; its specs resolve locally"
+                    );
+                }
+                Err(e) => return Err(e),
+            }
+        }
+        tracing::debug!(
+            "cache index: closure union ready, {} entries",
+            rc.index.len()
+        );
+        Ok(rc)
+    }
 }
 
 impl<B: FetchBackend> RemoteCache<B> {
@@ -924,6 +1005,76 @@ mod tests {
         assert_eq!(rc.sha256(&spec_hash), Some(sha256));
     }
 
+    #[tokio::test]
+    async fn configured_closure_union_merges_per_link_closures() {
+        const OBJ_A: &str = "github.com/gominimal/pkgs/aaaa.closure.shisha";
+        const OBJ_B: &str = "github.com/acme/extra/bbbb.closure.shisha";
+        let spec_a = SpecHash::from_bytes([0x0A; 32]);
+        let spec_b = SpecHash::from_bytes([0x0B; 32]);
+        let base = serve_objects(vec![
+            (OBJ_A.to_string(), index_bytes_for(&spec_a, [0xA1; 32])),
+            (OBJ_B.to_string(), index_bytes_for(&spec_b, [0xB1; 32])),
+        ]);
+
+        let config = mfile::CacheConfig::CommitClosures {
+            upstream: OBJ_A.to_string(),
+            sideloads: vec![OBJ_B.to_string()],
+        };
+        let rc = RemoteCache::new_any_configured(https_url(&base), None, None, None, &config)
+            .await
+            .unwrap();
+        // Entries from BOTH links resolve through the one merged index.
+        assert_eq!(rc.sha256(&spec_a), Some([0xA1; 32]));
+        assert_eq!(rc.sha256(&spec_b), Some([0xB1; 32]));
+        assert_eq!(rc.sha256(&SpecHash::from_bytes([0x0C; 32])), None);
+    }
+
+    #[tokio::test]
+    async fn configured_closure_missing_upstream_is_a_loud_error() {
+        const OBJ_UP: &str = "github.com/gominimal/pkgs/aaaa.closure.shisha";
+        // The root index exists and must NOT be silently used: the upstream
+        // closure is the required (signed) catalog.
+        let spec_a = SpecHash::from_bytes([0x0A; 32]);
+        let base = serve_index(Some(index_bytes_for(&spec_a, [0xA1; 32])));
+
+        let config = mfile::CacheConfig::CommitClosures {
+            upstream: OBJ_UP.to_string(),
+            sideloads: vec![],
+        };
+        let err = RemoteCache::new_any_configured(https_url(&base), None, None, None, &config)
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(&err, Error::SnapshotMissing { object } if object == OBJ_UP),
+            "expected SnapshotMissing for {OBJ_UP}, got {err:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn configured_closure_missing_sideload_is_skipped_not_fatal() {
+        const OBJ_UP: &str = "github.com/gominimal/pkgs/aaaa.closure.shisha";
+        const OBJ_SIDE: &str = "github.com/acme/extra/bbbb.closure.shisha";
+        // Only the upstream closure exists — the sideload repo publishes no
+        // index (the common case: not covered by build infrastructure).
+        let spec_a = SpecHash::from_bytes([0x0A; 32]);
+        let base = serve_objects(vec![(
+            OBJ_UP.to_string(),
+            index_bytes_for(&spec_a, [0xA1; 32]),
+        )]);
+
+        let config = mfile::CacheConfig::CommitClosures {
+            upstream: OBJ_UP.to_string(),
+            sideloads: vec![OBJ_SIDE.to_string()],
+        };
+        let rc = RemoteCache::new_any_configured(https_url(&base), None, None, None, &config)
+            .await
+            .unwrap();
+        // Upstream entries resolve; the sideload's specs simply miss (and
+        // would build locally), exactly as they do today.
+        assert_eq!(rc.sha256(&spec_a), Some([0xA1; 32]));
+        assert_eq!(rc.sha256(&SpecHash::from_bytes([0x0B; 32])), None);
+    }
+
     #[tokio::test]
     async fn configured_pinned_missing_snapshot_is_an_error_not_a_fallback() {
         const OBJECT: &str = "github.com/gominimal/pkgs/0123abcd.shisha";