dmux · Aug 8, 2026 · Aug 8, 2026 · Aug 8, 2026
diff --git a/Dockerfile b/Dockerfile
@@ -1,3 +1,29 @@
+# Resolve the Delta and Unity Catalog connector jars at build time, using the same Ivy
+# resolver spark-submit uses at runtime, so a job container never needs Maven Central.
+# Reusing spark-submit (rather than curl-ing jars by hand) is deliberate: it writes the
+# ivy-*.xml metadata alongside the jars, and without that metadata Ivy goes back to the
+# network even when the jar is already on disk.
+#
+# --platform=$BUILDPLATFORM: jars are architecture-independent, so this stage does not need
+# to run under QEMU during the multi-arch release build.
+FROM --platform=$BUILDPLATFORM apache/spark:3.5.3-scala2.12-java17-python3-ubuntu AS sparkjars
+
+USER root
+
+# Keep these in step with docker_executor.DEFAULT_DELTA_PACKAGE and the connector coordinate
+# documented for spark.table() resolution.
+ARG DELTA_PACKAGE=io.delta:delta-spark_2.12:3.2.1
+ARG UC_PACKAGE=io.unitycatalog:unitycatalog-spark_2.12:0.2.1
+
+RUN mkdir -p /opt/ivy/cache /opt/ivy/jars && \
+    echo 'pass' > /tmp/warmup.py && \
+    /opt/spark/bin/spark-submit \
+        --packages "${DELTA_PACKAGE},${UC_PACKAGE}" \
+        --conf spark.jars.ivy=/opt/ivy \
+        /tmp/warmup.py && \
+    ls /opt/ivy/jars/*.jar > /dev/null
+
+
 FROM python:3.11-slim
 
 WORKDIR /opt/minilake
@@ -17,6 +43,21 @@ COPY README.md README.md
 # needed to turn the MCP server on; the extra stays optional for PyPI installs.
 RUN pip install --no-cache-dir -e ".[mcp]"
 
+# Pre-install DuckDB's `delta` extension. Without this the server downloads it from
+# extensions.duckdb.org on every first boot, into a $HOME path that is not a volume — so it
+# is re-fetched for each new container, and fails outright with no internet access.
+# The directory is explicit rather than $HOME/.duckdb so it survives `docker run --user`, and
+# sits outside WORKDIR so a bind-mount over the source tree cannot shadow it.
+ENV MINILAKE_DUCKDB_EXTENSION_DIR=/opt/duckdb-extensions
+RUN python -c "import duckdb, os; duckdb.connect(config={'extension_directory': os.environ['MINILAKE_DUCKDB_EXTENSION_DIR']}).execute('INSTALL delta')" && \
+    find "$MINILAKE_DUCKDB_EXTENSION_DIR" -name 'delta.duckdb_extension' | grep -q . && \
+    chmod -R a+rX "$MINILAKE_DUCKDB_EXTENSION_DIR"
+
+# Delta / Unity Catalog jars resolved in the sparkjars stage above. Copied into the volume
+# shared with job containers on first use (see docker_executor._seed_ivy_cache).
+COPY --from=sparkjars /opt/ivy /opt/ivy-cache
+ENV MINILAKE_IVY_SEED=/opt/ivy-cache
+
 # Create data directory
 RUN mkdir -p /data
 

diff --git a/Dockerfile.test b/Dockerfile.test
@@ -1,3 +1,20 @@
+# Same build-time jar resolution as the runtime image — see Dockerfile for the rationale.
+FROM --platform=$BUILDPLATFORM apache/spark:3.5.3-scala2.12-java17-python3-ubuntu AS sparkjars
+
+USER root
+
+ARG DELTA_PACKAGE=io.delta:delta-spark_2.12:3.2.1
+ARG UC_PACKAGE=io.unitycatalog:unitycatalog-spark_2.12:0.2.1
+
+RUN mkdir -p /opt/ivy/cache /opt/ivy/jars && \
+    echo 'pass' > /tmp/warmup.py && \
+    /opt/spark/bin/spark-submit \
+        --packages "${DELTA_PACKAGE},${UC_PACKAGE}" \
+        --conf spark.jars.ivy=/opt/ivy \
+        /tmp/warmup.py && \
+    ls /opt/ivy/jars/*.jar > /dev/null
+
+
 FROM python:3.11-slim
 
 WORKDIR /opt/minilake
@@ -30,6 +47,19 @@ RUN uv pip install -e ".[mcp]" --python /usr/local/bin/python && \
     pyarrow>=14.0 \
     --python /usr/local/bin/python
 
+# Mirror the runtime image's pre-baked DuckDB extension and Ivy cache. The test runner does
+# not execute jobs itself, but it shares the data volume with the server container, and
+# keeping the two images symmetric is what stops one of them from silently drifting offline.
+# Both live outside WORKDIR: docker-compose.test.yml bind-mounts the repo over /opt/minilake,
+# which would otherwise hide them at runtime.
+ENV MINILAKE_DUCKDB_EXTENSION_DIR=/opt/duckdb-extensions
+RUN python -c "import duckdb, os; duckdb.connect(config={'extension_directory': os.environ['MINILAKE_DUCKDB_EXTENSION_DIR']}).execute('INSTALL delta')" && \
+    find "$MINILAKE_DUCKDB_EXTENSION_DIR" -name 'delta.duckdb_extension' | grep -q . && \
+    chmod -R a+rX "$MINILAKE_DUCKDB_EXTENSION_DIR"
+
+COPY --from=sparkjars /opt/ivy /opt/ivy-cache
+ENV MINILAKE_IVY_SEED=/opt/ivy-cache
+
 # Create data directory for tests
 RUN mkdir -p /data
 

diff --git a/FEATURES.md b/FEATURES.md
@@ -4,8 +4,8 @@
 
 **minilake** is a local Databricks API emulator backed by DuckDB for real SQL execution. This document details all implemented features, APIs, and their current status.
 
-**Latest Update:** 2026-07-30
-**Version:** 1.5.1
+**Latest Update:** 2026-08-08
+**Version:** 1.6.0
 **Project Status:** Core SQL + UC (per-catalog isolation) + Jobs (real DAG scheduling) + Workspace + DBFS + Files + Secrets + Clusters + Permissions + real Spark/Delta execution all working and tested; `MINILAKE_PERSIST` is now actually wired in. See [Known Limitations](#known-limitations) for what's intentionally not built (this is a single-dev local tool, not a multi-tenant server)
 
 ---
@@ -315,7 +315,7 @@ after each item, run via
 
 1. Create the table with `table_type=EXTERNAL`, `data_source_format=DELTA`, and an explicit `storage_location` (a path under the shared data volume, e.g. `/data/delta/<catalog>/<schema>/<table>`). minilake does **not** create a DuckDB-native table for this — it only registers the metadata and pre-creates `storage_location` as a world-writable directory (`0777`), because sibling containers that write to it (Spark job images, JupyterLab) each run as their own non-root UID/GID convention and there is no single group that covers all of them on a local-dev machine.
 2. Anything writes real Delta files to `storage_location` — a real Spark session (`df.write.format("delta").save(...)`), the notebook service, or the lightweight `deltalake` (delta-rs) package.
-3. `SELECT * FROM catalog.schema.table` is rewritten internally to `delta_scan('storage_location')` (DuckDB's `delta` extension, `INSTALL`/`LOAD`ed at startup) instead of the native-DuckDB-table path used for MANAGED tables.
+3. `SELECT * FROM catalog.schema.table` is rewritten internally to `delta_scan('storage_location')` (DuckDB's `delta` extension, `LOAD`ed at startup) instead of the native-DuckDB-table path used for MANAGED tables.
 
 **Request Format:**
 
@@ -344,7 +344,7 @@ after each item, run via
   - job containers join minilake's Docker network (`docker_executor._resolve_network`), or the connector cannot reach the API at all
 
 - ⚠️ **Only EXTERNAL Delta tables are visible to Spark**: a MANAGED table is a DuckDB table with no files, so it has nothing for Spark to read. This is not a gap to close — the alternative, Unity Catalog's own managed tables, use the `catalogManaged` Delta feature whose commit log lives partly in the catalog, and DuckDB's `delta_scan` cannot read those at all (`Catalog-managed table requires max_catalog_version to be set`). Keeping MANAGED on DuckDB is what preserves the fast SQL path.
-- ⚠️ First `delta_scan()` requires network access once, to `INSTALL` DuckDB's `delta` extension (fails loudly, not silently, if unavailable)
+- ✅ **No network needed in Docker**: the image installs DuckDB's `delta` extension at build time into `MINILAKE_DUCKDB_EXTENSION_DIR` (`/opt/duckdb-extensions`) and the server only `LOAD`s it, with `autoinstall_known_extensions` off so nothing can be fetched behind your back. A `pip install minilake` still `INSTALL`s the extension from `extensions.duckdb.org` on first run, and so needs network access once. Verify the image with `./scripts/verify-offline.sh`.
 
 **Status:** ✅ Complete and tested
 
@@ -445,7 +445,8 @@ after each item, run via
 
 **Known limitations:**
 
-- ⚠️ First run pulls the Spark image if not already cached (can take a while; subsequent runs are fast — `prewarm_spark_image()` pre-pulls it best-effort at server startup)
+- ⚠️ First run pulls the Spark image if not already cached (can take a while; subsequent runs are fast — `prewarm_spark_image()` pre-pulls it best-effort at server startup, and `MINILAKE_SPARK_PREWARM=0` skips the attempt on an offline machine). This is the one dependency an image cannot bake in — a Docker image cannot contain another Docker image — so offline use means pulling `apache/spark:3.5.3-scala2.12-java17-python3-ubuntu` once while you still have network
+- ✅ The Delta and Unity Catalog connector jars *are* baked in: the image resolves them at build time with `spark-submit` and copies that Ivy cache onto the shared data volume on first use (`docker_executor._seed_ivy_cache`), so `spark-submit --packages` resolves locally. Only extra coordinates you pass yourself (`libraries` on a task, `packages=[...]` on the MCP tool) still need Maven Central
 - ⚠️ Job/run history survives a restart only when `MINILAKE_PERSIST=1` (see Persistence below) — running job containers are not resumed, only completed run records are restored
 
 **Status:** ✅ Complete and tested (`tests/test_jobs.py`, `tests/test_docker_executor.py` — real container/subprocess execution, DAG scheduling, secret injection, `sql_task`)

diff --git a/README.md b/README.md
@@ -43,7 +43,11 @@ docker compose up -d
 curl http://localhost:8000/_minilake/health
 ```
 
-No account, no API key, no sign-up. Then point any Databricks client at it:
+No account, no API key, no sign-up — and the container image downloads nothing at runtime:
+DuckDB's `delta` extension and the Delta / Unity Catalog Spark jars are baked in at build
+time, so it works air-gapped ([details](https://github.com/dmux/minilake/blob/main/docs/getting-started.md#offline-use)).
+
+Then point any Databricks client at it:
 
 ```python
 from databricks.sdk import WorkspaceClient

diff --git a/docs/configuration.md b/docs/configuration.md
@@ -13,7 +13,8 @@ Everything is set through environment variables. Nothing needs a config file.
 | `MINILAKE_DATA_DIR` | `./data` | Where catalogs, workspace files, DBFS and Delta data live |
 | `MINILAKE_VERBOSE` | unset | `1` for full `INFO` logs (per-request access logs, SQL and job traces). Off by default — only the startup banner and warnings appear |
 | `MINILAKE_SERVICES` | (all) | Comma-separated allowlist, e.g. `unity_catalog,sql_statements,sql_warehouses` |
-| `MINILAKE_DUCKDB_MEMORY_LIMIT` | `4GB` | Passed to DuckDB |
+| `MINILAKE_DUCKDB_MEMORY_LIMIT` | `4GB` | Declared but not yet applied to any connection — currently a no-op |
+| `MINILAKE_DUCKDB_EXTENSION_DIR` | unset (`/opt/duckdb-extensions` in the image) | Directory of pre-installed DuckDB extensions. When set, the server only `LOAD`s the `delta` extension and never downloads it, which is what makes the image work with no internet access |
 
 ### Persistence
 
@@ -31,6 +32,8 @@ Everything is set through environment variables. Nothing needs a config file.
 | `MINILAKE_DELTA_PACKAGE` | `io.delta:delta-spark_2.12:3.2.1` | Maven coordinate for the Delta jars. Must match the Spark version in the image |
 | `MINILAKE_DOCKER_VOLUME` | unset | Named volume backing `MINILAKE_DATA_DIR`, shared with spawned job containers. Falls back to introspecting minilake's own mounts |
 | `MINILAKE_DOCKER_NETWORK` | unset | Network to attach job containers to, so a task can call back into the API. Falls back to introspection when minilake is on exactly one network |
+| `MINILAKE_IVY_SEED` | unset (`/opt/ivy-cache` in the image) | Pre-resolved Delta / Unity Catalog jars, copied onto the shared data volume on first job run so `spark-submit --packages` resolves without Maven Central |
+| `MINILAKE_SPARK_PREWARM` | unset | `0` skips the best-
8
effort `
10BC8
docker pull` of the Spark image at startup — worth setting on a machine with no network, where the attempt only costs a timeout |
 
 ### TLS
 

diff --git a/docs/getting-started.md b/docs/getting-started.md
@@ -44,6 +44,24 @@ docker compose up -d
 `docker compose` wires the socket, the volume and the environment for you, and is the
 closest thing to a reference deployment.
 
+### Offline use
+
+The image downloads nothing at runtime. DuckDB's `delta` extension and the Delta / Unity
+Catalog Spark jars are both resolved at build time and shipped inside it, so a container
+reads EXTERNAL Delta tables on an air-gapped machine. `./scripts/verify-offline.sh` proves
+it by running the image under `docker run --network none`.
+
+One thing an image cannot contain is another image. If you want Jobs to run on real Spark
+offline, pull the executor image once while you still have network:
+
+```bash
+docker pull apache/spark:3.5.3-scala2.12-java17-python3-ubuntu
+```
+
+Then set `MINILAKE_SPARK_PREWARM=0` to stop minilake from re-checking for it at startup.
+Maven coordinates you supply yourself — a task's `libraries`, or `packages=[...]` on the
+MCP job tool — are still fetched from Maven Central on first use.
+
 ## Verify
 
 ```bash

diff --git a/docs/releases.md b/docs/releases.md
@@ -26,7 +26,12 @@ That triggers [`release.yml`](../.github/workflows/release.yml), which:
    everything downstream — a mismatch would ship an image labelled one version and a wheel
    containing another, and a PyPI upload cannot be replaced afterwards.
 2. Builds a multi-arch image (`amd64` + `arm64`) and pushes it to GHCR as
-   `ghcr.io/dmux/minilake:X.Y.Z`, `:X.Y` and `:latest`
+   `ghcr.io/dmux/minilake:X.Y.Z`, `:X.Y` and `:latest`. The build needs network for more
+   than pip: it also installs DuckDB's `delta` extension and resolves the Delta / Unity
+   Catalog Spark jars, which is what lets the resulting image run with none. That costs
+   roughly 100 MB of image size and one extra `spark-submit` stage, and the build fails
+   outright if either fetch does not land — better than silently publishing an image that
+   goes back to downloading at boot.
 3. Builds the sdist and wheel and **publishes them to PyPI**
 4. Creates a [GitHub Release](https://github.com/dmux/minilake/releases) with generated notes
 

diff --git a/docs/spark-and-delta.md b/docs/spark-and-delta.md
@@ -78,6 +78,11 @@ Anything touching Delta needs the jars, which the base image does not ship. Over
 is `delta=True`; over the Jobs API it is a `libraries` entry with the Maven coordinate
 `io.delta:delta-spark_2.12:3.2.1`.
 
+Those jars — and the Unity Catalog connector's — are resolved when the minilake image is
+built and copied onto the shared data volume on the first job run, so `--packages` resolves
+from the local Ivy cache and no job needs Maven Central. Any *other* coordinate you pass
+still does.
+
 ## Reading by name with Unity Catalog
 
 `spark.table("catalog.schema.table")` resolves against minilake. Spark's

diff --git a/pyproject.toml b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "minilake"
-version = "1.5.1"
+version = "1.6.0"
 description = "A local Databricks API emulator backed by DuckDB."
 readme = "README.md"
 requires-python = ">=3.11"

diff --git a/scripts/verify-offline.sh b/scripts/verify-offline.sh
@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+#
+# Proves the published image needs no network at runtime.
+#
+# `docker run --network none` leaves loopback up (so the server still boots and its health
+# check still works) while making every outbound request fail immediately. Anything that
+# would have been downloaded on first boot shows up here as a hard failure instead of a
+# slow, silent degradation.
+#
+# Usage: ./scripts/verify-offline.sh [image-tag]
+
+set -euo pipefail
+
+IMAGE="${1:-minilake:offline-check}"
+
+if [ -z "${1:-}" ]; then
+    echo "==> Building ${IMAGE}"
+    docker build -t "${IMAGE}" .
+fi
+
+echo "==> Checking the DuckDB delta extension loads with no network"
+# -i is load-bearing: without stdin attached, `python -` reads nothing and exits 0, and the
+# check passes without having checked anything.
+docker run --rm -i --network none "${IMAGE}" python - <<'PY'
+import os
+import sys
+
+import duckdb
+
+extension_dir = os.environ["MINILAKE_DUCKDB_EXTENSION_DIR"]
+conn = duckdb.connect(config={"extension_directory": extension_dir, "autoinstall_known_extensions": False})
+conn.execute("LOAD delta")
+name, loaded, installed, path = conn.execute(
+    "SELECT extension_name, loaded, installed, install_path FROM duckdb_extensions() "
+    "WHERE extension_name = 'delta'"
+).fetchone()
+if not (loaded and installed and path.startswith(extension_dir)):
+    sys.exit(f"delta extension not served from the image: loaded={loaded} installed={installed} path={path}")
+print(f"    delta {duckdb.__version__} loaded from {path}")
+PY
+
+echo "==> Checking the Spark Delta / Unity Catalog jars are in the image"
+docker run --rm --network none "${IMAGE}" sh -c '
+    set -e
+    ls "$MINILAKE_IVY_SEED"/jars/*.jar > /dev/null
+    printf "    %s jars under %s\n" "$(ls "$MINILAKE_IVY_SEED"/jars/*.jar | wc -l)" "$MINILAKE_IVY_SEED"
+    # Ivy names jars <group>_<artifact>-<version>.jar in its jars/ dir.
+    ls "$MINILAKE_IVY_SEED"/jars/ | grep -q "^io.delta_delta-spark" || { echo "delta-spark jar missing"; exit 1; }
+    ls "$MINILAKE_IVY_SEED"/jars/ | grep -q "^io.unitycatalog_unitycatalog-spark" || { echo "unitycatalog-spark jar missing"; exit 1; }
+'
+
+echo "==> Booting the server with no network and checking it reports no extension failure"
+container=$(docker run -d --rm --network none "${IMAGE}")
+trap 'docker rm -f "${container}" > /dev/null 2>&1 || true' EXIT
+
+for _ in $(seq 1 30); do
+    if docker exec "${container}" curl -sf http://localhost:8000/_minilake/health > /dev/null 2>&1; then
+        break
+    fi
+    sleep 1
+done
+
+docker exec "${container}" curl -sf http://localhost:8000/_minilake/health > /dev/null
+logs=$(docker logs "${container}" 2>&1)
+if echo "${logs}" | grep -iE "failed to (load|install).*delta extension"; then
+    echo "The server could not load the delta extension offline." >&2
+    exit 1
+fi
+
+echo
+echo "OK — ${IMAGE} boots and serves Delta reads with no network access."
+echo "Note: running a Spark job still needs the Spark image locally:"
+echo "  docker pull apache/spark:3.5.3-scala2.12-java17-python3-ubuntu"
diff --git a/src/minilake/config.py b/src/minilake/config.py
@@ -37,6 +37,13 @@ class Settings(BaseSettings):
 
     # DuckDB settings
     duckdb_memory_limit: str = os.getenv("MINILAKE_DUCKDB_MEMORY_LIMIT", "4GB")
+    # Directory holding pre-installed DuckDB extensions. The Docker image sets this and
+    # populates it at build time, which is what makes the container work with no network:
+    # when it is set, startup only LOADs the `delta` extension and never INSTALLs it. Unset
+    # (a plain `pip install minilake`) keeps the download-on-first-run behaviour.
+    duckdb_extension_dir: Optional[Path] = (
+        Path(os.environ["MINILAKE_DUCKDB_EXTENSION_DIR"]) if os.getenv("MINILAKE_DUCKDB_EXTENSION_DIR") else None
+    )
 
     # TLS / native HTTPS — lets the Databricks CLI (which wants an https:// host)
     # talk to minilake without a separate TLS proxy. When enabled, minilake serves