Repository navigation
Expand file tree
/
Copy pathDockerfile.test
More file actions
82 lines (67 loc) · 3.02 KB
/
Copy pathDockerfile.test
File metadata and controls
82 lines (67 loc) · 3.02 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
# Same build-time jar resolution as the runtime image — see Dockerfile for the rationale.
FROM --platform=$BUILDPLATFORM apache/spark:3.5.3-scala2.12-java17-python3-ubuntu AS sparkjars
USER root
ARG DELTA_PACKAGE=io.delta:delta-spark_2.12:3.2.1
ARG UC_PACKAGE=io.unitycatalog:unitycatalog-spark_2.12:0.2.1
RUN mkdir -p /opt/ivy/cache /opt/ivy/jars && \
echo 'pass' > /tmp/warmup.py && \
/opt/spark/bin/spark-submit \
--packages "${DELTA_PACKAGE},${UC_PACKAGE}" \
--conf spark.jars.ivy=/opt/ivy \
/tmp/warmup.py && \
ls /opt/ivy/jars/*.jar > /dev/null
FROM python:3.14-slim
WORKDIR /opt/minilake
# Install system dependencies + uv
RUN apt-get update && apt-get install -y \
build-essential \
curl \
&& rm -rf /var/lib/apt/lists/* && \
pip install --no-cache-dir uv
# Databricks CLI, for scripts/cli_coverage.py. Pinned like the Spark/Delta packages above:
# the coverage numbers in docs/CLI_COVERAGE.md are only comparable across runs if the CLI
# that produced them is the same one.
ARG DATABRICKS_CLI_VERSION=1.16.1
RUN curl -fsSL -o /tmp/databricks.zip \
"https://github.com/databricks/cli/releases/download/v${DATABRICKS_CLI_VERSION}/databricks_cli_${DATABRICKS_CLI_VERSION}_linux_amd64.zip" && \
python -c "import zipfile; zipfile.ZipFile('/tmp/databricks.zip').extract('databricks', '/usr/local/bin')" && \
chmod 755 /usr/local/bin/databricks && \
rm /tmp/databricks.zip && \
databricks --version
# Copy project files
COPY pyproject.toml pyproject.toml
COPY src/ src/
COPY tests/ tests/
COPY README.md README.md
# Install minilake + all dev dependencies
RUN uv pip install -e ".[mcp,notebook]" --python /usr/local/bin/python && \
uv pip install \
pytest>=8.0 \
pytest-xdist>=3.6 \
httpx>=0.27 \
databricks-sdk>=0.40 \
pytest-cov \
pytest-json-report \
pytest-asyncio \
deltalake>=0.19 \
pandas>=2.0 \
pyarrow>=14.0 \
--python /usr/local/bin/python
# Mirror the runtime image's pre-baked DuckDB extension and Ivy cache. The test runner does
# not execute jobs itself, but it shares the data volume with the server container, and
# keeping the two images symmetric is what stops one of them from silently drifting offline.
# Both live outside WORKDIR: docker-compose.test.yml bind-mounts the repo over /opt/minilake,
# which would otherwise hide them at runtime.
ENV MINILAKE_DUCKDB_EXTENSION_DIR=/opt/duckdb-extensions
RUN python -c "import duckdb, os; duckdb.connect(config={'extension_directory': os.environ['MINILAKE_DUCKDB_EXTENSION_DIR']}).execute('INSTALL delta')" && \
find "$MINILAKE_DUCKDB_EXTENSION_DIR" -name 'delta.duckdb_extension' | grep -q . && \
chmod -R a+rX "$MINILAKE_DUCKDB_EXTENSION_DIR"
COPY --from=sparkjars /opt/ivy /opt/ivy-cache
ENV MINILAKE_IVY_SEED=/opt/ivy-cache
# Create data directory for tests
RUN mkdir -p /data
# Set test environment
ENV MINILAKE_DATA_DIR=/data
ENV PYTHONUNBUFFERED=1
# Run tests
CMD ["pytest", "tests/", "-v", "--cov=minilake", "--cov-report=html", "--cov-report=term-missing", "--tb=short"]