Repository navigation
Expand file tree
/
Copy pathDockerfile.test
More file actions
112 lines (96 loc) · 4.96 KB
/
Copy pathDockerfile.test
File metadata and controls
112 lines (96 loc) · 4.96 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
# Same build-time jar resolution as the runtime image — see Dockerfile for the rationale.
FROM --platform=$BUILDPLATFORM apache/spark:3.5.3-scala2.12-java17-python3-ubuntu AS sparkjars
USER root
ARG DELTA_PACKAGE=io.delta:delta-spark_2.12:3.2.1
ARG UC_PACKAGE=io.unitycatalog:unitycatalog-spark_2.12:0.2.1
RUN mkdir -p /opt/ivy/cache /opt/ivy/jars && \
echo 'pass' > /tmp/warmup.py && \
/opt/spark/bin/spark-submit \
--packages "${DELTA_PACKAGE},${UC_PACKAGE}" \
--conf spark.jars.ivy=/opt/ivy \
/tmp/warmup.py && \
ls /opt/ivy/jars/*.jar > /dev/null
FROM python:3.14-slim
WORKDIR /opt/minilake
# Install system dependencies + uv
RUN apt-get update && apt-get install -y \
build-essential \
curl \
&& rm -rf /var/lib/apt/lists/* && \
pip install --no-cache-dir uv
# Databricks CLI, for scripts/cli_coverage.py. Pinned like the Spark/Delta packages above:
# the coverage numbers in docs/CLI_COVERAGE.md are only comparable across runs if the CLI
# that produced them is the same one.
ARG DATABRICKS_CLI_VERSION=1.16.1
RUN curl -fsSL -o /tmp/databricks.zip \
"https://github.com/databricks/cli/releases/download/v${DATABRICKS_CLI_VERSION}/databricks_cli_${DATABRICKS_CLI_VERSION}_linux_amd64.zip" && \
python -c "import zipfile; zipfile.ZipFile('/tmp/databricks.zip').extract('databricks', '/usr/local/bin')" && \
chmod 755 /usr/local/bin/databricks && \
rm /tmp/databricks.zip && \
databricks --version
# Terraform plus a local mirror of the Databricks provider, for
# tests/terraform/test_terraform_conformance.py. Both are pinned: the conformance test
# is only meaningful if it keeps testing the same provider, and a floating version would
# turn an upstream release into a mysterious CI failure.
#
# `terraform init` normally reaches registry.terraform.io. Warming the plugin cache here
# — where the build already has network — lets the test run `init -plugin-dir` against a
# local mirror instead, so the suite needs no network for this.
ARG TERRAFORM_VERSION=1.16.2
ARG DATABRICKS_PROVIDER_VERSION=1.132.0
# Deliberately NOT TF_PLUGIN_CACHE_DIR at runtime: the test passes this directory to
# `terraform init -plugin-dir`, and if it were also the plugin cache, terraform would try
# to install the provider from the directory into itself and fail. The cache variable is
# set only for the warm-up below, which is what populates the mirror.
ENV MINILAKE_TF_PROVIDER_MIRROR=/opt/tf-providers
RUN curl -fsSL -o /tmp/terraform.zip \
"https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" && \
python -c "import zipfile; zipfile.ZipFile('/tmp/terraform.zip').extract('terraform', '/usr/local/bin')" && \
chmod 755 /usr/local/bin/terraform && \
rm /tmp/terraform.zip && \
terraform version && \
mkdir -p "$MINILAKE_TF_PROVIDER_MIRROR" /tmp/tf-warmup && \
printf 'terraform {\n required_providers {\n databricks = {\n source = "databricks/databricks"\n version = "%s"\n }\n }\n}\n' \
"${DATABRICKS_PROVIDER_VERSION}" > /tmp/tf-warmup/main.tf && \
cd /tmp/tf-warmup && \
TF_PLUGIN_CACHE_DIR="$MINILAKE_TF_PROVIDER_MIRROR" terraform init -input=false -no-color > /dev/null && \
rm -rf /tmp/tf-warmup && \
find "$MINILAKE_TF_PROVIDER_MIRROR" -name 'terraform-provider-databricks*' | grep -q . && \
chmod -R a+rX "$MINILAKE_TF_PROVIDER_MIRROR"
# Copy project files
COPY pyproject.toml pyproject.toml
COPY src/ src/
COPY tests/ tests/
COPY README.md README.md
# Install minilake + all dev dependencies
RUN uv pip install -e ".[mcp,notebook]" --python /usr/local/bin/python && \
uv pip install \
pytest>=8.0 \
pytest-xdist>=3.6 \
httpx>=0.27 \
databricks-sdk>=0.40 \
pytest-cov \
pytest-json-report \
pytest-asyncio \
deltalake>=0.19 \
pandas>=2.0 \
pyarrow>=14.0 \
--python /usr/local/bin/python
# Mirror the runtime image's pre-baked DuckDB extension and Ivy cache. The test runner does
# not execute jobs itself, but it shares the data volume with the server container, and
# keeping the two images symmetric is what stops one of them from silently drifting offline.
# Both live outside WORKDIR: docker-compose.test.yml bind-mounts the repo over /opt/minilake,
# which would otherwise hide them at runtime.
ENV MINILAKE_DUCKDB_EXTENSION_DIR=/opt/duckdb-extensions
RUN python -c "import duckdb, os; duckdb.connect(config={'extension_directory': os.environ['MINILAKE_DUCKDB_EXTENSION_DIR']}).execute('INSTALL delta')" && \
find "$MINILAKE_DUCKDB_EXTENSION_DIR" -name 'delta.duckdb_extension' | grep -q . && \
chmod -R a+rX "$MINILAKE_DUCKDB_EXTENSION_DIR"
COPY --from=sparkjars /opt/ivy /opt/ivy-cache
ENV MINILAKE_IVY_SEED=/opt/ivy-cache
# Create data directory for tests
RUN mkdir -p /data
# Set test environment
ENV MINILAKE_DATA_DIR=/data
ENV PYTHONUNBUFFERED=1
# Run tests
CMD ["pytest", "tests/", "-v", "--cov=minilake", "--cov-report=html", "--cov-report=term-missing", "--tb=short"]