8000 feat(dq): deterministic check engine + cross-dialect portability + churn fix by BaraaZ95 · Pull Request #21 · zaherab/DQOPS_API · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 17 additions & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Multi-stage Dockerfile for DQ Platform
# Stage 1: Builder - Install dependencies
FROM python:3.14-slim AS builder
FROM python:3.12-slim AS builder

# Install build-time system dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
Expand All @@ -20,13 +20,23 @@ COPY pyproject.toml README.md ./
RUN pip install --no-cache-dir -e .

# Stage 2: Runtime - Minimal image with application code
FROM python:3.14-slim AS runtime
FROM python:3.12-slim AS runtime

# Install runtime system dependencies only (no compilers)
# Install runtime system dependencies only (no compilers).
# msodbcsql18 is the actual SQL Server ODBC driver — unixodbc alone is
# only the driver *manager* and pyodbc fails with "Can't open lib" without
# the vendor driver. Pulled from Microsoft's apt repo.
RUN apt-get update && apt-get install -y --no-install-recommends \
libpq5 \
unixodbc \
curl \
gnupg \
&& curl -fsSL https://packages.microsoft.com/keys/microsoft.asc \
| gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg \
&& echo "deb [signed-by=/usr/share/keyrings/microsoft-prod.gpg] https://packages.microsoft.com/debian/12/pro 8000 d bookworm main" \
> /etc/apt/sources.list.d/mssql-release.list \
&& apt-get update \
&& ACCEPT_EULA=Y apt-get install -y --no-install-recommends msodbcsql18 \
&& rm -rf /var/lib/apt/lists/*

# Copy virtual environment from builder stage
Expand All @@ -41,6 +51,10 @@ ENV PYTHONPATH=/app/src
# Create non-root user
RUN groupadd -r appuser && useradd -r -g appuser -u 1000 appuser

# Shared-data mountpoint, owned by appuser so file-based connectors
# (DuckDB) can read/write when this path is backed by a named volume.
RUN mkdir -p /shared && chown appuser:appuser /shared

# Copy application code
COPY --chown=appuser:appuser src/ /app/src/
COPY --chown=appuser:appuser alembic.ini /app/
Expand Down
14 changes: 11 additions & 3 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,11 @@ services:
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
test: ["CMD-SHELL", "pg_isready -U postgres -d dq_platform -h 127.0.0.1 -p 5432 -q"]
interval: 5s
timeout: 5s
retries: 5
timeout: 3s
retries: 20
start_period: 30s
networks:
- dq-platform-network

Expand Down Expand Up @@ -61,6 +62,10 @@ services:
command: uvicorn dq_platform.main:app --host 0.0.0.0 --port 9000 --workers 4
ports:
- "9000:9000"
volumes:
# Shared sample-data volume — lets file-based connectors (DuckDB)
# be reached by both api (profile) and worker (check execution).
- dq_shared_data:/shared
environment:
DATABASE_URL: postgresql+asyncpg://postgres:postgres@postgres:5432/dq_platform
REDIS_URL: redis://redis:6379/0
Expand Down Expand Up @@ -94,6 +99,8 @@ services:
command: celery -A dq_platform.workers.celery_app worker --loglevel=info
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
- dq_shared_data:/shared
environment:
DATABASE_URL: postgresql+asyncpg://postgres:postgres@postgres:5432/dq_platform
REDIS_URL: redis://redis:6379/0
Expand Down Expand Up @@ -153,6 +160,7 @@ services:
volumes:
postgres_data:
redis_data:
dq_shared_data:

networks:
dq-platform-network:
Expand Down
3 changes: 3 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ dependencies = [
"oracledb>=2.0.0",
"databricks-sql-connector>=4.2.5",

# SQL dialect transpilation (sensor SQL portability across engines)
"sqlglot>=25.0.0",

# HTTP client (webhook notifications)
"httpx>=0.25.0",

Expand Down
5 changes: 5 additions & 0 deletions src/dq_platform/api/v1/checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
)
from dq_platform.core.security import verify_api_key
from dq_platform.models.check import CheckMode, CheckTimeScale, CheckType
from dq_platform.odps.dimension_mapping import get_dimension_for_check_type
from dq_platform.schemas.check import (
BatchRunRequest,
CheckCreate,
Expand Down Expand Up @@ -122,12 +123,14 @@ async def list_check_types(

# Add DQOps checks
for check in dqops_checks:
resolved_dim = get_dimension_for_check_type(check.name)
check_types.append(
CheckTypeInfo(
type=check.name,
description=check.description,
is_column_level=check.is_column_level,
category=check.category,
dimension=resolved_dim.value if resolved_dim else None,
rule_type=check.rule_type.value if check.rule_type else None,
default_params=check.default_params or None,
)
Expand All @@ -144,12 +147,14 @@ async def list_check_types(
try:
is_col_level = is_column_level_check(check_type)
description = get_check_description(check_type)
resolved_dim = get_dimension_for_check_type(check_type.value)
check_types.append(
CheckTypeInfo(
type=check_type.value,
description=description,
is_column_level=is_col_level,
category=None,
dimension=resolved_dim.value if resolved_dim else None,
)
)
except ValueError:
Expand Down
Loading
Loading
0