8000 fix: reject unrunnable check configs and harden sensor/runner error paths by BaraaZ95 · Pull Request #19 · zaherab/DQOPS_API · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@ services:
dockerfile: Dockerfile
container_name: dq-platform-worker
command: celery -A dq_platform.workers.celery_app worker --loglevel=info
extra_hosts:
- "host.docker.internal:host-gateway"
environment:
DATABASE_URL: postgresql+asyncpg://postgres:postgres@postgres:5432/dq_platform
REDIS_URL: redis://redis:6379/0
Expand Down
50 changes: 31 additions & 19 deletions src/dq_platform/api/v1/checks.py
8000
Original file line number Diff line number Diff line change
Expand Up @@ -128,27 +128,33 @@ async def list_check_types(
description=check.description,
is_column_level=check.is_column_level,
category=check.category,
rule_type=check.rule_type.value if check.rule_type else None,
default_params=check.default_params or None,
)
)

# Add legacy Great Expectations checks not yet in DQOps format
existing_types = {c.name for c in dqops_checks}
for check_type in CheckType:
if check_type.value not in existing_types:
try:
is_col_level = is_column_level_check(check_type)
description = get_check_description(check_type)
check_types.append(
CheckTypeInfo(
type=check_type.value,
description=description,
is_column_level=is_col_level,
category=None,
# Add legacy Great Expectations checks not yet in DQOps format.
# Skip when a category filter is active — GX-only checks have no
# category, so including them would leak unrelated results into a
# filtered response.
if not category:
existing_types = {c.name for c in dqops_checks}
for check_type in CheckType:
if check_type.value not in existing_types:
try:
is_col_level = is_column_level_check(check_type)
description = get_check_description(check_type)
check_types.append(
CheckTypeInfo(
type=check_type.value,
description=description,
is_column_level=is_col_level,
category=None,
)
)
)
except ValueError:
# Skip if not in GX registry
pass
except ValueError:
# Skip if not in GX registry
pass

return check_types

Expand Down Expand Up @@ -306,15 +312,21 @@ async def delete_check(
raise NotFoundError("Check", str(check_id))


@router.post("/{check_id}/run", response_model=dict[str, Any])
@router.post(
"/{check_id}/run",
response_model=dict[str, Any],
status_code=status.HTTP_202_ACCEPTED,
)
async def run_check(
check_id: UUID,
request: RunCheckRequest | None = None,
db: AsyncSession = Depends(get_db),
) -> dict[str, Any]:
"""Run a check asynchronously.

Returns a job ID that can be used to track execution.
Returns a job ID that can be used to track execution. 202 Accepted is
the correct status for dispatching async work — the job is queued but
not yet complete.
"""
request = request or RunCheckRequest()
execution_service = ExecutionService(db)
Expand Down
100 changes: 60 additions & 40 deletions src/dq_platform/checks/check_runner.py
10000
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Unified check execution logic shared between API preview and Celery worker paths."""

import time
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Any
Expand Down Expand Up @@ -30,6 +31,8 @@ class CheckRunResult:
message: str
executed_sql: str | None
executed_at: datetime
execution_time_ms: int | None = None
rows_scanned: int | None = None


async def run_check(
Expand All @@ -51,54 +54,66 @@ async def run_check(
Check execution result.
"""
executed_at = datetime.now(UTC)
t0 = time.perf_counter()

# Try DQOps-style execution first
# Resolve the check definition. Only an unknown check type falls back to
# GX — any error beyond this point (sensor render, SQL execution, rule
# evaluation) is a real execution error and must propagate so the job's
# error_message reflects the actual cause, not a misleading "GX not
# implemented" from the fallback path.
try:
dqops_check_type = DQOpsCheckType(check.check_type.value)
dqops_check_def = get_dqops_check_def(dqops_check_type)
except (ValueError, KeyError):
return await _run_gx_fallback(check, connection_config, executed_at, t0)

# Build rule parameters
rule_params = _build_rule_params(check)

# Anomaly: inject historical values
if dqops_check_def.rule_type == RuleType.ANOMALY_PERCENTILE and db is not None:
rule_params["_historical_values"] = await _get_historical_values(db, check)

# Cross-source: dual-connection execution
if "reference_connection_id" in (check.parameters or {}):
return await _run_cross_source(check, connection_config, dqops_check_def, rule_params, executed_at, db)

# Determine quote char from connection type
conn_type = connection_config.get("type", "")
quote_char = QUOTE_CHARS.get(conn_type, '"')

# Execute DQOps check
result = await run_dqops_check(
check_type=dqops_check_type,
connection_config=connection_config,
schema_name=check.target_schema or "public",
table_name=check.target_table,
column_name=check.target_column,
rule_params=rule_params,
quote_char=quote_char,
)
# Build rule parameters
rule_params = _build_rule_params(check)

return CheckRunResult(
passed=result.passed,
severity=result.severity.value,
sensor_value=result.sensor_value,
expected=result.expected,
actual=result.actual,
message=result.message,
executed_sql=result.executed_sql,
executed_at=executed_at,
)
# Anomaly: inject historical values
if dqops_check_def.rule_type == RuleType.ANOMALY_PERCENTILE and db is not None:
rule_params["_historical_values"] = await _get_historical_values(db, check)

except (ValueError, KeyError):
# Fall back to Great Expectations execution
pass
# Cross-source: dual-connection execution
if "reference_connection_id" in (check.parameters or {}):
return await _run_cross_source(check, connection_config, dqops_check_def, rule_params, executed_at, db, t0)

# Execute via Great Expectations
# Determine quote char from connection type
conn_type = connection_config.get("type", "")
quote_char = QUOTE_CHARS.get(conn_type, '"')

# Execute DQOps check
result = await run_dqops_check(
check_type=dqops_check_type,
connection_config=connection_config,
schema_name=check.target_schema or "public",
table_name=check.target_table,
column_name=check.target_column,
rule_params=rule_params,
quote_char=quote_char,
)

return CheckRunResult(
passed=result.passed,
severity=result.severity.value,
sensor_value=result.sensor_value,
expected=result.expected,
actual=result.actual,
message=result.message,
executed_sql=result.executed_sql,
executed_at=executed_at,
execution_time_ms=int((time.perf_counter() - t0) * 1000),
rows_scanned=result.rows_scanned,
)


async def _run_gx_fallback(
check: Check,
connection_config: dict[str, Any],
executed_at: datetime,
t0: float,
) -> CheckRunResult:
"""Fallback execution path for check types not in the DQOps registry."""
gx_result = await run_gx_check(
check_type=check.check_type,
connection_config=connection_config,
Expand All @@ -119,6 +134,7 @@ async def run_check(
message=gx_result.get("result", {}).get("comment", "Check executed"),
executed_sql=None,
executed_at=executed_at,
execution_time_ms=int((time.perf_counter() - t0) * 1000),
)


Expand Down Expand Up @@ -159,6 +175,7 @@ async def _run_cross_source(
rule_params: dict[str, Any],
executed_at: datetime,
db: AsyncSession | None,
t0: float,
) -> CheckRunResult:
"""Run cross-source comparison check."""
from uuid import UUID
Expand All @@ -178,6 +195,7 @@ async def _run_cross_source(
message="Cross-source check requires a database session",
execu 10BC8 ted_sql=None,
executed_at=executed_at,
execution_time_ms=int((time.perf_counter() - t0) * 1000),
)

conn_service = ConnectionService(db)
Expand All @@ -192,6 +210,7 @@ async def _run_cross_source(
message=f"Reference connection {ref_conn_id} not found",
executed_sql=None,
executed_at=executed_at,
execution_time_ms=int((time.perf_counter() - t0) * 1000),
)

ref_config = ref_connection.decrypted_config
Expand Down Expand Up @@ -246,4 +265,5 @@ async def _run_cross_source(
message=f"Source={source_value}, Reference={ref_value}. {rule_result.message}",
executed_sql=f"-- Source:\n{source_sql}\n-- Reference:\n{ref_sql}",
executed_at=executed_at,
execution_time_ms=int((time.perf_counter() - t0) * 1000),
)
41 changes: 41 additions & 0 deletions src/dq_platform/checks/dqops_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ class CheckExecutionResult:
actual: Any
message: str
executed_sql: str
rows_scanned: int | None = None


class DQOpsExecutor:
Expand Down Expand Up @@ -107,6 +108,18 @@ async def execute_check(
sql=sql,
)

# Capture rows_scanned by running a paired COUNT(*) on the target
# table with the same partition filter. Cheap for most engines
# (one aggregate query), and gives a consistent count-of-rows
# denominator across every sensor type.
rows_scanned = await self._count_rows(
connection_config=connection_config,
schema_name=schema_name,
table_name=table_name,
partition_filter=partition_filter,
quote_char=quote_char,
)

# Build rule parameters
final_rule_params = {}
if check.default_params:
Expand All @@ -121,6 +134,7 @@ async def execute_check(
final_rule_params,
category=check.category,
description=check.description,
sensor_type=check.sensor_type.value,
)

return CheckExecutionResult(
Expand All @@ -132,6 +146,7 @@ async def execute_check(
actual=rule_result.actual,
message=rule_result.message,
executed_sql=sql,
rows_scanned=rows_scanned,
)

async def _execute_sensor_sql(
Expand Down Expand Up @@ -178,6 +1 10BC8 93,32 @@ async def _execute_sensor_sql(
if own_connector and connector:
await connector.disconnect_async()

async def _count_rows(
self,
connection_config: dict[str, Any],
schema_name: str,
table_name: str,
partition_filter: str | None,
quote_char: str,
) -> int | None:
"""Return row count of the target table (with partition filter if any).

Best-effort: returns None on any failure so a COUNT(*) hiccup never
fails the check itself. Used to populate rows_scanned on results.
"""
q = quote_char
where = f" WHERE {partition_filter}" if partition_filter else ""
sql = f"SELECT COUNT(*) AS sensor_value FROM {q}{schema_name}{q}.{q}{table_name}{q}{where}"
try:
value = await self._execute_sensor_sql(
connection_config=connection_config,
sql=sql,
)
return int(value) if value is not None else None
except Exception:
logger.warning("rows_scanned count query failed", extra={"sql": sql})
return None


class DQOpsLocalExecutor:
"""Executor for DQOps checks using local database session.
Expand Down
10 changes: 3 additions & 7 deletions src/dq_platform/checks/gx_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -429,10 +429,6 @@ async def run_gx_check(
Returns:
Check result dictionary.
"""
# This is a placeholder implementation
# In production, this would execute the actual GX check
return {
"success": True,
"observed_value": None,
"result": {"comment": "GX check execution placeholder"},
}
raise NotImplementedError(
f"GX executor not implemented for check type '{check_type.value}'. Use DQOps-style checks instead."
)
21 changes: 17 additions & 4 deletions src/dq_platform/checks/rules.py
Original file line number Diff line number Diff line change
Expand Up @@ -622,6 +622,7 @@ def _humanize_message(
result: RuleResult,
category: str,
description: str | None = None,
sensor_type: str | None = None,
) -> RuleResult:
"""Rewrite rule message to be human-readable based on check category and description.

Expand Down Expand Up @@ -651,12 +652,20 @@ def _humanize_message(
elif category == "nulls":
result.message = f"{_fmt_percent(float(actual))} null values (limit: {result.expected})"

# ── Uniqueness: duplicate percentage/count ───────────────────────────
# ── Uniqueness: distinct vs duplicate wording per sensor type ─────────
elif category == "uniqueness":
# Prefer sensor_type (authoritative: DISTINCT_COUNT, DUPLICATE_COUNT,
# etc.) over description string matching. Description is only the
# fallback for callers that haven't threaded sensor_type yet.
if sensor_type is not None:
is_distinct = sensor_type.startswith("distinct")
else:
is_distinct = description is not None and "distinct" in description.lower()
label = "distinct values" if is_distinct else "duplicates"
if "%" in str(result.expected):
result.message = f"{_fmt_percent(float(actual))} duplicates (limit: {result.expected})"
result.message = f"{_fmt_percent(float(actual))} {label} (limit: {result.expected})"
else:
result.message = f"{_fmt_number(float(actual))} duplicates (limit: {result.expected})"
result.message = f"{_fmt_number(float(actual))} {label} (limit: {result.expected})"

# ── Numeric/Statistical: value ranges ────────────────────────────────
elif category in ("numeric", "statistical"):
Expand Down Expand Up @@ -813,6 +822,7 @@ def evaluate_rule(
params: dict[str, Any],
category: str | None = None,
description: str | None = None,
sensor_type: str | None = None,
) -> RuleResult:
"""Evaluate a sensor value against a rule.

Expand All @@ -822,14 +832,17 @@ def evaluate_rule(
params: Rule parameters (thresholds, etc.).
category: Check category for human-readable message formatting.
description: Check description for dynamic fallback messages.
sensor_type: Sensor type name (e.g. "distinct_count",
"duplicate_percent"); preferred over description for
disambiguating messages within a category.

Returns:
The rule evaluation result.
"""
rule_func = get_rule(rule_type)
result = rule_func(sensor_value, params)
if category:
result = _humanize_message(result, category, description)
result = _humanize_message(result, category, description, sensor_type)
return result


Expand Down
Loading
Loading
0