8000 fix(dq): seven bug fixes from end-to-end DQ flow audit by BaraaZ95 · Pull Request #20 · zaherab/DQOPS_API · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions src/dq_platform/checks/check_runner.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Unified check execution logic shared between API preview and Celery worker paths."""

import time
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Any
Expand Down Expand Up @@ -30,6 +31,8 @@ class CheckRunResult:
message: str
executed_sql: str | None
executed_at: datetime
execution_time_ms: int | None = None
rows_scanned: int | None = None


async def run_check(
Expand All @@ -51,6 +54,7 @@ async def run_check(
Check execution result.
"""
executed_at = datetime.now(UTC)
t0 = time.perf_counter()

# Resolve the check definition. Only an unknown check type falls back to
# GX — any error beyond this point (sensor render, SQL execution, rule
Expand All @@ -61,7 +65,7 @@ async def run_check(
dqops_check_type = DQOpsCheckType(check.check_type.value)
dqops_check_def = get_dqops_check_def(dqops_check_type)
except (ValueError, KeyError):
return await _run_gx_fallback(check, connection_config, executed_at)
return await _run_gx_fallback(check, connection_config, executed_at, t0)

# Build rule parameters
rule_params = _build_rule_params(check)
Expand All @@ -72,7 +76,7 @@ async def run_check(

# Cross-source: dual-connection execution
if "reference_connection_id" in (check.parameters or {}):
return await _run_cross_source(check, connection_config, dqops_check_def, rule_params, executed_at, db)
return await _run_cross_source(check, connection_config, dqops_check_def, rule_params, executed_at, db, t0)

# Determine quote char from connection type
conn_type = connection_config.get("type", "")
Expand All @@ -98,13 +102,16 @@ async def run_check(
message=result.message,
executed_sql=result.executed_sql,
executed_at=executed_at,
execution_time_ms=int((time.perf_counter() - t0) * 1000),
rows_scanned=result.rows_scanned,
)


async def _run_gx_fallback(
check: Check,
connection_config: dict[str, Any],
executed_at: datetime,
t0: float,
) -> CheckRunResult:
"""Fallback execution path for check types not in the DQOps registry."""
gx_result = await run_gx_check(
Expand All @@ -127,6 +134,7 @@ async def _run_gx_fallback(
message=gx_result.get("result", {}).get("comment", "Check executed"),
executed_sql=None,
executed_at=executed_at,
execution_time_ms=int((time.perf_counter() - t0) * 1000),
)


Expand Down Expand Up @@ -167,6 +175,7 @@ async def _run_cross_source(
rule_params: dict[str, Any],
executed_at: datetime,
db: AsyncSession | None,
t0: float,
) -> CheckRunResult:
"""Run cross-source comparison check."""
from uuid import UUID
Expand All @@ -186,6 +195,7 @@ async def _run_cross_source(
message="Cross-source check requires a database session",
executed_sql=None,
executed_at=executed_at,
execution_time_ms=int((time.perf_counter() - t0) * 1000),
)

conn_service = ConnectionService(db)
Expand All @@ -200,6 +210,7 @@ async def _run_cross_source(
message=f"Reference connection {ref_conn_id} not found",
executed_sql=None,
executed_at=executed_at,
execution_time_ms=int((time.perf_counter() - t0) * 1000),
)

ref_config = ref_connection.decrypted_config
Expand Down Expand Up @@ -254,4 +265,5 @@ async def _run_cross_source(
message=f"Source={source_value}, Reference={ref_value}. {rule_result.message}",
executed_sql=f"-- Source:\n{source_sql}\n-- Reference:\n{ref_sql}",
executed_at=executed_at,
execution_time_ms=int((time.perf_counter() - t0) * 1000),
)
41 changes: 41 additions & 0 deletions src/dq_platform/checks/dqops_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ class CheckExecutionResult:
actual: Any
message: str
executed_sql: str
rows_scanned: int | None = None


class DQOpsExecutor:
10BC8 Expand Down Expand Up @@ -107,6 +108,18 @@ async def execute_check(
sql=sql,
)

# Capture rows_scanned by running a paired COUNT(*) on the target
# table with the same partition filter. Cheap for most engines
# (one aggregate query), and gives a consistent count-of-rows
# denominator across every sensor type.
rows_scanned = await self._count_rows(
connection_config=connection_config,
schema_name=schema_name,
table_name=table_name,
partition_filter=partition_filter,
quote_char=quote_char,
)

# Build rule parameters
final_rule_params = {}
if check.default_params:
Expand All @@ -121,6 +134,7 @@ async def execute_check(
final_rule_params,
category=check.category,
description=check.description,
sensor_type=check.sensor_type.value,
)

return CheckExecutionResult(
Expand All @@ -132,6 +146,7 @@ async def execute_check(
actual=rule_result.actual,
message=rule_result.message,
executed_sql=sql,
rows_scanned=rows_scanned,
)

async def _execute_sensor_sql(
Expand Down Expand Up @@ -178,6 +193,32 @@ async def _execute_sensor_sql(
if own_connector and connector:
await connector.disconnect_async()

async def _count_rows(
self,
connection_config: dict[str, Any],
schema_name: str,
table_name: str,
partition_filter: str | None,
quote_char: str,
) -> int | None:
"""Return row count of the target table (with partition filter if any).

Best-effort: returns None on any failure so a COUNT(*) hiccup never
fails the check itself. Used to populate rows_scanned on results.
"""
q = quote_char
where = f" WHERE {partition_filter}" if partition_filter else ""
sql = f"SELECT COUNT(*) AS sensor_value FROM {q}{schema_name}{q}.{q}{table_name}{q}{where}"
try:
value = await self._execute_sensor_sql(
connection_config=connection_config,
sql=sql,
)
return int(value) if value is not None else None
except Exception:
logger.warning("rows_scanned count query failed", extra={"sql": sql})
return None


class DQOpsLocalExecutor:
"""Executor for DQOps checks using local database session.
Expand Down
21 changes: 17 additions & 4 deletions src/dq_platform/checks/rules.py
Original file line number Diff line number Diff line change
Expand Up @@ -622,6 +622,7 @@ def _humanize_message(
result: RuleResult,
category: str,
description: str | None = None,
sensor_type: str | None = None,
) -> RuleResult:
"""Rewrite rule message to be human-readable based on check category and description.

Expand Down Expand Up @@ -651,12 +652,20 @@ def _humanize_message(
elif category == "nulls":
result.message = f"{_fmt_percent(float(actual))} null values (limit: {result.expected})"

# ── Uniqueness: duplicate percentage/count ───────────────────────────
# ── Uniqueness: distinct vs duplicate wording per sensor type ─────────
elif category == "uniqueness":
# Prefer sensor_type (authoritative: DISTINCT_COUNT, DUPLICATE_COUNT,
# etc.) over description string matching. Description is only the
# fallback for callers that haven't threaded sensor_type yet.
if sensor_type is not None:
is_distinct = sensor_type.startswith("distinct")
else:
is_distinct = description is not None and "distinct" in description.lower()
label = "distinct values" if is_distinct else "duplicates"
if "%" in str(result.expected):
result.message = f"{_fmt_percent(float(actual))} duplicates (limit: {result.expected})"
result.message = f"{_fmt_percent(float(actual))} {label} (limit: {result.expected})"
else:
result.message = f"{_fmt_number(float(actual))} duplicates (limit: {result.expected})"
result.message = f"{_fmt_number(float(actual))} {label} (limit: {result.expected})"

# ── Numeric/Statistical: value ranges ────────────────────────────────
elif category in ("numeric", "statistical"):
Expand Down Expand Up @@ -813,6 +822,7 @@ def evaluate_rule(
params: dict[str, Any],
category: str | None = None,
description: str | None = None,
sensor_type: str | None = None,
) -> RuleResult:
"""Evaluate a sensor value against a rule.

Expand All @@ -822,14 +832,17 @@ def evaluate_rule(
params: Rule parameters (thresholds, etc.).
category: Check category for human-readable message formatting.
description: Check description for dynamic fallback messages.
sensor_type: Sensor type name (e.g. "distinct_count",
"duplicate_percent"); preferred over description for
disambiguating messages within a category.

Returns:
The rule evaluation result.
"""
rule_func = get_rule(rule_type)
result = rule_func(sensor_value, params)
if category:
result = _humanize_message(result, category, description)
result = _humanize_message(result, category, description, sensor_type)
return result


Expand Down
15 changes: 9 additions & 6 deletions src/dq_platform/checks/sensors/_column_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@

DISTINCT_PERCENT_SENSOR = Sensor(
name=SensorType.DISTINCT_PERCENT,
description="Percentage of distinct values in the column",
description="Percentage of distinct values in the column (among non-null rows)",
is_column_level=True,
template="""
SELECT
Expand All @@ -96,28 +96,30 @@
ELSE (COUNT(DISTINCT {{ column_name }})::FLOAT / COUNT(*)) * 100
END as sensor_value
FROM {{ schema_name }}.{{ table_name }}
WHERE {{ column_name }} IS NOT NULL
{% if partition_filter %}
WHERE {{ partition_filter }}
AND {{ partition_filter }}
{% endif %}
""",
)

DUPLICATE_COUNT_SENSOR = Sensor(
name=SensorType.DUPLICATE_COUNT,
description="Count of duplicate values in the column",
description="Count of duplicate values in the column (excluding nulls)",
is_column_level=True,
template="""
SELECT COUNT(*) - COUNT(DISTINCT {{ column_name }}) as sensor_value
FROM {{ schema_name }}.{{ table_name }}
WHERE {{ column_name }} IS NOT NULL
{% if partition_filter %}
WHERE {{ partition_filter }}
AND {{ partition_filter }}
{% endif %}
""",
)

DUPLICATE_PERCENT_SENSOR = Sensor(
name=SensorType.DUPLICATE_PERCENT,
description="Percentage of duplicate values in the column",
description="Percentage of duplicate values in the column (among non-null rows)",
is_column_level=True,
template="""
SELECT
Expand All @@ -126,8 +128,9 @@
ELSE ((COUNT(*) - COUNT(DISTINCT {{ column_name }}))::FLOAT / COUNT(*)) * 100
END as sensor_value
FROM {{ schema_name }}.{{ table_name }}
WHERE {{ column_name }} IS NOT NULL
{% if partition_filter %}
WHERE {{ partition_filter }}
AND {{ partition_filter }}
{% endif %}
""",
)
Expand Down
1 change: 1 addition & 0 deletions src/dq_platform/schemas/result.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ class ResultResponse(BaseModel):
actual_value: float | None
expected_value: float | None
passed: bool
severity: str
execution_time_ms: int | None
rows_scanned: int | None
result_details: dict[str, Any] | None
Expand Down
20 changes: 18 additions & 2 deletions src/dq_platform/services/connection_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,14 @@
from concurrent.futures import ThreadPoolExecutor
from typing import Any

from sqlalchemy import func, select
from sqlalchemy import func, select, update
from sqlalchemy.ext.asyncio import AsyncSession

from dq_platform.api.errors import NotFoundError
from dq_platform.connectors.base import ColumnInfo, TableInfo
from dq_platform.connectors.factory import get_connector
from dq_platform.core.encryption import decrypt_config, encrypt_config
from dq_platform.models.check import Check
from dq_platform.models.connection import Connection, ConnectionType

_connector_executor = ThreadPoolExecutor(max_workers=10, thread_name_prefix="db-connector")
Expand Down Expand Up @@ -170,13 +171,28 @@ async def update(
return connection

async def delete(self, connection_id: uuid.UUID) -> None:
"""Soft delete a connection.
"""Soft delete a connection and cascade-deactivate its checks.

Without cascading, soft-deleting a connection left its checks
active — they'd keep running (or orphan, if the connection was
hard-deleted later) with no governance counterpart. Schedulers,
dimension scores, and incident lifecycle all treated those
checks as live, which is wrong.

Args:
connection_id: Connection UUID.
"""
connection = await self.get(connection_id)
connection.is_active = False

# Cascade: deactivate all checks on this connection in a single
# statement. Results/incidents/jobs remain for history — only
# active checks stop producing new work.
await self.db.execute(
update(Check)
.where(Check.connection_id == connection_id, Check.is_active == True) # noqa: E712
.values(is_active=False)
)
await self.db.flush()

async def test_connection(self, connection_id: uuid.UUID) -> bool:
Expand Down
11 changes: 8 additions & 3 deletions src/dq_platform/services/execution_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,10 @@ async def cancel_job(self, job_id: uuid.UUID) -> Job:
async def submit_job(self, job_id: uuid.UUID) -> str:
"""Submit a job to the Celery queue.

Commits the Job row before enqueueing so a fast worker can't pick up
the task before the row is visible. Without this, batch runs race and
a fraction of jobs end up stuck pending with celery_task_id = NULL.

Args:
job_id: Job UUID.

Expand All @@ -193,13 +197,14 @@ async def submit_job(self, job_id: uuid.UUID) -> str:
"""
from dq_platform.workers.tasks import execute_check

# Submit to Celery
# Ensure the Job row is durably visible to the worker before enqueue.
await self.db.commit()

task = execute_check.delay(str(job_id))

# Update job with task ID
job = await self.get_job(job_id)
task_id: str = task.id
job.celery_task_id = task_id
await self.db.flush()
await self.db.commit()

return task_id
4 changes: 4 additions & 0 deletions src/dq_platform/workers/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,8 @@ async def _execute_check_async(task: Any, job_id: str) -> dict[str, Any]:
actual=execution_result.get("actual"),
message=execution_result.get("message"),
executed_sql=execution_result.get("executed_sql"),
execution_time_ms=execution_result.get("execution_time_ms"),
rows_scanned=execution_result.get("rows_scanned"),
)

# Create/update incident if failed
Expand Down Expand Up @@ -206,6 +208,8 @@ async def _run_check_execution(
"message": result.message,
"executed_sql": result.executed_sql,
"executed_at": result.executed_at.isoformat(),
"execution_time_ms": result.execution_time_ms,
"rows_scanned": result.rows_scanned,
}


Expand Down
Loading
Loading
0