8000 Monthly PSI and ROS_AUC for ensemble score report (BACK-4287) by sunmarinup · Pull Request #370 · upgini/upgini · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/upgini/__about__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "1.2.170"
__version__ = "1.2.171"
10 changes: 7 additions & 3 deletions src/upgini/features_enricher.py
Original file line number Diff line number Diff line change
Expand Up @@ -2590,7 +2590,9 @@ def _peek_cached_sampled(self):
return max(self.__cached_sampled_datasets.values(), key=lambda item: len(item[0]) if item[0] is not None else 0)

def _assemble_report_data(self):
features, sources, model_shap, summary, autofe = self._report_search_results()
features, sources, model_shap, summary, autofe, feature_monthly_psi, feature_monthly_auc = (
self._report_search_results()
)
search_id = self._search_task.search_task_id if self._search_task is not None else (self.search_id or "")
return assemble_report_data(
search_id=search_id,
Expand All @@ -2610,13 +2612,15 @@ def _assemble_report_data(self):
sources=sources,
autofe=autofe,
model_feature_shap=model_shap,
feature_monthly_psi=feature_monthly_psi,
feature_monthly_auc=feature_monthly_auc,
)

def _report_search_results(self):
features_meta = []
if self._search_task is not None:
features_meta = list(self._search_task.get_all_features_metadata_v2() or [])
features, sources, model_shap, summary = build_search_results(
features, sources, model_shap, summary, feature_monthly_psi, feature_monthly_auc = build_search_results(
features_meta=features_meta,
is_ensemble=self._is_ensemble_feature,
generated_names=self._column_name_aliases(
Expand All @@ -2632,7 +2636,7 @@ def _report_search_results(self):
self.get_autofe_features_description(features_meta=features_meta),
self.bundle,
)
return features, sources, model_shap, summary, autofe
return features, sources, model_shap, summary, autofe, feature_monthly_psi, feature_monthly_auc

def _score_report_html(self) -> str | None:
if self._search_task is None or not self._has_single_ensemble_score():
Expand Down
5 changes: 5 additions & 0 deletions src/upgini/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,11 @@ class FeaturesMetadataV2(BaseModel):
from_online_api: Optional[bool] = None
psi_value: Optional[float] = None
drift_score: Optional[float] = None
# Sample id ("train", "eval_1") -> calendar month ("2024-01") -> PSI.
# The earliest train month is the reference and is 0.
monthly_psi: Optional[Dict[str, Dict[str, float]]] = None
# Sample id ("train", "eval_1") -> calendar month -> univariate ROC AUC.
monthly_roc_auc: Optional[Dict[str, Dict[str, float]]] = None


class HitRateMetrics(BaseModel):
Expand Down
34 changes: 29 additions & 5 deletions src/upgini/report/assemble.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
from upgini.autofe.vector import CatboostModel, EnsembleModel, OnnxModel
from upgini.metadata import BaseColumnMetadata, FeaturesMetadataV2, GeneratedFeatureMetadata
from upgini.report.data import (
FeatureMonthlyAuc,
FeatureMonthlyPsi,
FeatureRow,
ModelFeatureShap,
QualitySample,
Expand Down Expand Up @@ -62,6 +64,8 @@ def assemble_report_data(
sources: Optional[list[SourceRow]] = None,
autofe: Optional[list[dict[str, str]]] = None,
model_feature_shap: Optional[list[ModelFeatureShap]] = None,
feature_monthly_psi: Optional[list[FeatureMonthlyPsi]] = None,
feature_monthly_auc: Optional[list[FeatureMonthlyAuc]] = None,
) -> ReportData:
generated_at = generated_at or datetime.now(timezone.utc)
sample_names = list(samples or [])
Expand Down Expand Up @@ -94,6 +98,8 @@ def assemble_report_data(
sources=list(sources or []),
autofe=list(autofe or []),
model_feature_shap=list(model_feature_shap or []),
feature_monthly_psi=list(feature_monthly_psi or []),
feature_monthly_auc=list(feature_monthly_auc or []),
charts=charts,
)

Expand All @@ -106,18 +112,26 @@ def build_search_results(
generated_features: Optional[list[GeneratedFeatureMetadata]] = None,
joined_ads_features_count: Optional[int] = None,
joined_ads_count: Optional[int] = None,
) -> tuple[list[FeatureRow], list[SourceRow], list[ModelFeatureShap], SearchResultsSummary]:
) -> tuple[
list[FeatureRow],
list[SourceRow],
list[ModelFeatureShap],
SearchResultsSummary,
list[FeatureMonthlyPsi],
list[FeatureMonthlyAuc],
]:
generated_names = generated_names or set()
model_rows: list[FeatureRow] = []
paired: list[tuple[FeatureRow, FeaturesMetadataV2]] = []
found_rows: list[FeatureRow] = []
for meta in features_meta:
if is_ensemble(meta.name) or not _has_nonzero_shap_value(meta.shap_value):
continue
row = _feature_row(meta.name, meta, generated_names)
model_rows.append(row)
paired.append((row, meta))
if meta.source != "etalon":
found_rows.append(row)
model_rows.sort(key=lambda row: (row.shap is None, -abs(row.shap or 0.0), row.name))
paired.sort(key=lambda item: (item[0].shap is None, -abs(item[0].shap or 0.0), item[0].name))
model_rows = [row for row, _meta in paired]
model_shap = [
ModelFeatureShap(
rank=index,
Expand All @@ -137,7 +151,16 @@ def build_search_results(
joined_ads_features_count=joined_ads_features_count,
joined_ads_count=joined_ads_count,
)
return model_rows, sources, model_shap, summary
feature_monthly_psi: list[FeatureMonthlyPsi] = []
feature_monthly_auc: list[FeatureMonthlyAuc] = []
for row, meta in paired:
psi = meta.monthly_psi or {}
auc = meta.monthly_roc_auc or {}
if psi:
feature_monthly_psi.append(FeatureMonthlyPsi(feature=row.name, psi=psi))
if auc:
feature_monthly_auc.append(FeatureMonthlyAuc(feature=row.name, auc=auc))
return model_rows, sources, model_shap, summary, feature_monthly_psi, feature_monthly_auc


def autofe_rows_from_description(df: Optional[pd.DataFrame], bundle: ResourceBundle) -> list[dict[str, str]]:
Expand Down Expand Up @@ -256,6 +279,7 @@ def _feature_row(
provider=info.internal_provider,
source=_feature_source(meta, is_generated, info.internal_source),
stability_status=_psi_status(meta.psi_value),
missing_rate=None if meta.hit_rate is None else 100.0 - meta.hit_rate,
)


Expand Down
19 changes: 19 additions & 0 deletions src/upgini/report/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@ class FeatureRow:
provider: str = ""
source: str = ""
stability_status: Optional[str] = None
rows: Optional[int] = None
labeled: Optional[int] = None
missing_rate: Optional[float] = None


@dataclass
Expand Down Expand Up @@ -132,6 +135,20 @@ class ScorePsi:
coverage: list[Optional[float]] = field(default_factory=list)


@dataclass
class FeatureMonthlyPsi:
feature: str
# Sample id ("train", "eval_1") -> calendar month ("2024-01") -> PSI.
psi: dict[str, dict[str, float]] = field(default_factory=dict)


@dataclass
class FeatureMonthlyAuc:
feature: str
# Sample id ("train", "eval_1") -> calendar month ("2024-01") -> ROC AUC.
auc: dict[str, dict[str, float]] = field(default_factory=dict)


@dataclass
class ReportCharts:
timeline_months: list[str] = field(default_factory=list)
Expand Down Expand Up @@ -162,4 +179,6 @@ class ReportData:
sources: list[SourceRow] = field(default_factory=list)
autofe: list[dict[str, str]] = field(default_factory=list)
model_feature_shap: list[ModelFeatureShap] = field(default_factory=list)
feature_monthly_psi: list[FeatureMonthlyPsi] = field(default_factory=list)
feature_monthly_auc: list[FeatureMonthlyAuc] = field(default_factory=list)
charts: ReportCharts = field(default_factory=ReportCharts)
42 changes: 41 additions & 1 deletion src/upgini/report/html.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from typing import Optional

from upgini.report.data import FeatureRow, ReportCharts, ReportData, SampleStats, SearchResultsSummary, SourceRow
from upgini.report.stats import MEAN_AXIS_PADDING
from upgini.report.stats import MEAN_AXIS_PADDING, psi_axis_max
from upgini.utils.feature_info import CLIENT_SOURCE, GENERATED_SOURCE

_TEMPLATE_PATH = Path(__file__).with_name("template.html")
Expand Down Expand Up @@ -38,6 +38,7 @@ def _report_payload(data: ReportData) -> dict:
"scoreAnalysis": _score_analysis_payload(charts, samples),
"scoreDistribution": _score_distribution_payload(charts, samples),
"scoreStability": _score_stability_payload(charts, samples),
"featureStability": _feature_stability_payload(data, charts),
"features": _features_payload(data.features),
"shap": {"topN": 5},
"searchResults": {
Expand Down Expand Up @@ -185,6 +186,9 @@ def _features_payload(features: list[FeatureRow]) -> list[dict]:
"importance": abs(row.shap) if row.shap is not None else 0,
"shap": row.shap,
"coverage": row.coverage,
"rows": row.rows,
"labeled": row.labeled,
"missingRate": row.missing_rate,
"status": row.stability_status,
"psi": row.psi,
"drift": row.drift,
Expand Down Expand Up @@ -304,6 +308,42 @@ def _score_stability_payload(charts: ReportCharts, samples: list[dict]) -> dict:
}


def _feature_stability_payload(data: ReportData, charts: ReportCharts) -> dict:
psi_rows = data.feature_monthly_psi
auc_rows = data.feature_monthly_auc
psi_values = [value for row in psi_rows for months in row.psi.values() for value in months.values()]
sample_ids: list[str] = []
for row in psi_rows:
for sample_id in row.psi:
if sample_id not in sample_ids:
sample_ids.append(sample_id)
for row in auc_rows:
for sample_id in row.auc:
if sample_id not in sample_ids:
sample_ids.append(sample_id)
return {
"psiAxisMax": psi_axis_max(psi_values, charts.psi_critical),
"thresholds": {"warning": charts.psi_warning, "critical": charts.psi_critical},
"features": [{"feature": row.feature, "provider": row.provider} for row in data.model_feature_shap],
"bySample": {
sample_id: {
"psi": _feature_series(psi_rows, "psi", sample_id),
"auc": _feature_series(auc_rows, "auc", sample_id),
}
for sample_id in sample_ids
},
}


def _feature_series(rows, field: str, sample_id: str) -> list[dict]:
series = []
for row in rows:
values = getattr(row, field).get(sample_id)
if values:
series.append({"feature": row.feature, "values": values})
return series


def _stat_row(label: str, samples: list[dict], by_id: dict[str, SampleStats], value) -> dict:
values = {}
for sample in samples:
Expand Down
37 changes: 34 additions & 3 deletions src/upgini/report/stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,12 +87,43 @@ def compute_report_charts(
densities = [value for hist in charts.histograms for value in hist.target_0 + hist.target_1]
charts.histogram_density_max = max(densities, default=0.0)
psis = [value for row in charts.score_psi for value in row.psi if value is not None]
charts.psi_axis_max = max([PSI_CRITICAL, *psis], default=PSI_CRITICAL)
if charts.psi_axis_max == PSI_CRITICAL and psis:
charts.psi_axis_max = max(PSI_CRITICAL, max(psis) * 1.15)
charts.psi_axis_max = psi_axis_max(psis)
return sample_stats, charts


def psi_axis_max(values: list[Optional[float]], critical: float = PSI_CRITICAL) -> float:
finite = [float(value) for value in values if value is not None and not pd.isna(value)]
axis = max([critical, *finite])
if axis == critical and finite:
axis = max(critical, max(finite) * 1.15)
return axis


def normalize_month_label(value) -> Optional[str]:
if value is None:
return None
text = str(value).strip()
if not text or text.lower() == "nat":
return None
parts = text.split()
if len(parts) == 2 and parts[0] in _MONTHS and parts[1].isdigit():
return f"{parts[0]} {int(parts[1])}"
parsed = pd.to_datetime(text, errors="coerce")
if pd.isna(parsed):
return None
return _month_label(parsed.to_period("M"))


def sorted_month_labels(months) -> list[str]:
unique = list(dict.fromkeys(month for month in months if month))

def sort_key(label: str):
parsed = pd.to_datetime(label, errors="coerce")
return (bool(pd.isna(parsed)), parsed)

return sorted(unique, key=sort_key)


def _timeline_months(
names: list[str], dataset_samples: dict[str, pd.DataFrame], scored_samples: dict[str, pd.DataFrame]
) -> list[str]:
Expand Down
Loading
Loading
0