Repository navigation
Expand file tree
/
Copy pathmetadata.py
More file actions
338 lines (280 loc) · 11 KB
/
Copy pathmetadata.py
File metadata and controls
338 lines (280 loc) · 11 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
from __future__ import annotations
from enum import Enum
from typing import Any, Dict, List, Optional, Set, Union
from pydantic import BaseModel
SYSTEM_RECORD_ID = "system_record_id"
ENTITY_SYSTEM_RECORD_ID = "entity_system_record_id"
SEARCH_KEY_UNNEST = "search_key_unnest"
SORT_ID = "sort_id"
EVAL_SET_INDEX = "eval_set_index"
TARGET = "target"
COUNTRY = "country_iso_code"
RENAMED_INDEX = "index_col"
DEFAULT_INDEX = "index"
ORIGINAL_INDEX = "original_index"
SYSTEM_COLUMNS = {SYSTEM_RECORD_ID, ENTITY_SYSTEM_RECORD_ID, SEARCH_KEY_UNNEST, EVAL_SET_INDEX, TARGET, COUNTRY}
class FileColumnMeaningType(Enum):
MSISDN = "MSISDN"
MSISDN_RANGE_FROM = "MSISDN_RANGE_FROM"
MSISDN_RANGE_TO = "MSISDN_RANGE_TO"
EMAIL = "EMAIL"
EMAIL_ONE_DOMAIN = "EMAIL_ONE_DOMAIN"
IP_ADDRESS = "IP_ADDRESS"
IPV6_ADDRESS = "IPV6_ADDRESS"
IPV6_RANGE_FROM = "IPV6_RANGE_FROM"
IPV6_RANGE_TO = "IPV6_RANGE_TO"
IP_RANGE_FROM = "IP_RANGE_FROM"
IP_RANGE_TO = "IP_RANGE_TO"
HEM = "HEM"
DATE = "DATE"
DATETIME = "DATETIME"
SCORE = "SCORE"
TARGET = "TARGET"
FEATURE = "FEATURE"
CUSTOM_KEY = "CUSTOM_KEY"
COUNTRY = "COUNTRY"
POSTAL_CODE = "POSTAL_CODE"
SYSTEM_RECORD_ID = "SYSTEM_RECORD_ID"
EVAL_SET_INDEX = "EVAL_SET_INDEX"
ENTITY_SYSTEM_RECORD_ID = "ENTITY_SYSTEM_RECORD_ID"
UNNEST_KEY = "UNNEST_KEY"
class SearchKey(Enum):
EMAIL = FileColumnMeaningType.EMAIL
HEM = FileColumnMeaningType.HEM
IP = FileColumnMeaningType.IP_ADDRESS
PHONE = FileColumnMeaningType.MSISDN
DATE = FileColumnMeaningType.DATE
DATETIME = FileColumnMeaningType.DATETIME
CUSTOM_KEY = FileColumnMeaningType.CUSTOM_KEY
COUNTRY = FileColumnMeaningType.COUNTRY
POSTAL_CODE = FileColumnMeaningType.POSTAL_CODE
# For data source registration. Don't use it for FeaturesEnricher
IPV6_ADDRESS = FileColumnMeaningType.IPV6_ADDRESS
IPV6_RANGE_FROM = FileColumnMeaningType.IPV6_RANGE_FROM
IPV6_RANGE_TO = FileColumnMeaningType.IPV6_RANGE_TO
# For data source registration. Don't use it for FeaturesEnricher
EMAIL_ONE_DOMAIN = FileColumnMeaningType.EMAIL_ONE_DOMAIN
# For data source registration. Don't use it for FeaturesEnricher
IP_RANGE_FROM = FileColumnMeaningType.IP_RANGE_FROM
# For data source registration. Don't use it for FeaturesEnricher
IP_RANGE_TO = FileColumnMeaningType.IP_RANGE_TO
# For data source registration. Don't use it for FeaturesEnricher
MSISDN_RANGE_FROM = FileColumnMeaningType.MSISDN_RANGE_FROM
# For data source registration. Don't use it for FeaturesEnricher
MSISDN_RANGE_TO = FileColumnMeaningType.MSISDN_RANGE_TO
@staticmethod
def personal_keys() -> List["SearchKey"]:
return [SearchKey.EMAIL, SearchKey.HEM, SearchKey.IP, SearchKey.PHONE]
@staticmethod
def from_meaning_type(meaning_type: FileColumnMeaningType) -> "SearchKey":
if meaning_type == FileColumnMeaningType.EMAIL:
return SearchKey.EMAIL
if meaning_type == FileColumnMeaningType.HEM:
return SearchKey.HEM
if meaning_type == FileColumnMeaningType.IP_ADDRESS:
return SearchKey.IP
if meaning_type == FileColumnMeaningType.MSISDN:
return SearchKey.PHONE
if meaning_type == FileColumnMeaningType.DATE:
return SearchKey.DATE
if meaning_type == FileColumnMeaningType.DATETIME:
return SearchKey.DATETIME
if meaning_type == FileColumnMeaningType.CUSTOM_KEY:
return SearchKey.CUSTOM_KEY
if meaning_type == FileColumnMeaningType.COUNTRY:
return SearchKey.COUNTRY
if meaning_type == FileColumnMeaningType.POSTAL_CODE:
return SearchKey.POSTAL_CODE
if meaning_type == FileColumnMeaningType.IPV6_ADDRESS:
return SearchKey.IPV6_ADDRESS
if meaning_type == FileColumnMeaningType.IPV6_RANGE_FROM:
return SearchKey.IPV6_RANGE_FROM
if meaning_type == FileColumnMeaningType.IPV6_RANGE_TO:
return SearchKey.IPV6_RANGE_TO
if meaning_type == FileColumnMeaningType.EMAIL_ONE_DOMAIN:
return SearchKey.EMAIL_ONE_DOMAIN
if meaning_type == FileColumnMeaningType.IP_RANGE_FROM:
return SearchKey.IP_RANGE_FROM
if meaning_type == FileColumnMeaningType.IP_RANGE_TO:
return SearchKey.IP_RANGE_TO
if meaning_type == FileColumnMeaningType.MSISDN_RANGE_FROM:
return SearchKey.MSISDN_RANGE_FROM
if meaning_type == FileColumnMeaningType.MSISDN_RANGE_TO:
return SearchKey.MSISDN_RANGE_TO
@staticmethod
def find_key(search_keys: Dict[str, SearchKey], keys: Union[SearchKey, List[SearchKey]]) -> Optional[SearchKey]:
if isinstance(keys, SearchKey):
keys = [keys]
for col, key_type in search_keys.items():
if key_type in keys:
return col
return None
@staticmethod
def find_all_keys(search_keys: Dict[str, SearchKey], keys: Union[SearchKey, List[SearchKey]]) -> List[SearchKey]:
if isinstance(keys, SearchKey):
keys = [keys]
return [col for col, key_type in search_keys.items() if key_type in keys]
class DataType(Enum):
INT = "INT"
DECIMAL = "DECIMAL"
BIG_NUMERIC = "BIG_NUMERIC"
DATE_TIME = "DATE_TIME"
STRING = "STRING"
BOOLEAN = "BOOLEAN"
class ModelTaskType(Enum):
BINARY = "BINARY"
MULTICLASS = "MULTICLASS"
REGRESSION = "REGRESSION"
TIMESERIES = "TIMESERIES"
class ModelLabelType(Enum):
GINI = "gini"
AUC = "auc"
RMSE = "rmse"
ACCURACY = "accuracy"
class RegressionTask(BaseModel):
mse: float
rmse: float
msle: float
rmsle: float
class BinaryTask(BaseModel):
auc: float
gini: float
class MulticlassTask(BaseModel):
accuracy: float
class FileMetricsInterval(BaseModel):
date_cut: float
count: float
valid_count: float
avg_target: Optional[float] = None # not for multiclass
avg_score_etalon: Optional[float] = None
class FileMetrics(BaseModel):
# etalon metadata
task_type: Optional[ModelTaskType] = None
label: Optional[ModelLabelType] = None
count: Optional[int] = None
valid_count: Optional[int] = None
valid_rate: Optional[float] = None
avg_target: Optional[float] = None
metrics_binary_etalon: Optional[BinaryTask] = None
metrics_regression_etalon: Optional[RegressionTask] = None
metrics_multiclass_etalon: Optional[MulticlassTask] = None
cuts: Optional[List[float]] = None
interval: Optional[List[FileMetricsInterval]] = None
class NumericInterval(BaseModel):
minValue: int
maxValue: int
class FileColumnMetadata(BaseModel):
index: int
name: str
dataType: DataType
meaningType: FileColumnMeaningType
minMaxValues: Optional[NumericInterval] = None
originalName: Optional[str] = None
# is this column contains keys from multiple key columns like msisdn1, msisdn2
isUnnest: bool = False
# list of original etalon key column names like msisdn1, msisdn2
unnestKeyNames: Optional[List[str]] = None
class FileMetadata(BaseModel):
name: str
description: Optional[str] = None
co
8000
lumns: List[FileColumnMetadata]
searchKeys: List[List[str]]
excludeFeaturesSources: Optional[List[str]] = None
hierarchicalGroupKeys: Optional[List[str]] = None
hierarchicalSubgroupKeys: Optional[List[str]] = None
taskType: Optional[ModelTaskType] = None
rowsCount: Optional[int] = None
checksumMD5: Optional[str] = None
digest: Optional[str] = None
def column_by_name(self, name: str) -> Optional[FileColumnMetadata]:
for c in self.columns:
if c.name == name:
return c
return None
def search_types(self) -> Set[SearchKey]:
search_keys = set()
for keys_group in self.searchKeys:
for key in keys_group:
column = self.column_by_name(key)
if column:
search_keys.add(SearchKey.from_meaning_type(column.meaningType))
return search_keys
class FeaturesMetadataV2(BaseModel):
name: str
type: str
source: str
hit_rate: float
shap_value: float
commercial_schema: Optional[str] = None
data_provider: Optional[str] = None
data_providers: Optional[List[str]] = None
data_provider_link: Optional[str] = None
data_provider_links: Optional[List[str]] = None
data_source: Optional[str] = None
data_sources: Optional[List[str]] = None
data_source_link: Optional[str] = None
data_source_links: Optional[List[str]] = None
doc_link: Optional[str] = None
update_frequency: Optional[str] = None
class HitRateMetrics(BaseModel):
etalon_row_count: int
max_hit_count: int
join_flags_count: Dict[str, int] = {}
hit_rate: float
hit_rate_percent: float
class ModelEvalSet(BaseModel):
eval_set_index: int
hit_rate: float
hit_rate_metrics: HitRateMetrics
class BaseColumnMetadata(BaseModel):
original_name: str
hashed_name: str
ads_definition_id: Optional[str] = None
is_augmented: bool
class GeneratedFeatureMetadata(BaseModel):
alias: Optional[str] = None
formula: str
display_index: str
base_columns: List[BaseColumnMetadata]
operator_params: Optional[Dict[str, str]] = None
class ProviderTaskMetadataV2(BaseModel):
features: List[FeaturesMetadataV2]
hit_rate_metrics: Optional[HitRateMetrics] = None
eval_set_metrics: Optional[List[ModelEvalSet]] = None
zero_hit_rate_search_keys: Optional[List[str]] = None
features_used_for_embeddings: Optional[List[str]] = None
shuffle_kfold: Optional[bool] = None
generated_features: Optional[List[GeneratedFeatureMetadata]] = None
class FeaturesFilter(BaseModel):
minImportance: Optional[float] = None
maxPSI: Optional[float] = None
maxCount: Optional[int] = None
selectedFeatures: Optional[List[str]] = None
class RuntimeParameters(BaseModel):
properties: Dict[str, Any] = {}
class SearchCustomization(BaseModel):
featuresFilter: Optional[FeaturesFilter] = None
extractFeatures: Optional[bool] = None
accurateModel: Optional[bool] = None
importanceThreshold: Optional[float] = None
maxFeatures: Optional[int] = None
returnScores: Optional[bool] = None
runtimeParameters: Optional[RuntimeParameters] = None
metricsCalculation: Optional[bool] = None
def __repr__(self):
return (
f"Features filter: {self.featuresFilter}, "
f"extract features: {self.extractFeatures}, "
f"accurate model: {self.accurateModel}, "
f"importance threshold: {self.importanceThreshold}, "
f"max features: {self.maxFeatures}, "
f"return scores: {self.returnScores}, "
f"runtimeParameters: {self.runtimeParameters}, "
f"metricsCalculation: {self.metricsCalculation}"
)
class CVType(Enum):
k_fold = "k_fold"
group_k_fold = "group_k_fold"
time_series = "time_series"
blocked_time_series = "blocked_time_series"
not_set = "not_set"