8000 qlib/qlib/data/ops.py at online-hotfix · jrehm135/qlib · GitHub
[go: up one dir, main page]

Skip to content

Latest commit

 

History

History
1547 lines (1209 loc) · 38.7 KB

ops.py

File metadata and controls

1547 lines (1209 loc) · 38.7 KB
8000
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
from __future__ import division
from __future__ import print_function
import sys
import abc
import numpy as np
import pandas as pd
from typing import Union, List, Type
from scipy.stats import percentileofscore
from .base import Expression, ExpressionOps
from ..log import get_module_logger
from ..utils import get_cls_kwargs
try:
from ._libs.rolling import rolling_slope, rolling_rsquare, rolling_resi
from ._libs.expanding import expanding_slope, expanding_rsquare, expanding_resi
except ImportError:
print(
"#### Do not import qlib package in the repository directory in case of importing qlib from . without compiling #####"
)
raise
np.seterr(invalid="ignore")
#################### Element-Wise Operator ####################
class ElemOperator(ExpressionOps):
"""Element-wise Operator
Parameters
----------
feature : Expression
feature instance
Returns
----------
Expression
feature operation output
"""
def __init__(self, feature):
self.feature = feature
def __str__(self):
return "{}({})".format(type(self).__name__, self.feature)
def get_longest_back_rolling(self):
return self.feature.get_longest_back_rolling()
def get_extended_window_size(self):
return self.feature.get_extended_window_size()
class NpElemOperator(ElemOperator):
"""Numpy Element-wise Operator
Parameters
----------
feature : Expression
feature instance
func : str
numpy feature operation method
Returns
----------
Expression
feature operation output
"""
def __init__(self, feature, func):
self.func = func
super(NpElemOperator, self).__init__(feature)
def _load_internal(self, instrument, start_index, end_index, freq):
series = self.feature.load(instrument, start_index, end_index, freq)
return getattr(np, self.func)(series)
class Abs(NpElemOperator):
"""Feature Absolute Value
Parameters
----------
feature : Expression
feature instance
Returns
----------
Expression
a feature instance with absolute output
"""
def __init__(self, feature):
super(Abs, self).__init__(feature, "abs")
class Sign(NpElemOperator):
"""Feature Sign
Parameters
----------
feature : Expression
feature instance
Returns
----------
Expression
a feature instance with sign
"""
def __init__(self, feature):
super(Sign, self).__init__(feature, "sign")
def _load_internal(self, instrument, start_index, end_index, freq):
"""
To avoid error raised by bool type input, we transform the data into float32.
"""
series = self.feature.load(instrument, start_index, end_index, freq)
# TODO: More precision types should be configurable
series = series.astype(np.float32)
return getattr(np, self.func)(series)
class Log(NpElemOperator):
"""Feature Log
Parameters
----------
feature : Expression
feature instance
Returns
----------
Expression
a feature instance with log
"""
def __init__(self, feature):
super(Log, self).__init__(feature, "log")
class Power(NpElemOperator):
"""Feature Power
Parameters
----------
feature : Expression
feature instance
Returns
----------
Expression
a feature instance with power
"""
def __init__(self, feature, exponent):
super(Power, self).__init__(feature, "power")
self.exponent = exponent
def __str__(self):
return "{}({},{})".format(type(self).__name__, self.feature, self.exponent)
def _load_internal(self, instrument, start_index, end_index, freq):
series = self.feature.load(instrument, start_index, end_index, freq)
return getattr(np, self.func)(series, self.exponent)
class Mask(NpElemOperator):
"""Feature Mask
Parameters
----------
feature : Expression
feature instance
instrument : str
instrument mask
Returns
----------
Expression
a feature instance with masked instrument
"""
def __init__(self, feature, instrument):
super(Mask, self).__init__(feature, "mask")
self.instrument = instrument
def __str__(self):
return "{}({},{})".format(type(self).__name__, self.feature, self.instrument.lower())
def _load_internal(self, instrument, start_index, end_index, freq):
return self.feature.load(self.instrument, start_index, end_index, freq)
class Not(NpElemOperator):
"""Not Operator
Parameters
----------
feature_left : Expression
feature instance
feature_right : Expression
feature instance
Returns
----------
Feature:
feature elementwise not output
"""
def __init__(self, feature):
super(Not, self).__init__(feature, "bitwise_not")
#################### Pair-Wise Operator ####################
class PairOperator(ExpressionOps):
"""Pair-wise operator
Parameters
----------
feature_left : Expression
feature instance or numeric value
feature_right : Expression
feature instance or numeric value
func : str
operator function
Returns
----------
Feature:
two features' operation output
"""
def __init__(self, feature_left, feature_right):
self.feature_left = feature_left
self.feature_right = feature_right
def __str__(self):
return "{}({},{})".format(type(self).__name__, self.feature_left, self.feature_right)
def get_longest_back_rolling(self):
if isinstance(self.feature_left, Expression):
left_br = self.feature_left.get_longest_back_rolling()
else:
left_br = 0
if isinstance(self.feature_right, Expression):
right_br = self.feature_right.get_longest_back_rolling()
else:
right_br = 0
return max(left_br, right_br)
def get_extended_window_size(self):
if isinstance(self.feature_left, Expression):
ll, lr = self.feature_left.get_extended_window_size()
else:
ll, lr = 0, 0
if isinstance(self.feature_right, Expression):
rl, rr = self.feature_right.get_extended_window_size()
else:
rl, rr = 0, 0
return max(ll, rl), max(lr, rr)
class NpPairOperator(PairOperator):
"""Numpy Pair-wise operator
Parameters
----------
feature_left : Expression
feature instance or numeric value
feature_right : Expression
feature instance or numeric value
func : str
operator function
Returns
----------
Feature:
two features' operation output
"""
def __init__(self, feature_left, feature_right, func):
self.func = func
super(NpPairOperator, self).__init__(feature_left, feature_right)
def _load_internal(self, instrument, start_index, end_index, freq):
assert any(
[isinstance(self.feature_left, Expression), self.feature_right, Expression]
), "at least one of two inputs is Expression instance"
if isinstance(self.feature_left, Expression):
series_left = self.feature_left.load(instrument, start_index, end_index, freq)
else:
series_left = self.feature_left # numeric value
if isinstance(self.feature_right, Expression):
series_right = self.feature_right.load(instrument, start_index, end_index, freq)
else:
series_right = self.feature_right
return getattr(np, self.func)(series_left, series_right)
class Add(NpPairOperator):
"""Add Operator
Parameters
----------
feature_left : Expression
feature instance
feature_right : Expression
feature instance
Returns
----------
Feature:
two features' sum
"""
def __init__(self, feature_left, feature_right):
super(Add, self).__init__(feature_left, feature_right, "add")
class Sub(NpPairOperator):
"""Subtract Operator
Parameters
----------
feature_left : Expression
feature instance
feature_right : Expression
feature instance
Returns
----------
Feature:
two features' subtraction
"""
def __init__(self, feature_left, feature_right):
super(Sub, self).__init__(feature_left, feature_right, "subtract")
class Mul(NpPairOperator):
"""Multiply Operator
Parameters
----------
feature_left : Expression
feature instance
feature_right : Expression
feature instance
Returns
----------
Feature:
two features' product
"""
def __init__(self, feature_left, feature_right):
super(Mul, self).__init__(feature_left, feature_right, "multiply")
class Div(NpPairOperator):
"""Division Operator
Parameters
----------
feature_left : Expression
feature instance
feature_right : Expression
feature instance
Returns
----------
Feature:
two features' division
"""
def __init__(self, feature_left, feature_right):
super(Div, self).__init__(feature_left, feature_right, "divide")
class Greater(NpPairOperator):
"""Greater Operator
Parameters
----------
feature_left : Expression
feature instance
feature_right : Expression
feature instance
Returns
----------
Feature:
greater elements taken from the input two features
"""
def __init__(self, feature_left, feature_right):
super(Greater, self).__init__(feature_left, feature_right, "maximum")
class Less(NpPairOperator):
"""Less Operator
Parameters
----------
feature_left : Expression
feature instance
feature_right : Expression
feature instance
Returns
----------
Feature:
smaller elements taken from the input two features
"""
def __init__(self, feature_left, feature_right):
super(Less, self).__init__(feature_left, feature_right, "minimum")
class Gt(NpPairOperator):
"""Greater Than Operator
Parameters
----------
feature_left : Expression
feature instance
feature_right : Expression
feature instance
Returns
----------
Feature:
bool series indicate `left > right`
"""
def __init__(self, feature_left, feature_right):
super(Gt, self).__init__(feature_left, feature_right, "greater")
class Ge(NpPairOperator):
"""Greater Equal Than Operator
Parameters
----------
feature_left : Expression
feature instance
feature_right : Expression
feature instance
Returns
----------
Feature:
bool series indicate `left >= right`
"""
def __init__(self, feature_left, feature_right):
super(Ge, self).__init__(feature_left, feature_right, "greater_equal")
class Lt(NpPairOperator):
"""Less Than Operator
Parameters
----------
feature_left : Expression
feature instance
feature_right : Expression
feature instance
Returns
----------
Feature:
bool series indicate `left < right`
"""
def __init__(self, feature_left, feature_right):
super(Lt, self).__init__(feature_left, feature_right, "less")
class Le(NpPairOperator):
"""Less Equal Than Operator
Parameters
----------
feature_left : Expression
feature instance
feature_right : Expression
feature instance
Returns
----------
Feature:
bool series indicate `left <= right`
"""
def __init__(self, feature_left, feature_right):
super(Le, self).__init__(feature_left, feature_right, "less_equal")
class Eq(NpPairOperator):
"""Equal Operator
Parameters
----------
feature_left : Expression
feature instance
feature_right : Expression
feature instance
Returns
----------
Feature:
bool series indicate `left == right`
"""
def __init__(self, feature_left, feature_right):
super(Eq, self).__init__(feature_left, feature_right, "equal")
class Ne(NpPairOperator):
"""Not Equal Operator
Parameters
----------
feature_left : Expression
feature instance
feature_right : Expression
feature instance
Returns
----------
Feature:
bool series indicate `left != right`
"""
def __init__(self, feature_left, feature_right):
super(Ne, self).__init__(feature_left, feature_right, "not_equal")
class And(NpPairOperator):
"""And Operator
Parameters
----------
feature_left : Expression
feature instance
feature_right : Expression
feature instance
Returns
----------
Feature:
two features' row by row & output
"""
def __init__(self, feature_left, feature_right):
super(And, self).__init__(feature_left, feature_right, "bitwise_and")
class Or(NpPairOperator):
"""Or Operator
Parameters
----------
feature_left : Expression
feature instance
feature_right : Expression
feature instance
Returns
----------
Feature:
two features' row by row | outputs
"""
def __init__(self, feature_left, feature_right):
super(Or, self).__init__(feature_left, feature_right, "bitwise_or")
#################### Triple-wise Operator ####################
class If(ExpressionOps):
"""If Operator
Parameters
----------
condition : Expression
feature instance with bool values as condition
feature_left : Expression
feature instance
feature_right : Expression
feature instance
"""
def __init__(self, condition, feature_left, feature_right):
self.condition = condition
self.feature_left = feature_left
self.feature_right = feature_right
def __str__(self):
return "If({},{},{})".format(self.condition, self.feature_left, self.feature_right)
def _load_internal(self, instrument, start_index, end_index, freq):
series_cond = self.condition.load(instrument, start_index, end_index, freq)
if isinstance(self.feature_left, Expression):
series_left = self.feature_left.load(instrument, start_index, end_index, freq)
else:
series_left = self.feature_left
if isinstance(self.feature_right, Expression):
series_right = self.feature_right.load(instrument, start_index, end_index, freq)
else:
series_right = self.feature_right
series = pd.Series(np.where(series_cond, series_left, series_right), index=series_cond.index)
return series
def get_longest_back_rolling(self):
if isinstance(self.feature_left, Expression):
left_br = self.feature_left.get_longest_back_rolling()
else:
left_br = 0
if isinstance(self.feature_right, Expression):
right_br = self.feature_right.get_longest_back_rolling()
else:
right_br = 0
if isinstance(self.condition, Expression):
c_br = self.condition.get_longest_back_rolling()
else:
c_br = 0
return max(left_br, right_br, c_br)
def get_extended_window_size(self):
if isinstance(self.feature_left, Expression):
ll, lr = self.feature_left.get_extended_window_size()
else:
ll, lr = 0, 0
if isinstance(self.feature_right, Expression):
rl, rr = self.feature_right.get_extended_window_size()
else:
rl, rr = 0, 0
if isinstance(self.condition, Expression):
cl, cr = self.condition.get_extended_window_size()
else:
cl, cr = 0, 0
return max(ll, rl, cl), max(lr, rr, cr)
#################### Rolling ####################
# NOTE: methods like `rolling.mean` are optimized with cython,
# and are super faster than `rolling.apply(np.mean)`
class Rolling(ExpressionOps):
"""Rolling Operator
Parameters
----------
feature : Expression
feature instance
N : int
rolling window size
func : str
rolling method
Returns
----------
Expression
rolling outputs
"""
def __init__(self, feature, N, func):
self.feature = feature
self.N = N
self.func = func
def __str__(self):
return "{}({},{})".format(type(self).__name__, self.feature, self.N)
def _load_internal(self, instrument, start_index, end_index, freq):
series = self.feature.load(instrument, start_index, end_index, freq)
# NOTE: remove all null check,
# now it's user's responsibility to decide whether use features in null days
# isnull = series.isnull() # NOTE: isnull = NaN, inf is not null
if self.N == 0:
series = getattr(series.expanding(min_periods=1), self.func)()
elif 0 < self.N < 1:
series = series.ewm(alpha=self.N, min_periods=1).mean()
else:
series = getattr(series.rolling(self.N, min_periods=1), self.func)()
# series.iloc[:self.N-1] = np.nan
# series[isnull] = np.nan
return series
def get_longest_back_rolling(self):
if self.N == 0:
return np.inf
if 0 < self.N < 1:
return int(np.log(1e-6) / np.log(1 - self.N)) # (1 - N)**window == 1e-6
return self.feature.get_longest_back_rolling() + self.N - 1
def get_extended_window_size(self):
if self.N == 0:
# FIXME: How to make this accurate and efficiently? Or should we
# remove such support for N == 0?
get_module_logger(self.__class__.__name__).warning("The Rolling(ATTR, 0) will not be accurately calculated")
return self.feature.get_extended_window_size()
elif 0 < self.N < 1:
lft_etd, rght_etd = self.feature.get_extended_window_size()
size = int(np.log(1e-6) / np.log(1 - self.N))
lft_etd = max(lft_etd + size - 1, lft_etd)
return lft_etd, rght_etd
else:
lft_etd, rght_etd = self.feature.get_extended_window_size()
lft_etd = max(lft_etd + self.N - 1, lft_etd)
return lft_etd, rght_etd
class Ref(Rolling):
"""Feature Reference
Parameters
----------
feature : Expression
feature instance
N : int
N = 0, retrieve the first data; N > 0, retrieve data of N periods ago; N < 0, future data
Returns
----------
Expression
a feature instance with target reference
"""
def __init__(self, feature, N):
super(Ref, self).__init__(feature, N, "ref")
def _load_internal(self, instrument, start_index, end_index, freq):
series = self.feature.load(instrument, start_index, end_index, freq)
# N = 0, return first day
if series.empty:
return series # Pandas bug, see: https://github.com/pandas-dev/pandas/issues/21049
elif self.N == 0:
series = pd.Series(series.iloc[0], index=series.index)
else:
series = series.shift(self.N) # copy
return series
def get_longest_back_rolling(self):
if self.N == 0:
return np.inf
return self.feature.get_longest_back_rolling() + self.N
def get_extended_window_size(self):
if self.N == 0:
get_module_logger(self.__class__.__name__).warning("The Ref(ATTR, 0) will not be accurately calculated")
return self.feature.get_extended_window_size()
else:
lft_etd, rght_etd = self.feature.get_extended_window_size()
lft_etd = max(lft_etd + self.N, lft_etd)
rght_etd = max(rght_etd - self.N, rght_etd)
return lft_etd, rght_etd
class Mean(Rolling):
"""Rolling Mean (MA)
Parameters
----------
feature : Expression
feature instance
N : int
rolling window size
Returns
----------
Expression
a feature instance with rolling average
"""
def __init__(self, feature, N):
super(Mean, self).__init__(feature, N, "mean")
class Sum(Rolling):
"""Rolling Sum
Parameters
----------
feature : Expression
feature instance
N : int
rolling window size
Returns
----------
Expression
a feature instance with rolling sum
"""
def __init__(self, feature, N):
super(Sum, self).__init__(feature, N, "sum")
class Std(Rolling):
"""Rolling Std
Parameters
----------
feature : Expression
feature instance
N : int
rolling window size
Returns
----------
Expression
a feature instance with rolling std
"""
def __init__(self, feature, N):
super(Std, self).__init__(feature, N, "std")
class Var(Rolling):
"""Rolling Variance
Parameters
----------
feature : Expression
feature instance
N : int
rolling window size
Returns
----------
Expression
a feature instance with rolling variance
"""
def __init__(self, feature, N):
super(Var, self).__init__(feature, N, "var")
class Skew(Rolling):
"""Rolling Skewness
Parameters
----------
feature : Expression
feature instance
N : int
rolling window size
Returns
----------
Expression
a feature instance with rolling skewness
"""
def __init__(self, feature, N):
if N != 0 and N < 3:
raise ValueError("The rolling window size of Skewness operation should >= 3")
super(Skew, self).__init__(feature, N, "skew")
class Kurt(Rolling):
"""Rolling Kurtosis
Parameters
----------
feature : Expression
feature instance
N : int
rolling window size
Returns
----------
Expression
a feature instance with rolling kurtosis
"""
def __init__(self, feature, N):
if N != 0 and N < 4:
raise ValueError("The rolling window size of Kurtosis operation should >= 5")
super(Kurt, self).__init__(feature, N, "kurt")
class Max(Rolling):
"""Rolling Max
Parameters
----------
feature : Expression
feature instance
N : int
rolling window size
Returns
----------
Expression
a feature instance with rolling max
"""
def __init__(self, feature, N):
super(Max, self).__init__(feature, N, "max")
class IdxMax(Rolling):
"""Rolling Max Index
Parameters
----------
feature : Expression
feature instance
N : int
rolling window size
Returns
----------
Expression
a feature instance with rolling max index
"""
def __init__(self, feature, N):
super(IdxMax, self).__init__(feature, N, "idxmax")
def _load_internal(self, instrument, start_index, end_index, freq):
series = self.feature.load(instrument, start_index, end_index, freq)
if self.N == 0:
series = series.expanding(min_periods=1).apply(lambda x: x.argmax() + 1, raw=True)
else:
series = series.rolling(self.N, min_periods=1).apply(lambda x: x.argmax() + 1, raw=True)
return series
class Min(Rolling):
"""Rolling Min
Parameters
----------
feature : Expression
feature instance
N : int
rolling window size
Returns
----------
Expression
a feature instance with rolling min
"""
def __init__(self, feature, N):
super(Min, self).__init__(feature, N, "min")
class IdxMin(Rolling):
"""Rolling Min Index
Parameters
----------
feature : Expression
feature instance
N : int
rolling window size
Returns
----------
Expression
a feature instance with rolling min index
"""
def __init__(self, feature, N):
super(IdxMin, self).__init__(feature, N, "idxmin")
def _load_internal(self, instrument, start_index, end_index, freq):
series = self.feature.load(instrument, start_index, end_index, freq)
if self.N == 0:
series = series.expanding(min_periods=1).apply(lambda x: x.argmin() + 1, raw=True)
else:
series = series.rolling(self.N, min_periods=1).apply(lambda x: x.argmin() + 1, raw=True)
return series
class Quantile(Rolling):
"""Rolling Quantile
Parameters
0