52 lines
1.3 KiB
Python
52 lines
1.3 KiB
Python
# -*- coding: UTF-8 -*-
|
||
"""
|
||
@Project -> File ：IoD_data_analysis_tool -> outlierprocessing
|
||
@IDE ：PyCharm
|
||
@Author ：rengengchen
|
||
@Date ：2022/4/26 10:24
|
||
@Desc ：
|
||
"""
|
||
from typing import Union
|
||
|
||
import pandas as pd
|
||
|
||
|
||
def MAD(data: pd.Series, n: float = 2.5, constant=1.4826, axis=0):
|
||
"""
|
||
MAD = median(|Xi - median(X)|)
|
||
@return pandas.Index
|
||
"""
|
||
x = data.median()
|
||
MC = (data - x).abs().median()
|
||
MAD = MC * constant
|
||
offset = n * MAD
|
||
if isinstance(data, pd.DataFrame):
|
||
return data.clip(lower=x - offset, upper=x + offset, axis=axis)
|
||
else:
|
||
return data.clip(lower=x - offset, upper=x + offset)
|
||
|
||
|
||
def three_sigma(data: pd.Series):
|
||
miu = data.mean()
|
||
sigma = data.std()
|
||
low = miu - 3 * sigma
|
||
up = miu + 3 * sigma
|
||
return data.index[(data < low) | (data > up)]
|
||
|
||
|
||
def box_plot(data: pd.Series, q1: float = 0.25, q3: float = 0.75, k: float = 1.5):
|
||
q = data.quantile(q=[q1, q3])
|
||
IQR = q[q3] - q[q1]
|
||
lower_whisker_limit = q[q1] - k * IQR
|
||
upper_whisker_limit = q[q3] + k * IQR
|
||
return data.index[(data < lower_whisker_limit) | (data > upper_whisker_limit)]
|
||
|
||
|
||
def regex_match(data: pd.Series, *patterns):
|
||
pattern = '|'.join(patterns)
|
||
return data.index[data.astype(str).str.contains(pattern, regex=True)]
|
||
|
||
|
||
def empty(data: Union[pd.Series, pd.DataFrame]):
|
||
return any(data.isnull())
|