如何在模拟数据集（Dummy Dataset）中手动插入Temp与Humidity异常值

已通过以下代码生成模拟数据集：

from datetime import datetime
import numpy as np
import pandas as pd
from faker import Faker

fake = Faker()

def make_workers() -> list:
    status_list = ['in', 'out']
    room_list = ['FL1_RM1', 'FL1_RM2', 'FL1_RM3', 'FL1_RM4', 'FL2_RM1', 'FL2_RM2', 'FL2_RM3', 'FL2_RM4', 'FL3_RM1',
                 'FL3_RM2', 'FL3_RM3', 'FL3_RM4', 'FL4_RM1', 'FL4_RM2', 'FL4_RM3', 'FL4_RM4']
    Property = ['B1', 'B2', 'B3', 'B4']
    d1 = datetime.strptime('03/01/2022', '%m/%d/%Y')
    d2 = datetime.strptime('08/08/2022', '%m/%d/%Y')
    timestamps = pd.date_range(d1, d2, freq="1min")
    return [{**elem, **{"Floor_Number": elem.get("room_id")[2]}} for elem in [
        {'ID'          : fake.random_number(digits=6),
         'Property num': np.random.choice(Property, p=[0.25, 0.25, 0.25, 0.25]),
         'room_id'     : np.random.choice(room_list),
         'Temp'        : np.random.randint(low=35, high=50),
         'noted Date'  : timestamps[x],
         'Status'      : np.random.choice(status_list),
         'Humidity'    : np.random.uniform(low=-35.09, high=70.00),
         'Dust'        : np.random.randint(low=2, high=5),
         'CO2 level'   : np.random.uniform(low=350.09, high=450.00)
         } for x in range(len(timestamps))]]

worker_df = pd.DataFrame(make_workers())
worker_df.head(30)

需求是为Temp和Humidity列插入异常值：

• Temp：异常值需大于50或小于35（原范围35-50）
• Humidity：异常值需超出-35.09至70.00的原范围

方法一：生成数据时直接插入异常值 #

修改生成函数，按指定概率在生成数据时插入异常值（示例设为5%异常概率）：

from datetime import datetime
import numpy as np
import pandas as pd
from faker import Faker

fake = Faker()

def make_workers_with_outliers() -> list:
    status_list = ['in', 'out']
    room_list = ['FL1_RM1', 'FL1_RM2', 'FL1_RM3', 'FL1_RM4', 'FL2_RM1', 'FL2_RM2', 'FL2_RM3', 'FL2_RM4', 'FL3_RM1',
                 'FL3_RM2', 'FL3_RM3', 'FL3_RM4', 'FL4_RM1', 'FL4_RM2', 'FL4_RM3', 'FL4_RM4']
    Property = ['B1', 'B2', 'B3', 'B4']
    d1 = datetime.strptime('03/01/2022', '%m/%d/%Y')
    d2 = datetime.strptime('08/08/2022', '%m/%d/%Y')
    timestamps = pd.date_range(d1, d2, freq="1min")
    
    # 异常值出现概率
    outlier_prob = 0.05
    
    return [{**elem, **{"Floor_Number": elem.get("room_id")[2]}} for elem in [
        {
            'ID'          : fake.random_number(digits=6),
            'Property num': np.random.choice(Property, p=[0.25, 0.25, 0.25, 0.25]),
            'room_id'     : np.random.choice(room_list),
            # Temp异常值：一半概率生成<35的值，一半概率生成>50的值
            'Temp'        : np.random.choice(
                [np.random.randint(0,35), np.random.randint(51,70), np.random.randint(35,50)],
                p=[outlier_prob/2, outlier_prob/2, 1-outlier_prob]
            ),
            'noted Date'  : timestamps[x],
            'Status'      : np.random.choice(status_list),
            # Humidity异常值：一半概率生成<-35.09的值，一半概率生成>70.00的值
            'Humidity'    : np.random.choice(
                [np.random.uniform(-50.0, -35.1), np.random.uniform(70.1, 90.0), np.random.uniform(-35.09,70.00)],
                p=[outlier_prob/2, outlier_prob/2, 1-outlier_prob]
            ),
            'Dust'        : np.random.randint(low=2, high=5),
            'CO2 level'   : np.random.uniform(low=350.09, high=450.00)
        } for x in range(len(timestamps))]]

# 生成带异常值的数据集
worker_df_with_outliers = pd.DataFrame(make_workers_with_outliers())
worker_df_with_outliers.head(30)

方法二：在已有数据集上插入异常值 #

如果已生成原始数据集worker_df，可以随机替换部分行的值为异常值：

# 设置异常值占比
outlier_ratio = 0.05

# 为Temp列插入异常值
temp_outlier_indices = np.random.choice(worker_df.index, size=int(len(worker_df)*outlier_ratio), replace=False)
worker_df.loc[temp_outlier_indices, 'Temp'] = np.where(
    np.random.rand(len(temp_outlier_indices)) > 0.5,
    np.random.randint(51,70, size=len(temp_outlier_indices)),
    np.random.randint(0,35, size=len(temp_outlier_indices))
)

# 为Humidity列插入异常值
humidity_outlier_indices = np.random.choice(worker_df.index, size=int(len(worker_df)*outlier_ratio), replace=False)
worker_df.loc[humidity_outlier_indices, 'Humidity'] = np.where(
    np.random.rand(len(humidity_outlier_indices)) > 0.5,
    np.random.uniform(70.1,90.0, size=len(humidity_outlier_indices)),
    np.random.uniform(-50.0,-35.1, size=len(humidity_outlier_indices))
)

# 验证异常值插入结果
worker_df[['Temp','Humidity']].describe()

内容的提问来源于stack exchange，提问作者Jesse ansari