使用Python进行数据清洗和预处理的实现代码

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

# 读取CSV文件
data = pd.read_csv('data.csv')

# 查看数据的前几行
print(data.head())

# 查看数据的基本信息
print(data.info())

# 查看数据的统计描述
print(data.describe())

# 查看缺失值情况
print(data.isnull().sum())

# 删除含有缺失值的行
data_cleaned = data.dropna()

# 用均值填充缺失值
data_filled = data.fillna(data.mean())

# 使用插值法填充缺失值
data_interpolated = data.interpolate()

# 删除重复值
data_deduplicated = data.drop_duplicates()

# 将字符串类型的日期转换为日期类型
data['date'] = pd.to_datetime(data['date'])

# 将分类数据转换为数值类型
data['category'] = data['category'].astype('category').cat.codes

from sklearn.preprocessing import StandardScaler, MinMaxScaler

# 标准化
scaler = StandardScaler()
data_standardized = scaler.fit_transform(data)

# 归一化
scaler = MinMaxScaler()
data_normalized = scaler.fit_transform(data)

# 使用箱线图法检测和处理异常值
Q1 = data.quantile(0.25)
Q3 = data.quantile(0.75)
IQR = Q3 - Q1
data_outlier_removed = data[~((data < (Q1 - 1.5 * IQR)) |(data > (Q3 + 1.5 * IQR))).any(axis=1)]

# 使用Z分数法检测和处理异常值
from scipy import stats
data_zscore = data[(np.abs(stats.zscore(data)) < 3).all(axis=1)]

# 创建新特征：日期特征分解
data['year'] = data['date'].dt.year
data['month'] = data['date'].dt.month
data['day'] = data['date'].dt.day

# 特征组合
data['total_amount'] = data['quantity'] * data['price']

# 绘制直方图
data['column_name'].hist()
plt.show()

# 绘制箱线图
data.boxplot(column='column_name')
plt.show()

# 绘制散点图
plt.scatter(data['column1'], data['column2'])
plt.show()

# 计算与目标变量的相关系数
correlation = data.corr()
print(correlation['target_variable'].sort_values(ascending=False))

from sklearn.linear_model import Lasso

# 使用Lasso进行特征选择
lasso = Lasso(alpha=0.1)
lasso.fit(data.drop('target_variable', axis=1), data['target_variable'])
selected_features = data.columns[lasso.coef_ != 0]
print(selected_features)

from sklearn.feature_selection import RFE
from sklearn.linear_model import LogisticRegression

# 使用RFE进行特征选择
model = LogisticRegression()
rfe = RFE(model, 5)
fit = rfe.fit(data.drop('target_variable', axis=1), data['target_variable'])
selected_features = data.columns[fit.support_]
print(selected_features)

from sklearn.model_selection import train_test_split

# 数据分割
X_train, X_test, y_train, y_test = train_test_split(data.drop('target_variable', axis=1), data['target_variable'], test_size=0.2, random_state=42)

import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split

# 读取数据
data = pd.read_csv('data.csv')

# 数据探索
print(data.info())
print(data.describe())

# 处理缺失值
data = data.fillna(data.mean())

# 删除重复值
data = data.drop_duplicates()

# 数据类型转换
data['date'] = pd.to_datetime(data['date'])
data['category'] = data['category'].astype('category').cat.codes

# 特征工程
data['year'] = data['date'].dt.year
data['month'] = data['date'].dt.month
data['day'] = data['date'].dt.day
data['total_amount'] = data['quantity'] * data['price']

# 处理异常值
Q1 = data.quantile(0.25)
Q3 = data.quantile(0.75)
IQR = Q3 - Q1
data = data[~((data < (Q1 - 1.5 * IQR)) | (data > (Q3 + 1.5 * IQR))).any(axis=1)]

# 数据标准化
scaler = StandardScaler()
data_scaled = scaler.fit_transform(data.drop(['date', 'target_variable'], axis=1))

# 数据分割
X_train, X_test, y_train, y_test = train_test_split(data_scaled, data['target_variable'], test_size=0.2, random_state=42)