1.读入训练集与验证集数据train,test
train = pd.read_csv('../input/train.csv') test = pd.read_csv('../input/test.csv')
2.将训练与验证集整合到一起，进行特征工程
具体实施：将这个数据集合整合到一个列表里面，data_full=[train,test]。通过列表的遍历操作对两个数据进行统一处理。泰坦尼克特征工程处理参考：
full_data = [train, test] # Some features of my own that I have added in # Gives the length of the name train['Name_length'] = train['Name'].apply(len) test['Name_length'] = test['Name'].apply(len) # Feature that tells whether a passenger had a cabin on the Titanic train['Has_Cabin'] = train["Cabin"].apply(lambda x: 0 if type(x) == float else 1) test['Has_Cabin'] = test["Cabin"].apply(lambda x: 0 if type(x) == float else 1) # Feature engineering steps taken from Sina # Create new feature FamilySize as a combination of SibSp and Parch for dataset in full_data: dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1 # Create new feature IsAlone from FamilySize for dataset in full_data: dataset['IsAlone'] = 0 dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1 # Remove all NULLS in the Embarked column for dataset in full_data: dataset['Embarked'] = dataset['Embarked'].fillna('S') # Remove all NULLS in the Fare column and create a new feature CategoricalFare for dataset in full_data: dataset['Fare'] = dataset['Fare'].fillna(train['Fare'].median()) train['CategoricalFare'] = pd.qcut(train['Fare'], 4) # Create a New feature CategoricalAge for dataset in full_data: age_avg = dataset['Age'].mean() age_std = dataset['Age'].std() age_null_count = dataset['Age'].isnull().sum() age_null_random_list = np.random.randint(age_avg - age_std, age_avg + age_std, size=age_null_count) dataset['Age'][np.isnan(dataset['Age'])] = age_null_random_list dataset['Age'] = dataset['Age'].astype(int) train['CategoricalAge'] = pd.cut(train['Age'], 5) # Define function to extract titles from passenger names def get_title(name): title_search = re.search(' ([A-Za-z]+).', name) # If the title exists, extract and return it. if title_search: return title_search.group(1) return "" # Create a new feature Title, containing the titles of passenger names for dataset in full_data: dataset['Title'] = dataset['Name'].apply(get_title) # Group all non-common titles into one single grouping "Rare" for dataset in full_data: dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare') dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss') dataset['Title'] = dataset['Title'].replace('Ms', 'Miss') dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs') for dataset in full_data: # Mapping Sex dataset['Sex'] = dataset['Sex'].map( {'female': 0, 'male': 1} ).astype(int) # Mapping titles title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5} dataset['Title'] = dataset['Title'].map(title_mapping) dataset['Title'] = dataset['Title'].fillna(0) # Mapping Embarked dataset['Embarked'] = dataset['Embarked'].map( {'S': 0, 'C': 1, 'Q': 2} ).astype(int) # Mapping Fare dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0 dataset.loc[(dataset['Fare'] > 7.91) & (dataset['Fare'] <= 14.454), 'Fare'] = 1 dataset.loc[(dataset['Fare'] > 14.454) & (dataset['Fare'] <= 31), 'Fare'] = 2 dataset.loc[ dataset['Fare'] > 31, 'Fare'] = 3 dataset['Fare'] = dataset['Fare'].astype(int) # Mapping Age dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0 dataset.loc[(dataset['Age'] > 16) & (dataset['Age'] <= 32), 'Age'] = 1 dataset.loc[(dataset['Age'] > 32) & (dataset['Age'] <= 48), 'Age'] = 2 dataset.loc[(dataset['Age'] > 48) & (dataset['Age'] <= 64), 'Age'] = 3 dataset.loc[ dataset['Age'] > 64, 'Age'] = 4 ;
3.具体的分开使用
例子：
a=[1,2,3,4,5,6] b=[7,8,9,10,11,12] c=[a,b] c[0][1]=222 #c=[[1, 222, 3, 4, 5, 6], [7, 8, 9, 10, 11, 12]] #a=[1, 222, 3, 4, 5, 6]
# Feature selection drop_elements = ['PassengerId', 'Name', 'Ticket', 'Cabin', 'SibSp'] train = train.drop(drop_elements, axis = 1) train = train.drop(['CategoricalAge', 'CategoricalFare'], axis = 1) test = test.drop(drop_elements, axis = 1)
4.样本不均衡数据处理(SMOTE)
from imblearn.over_sampling import SMOTE X_train_res,y_train_res=SMOTE().fit_sample(X_train,y_train)
5.各个特征之间的相关性
data.corr()
6.时间片段
Now=pd.to_datetime("2019-07-20") data["timeperiod"]=(Now-data["CustomerSince"]).apply(lambda x:x.days)
7.查看特征值是否符合正态分布，如果偏度过大进行boxcox变换
from scipy.stats import norm, skew numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index # Check the skew of all numerical features skewed_feats = all_data[numeric_feats].apply(lambda x: skew(x.dropna())).sort_values(ascending=False) print(" Skew in numerical features: ") skewness = pd.DataFrame({'Skew' :skewed_feats}) skewness.head(10)
skewness = skewness[abs(skewness) > 0.75] print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0])) from scipy.special import boxcox1p skewed_features = skewness.index lam = 0.15 for feat in skewed_features: all_data[feat] = boxcox1p(all_data[feat], lam)
8.针对特定列，查看列的每一个值的count的可视化。
sns.barplot(data_init["PhoneType"].value_counts().index[0:10],data_init["PhoneType"].value_counts()[0:10])
sns.countplot(order=data_init["PhoneType"].value_counts().index,y=data_init["PhoneType"])