# data is a DataFrame type data.sample(nums) # 随机取nums个值 data.col.unique() # 返回col取的所有值 #对于变量（不论连续或者离散或者类型变量）， 得到其col的取值直方图 fig = data.loan_amnt.hist(bins=50) # loan_amnt is a col in data fig.set_title('loan Amount hist') fig.set_xlabel('loan Amount') fig.set_ylabel('Number of Loans')
data.open_acc.dropna().unique() # col dropna() 删去所有含有缺失值的 np.where(binary_array, 1, 0) # binary_array中为True的用1代替，其他的用0代替 data['defaulted'] = np.where(data.loan_status.isin(['Default']), 1, 0) data.loan_status.isin(['Default']) # 判断loan_status是否是‘Default’，返回列表
data['col'].value_counts() # for each category of home ownership fig = data['home_ownership'].value_counts().plot.bar() fig.set_title('Home Ownership') fig.set_ylabel('Number of customers')
data.groupby([col])['y'].sum()
缺失值
data.isnull().sum() # 计算每个col中为null的数目 data.isnull().mean() #计算每个col中null数据所占比例 data['cabin_null'] = np.where(data.Cabin.isnull(), 1, 0) data.groupby(['Survived'])['cabin_null'].mean() data['emp_length_redefined'] = data.emp_length.map(length_dict) data.emp_length_redefined.unique() data[data.emp_title.isnull()].groupby(['emp_length_redefined'])['emp_length'].count().sort_values() / value
Outliers
import seaborn as sns sns.distplot(data.Age)
# another way of visualising outliers is using boxplots and whiskers, # which provides the quantiles (box) and inter-quantile range (whiskers), # with the outliers sitting outside the error bars (whiskers). # All the dots in the plot below are outliers according to the quantiles + 1.5 IQR rule fig = data.boxplot(column='Fare') fig.set_title('') fig.set_xlabel('Survived') fig.set_ylabel('Fare')
# let's look at the values of the quantiles so we can # calculate the upper and lower boundaries for the outliers # 25%, 50% and 75% in the output below indicate the # 25th quantile, median and 75th quantile respectively data.Fare.describe()
# Let's calculate the upper and lower boundaries # to identify outliers according # to interquantile proximity rule IQR = data.Fare.quantile(0.75) - data.Fare.quantile(0.25) Lower_fence = data.Fare.quantile(0.25) - (IQR * 1.5) Upper_fence = data.Fare.quantile(0.75) + (IQR * 1.5) Upper_fence, Lower_fence, IQR
multiple_tickets = pd.concat( [ high_fare_df.groupby('Ticket')['Fare'].count(), high_fare_df.groupby('Ticket')['Fare'].mean() ], axis=1) multiple_tickets.columns = ['Ticket', 'Fare'] multiple_tickets.head(10)
Rare Values
Let's make a combined plot of the label frequency and # the time to pass testing. # This will help us visualise the relationship between the # target and the labels of X3 fig, ax = plt.subplots(figsize=(8, 4)) plt.xticks(temp_df.index, temp_df['X3'], rotation=0) ax2 = ax.twinx() ax.bar(temp_df.index, temp_df["X3_perc_cars"], color='lightgrey') ax2.plot(temp_df.index, temp_df["y"], color='green', label='Seconds') ax.set_ylabel('percentage of cars per category') ax2.set_ylabel('Seconds')
# let's automate the above process for all the categorical variables for col in cols_to_use: # calculate the frequency of the different labels in the variable temp_df = pd.Series(data[col].value_counts() / total_cars).reset_index() # rename the columns temp_df.columns = [col, col + '_perc_cars'] # merge onto the mean time to pass the test temp_df = temp_df.merge( data.groupby([col])['y'].mean().reset_index(), on=col, how='left') # plot the figure as shown above fig, ax = plt.subplots(figsize=(8, 4)) plt.xticks(temp_df.index, temp_df[col], rotation=0) ax2 = ax.twinx() ax.bar( temp_df.index, temp_df[col + '_perc_cars'], color='lightgrey', label=col) ax2.plot( temp_df.index, temp_df["y"], color='green', ) ax.set_ylabel('percentage of cars per category') ax2.set_ylabel('Seconds') ax.legend() plt.show()
# let's automate the replacement of infrequent categories # by the label 'rare' in the remaining categorical variables # I start from 1 because I already replaced the first variable in # the list for col in cols_to_use[1:]: # calculate the % of cars in each category temp_df = pd.Series(data[col].value_counts() / total_cars) # create a dictionary to replace the rare labels with the # string 'rare' grouping_dict = { k: ('rare' if k not in temp_df[temp_df >= 0.1].index else k) for k in temp_df.index } # replace the rare labels data[col + '_grouped'] = data[col].map(grouping_dict) data.head()
# In order to use this variables to build machine learning using sklearn # first we need to replace the labels by numbers. # The correct way to do this, is to first separate into training and test # sets. And then create a replacing dictionary using the train set # and replace the strings both in train and test using the dictionary # created. # This will lead to the introduction of missing values / NaN in the # test set, for those labels that are not present in the train set # we saw this effect in the previous lecture # in the section dedicated to rare values later in the course, I will # show you how to avoid this problem # now, in order to speed up the demonstration, I will replace the # labels by strings in the entire dataset, and then divide into # train and test. # but remember: THIS IS NOT GOOD PRACTICE! # original variables for col in cols_to_use: # create the dic and replace the strings in one line data.loc[:, col] = data.loc[:, col].map( {k: i for i, k in enumerate(data[col].unique(), 0)}) # variables with grouped categories for col in ['X1_grouped', 'X6_grouped', 'X3_grouped', 'X2_grouped']: # create the dic and replace the strings in one line data.loc[:, col] = data.loc[:, col].map( {k: i for i, k in enumerate(data[col].unique(), 0)})
# let's capture the first letter data['Cabin_reduced'] = data['Cabin'].astype(str).str[0]
# Now I will replace the letters in the reduced cabin variable # create replace dictionary cabin_dict = {k: i for i, k in enumerate(X_train['Cabin_reduced'].unique(), 0)} # replace labels by numbers with dictionary X_train.loc[:, 'Cabin_reduced'] = X_train.loc[:, 'Cabin_reduced'].map(cabin_dict) X_test.loc[:, 'Cabin_reduced'] = X_test.loc[:, 'Cabin_reduced'].map(cabin_dict)