"soil_bayesian_merge_padel_reduced.drop(columns=['Unnamed: 0', 'topoShape'], inplace=True) #contains subset of all target variable (without pH), and reduced descriptors\n",
"soil_bayesian_merge_padel_reduced"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[]"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# find the difference descriptor in both datasets\n",
" cluster_ids = hierarchy.fcluster(dist_linkage, CLUSTER_THRESHOLD, criterion=\"distance\") # return a list\n",
" print(\"Cludter_ids:\", cluster_ids)\n",
" cluster_id_to_feature_ids = defaultdict(list) # create a default dict\n",
"\n",
" for idx, cluster_id in enumerate(cluster_ids): #generate the tuple [(0, \"\"), (1, \"\"), (2, \"\")]\n",
" cluster_id_to_feature_ids[cluster_id].append(idx) # use the 'cluster_id' as the key\n",
" selected_feature_ids = [v[0] for v in cluster_id_to_feature_ids.values()] # Each v is a list that might contins single or multiple cluster_id, here only thake the first element v[0] as the representative cludter_id \n",
" \n",
" # Transform the inupt_df into np.array and tranpose so that the list in each row is the data of each features.\n",
" # We use the [i] to select the features that list in selected_features_ids.\n",
" # After the List Comprehension, you need to convert the list back to the np.array so that you can transpose to original direction of the column and row.\n",
" new_X = np.array([np.array(input_df).T[i] for i in selected_feature_ids]).T \n",
" selected_features = [feature_names[i] for i in selected_feature_ids]\n",
"\n",
" print('Descriptors after applying cluster threshold of {}: {}'.format(CLUSTER_THRESHOLD, len(selected_features)))\n",
" D = {'new_X': new_X, 'selected_features': selected_features}\n",
" # When you run the combined learning, you first check the collinearity on soil and sludge together since the concatenated dataset might have some undiscover relationship. \n",
" selected_X_combined_padel_array, selected_features_combined = check_collinearity(X_combined_padel) # return the nd.array and list of selected features # cluster threshold: 0.02, Descriptors: 1085 \n",
"c:\\Users\\leetseng\\Anaconda3\\envs\\envipath\\lib\\site-packages\\statsmodels\\regression\\linear_model.py:1736: RuntimeWarning: invalid value encountered in double_scalars\n",
" return 1 - self.ssr/self.centered_tss\n",
"c:\\Users\\leetseng\\Anaconda3\\envs\\envipath\\lib\\site-packages\\statsmodels\\stats\\outliers_influence.py:195: RuntimeWarning: divide by zero encountered in double_scalars\n",
"# # X_train_CL = pd.concat([X_train_sludge, soil_bayesian_merge_padel_reduced.iloc[:, 2:]], join='inner', ignore_index=True) # We lost 30 cpds in this step\n",
" # Concate the split sludge data with soil data\n",
" X_train_CL = pd.concat([X_train_sludge, soil_bayesian_merge_padel_reduced.iloc[:, 2:]], join='inner', ignore_index=True) # We lost 30 cpds in this step\n",
"C:\\Users\\leetseng\\AppData\\Local\\Temp\\ipykernel_37560\\2690038075.py:33: DeprecationWarning: elementwise comparison failed; this will raise an error in the future.\n",
" if np.all(l.get_xdata() == [xmin, xmax]):\n"
]
},
{
"data": {
"text/plain": [
"Text(0.5, 1.0, 'Combined Learning & top 40 features of PaDEL')"
"plt.title('Combined Learning & top 40 features of PaDEL', fontsize=20)"
]
},
{
"cell_type": "code",
"execution_count": 178,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\leetseng\\AppData\\Local\\Temp\\ipykernel_37560\\2690038075.py:33: DeprecationWarning: elementwise comparison failed; this will raise an error in the future.\n",
" if np.all(l.get_xdata() == [xmin, xmax]):\n"
]
},
{
"data": {
"text/plain": [
"Text(0.5, 1.0, 'Combined Learning & features of PaDEL')"