Skip to content

Commit

Permalink
batch evaluation for test set
Browse files Browse the repository at this point in the history
  • Loading branch information
@sohompaul
sohompaul committed Jun 13, 2020
1 parent c480466 commit c4e2b01
Showing 1 changed file with 52 additions and 80 deletions.
132 changes: 52 additions & 80 deletions multitask/QM9GNN2_Multitask.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
"source": [
"import pickle\n",
"from os import path\n",
"from time import time\n",
"import itertools\n",
"\n",
"import numpy as np\n",
"from sklearn.model_selection import train_test_split\n",
Expand Down Expand Up @@ -43,7 +45,7 @@
" X_uniq = X_uniq[X_uniq != 0]\n",
" E_uniq = np.unique(E_all)\n",
" E_uniq = E_uniq[E_uniq != 0]\n",
"\n",
" \n",
" X_all = label_to_one_hot(X_all, X_uniq)\n",
" E_all = label_to_one_hot(E_all, E_uniq)\n",
" \n",
Expand Down Expand Up @@ -75,7 +77,6 @@
"def standardize(y):\n",
" task_to_scaler = dict()\n",
" for task in list(y.columns)[1:]:\n",
" # scaler = StandardScaler()\n",
" scaler = PowerTransformer()\n",
" y.loc[:, task] = scaler.fit_transform(y[[task]])\n",
" task_to_scaler[task] = scaler\n",
Expand Down Expand Up @@ -140,16 +141,17 @@
"metadata": {},
"outputs": [],
"source": [
"def build_hard_sharing_model(*, A, X, E, num_tasks, learning_rate=1e-3, loss='mse'):\n",
"def build_hard_sharing_model(*, A, X, E, num_tasks, \n",
" learning_rate=1e-3, loss='mse'):\n",
" X_in, A_in, E_in = get_input_tensors(A=A, X=X, E=E)\n",
"\n",
" gc1 = EdgeConditionedConv(64, activation='relu')([X_in, A_in, E_in])\n",
" gc2 = EdgeConditionedConv(128, activation='relu')([gc1, A_in, E_in])\n",
" pool = GlobalAttentionPool(256)(gc2)\n",
" dense_list = [Dense(256, activation='relu')(pool) for i in range(num_tasks)]\n",
" dense_list = [Dense(256, activation='relu')(pool) \n",
" for i in range(num_tasks)]\n",
" output_list = [Dense(1)(dense_layer) for dense_layer in dense_list]\n",
"\n",
" # Build model\n",
" model = Model(inputs=[X_in, A_in, E_in], outputs=output_list)\n",
" optimizer = Adam(lr=learning_rate)\n",
" model.compile(optimizer=optimizer, loss=loss)\n",
Expand All @@ -163,25 +165,27 @@
"metadata": {},
"outputs": [],
"source": [
"def build_soft_sharing_model(*, A, X, E, num_tasks, share_param, learning_rate=1e-3, loss='mse'):\n",
"def build_soft_sharing_model(*, A, X, E, num_tasks, share_param, \n",
" learning_rate=1e-3, loss='mse'):\n",
" X_in, A_in, E_in = get_input_tensors(A=A, X=X, E=E)\n",
"\n",
" gc1_list = [EdgeConditionedConv(64, activation='relu')([X_in, A_in, E_in]) for i in range(num_tasks)]\n",
" gc2_list = [EdgeConditionedConv(128, activation='relu')([gc1, A_in, E_in]) for gc1 in gc1_list]\n",
" gc1_list = [EdgeConditionedConv(64, activation='relu')([X_in, A_in, E_in]) \n",
" for i in range(num_tasks)]\n",
" gc2_list = [EdgeConditionedConv(128, activation='relu')([gc1, A_in, E_in]) \n",
" for gc1 in gc1_list]\n",
" pool_list = [GlobalAttentionPool(256)(gc2) for gc2 in gc2_list]\n",
" dense_list = [Dense(256, activation='relu')(pool) for pool in pool_list]\n",
" output_list = [Dense(1)(dense) for dense in dense_list]\n",
"\n",
" def loss(y_actual, y_pred):\n",
" avg_layer_diff = 0\n",
" for i in range(num_tasks):\n",
" for j in range(i):\n",
" for gc in [gc1_list, gc2_list]:\n",
" avg_layer_diff += mean(square(gc[i].trainable_weights - gc[j].trainable_weights))\n",
" for i, j in itertools.combinations(range(num_tasks), 2):\n",
" for gc in [gc1_list, gc2_list]:\n",
" diff = gc[i].trainable_weights - gc[j].trainable_weights\n",
" avg_layer_diff += mean(square(diff))\n",
" avg_layer_diff /= (num_tasks)*(num_tasks-1)/2 \n",
" return mean(square(y_actual - y_pred)) + share_param*avg_layer_diff\n",
"\n",
" # Build model\n",
" model = Model(inputs=[X_in, A_in, E_in], outputs=output_list)\n",
" optimizer = Adam(lr=learning_rate)\n",
" model.compile(optimizer=optimizer, loss=loss)\n",
Expand Down Expand Up @@ -214,8 +218,6 @@
" for task in tasks:\n",
" scaler_filename = generate_task_scaler_filename(task)\n",
" with open(scaler_filename, 'wb') as f:\n",
"# print(task_to_scaler[task].mean_[0], file=f)\n",
"# print(task_to_scaler[task].scale_[0], file=f)\n",
" scaler = task_to_scaler[task]\n",
" pickle.dump(obj=scaler, file=f)\n",
"\n",
Expand All @@ -235,32 +237,25 @@
"metadata": {},
"outputs": [],
"source": [
"def predict_property(prop, mol_id, clusters, task_to_scaler=dict(), *, X_all, A_all, E_all):\n",
" for cluster in clusters:\n",
" if prop in cluster:\n",
" model, task_to_scaler = load_hard_sharing_model(A=A_all, X=X_all, E=E_all, tasks=cluster, task_to_scaler=task_to_scaler)\n",
" i = mol_id - 1\n",
"def predict_property(prop, mol_id, clusters, *, X_all, A_all, E_all, \n",
" model=None, task_to_scaler=dict()):\n",
" cluster = [c for c in clusters if prop in c][0]\n",
" if model is None:\n",
" model, task_to_scaler = load_hard_sharing_model(\n",
" A=A_all, X=X_all, E=E_all, tasks=cluster, \n",
" task_to_scaler=task_to_scaler\n",
" )\n",
" i = mol_id - 1\n",
"\n",
" # convert shape for batch mode\n",
" def wrap(a):\n",
" return a.reshape([1] + list(a.shape))\n",
" x = list(map(wrap, [X_all[i], A_all[i], E_all[i]]))\n",
" \n",
" cluster_prediction = model.predict(x)\n",
" prediction = cluster_prediction[cluster.index(prop)]\n",
" prediction = task_to_scaler[prop].inverse_transform(prediction)\n",
" return prediction[0][0]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"learning_rate = 1e-3\n",
"epochs = 10\n",
"batch_size = 32"
" # convert shape for batch mode\n",
" def wrap(a):\n",
" return a.reshape([1] + list(a.shape))\n",
" x = list(map(wrap, [X_all[i], A_all[i], E_all[i]]))\n",
"\n",
" cluster_prediction = model.predict(x) \n",
" prediction = cluster_prediction[cluster.index(prop)]\n",
" prediction = task_to_scaler[prop].inverse_transform(prediction)\n",
" return prediction[0][0]"
]
},
{
Expand Down Expand Up @@ -326,13 +321,15 @@
" print('begin training models')\n",
" for cluster in clusters:\n",
" print(f'training {cluster}')\n",
" model = build_hard_sharing_model(A=A_train, X=X_train, E=E_train, num_tasks=len(cluster))\n",
" model = build_hard_sharing_model(\n",
" A=A_train, X=X_train, E=E_train, num_tasks=len(cluster)\n",
" )\n",
" y_train_cluster = np.hsplit(y_train[cluster].values, len(cluster))\n",
" model.fit(x=[X_train, A_train, E_train], \n",
" y=y_train_cluster,\n",
" batch_size=64,\n",
" batch_size=32,\n",
" validation_split=0.1,\n",
" epochs=3)\n",
" epochs=25)\n",
" save_model(model, cluster, task_to_scaler)"
]
},
Expand All @@ -344,50 +341,25 @@
"source": [
"if __name__ == '__main__' and '__file__' not in globals(): \n",
" for cluster in clusters:\n",
" model, _ = load_hard_sharing_model(A=A_test, X=X_test, E=E_test, tasks=cluster)\n",
" model, task_to_scaler = load_hard_sharing_model(\n",
" A=A_test, X=X_test, E=E_test, tasks=cluster, \n",
" task_to_scaler=task_to_scaler\n",
" )\n",
" y_test_cluster = np.hsplit(y_test[cluster].values, len(cluster))\n",
" model_loss = model.evaluate(x=[X_test, A_test, E_test],\n",
" y=y_test_cluster)\n",
" print(f\"Test loss: {model_loss}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"if __name__ == '__main__' and '__file__' not in globals():\n",
" print(predict_property('A', 1, clusters, task_to_scaler, X_all=X_all, A_all=A_all, E_all=E_all))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"if __name__ == '__main__' and '__file__' not in globals():\n",
" for cluster in clusters:\n",
" for prop in cluster:\n",
" print(prop)\n",
" print(f\"Test loss on {cluster}: {model_loss}\")\n",
" \n",
" cluster_pred = model.predict([X_test, A_test, E_test])\n",
" for prop, batch_pred in zip(cluster, cluster_pred):\n",
" batch_pred = task_to_scaler[prop].inverse_transform(batch_pred)\n",
" errors = list()\n",
" for index in np.random.choice(133885, 10):\n",
" mol_id = index+1\n",
" pred = predict_property(prop, mol_id, clusters, X_all=X_all, A_all=A_all, E_all=E_all)\n",
" for index, pred in zip(y_test.index.values, batch_pred):\n",
" actual = y_all.loc[index, prop]\n",
" err = abs((pred-actual)/actual)\n",
" print(f\"error on molecule {index+1} is {err:.2%}\")\n",
" errors.append(err)\n",
" print(f'avg error of {prop} is {sum(errors)/len(errors):.2%}')"
" errors.append(err[0])\n",
" print(f'Avg error of {prop} is {sum(errors)/len(errors):.2%}')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
Expand Down

0 comments on commit c4e2b01

Please sign in to comment.