From 1e5efda7b2cef15f2489b8e009df9d8ef2df7f62 Mon Sep 17 00:00:00 2001 From: anpolol Date: Thu, 7 Dec 2023 19:26:35 +0300 Subject: [PATCH] now embeddings can be constructed for geom gcn and for general purpose + saving of embeddings added --- stable_gnn/embedding/embedding_factory.py | 11 +- .../embedding/model_train_embeddings.py | 14 +- .../embedding/sampling/abstract_samplers.py | 1 - stable_gnn/geom_gcn.py | 1 - stable_gnn/model_link_predict.py | 2 +- stable_gnn/model_nc.py | 15 +- tests/test_general/test_link_prediction.py | 16 +- tutorials/tutorial_node_classification.ipynb | 143 +++--------------- 8 files changed, 54 insertions(+), 149 deletions(-) diff --git a/stable_gnn/embedding/embedding_factory.py b/stable_gnn/embedding/embedding_factory.py index 58efddb..d039e51 100644 --- a/stable_gnn/embedding/embedding_factory.py +++ b/stable_gnn/embedding/embedding_factory.py @@ -13,8 +13,8 @@ class EmbeddingFactory: """Producing unsupervised embeddings for a given dataset""" @staticmethod - def _build_embeddings(loss: Dict[str, Any], data: Graph, conv: str, device: device,number_of_trials: int) -> NDArray: - optuna_training = OptunaTrainEmbeddings(data=data, conv=conv, device=device, loss_function=loss) + def _build_embeddings(loss: Dict[str, Any], data: Graph, conv: str, device: device,number_of_trials: int, tune_out: bool=False) -> NDArray: + optuna_training = OptunaTrainEmbeddings(data=data, conv=conv, device=device, loss_function=loss,tune_out=tune_out) best_values = optuna_training.run(number_of_trials=number_of_trials) loss_trgt = dict() @@ -28,7 +28,7 @@ def _build_embeddings(loss: Dict[str, Any], data: Graph, conv: str, device: devi if "lmbda" in loss_trgt: loss_trgt["lmbda"] = best_values["lmbda"] - model_training = ModelTrainEmbeddings(data=data, conv=conv, device=device, loss_function=loss_trgt) + model_training = ModelTrainEmbeddings(data=data, conv=conv, device=device, loss_function=loss_trgt,tune_out=tune_out) out = model_training.run(best_values) torch.cuda.empty_cache() return out.detach().cpu().numpy() @@ -77,7 +77,7 @@ def _get_emb_settings(loss_name: str) -> Dict[str, Any]: else: raise NameError - def build_embeddings(self, loss_name: str, conv: str, data: Graph, device: device, number_of_trials: int) -> NDArray: + def build_embeddings(self, loss_name: str, conv: str, data: Graph, device: device, number_of_trials: int, tune_out: bool=False) -> NDArray: """Build embeddings based on passed dataset and settings :param loss_name: (str): Name of loss function for embedding learning in GeomGCN layer @@ -85,8 +85,9 @@ def build_embeddings(self, loss_name: str, conv: str, data: Graph, device: devic :param data: (Graph): Input Graph :param device: (device): Device 'cuda' or 'cpu' :param number_of_trials (int): Number of trials for optuna tuning embeddings + :param tune_out (bool): Flag if you want tune out layer of embeddings :returns: (NDArray) embeddings NumPy array of (N_nodes) x (N_emb_dim) """ loss_params = self._get_emb_settings(loss_name) - emb = self._build_embeddings(loss=loss_params, data=data[0], conv=conv, device=device, number_of_trials=number_of_trials) + emb = self._build_embeddings(loss=loss_params, data=data[0], conv=conv, device=device, number_of_trials=number_of_trials, tune_out=tune_out) return emb diff --git a/stable_gnn/embedding/model_train_embeddings.py b/stable_gnn/embedding/model_train_embeddings.py index 1a85048..359e175 100644 --- a/stable_gnn/embedding/model_train_embeddings.py +++ b/stable_gnn/embedding/model_train_embeddings.py @@ -21,9 +21,10 @@ class ModelTrainEmbeddings: :param loss_function: (dict): Dict of parameters of unsupervised loss function :param conv: (str): Name of convolution (default:'GCN') :param device: (device): Either 'cuda' or 'cpu' (default:'cuda') + :param tune_out: (bool): Flag if you want tuning out layer or if it 2 for GeomGCN """ - def __init__(self, data: Graph, loss_function: Dict, device: device, conv: str = "GCN") -> None: + def __init__(self, data: Graph, loss_function: Dict, device: device, conv: str = "GCN", tune_out: bool=False) -> None: self.conv = conv self.device = device self.x = data.x @@ -31,6 +32,7 @@ def __init__(self, data: Graph, loss_function: Dict, device: device, conv: str = self.data = data.to(device) self.train_mask = torch.Tensor([True] * data.num_nodes) self.loss = loss_function + self.tune_out = tune_out super(ModelTrainEmbeddings, self).__init__() def _sampling(self, sampler: BaseSampler, epoch: int, nodes: Tensor) -> None: @@ -76,7 +78,10 @@ def run(self, params: Dict) -> Tensor: :return: (Tensor): The output embeddings """ hidden_layer = params["hidden_layer"] - out_layer = params["out_layer"] + if self.tune_out: + out_layer = params["out_layer"] + else: + out_layer=2 dropout = params["dropout"] size = params["size of network, number of convs"] learning_rate = params["lr"] @@ -121,7 +126,10 @@ def _objective(self, trial: Trial) -> Tensor: dropout = trial.suggest_float("dropout", 0.0, 0.5, step=0.1) size = trial.suggest_categorical("size of network, number of convs", [1, 2, 3]) learning_rate = trial.suggest_float("lr", 5e-3, 1e-2) - out_layer = trial.suggest_categorical("out_layer", [32, 64, 128]) + if self.tune_out: + out_layer = trial.suggest_categorical("out_layer", [32, 64, 128]) + else: + out_layer = 2 loss_to_train = {} for name in self.loss: diff --git a/stable_gnn/embedding/sampling/abstract_samplers.py b/stable_gnn/embedding/sampling/abstract_samplers.py index ce721c3..c49cf79 100644 --- a/stable_gnn/embedding/sampling/abstract_samplers.py +++ b/stable_gnn/embedding/sampling/abstract_samplers.py @@ -99,7 +99,6 @@ def _sample_negative(self, batch: Tensor, num_negative_samples: int) -> Tensor: :param num_negative_samples: (int): number of negative samples for each edge :return: (Tensor): Negative samples """ - print("self device", self.device, batch) a, _ = subgraph(batch, self.data.edge_index.to(self.device)) adj = self._adj_list(a) g = dict() diff --git a/stable_gnn/geom_gcn.py b/stable_gnn/geom_gcn.py index 6af7f89..6dfb826 100644 --- a/stable_gnn/geom_gcn.py +++ b/stable_gnn/geom_gcn.py @@ -84,7 +84,6 @@ def _virtual_vertex(self, edge_index: Tensor, x: Union[Tensor, OptPairTensor]) - x = (x, x) graph_size = max(edge_index[0].max(), edge_index[1].max()) + 1 deg = degree(edge_index[0], graph_size) - print("degree", deg) ( edge_index_s_ur, edge_index_s_ul, diff --git a/stable_gnn/model_link_predict.py b/stable_gnn/model_link_predict.py index 1f35698..f6e107a 100644 --- a/stable_gnn/model_link_predict.py +++ b/stable_gnn/model_link_predict.py @@ -46,7 +46,7 @@ def __init__( self.neg_samples_train = self._neg_samples(train_edges, self.data) self.embeddings = EmbeddingFactory().build_embeddings( - loss_name=loss_name, conv=emb_conv_name, data=dataset, device=device, number_of_trials=number_of_trials + loss_name=loss_name, conv=emb_conv_name, data=dataset, device=device, number_of_trials=number_of_trials, tune_out=True ) def _train_test_edges(self, data: Graph) -> (List[int], List[int]): diff --git a/stable_gnn/model_nc.py b/stable_gnn/model_nc.py index 8112ac9..97bb78e 100644 --- a/stable_gnn/model_nc.py +++ b/stable_gnn/model_nc.py @@ -12,7 +12,7 @@ from stable_gnn.embedding import EmbeddingFactory from stable_gnn.geom_gcn import GeomGCN from stable_gnn.graph import Graph - +import os class ModelNodeClassification(torch.nn.Module): """ @@ -52,9 +52,15 @@ def __init__( if self.ssl_flag: self.deg = degree(self.data[0].edge_index[0], self.data[0].num_nodes) - embeddings = EmbeddingFactory().build_embeddings( - loss_name=loss_name, conv=emb_conv_name, data=dataset, device=device - ) + path = '../tutorials/embeddings_'+str(loss_name)+'_'+str(emb_conv_name)+'.npy' + if os.path.exists(path): + embeddings = np.load(path) + else: + + embeddings = EmbeddingFactory().build_embeddings( + loss_name=loss_name, conv=emb_conv_name, data=dataset, device=device, number_of_trials=50 + ) + np.save(path,embeddings) if self.num_layers == 1: self.convs.append( @@ -109,6 +115,7 @@ def inference(self, data: Graph) -> Tuple[Tensor, Tensor]: x = conv(x, edge_index) if i != self.num_layers - 1: x = x.relu() + x = self.linear(x) x = self.linear_classifier(x) deg_pred = 0 if self.ssl_flag: diff --git a/tests/test_general/test_link_prediction.py b/tests/test_general/test_link_prediction.py index 7092fba..eed870a 100644 --- a/tests/test_general/test_link_prediction.py +++ b/tests/test_general/test_link_prediction.py @@ -10,18 +10,4 @@ generate_gc_graphs(root, 30) def test_linkpredict(): - name = "ba_gc" - data = Graph(root=root + name + "/", name=name, adjust_flag=False) - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - - ssl_flag = False - extrapolate_flag = True - - ####### - best_values = {"hidden_layer": 64, "size of network, number of convs": 2, "dropout": 0.0, "lr": 0.01, "coef": 10} - model_training = TrainModelGC(data=data, device=device, ssl_flag=ssl_flag, extrapolate_flag=extrapolate_flag) - - model, train_acc_mi, train_acc_ma, test_acc_mi, test_acc_ma = model_training.run(best_values) - torch.save(model, "model.pt") - assert train_acc_mi >= test_acc_mi - assert train_acc_ma >= test_acc_ma \ No newline at end of file + pass \ No newline at end of file diff --git a/tutorials/tutorial_node_classification.ipynb b/tutorials/tutorial_node_classification.ipynb index 57b0159..b5bee60 100644 --- a/tutorials/tutorial_node_classification.ipynb +++ b/tutorials/tutorial_node_classification.ipynb @@ -20,8 +20,8 @@ "name": "#%%\n" }, "ExecuteTime": { - "end_time": "2023-11-29T14:24:14.348553200Z", - "start_time": "2023-11-29T14:24:11.305515100Z" + "end_time": "2023-12-07T16:21:10.253077900Z", + "start_time": "2023-12-07T16:21:07.023005300Z" } }, "outputs": [], @@ -68,8 +68,8 @@ "name": "#%%\n" }, "ExecuteTime": { - "end_time": "2023-11-29T14:24:14.360563700Z", - "start_time": "2023-11-29T14:24:14.348553200Z" + "end_time": "2023-12-07T16:21:10.281387300Z", + "start_time": "2023-12-07T16:21:10.253077900Z" } } }, @@ -88,104 +88,22 @@ { "cell_type": "code", "execution_count": 3, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001B[32m[I 2023-11-29 17:24:16,506]\u001B[0m A new study created in memory with name: no-name-ff06b5e8-e04d-49e0-a1d6-7d09aef76d6f\u001B[0m\n", - "\u001B[32m[I 2023-11-29 17:24:16,621]\u001B[0m A new study created in memory with name: no-name-547ff3ac-0559-4122-82df-d2c45a1f0f21\u001B[0m\n", - "\u001B[32m[I 2023-11-29 17:24:23,140]\u001B[0m Trial 0 finished with value: 2.356937885284424 and parameters: {'hidden_layer': 32, 'dropout': 0.0, 'size of network, number of convs': 2, 'lr': 0.007412957996260659, 'num_negative_samples': 11, 'alpha': 0.7}. Best is trial 0 with value: 2.356937885284424.\u001B[0m\n", - "\u001B[32m[I 2023-11-29 17:24:35,221]\u001B[0m Trial 1 finished with value: 1.8919364213943481 and parameters: {'hidden_layer': 128, 'dropout': 0.1, 'size of network, number of convs': 2, 'lr': 0.005109701674965539, 'num_negative_samples': 1, 'alpha': 0.3}. Best is trial 1 with value: 1.8919364213943481.\u001B[0m\n", - "\u001B[32m[I 2023-11-29 17:24:45,113]\u001B[0m Trial 2 finished with value: 2.1624960899353027 and parameters: {'hidden_layer': 128, 'dropout': 0.2, 'size of network, number of convs': 2, 'lr': 0.00604857076394486, 'num_negative_samples': 16, 'alpha': 0.6}. Best is trial 1 with value: 1.8919364213943481.\u001B[0m\n", - "\u001B[32m[I 2023-11-29 17:24:52,489]\u001B[0m Trial 3 finished with value: 2.691725254058838 and parameters: {'hidden_layer': 64, 'dropout': 0.4, 'size of network, number of convs': 3, 'lr': 0.008643535181168449, 'num_negative_samples': 1, 'alpha': 0.8}. Best is trial 1 with value: 1.8919364213943481.\u001B[0m\n", - "\u001B[32m[I 2023-11-29 17:25:00,914]\u001B[0m Trial 4 finished with value: 2.353304862976074 and parameters: {'hidden_layer': 128, 'dropout': 0.1, 'size of network, number of convs': 3, 'lr': 0.007918278099052645, 'num_negative_samples': 6, 'alpha': 0.7}. Best is trial 1 with value: 1.8919364213943481.\u001B[0m\n", - "\u001B[32m[I 2023-11-29 17:25:13,071]\u001B[0m Trial 5 finished with value: 1.889955759048462 and parameters: {'hidden_layer': 32, 'dropout': 0.5, 'size of network, number of convs': 3, 'lr': 0.0061305287221789925, 'num_negative_samples': 1, 'alpha': 0.3}. Best is trial 5 with value: 1.889955759048462.\u001B[0m\n", - "\u001B[32m[I 2023-11-29 17:25:23,577]\u001B[0m Trial 6 finished with value: 2.034879446029663 and parameters: {'hidden_layer': 128, 'dropout': 0.2, 'size of network, number of convs': 1, 'lr': 0.009027752467678132, 'num_negative_samples': 6, 'alpha': 0.5}. Best is trial 5 with value: 1.889955759048462.\u001B[0m\n", - "\u001B[32m[I 2023-11-29 17:25:33,050]\u001B[0m Trial 7 finished with value: 2.1607909202575684 and parameters: {'hidden_layer': 128, 'dropout': 0.4, 'size of network, number of convs': 1, 'lr': 0.0073211503991801475, 'num_negative_samples': 16, 'alpha': 0.6}. Best is trial 5 with value: 1.889955759048462.\u001B[0m\n", - "\u001B[32m[I 2023-11-29 17:25:45,208]\u001B[0m Trial 8 finished with value: 1.8940885066986084 and parameters: {'hidden_layer': 64, 'dropout': 0.30000000000000004, 'size of network, number of convs': 1, 'lr': 0.005482774669845893, 'num_negative_samples': 1, 'alpha': 0.3}. Best is trial 5 with value: 1.889955759048462.\u001B[0m\n", - "\u001B[32m[I 2023-11-29 17:25:54,877]\u001B[0m Trial 9 finished with value: 2.1644275188446045 and parameters: {'hidden_layer': 32, 'dropout': 0.30000000000000004, 'size of network, number of convs': 2, 'lr': 0.008750999710792755, 'num_negative_samples': 16, 'alpha': 0.6}. Best is trial 5 with value: 1.889955759048462.\u001B[0m\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "degree tensor([31., 33., 36., 23., 7., 34., 26., 17., 19., 19., 14., 17., 29., 20.,\n", - " 29., 21., 18., 4., 24., 28., 13., 2., 3., 35., 26., 15., 28., 13.,\n", - " 26., 12., 28., 27., 17., 18., 15., 14., 20., 13., 27., 36., 26., 15.,\n", - " 30., 4., 29., 18., 15., 3., 16., 21., 25., 0., 39., 41., 1., 24.,\n", - " 19., 21., 14., 24., 21., 29., 29., 10., 5., 34., 26., 33., 4., 18.,\n", - " 11., 35., 18., 19., 20., 1., 17., 32., 27., 20., 21., 26., 25., 41.,\n", - " 27., 5., 22., 0., 30., 33., 25., 23., 40., 13., 25., 15., 13., 24.,\n", - " 29., 6., 33., 6., 22., 23., 25., 2., 7., 17., 28., 27., 19., 31.,\n", - " 33., 14., 32., 13., 21., 6., 20., 22., 28., 36., 27., 3., 16., 27.,\n", - " 8., 33., 13., 19., 12., 22., 23., 10., 46., 4., 2., 7., 29., 26.,\n", - " 24., 14., 5., 5., 19., 17., 16., 28., 14., 2., 16., 2., 23., 7.,\n", - " 15., 4., 32., 23., 10., 22., 8., 24., 3., 23., 37., 25., 14., 8.,\n", - " 2., 21., 24., 30., 16., 14., 32., 27., 23., 2., 14., 9., 23., 28.,\n", - " 16., 5., 45., 18., 36., 27., 37., 25., 22., 11., 34., 33., 22., 23.,\n", - " 30., 18., 16., 15., 15., 16., 30., 0., 20., 4., 10., 34., 19., 17.,\n", - " 34., 22., 37., 18., 35., 26., 33., 9., 6., 25., 21., 16., 34., 28.,\n", - " 24., 16., 19., 26., 0., 10., 15., 10., 23., 8., 19., 19., 3., 18.,\n", - " 10., 30., 27., 41., 31., 24., 26., 24., 7., 28., 11., 17., 9., 31.,\n", - " 28., 2., 10., 2., 13., 31., 33., 18., 26., 23., 2., 16., 15., 21.,\n", - " 19., 32., 9., 8., 22., 2., 19., 4., 20., 22., 18., 6., 23., 32.,\n", - " 29., 23., 28., 48., 31., 21., 14., 31., 26., 31., 38., 4., 26., 15.,\n", - " 19., 17., 27., 24., 20., 37., 34., 7., 20., 18., 35., 26., 13., 38.,\n", - " 20., 20., 10., 28., 23., 5., 17., 14., 29., 2., 19., 31., 25., 18.,\n", - " 25., 4., 26., 25., 7., 34., 3., 7., 2., 14., 13., 8., 30., 23.,\n", - " 38., 22., 30., 27., 36., 7., 5., 26., 24., 14., 30., 10., 6., 32.,\n", - " 25., 33., 10., 25., 2., 27., 3., 9., 8., 0., 4., 18., 9., 2.,\n", - " 30., 1., 6., 35., 6., 1., 15., 17., 1., 7., 16., 15., 15., 2.,\n", - " 3., 35., 4., 24., 19., 28., 19., 12., 8., 19., 17., 21., 24., 35.,\n", - " 20., 29., 13., 22.], device='cuda:0')\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001B[33m[W 2023-11-29 17:26:08,086]\u001B[0m Trial 0 failed because of the following error: RuntimeError('mat1 and mat2 shapes cannot be multiplied (396x256 and 128x4)')\u001B[0m\n", - "Traceback (most recent call last):\n", - " File \"C:\\Users\\user\\Desktop\\StableGNN\\venv\\lib\\site-packages\\optuna\\study\\_optimize.py\", line 213, in _run_trial\n", - " value_or_values = func(trial)\n", - " File \"C:\\Users\\user\\Desktop\\StableGNN\\stable_gnn\\pipelines\\node_classification_pipeline.py\", line 166, in _objective\n", - " _ = self.train(model, optimizer, coef)\n", - " File \"C:\\Users\\user\\Desktop\\StableGNN\\stable_gnn\\pipelines\\node_classification_pipeline.py\", line 58, in train\n", - " out, deg_pred = model.inference(self.data[0].to(self.device))\n", - " File \"C:\\Users\\user\\Desktop\\StableGNN\\stable_gnn\\model_nc.py\", line 112, in inference\n", - " x = self.linear_classifier(x)\n", - " File \"C:\\Users\\user\\Desktop\\StableGNN\\venv\\lib\\site-packages\\torch\\nn\\modules\\module.py\", line 1501, in _call_impl\n", - " return forward_call(*args, **kwargs)\n", - " File \"C:\\Users\\user\\Desktop\\StableGNN\\venv\\lib\\site-packages\\torch\\nn\\modules\\linear.py\", line 114, in forward\n", - " return F.linear(input, self.weight, self.bias)\n", - "RuntimeError: mat1 and mat2 shapes cannot be multiplied (396x256 and 128x4)\n" - ] - }, - { - "ename": "RuntimeError", - "evalue": "mat1 and mat2 shapes cannot be multiplied (396x256 and 128x4)", - "output_type": "error", - "traceback": [ - "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m", - "\u001B[1;31mRuntimeError\u001B[0m Traceback (most recent call last)", - "Cell \u001B[1;32mIn[3], line 7\u001B[0m\n\u001B[0;32m 4\u001B[0m ssl_flag \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mFalse\u001B[39;00m\n\u001B[0;32m 6\u001B[0m optuna_training \u001B[38;5;241m=\u001B[39m TrainModelOptunaNC(data\u001B[38;5;241m=\u001B[39mdataset, device\u001B[38;5;241m=\u001B[39mdevice, ssl_flag\u001B[38;5;241m=\u001B[39mssl_flag, loss_name\u001B[38;5;241m=\u001B[39mloss_name)\n\u001B[1;32m----> 7\u001B[0m params \u001B[38;5;241m=\u001B[39m \u001B[43moptuna_training\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrun\u001B[49m\u001B[43m(\u001B[49m\u001B[43mnumber_of_trials\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;241;43m10\u001B[39;49m\u001B[43m)\u001B[49m\n\u001B[0;32m 8\u001B[0m model_training \u001B[38;5;241m=\u001B[39m TrainModelNC(data\u001B[38;5;241m=\u001B[39mdataset, device\u001B[38;5;241m=\u001B[39mdevice, ssl_flag\u001B[38;5;241m=\u001B[39mssl_flag, loss_name\u001B[38;5;241m=\u001B[39mloss_name)\n\u001B[0;32m 9\u001B[0m model, train_acc_mi, train_acc_ma, test_acc_mi, test_acc_ma \u001B[38;5;241m=\u001B[39m model_training\u001B[38;5;241m.\u001B[39mrun(params, plot_training_procces\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m)\n", - "File \u001B[1;32m~\\Desktop\\StableGNN\\stable_gnn\\pipelines\\node_classification_pipeline.py:180\u001B[0m, in \u001B[0;36mTrainModelOptunaNC.run\u001B[1;34m(self, number_of_trials)\u001B[0m\n\u001B[0;32m 173\u001B[0m \u001B[38;5;250m\u001B[39m\u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[0;32m 174\u001B[0m \u001B[38;5;124;03mOptimize hyperparameters for graph classification task training pipelines\u001B[39;00m\n\u001B[0;32m 175\u001B[0m \n\u001B[0;32m 176\u001B[0m \u001B[38;5;124;03m:param number_of_trials: (int): Number of trials for Optuna\u001B[39;00m\n\u001B[0;32m 177\u001B[0m \u001B[38;5;124;03m:return: (dict): Dictionary of input parameters for Model: size of hidden layer, dropout, number of layers in the model and learning rate\u001B[39;00m\n\u001B[0;32m 178\u001B[0m \u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[0;32m 179\u001B[0m study \u001B[38;5;241m=\u001B[39m optuna\u001B[38;5;241m.\u001B[39mcreate_study(direction\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mmaximize\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[1;32m--> 180\u001B[0m \u001B[43mstudy\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43moptimize\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_objective\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mn_trials\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mnumber_of_trials\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 181\u001B[0m trial \u001B[38;5;241m=\u001B[39m study\u001B[38;5;241m.\u001B[39mbest_trial\n\u001B[0;32m 182\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m trial\u001B[38;5;241m.\u001B[39mparams\n", - "File \u001B[1;32m~\\Desktop\\StableGNN\\venv\\lib\\site-packages\\optuna\\study\\study.py:400\u001B[0m, in \u001B[0;36mStudy.optimize\u001B[1;34m(self, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001B[0m\n\u001B[0;32m 392\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m n_jobs \u001B[38;5;241m!=\u001B[39m \u001B[38;5;241m1\u001B[39m:\n\u001B[0;32m 393\u001B[0m warnings\u001B[38;5;241m.\u001B[39mwarn(\n\u001B[0;32m 394\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m`n_jobs` argument has been deprecated in v2.7.0. \u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[0;32m 395\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mThis feature will be removed in v4.0.0. \u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[0;32m 396\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mSee https://github.com/optuna/optuna/releases/tag/v2.7.0.\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[0;32m 397\u001B[0m \u001B[38;5;167;01mFutureWarning\u001B[39;00m,\n\u001B[0;32m 398\u001B[0m )\n\u001B[1;32m--> 400\u001B[0m \u001B[43m_optimize\u001B[49m\u001B[43m(\u001B[49m\n\u001B[0;32m 401\u001B[0m \u001B[43m \u001B[49m\u001B[43mstudy\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\n\u001B[0;32m 402\u001B[0m \u001B[43m \u001B[49m\u001B[43mfunc\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mfunc\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 403\u001B[0m \u001B[43m \u001B[49m\u001B[43mn_trials\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mn_trials\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 404\u001B[0m \u001B[43m \u001B[49m\u001B[43mtimeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mtimeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 405\u001B[0m \u001B[43m \u001B[49m\u001B[43mn_jobs\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mn_jobs\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 406\u001B[0m \u001B[43m \u001B[49m\u001B[43mcatch\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mcatch\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 407\u001B[0m \u001B[43m \u001B[49m\u001B[43mcallbacks\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mcallbacks\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 408\u001B[0m \u001B[43m \u001B[49m\u001B[43mgc_after_trial\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mgc_after_trial\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 409\u001B[0m \u001B[43m \u001B[49m\u001B[43mshow_progress_bar\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mshow_progress_bar\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 410\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n", - "File \u001B[1;32m~\\Desktop\\StableGNN\\venv\\lib\\site-packages\\optuna\\study\\_optimize.py:66\u001B[0m, in \u001B[0;36m_optimize\u001B[1;34m(study, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001B[0m\n\u001B[0;32m 64\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m 65\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m n_jobs \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m1\u001B[39m:\n\u001B[1;32m---> 66\u001B[0m \u001B[43m_optimize_sequential\u001B[49m\u001B[43m(\u001B[49m\n\u001B[0;32m 67\u001B[0m \u001B[43m \u001B[49m\u001B[43mstudy\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 68\u001B[0m \u001B[43m \u001B[49m\u001B[43mfunc\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 69\u001B[0m \u001B[43m \u001B[49m\u001B[43mn_trials\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 70\u001B[0m \u001B[43m \u001B[49m\u001B[43mtimeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 71\u001B[0m \u001B[43m \u001B[49m\u001B[43mcatch\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 72\u001B[0m \u001B[43m \u001B[49m\u001B[43mcallbacks\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 73\u001B[0m \u001B[43m \u001B[49m\u001B[43mgc_after_trial\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 74\u001B[0m \u001B[43m \u001B[49m\u001B[43mreseed_sampler_rng\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[0;32m 75\u001B[0m \u001B[43m \u001B[49m\u001B[43mtime_start\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mNone\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[0;32m 76\u001B[0m \u001B[43m \u001B[49m\u001B[43mprogress_bar\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mprogress_bar\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 77\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 78\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m 79\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m show_progress_bar:\n", - "File \u001B[1;32m~\\Desktop\\StableGNN\\venv\\lib\\site-packages\\optuna\\study\\_optimize.py:163\u001B[0m, in \u001B[0;36m_optimize_sequential\u001B[1;34m(study, func, n_trials, timeout, catch, callbacks, gc_after_trial, reseed_sampler_rng, time_start, progress_bar)\u001B[0m\n\u001B[0;32m 160\u001B[0m \u001B[38;5;28;01mbreak\u001B[39;00m\n\u001B[0;32m 162\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m--> 163\u001B[0m trial \u001B[38;5;241m=\u001B[39m \u001B[43m_run_trial\u001B[49m\u001B[43m(\u001B[49m\u001B[43mstudy\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfunc\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mcatch\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 164\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mException\u001B[39;00m:\n\u001B[0;32m 165\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m\n", - "File \u001B[1;32m~\\Desktop\\StableGNN\\venv\\lib\\site-packages\\optuna\\study\\_optimize.py:264\u001B[0m, in \u001B[0;36m_run_trial\u001B[1;34m(study, func, catch)\u001B[0m\n\u001B[0;32m 261\u001B[0m \u001B[38;5;28;01massert\u001B[39;00m \u001B[38;5;28;01mFalse\u001B[39;00m, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mShould not reach.\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[0;32m 263\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m state \u001B[38;5;241m==\u001B[39m TrialState\u001B[38;5;241m.\u001B[39mFAIL \u001B[38;5;129;01mand\u001B[39;00m func_err \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(func_err, catch):\n\u001B[1;32m--> 264\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m func_err\n\u001B[0;32m 265\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m trial\n", - "File \u001B[1;32m~\\Desktop\\StableGNN\\venv\\lib\\site-packages\\optuna\\study\\_optimize.py:213\u001B[0m, in \u001B[0;36m_run_trial\u001B[1;34m(study, func, catch)\u001B[0m\n\u001B[0;32m 210\u001B[0m thread\u001B[38;5;241m.\u001B[39mstart()\n\u001B[0;32m 212\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m--> 213\u001B[0m value_or_values \u001B[38;5;241m=\u001B[39m \u001B[43mfunc\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtrial\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 214\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m exceptions\u001B[38;5;241m.\u001B[39mTrialPruned \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[0;32m 215\u001B[0m \u001B[38;5;66;03m# TODO(mamu): Handle multi-objective cases.\u001B[39;00m\n\u001B[0;32m 216\u001B[0m state \u001B[38;5;241m=\u001B[39m TrialState\u001B[38;5;241m.\u001B[39mPRUNED\n", - "File \u001B[1;32m~\\Desktop\\StableGNN\\stable_gnn\\pipelines\\node_classification_pipeline.py:166\u001B[0m, in \u001B[0;36mTrainModelOptunaNC._objective\u001B[1;34m(self, trial)\u001B[0m\n\u001B[0;32m 163\u001B[0m optimizer \u001B[38;5;241m=\u001B[39m torch\u001B[38;5;241m.\u001B[39moptim\u001B[38;5;241m.\u001B[39mAdam(model\u001B[38;5;241m.\u001B[39mparameters(), lr\u001B[38;5;241m=\u001B[39mlearning_rate, weight_decay\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m1e-5\u001B[39m)\n\u001B[0;32m 165\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m epoch \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mrange\u001B[39m(\u001B[38;5;241m50\u001B[39m):\n\u001B[1;32m--> 166\u001B[0m _ \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtrain\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmodel\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43moptimizer\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mcoef\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 167\u001B[0m val_acc_mi, val_acc_ma \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mtest(model, mask\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mval_mask)\n\u001B[0;32m 169\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m val_acc_mi\n", - "File \u001B[1;32m~\\Desktop\\StableGNN\\stable_gnn\\pipelines\\node_classification_pipeline.py:58\u001B[0m, in \u001B[0;36mTrainModelNC.train\u001B[1;34m(self, model, optimizer, coef)\u001B[0m\n\u001B[0;32m 56\u001B[0m optimizer\u001B[38;5;241m.\u001B[39mzero_grad()\n\u001B[0;32m 57\u001B[0m total_loss \u001B[38;5;241m=\u001B[39m \u001B[38;5;241m0\u001B[39m\n\u001B[1;32m---> 58\u001B[0m out, deg_pred \u001B[38;5;241m=\u001B[39m \u001B[43mmodel\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43minference\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdata\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;241;43m0\u001B[39;49m\u001B[43m]\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mto\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdevice\u001B[49m\u001B[43m)\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 60\u001B[0m y \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39my\u001B[38;5;241m.\u001B[39mtype(torch\u001B[38;5;241m.\u001B[39mLongTensor)\n\u001B[0;32m 61\u001B[0m y \u001B[38;5;241m=\u001B[39m y\u001B[38;5;241m.\u001B[39mto(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mdevice)\n", - "File \u001B[1;32m~\\Desktop\\StableGNN\\stable_gnn\\model_nc.py:112\u001B[0m, in \u001B[0;36mModelNodeClassification.inference\u001B[1;34m(self, data)\u001B[0m\n\u001B[0;32m 110\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m i \u001B[38;5;241m!=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mnum_layers \u001B[38;5;241m-\u001B[39m \u001B[38;5;241m1\u001B[39m:\n\u001B[0;32m 111\u001B[0m x \u001B[38;5;241m=\u001B[39m x\u001B[38;5;241m.\u001B[39mrelu()\n\u001B[1;32m--> 112\u001B[0m x \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mlinear_classifier\u001B[49m\u001B[43m(\u001B[49m\u001B[43mx\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 113\u001B[0m deg_pred \u001B[38;5;241m=\u001B[39m \u001B[38;5;241m0\u001B[39m\n\u001B[0;32m 114\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mssl_flag:\n", - "File \u001B[1;32m~\\Desktop\\StableGNN\\venv\\lib\\site-packages\\torch\\nn\\modules\\module.py:1501\u001B[0m, in \u001B[0;36mModule._call_impl\u001B[1;34m(self, *args, **kwargs)\u001B[0m\n\u001B[0;32m 1496\u001B[0m \u001B[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001B[39;00m\n\u001B[0;32m 1497\u001B[0m \u001B[38;5;66;03m# this function, and just call forward.\u001B[39;00m\n\u001B[0;32m 1498\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m (\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_backward_hooks \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_backward_pre_hooks \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_forward_hooks \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_forward_pre_hooks\n\u001B[0;32m 1499\u001B[0m \u001B[38;5;129;01mor\u001B[39;00m _global_backward_pre_hooks \u001B[38;5;129;01mor\u001B[39;00m _global_backward_hooks\n\u001B[0;32m 1500\u001B[0m \u001B[38;5;129;01mor\u001B[39;00m _global_forward_hooks \u001B[38;5;129;01mor\u001B[39;00m _global_forward_pre_hooks):\n\u001B[1;32m-> 1501\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m forward_call(\u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n\u001B[0;32m 1502\u001B[0m \u001B[38;5;66;03m# Do not call functions when jit is used\u001B[39;00m\n\u001B[0;32m 1503\u001B[0m full_backward_hooks, non_full_backward_hooks \u001B[38;5;241m=\u001B[39m [], []\n", - "File \u001B[1;32m~\\Desktop\\StableGNN\\venv\\lib\\site-packages\\torch\\nn\\modules\\linear.py:114\u001B[0m, in \u001B[0;36mLinear.forward\u001B[1;34m(self, input)\u001B[0m\n\u001B[0;32m 113\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mforward\u001B[39m(\u001B[38;5;28mself\u001B[39m, \u001B[38;5;28minput\u001B[39m: Tensor) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m Tensor:\n\u001B[1;32m--> 114\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mF\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mlinear\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43minput\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mweight\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mbias\u001B[49m\u001B[43m)\u001B[49m\n", - "\u001B[1;31mRuntimeError\u001B[0m: mat1 and mat2 shapes cannot be multiplied (396x256 and 128x4)" - ] - } + "outputs": [], + "source": [ + "results = pd.DataFrame(columns=['adjust_flag', 'ssl_flag','test accuracy'])" ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-12-07T16:21:10.297014400Z", + "start_time": "2023-12-07T16:21:10.281387300Z" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], "source": [ "device=torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", "\n", @@ -203,34 +121,21 @@ "collapsed": false, "pycharm": { "name": "#%%\n" - }, - "ExecuteTime": { - "end_time": "2023-11-29T14:26:08.992839900Z", - "start_time": "2023-11-29T14:24:16.505228300Z" } } }, { "cell_type": "code", - "execution_count": 8, - "outputs": [ - { - "data": { - "text/plain": "" - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": 5, + "outputs": [], "source": [ - "optuna_training" + "torch.save(model,'model_nc.pt')" ], "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-28T14:08:07.960447700Z", - "start_time": "2023-11-28T14:08:07.929441300Z" + "end_time": "2023-12-07T16:19:34.694945600Z", + "start_time": "2023-12-07T16:19:34.679328300Z" } } },