gcn_peermlp_comparison.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6yUIMXI3yMSC"
      },
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/snap-research/MLPInit-for-GNNs/blob/main/gcn_peermlp_comparison.ipynb)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! pip uninstall -y torch\n",
        "! pip uninstall -y torchvision torchtext torchdata torchaudio fastai"
      ],
      "metadata": {
        "id": "rWvAkFpkyO9x",
        "outputId": "d9cacc69-34ab-4333-e81a-d72b7c1bda94",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Found existing installation: torch 2.0.0+cu118\n",
            "Uninstalling torch-2.0.0+cu118:\n",
            "  Successfully uninstalled torch-2.0.0+cu118\n",
            "Found existing installation: torchvision 0.15.1+cu118\n",
            "Uninstalling torchvision-0.15.1+cu118:\n",
            "  Successfully uninstalled torchvision-0.15.1+cu118\n",
            "Found existing installation: torchtext 0.15.1\n",
            "Uninstalling torchtext-0.15.1:\n",
            "  Successfully uninstalled torchtext-0.15.1\n",
            "Found existing installation: torchdata 0.6.0\n",
            "Uninstalling torchdata-0.6.0:\n",
            "  Successfully uninstalled torchdata-0.6.0\n",
            "Found existing installation: torchaudio 2.0.1+cu118\n",
            "Uninstalling torchaudio-2.0.1+cu118:\n",
            "  Successfully uninstalled torchaudio-2.0.1+cu118\n",
            "Found existing installation: fastai 2.7.12\n",
            "Uninstalling fastai-2.7.12:\n",
            "  Successfully uninstalled fastai-2.7.12\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "VTl8U-ncyMSE",
        "outputId": "ed9f83bd-ff8a-46ad-8152-12fb458a1908",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Looking in links: https://download.pytorch.org/whl/torch_stable.html\n",
            "Collecting torch==1.9.0+cu111\n",
            "  Downloading https://download.pytorch.org/whl/cu111/torch-1.9.0%2Bcu111-cp39-cp39-linux_x86_64.whl (2041.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 GB\u001b[0m \u001b[31m819.6 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.9/dist-packages (from torch==1.9.0+cu111) (4.5.0)\n",
            "Installing collected packages: torch\n",
            "Successfully installed torch-1.9.0+cu111\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting torch-scatter==2.0.9\n",
            "  Downloading https://data.pyg.org/whl/torch-1.9.0%2Bcu111/torch_scatter-2.0.9-cp39-cp39-linux_x86_64.whl (10.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.4/10.4 MB\u001b[0m \u001b[31m59.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: torch-scatter\n",
            "Successfully installed torch-scatter-2.0.9\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting torch-sparse==0.6.12\n",
            "  Downloading https://data.pyg.org/whl/torch-1.9.0%2Bcu111/torch_sparse-0.6.12-cp39-cp39-linux_x86_64.whl (3.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.7/3.7 MB\u001b[0m \u001b[31m26.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.9/dist-packages (from torch-sparse==0.6.12) (1.10.1)\n",
            "Requirement already satisfied: numpy<1.27.0,>=1.19.5 in /usr/local/lib/python3.9/dist-packages (from scipy->torch-sparse==0.6.12) (1.22.4)\n",
            "Installing collected packages: torch-sparse\n",
            "Successfully installed torch-sparse-0.6.12\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting torch-geometric==2.0.4\n",
            "  Downloading torch_geometric-2.0.4.tar.gz (407 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m407.5/407.5 KB\u001b[0m \u001b[31m10.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.9/dist-packages (from torch-geometric==2.0.4) (4.65.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.9/dist-packages (from torch-geometric==2.0.4) (1.22.4)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.9/dist-packages (from torch-geometric==2.0.4) (1.10.1)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (from torch-geometric==2.0.4) (1.4.4)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.9/dist-packages (from torch-geometric==2.0.4) (3.1.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from torch-geometric==2.0.4) (2.27.1)\n",
            "Requirement already satisfied: pyparsing in /usr/local/lib/python3.9/dist-packages (from torch-geometric==2.0.4) (3.0.9)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.9/dist-packages (from torch-geometric==2.0.4) (1.2.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.9/dist-packages (from jinja2->torch-geometric==2.0.4) (2.1.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/dist-packages (from pandas->torch-geometric==2.0.4) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas->torch-geometric==2.0.4) (2022.7.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->torch-geometric==2.0.4) (3.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->torch-geometric==2.0.4) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests->torch-geometric==2.0.4) (2.0.12)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->torch-geometric==2.0.4) (1.26.15)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.9/dist-packages (from scikit-learn->torch-geometric==2.0.4) (1.1.1)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.9/dist-packages (from scikit-learn->torch-geometric==2.0.4) (3.1.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.9/dist-packages (from python-dateutil>=2.8.1->pandas->torch-geometric==2.0.4) (1.16.0)\n",
            "Building wheels for collected packages: torch-geometric\n",
            "  Building wheel for torch-geometric (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for torch-geometric: filename=torch_geometric-2.0.4-py3-none-any.whl size=616602 sha256=c8e3fb213b9eb0714970ddbc18c8dd0f242c4f96de3b6712f7ce741276561744\n",
            "  Stored in directory: /root/.cache/pip/wheels/ba/44/59/28a61b516dbc37ed93285752c61c2cf1724ba0a0dada6c1150\n",
            "Successfully built torch-geometric\n",
            "Installing collected packages: torch-geometric\n",
            "Successfully installed torch-geometric-2.0.4\n"
          ]
        }
      ],
      "source": [
        "! pip install torch==1.9.0+cu111 -f https://download.pytorch.org/whl/torch_stable.html\n",
        "! pip install https://data.pyg.org/whl/torch-1.9.0%2Bcu111/torch_scatter-2.0.9-cp39-cp39-linux_x86_64.whl\n",
        "! pip install https://data.pyg.org/whl/torch-1.9.0%2Bcu111/torch_sparse-0.6.12-cp39-cp39-linux_x86_64.whl\n",
        "! pip install torch-geometric==2.0.4\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "DXKGuxgkyMSE"
      },
      "outputs": [],
      "source": [
        "import os.path as osp\n",
        "\n",
        "import torch\n",
        "import torch.nn.functional as F\n",
        "from torch_geometric.datasets import Planetoid\n",
        "import torch_geometric.transforms as T\n",
        "from typing import Optional, Tuple\n",
        "from torch_geometric.typing import Adj, OptTensor, PairTensor\n",
        "\n",
        "import torch\n",
        "from torch import Tensor\n",
        "from torch.nn import Parameter\n",
        "from torch_scatter import scatter_add\n",
        "from torch_sparse import SparseTensor, matmul, fill_diag, sum as sparsesum, mul\n",
        "from torch_geometric.nn.conv import MessagePassing\n",
        "from torch_geometric.utils import add_remaining_self_loops\n",
        "from torch_geometric.utils.num_nodes import maybe_num_nodes\n",
        "import math"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "TwQI3pKdyMSF"
      },
      "outputs": [],
      "source": [
        "\n",
        "\n",
        "def glorot(tensor):\n",
        "    if tensor is not None:\n",
        "        stdv = math.sqrt(6.0 / (tensor.size(-2) + tensor.size(-1)))\n",
        "        tensor.data.uniform_(-stdv, stdv)\n",
        "\n",
        "\n",
        "def glorot_orthogonal(tensor, scale):\n",
        "    if tensor is not None:\n",
        "        torch.nn.init.orthogonal_(tensor.data)\n",
        "        scale /= ((tensor.size(-2) + tensor.size(-1)) * tensor.var())\n",
        "        tensor.data *= scale.sqrt()\n",
        "\n",
        "\n",
        "def zeros(tensor):\n",
        "    if tensor is not None:\n",
        "        tensor.data.fill_(0)\n",
        "\n",
        "\n",
        "\n",
        "@torch.jit._overload\n",
        "def gcn_norm(edge_index, edge_weight=None, num_nodes=None, improved=False,\n",
        "             add_self_loops=True, dtype=None):\n",
        "    # type: (Tensor, OptTensor, Optional[int], bool, bool, Optional[int]) -> PairTensor  # noqa\n",
        "    pass\n",
        "\n",
        "\n",
        "@torch.jit._overload\n",
        "def gcn_norm(edge_index, edge_weight=None, num_nodes=None, improved=False,\n",
        "             add_self_loops=True, dtype=None):\n",
        "    # type: (SparseTensor, OptTensor, Optional[int], bool, bool, Optional[int]) -> SparseTensor  # noqa\n",
        "    pass\n",
        "\n",
        "\n",
        "def gcn_norm(edge_index, edge_weight=None, num_nodes=None, improved=False,\n",
        "             add_self_loops=True, dtype=None):\n",
        "\n",
        "    fill_value = 2. if improved else 1.\n",
        "\n",
        "    if isinstance(edge_index, SparseTensor):\n",
        "        adj_t = edge_index\n",
        "        if not adj_t.has_value():\n",
        "            adj_t = adj_t.fill_value(1., dtype=dtype)\n",
        "        if add_self_loops:\n",
        "            adj_t = fill_diag(adj_t, fill_value)\n",
        "        deg = sparsesum(adj_t, dim=1)\n",
        "        deg_inv_sqrt = deg.pow_(-0.5)\n",
        "        deg_inv_sqrt.masked_fill_(deg_inv_sqrt == float('inf'), 0.)\n",
        "        adj_t = mul(adj_t, deg_inv_sqrt.view(-1, 1))\n",
        "        adj_t = mul(adj_t, deg_inv_sqrt.view(1, -1))\n",
        "        return adj_t\n",
        "\n",
        "    else:\n",
        "        num_nodes = maybe_num_nodes(edge_index, num_nodes)\n",
        "\n",
        "        if edge_weight is None:\n",
        "            edge_weight = torch.ones((edge_index.size(1), ), dtype=dtype,\n",
        "                                     device=edge_index.device)\n",
        "\n",
        "        if add_self_loops:\n",
        "            edge_index, tmp_edge_weight = add_remaining_self_loops(\n",
        "                edge_index, edge_weight, fill_value, num_nodes)\n",
        "            assert tmp_edge_weight is not None\n",
        "            edge_weight = tmp_edge_weight\n",
        "\n",
        "        row, col = edge_index[0], edge_index[1]\n",
        "        deg = scatter_add(edge_weight, col, dim=0, dim_size=num_nodes)\n",
        "        deg_inv_sqrt = deg.pow_(-0.5)\n",
        "        deg_inv_sqrt.masked_fill_(deg_inv_sqrt == float('inf'), 0)\n",
        "        return edge_index, deg_inv_sqrt[row] * edge_weight * deg_inv_sqrt[col]\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "class GCNConv(MessagePassing):\n",
        "    \n",
        "\n",
        "    _cached_edge_index: Optional[Tuple[Tensor, Tensor]]\n",
        "    _cached_adj_t: Optional[SparseTensor]\n",
        "\n",
        "    def __init__(self, in_channels: int, out_channels: int,\n",
        "                 improved: bool = False, cached: bool = False,\n",
        "                 add_self_loops: bool = True, normalize: bool = True,\n",
        "                 bias: bool = True, **kwargs):\n",
        "\n",
        "        kwargs.setdefault('aggr', 'add')\n",
        "        super(GCNConv, self).__init__(**kwargs)\n",
        "\n",
        "        self.in_channels = in_channels\n",
        "        self.out_channels = out_channels\n",
        "        self.improved = improved\n",
        "        self.cached = cached\n",
        "        self.add_self_loops = add_self_loops\n",
        "        self.normalize = normalize\n",
        "\n",
        "        self._cached_edge_index = None\n",
        "        self._cached_adj_t = None\n",
        "\n",
        "        self.weight = Parameter(torch.Tensor(in_channels, out_channels))\n",
        "\n",
        "        if bias:\n",
        "            self.bias = Parameter(torch.Tensor(out_channels))\n",
        "        else:\n",
        "            self.register_parameter('bias', None)\n",
        "\n",
        "        self.reset_parameters()\n",
        "\n",
        "    def reset_parameters(self):\n",
        "        glorot(self.weight)\n",
        "        zeros(self.bias)\n",
        "        self._cached_edge_index = None\n",
        "        self._cached_adj_t = None\n",
        "\n",
        "    def forward(self, x: Tensor, edge_index: Adj,\n",
        "                edge_weight: OptTensor = None) -> Tensor:\n",
        "        \"\"\"\"\"\"\n",
        "\n",
        "        if self.normalize:\n",
        "            if isinstance(edge_index, Tensor):\n",
        "                cache = self._cached_edge_index\n",
        "                if cache is None:\n",
        "                    edge_index, edge_weight = gcn_norm(  # yapf: disable\n",
        "                        edge_index, edge_weight, x.size(self.node_dim),\n",
        "                        self.improved, self.add_self_loops)\n",
        "                    if self.cached:\n",
        "                        self._cached_edge_index = (edge_index, edge_weight)\n",
        "                else:\n",
        "                    edge_index, edge_weight = cache[0], cache[1]\n",
        "\n",
        "            elif isinstance(edge_index, SparseTensor):\n",
        "                cache = self._cached_adj_t\n",
        "                if cache is None:\n",
        "                    edge_index = gcn_norm(  # yapf: disable\n",
        "                        edge_index, edge_weight, x.size(self.node_dim),\n",
        "                        self.improved, self.add_self_loops)\n",
        "                    if self.cached:\n",
        "                        self._cached_adj_t = edge_index\n",
        "                else:\n",
        "                    edge_index = cache\n",
        "\n",
        "        x = x @ self.weight\n",
        "\n",
        "        # propagate_type: (x: Tensor, edge_weight: OptTensor)\n",
        "        out = self.propagate(edge_index, x=x, edge_weight=edge_weight,\n",
        "                            size=None)\n",
        "\n",
        "        if self.bias is not None:\n",
        "            out += self.bias\n",
        "\n",
        "        return out\n",
        "\n",
        "    def message(self, x_j: Tensor, edge_weight: OptTensor) -> Tensor:\n",
        "        return x_j if edge_weight is None else edge_weight.view(-1, 1) * x_j\n",
        "\n",
        "    def message_and_aggregate(self, adj_t: SparseTensor, x: Tensor) -> Tensor:\n",
        "        return matmul(adj_t, x, reduce=self.aggr)\n",
        "\n",
        "    def __repr__(self):\n",
        "        return '{}({}, {})'.format(self.__class__.__name__, self.in_channels,\n",
        "                                   self.out_channels)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "class GCNConv_MLP(MessagePassing):\n",
        "    \n",
        "\n",
        "    _cached_edge_index: Optional[Tuple[Tensor, Tensor]]\n",
        "    _cached_adj_t: Optional[SparseTensor]\n",
        "\n",
        "    def __init__(self, in_channels: int, out_channels: int,\n",
        "                 improved: bool = False, cached: bool = False,\n",
        "                 add_self_loops: bool = True, normalize: bool = True,\n",
        "                 bias: bool = True, **kwargs):\n",
        "\n",
        "        kwargs.setdefault('aggr', 'add')\n",
        "        super(GCNConv_MLP, self).__init__(**kwargs)\n",
        "\n",
        "        self.in_channels = in_channels\n",
        "        self.out_channels = out_channels\n",
        "        self.improved = improved\n",
        "        self.cached = cached\n",
        "        self.add_self_loops = add_self_loops\n",
        "        self.normalize = normalize\n",
        "\n",
        "        self._cached_edge_index = None\n",
        "        self._cached_adj_t = None\n",
        "\n",
        "        self.weight = Parameter(torch.Tensor(in_channels, out_channels))\n",
        "\n",
        "        if bias:\n",
        "            self.bias = Parameter(torch.Tensor(out_channels))\n",
        "        else:\n",
        "            self.register_parameter('bias', None)\n",
        "\n",
        "        self.reset_parameters()\n",
        "\n",
        "    def reset_parameters(self):\n",
        "        glorot(self.weight)\n",
        "        zeros(self.bias)\n",
        "        self._cached_edge_index = None\n",
        "        self._cached_adj_t = None\n",
        "\n",
        "    def forward(self, x: Tensor, edge_index: Adj,\n",
        "                edge_weight: OptTensor = None) -> Tensor:\n",
        "        \"\"\"\"\"\"\n",
        "\n",
        "        if self.normalize:\n",
        "            if isinstance(edge_index, Tensor):\n",
        "                cache = self._cached_edge_index\n",
        "                if cache is None:\n",
        "                    edge_index, edge_weight = gcn_norm(  # yapf: disable\n",
        "                        edge_index, edge_weight, x.size(self.node_dim),\n",
        "                        self.improved, self.add_self_loops)\n",
        "                    if self.cached:\n",
        "                        self._cached_edge_index = (edge_index, edge_weight)\n",
        "                else:\n",
        "                    edge_index, edge_weight = cache[0], cache[1]\n",
        "\n",
        "            elif isinstance(edge_index, SparseTensor):\n",
        "                cache = self._cached_adj_t\n",
        "                if cache is None:\n",
        "                    edge_index = gcn_norm(  # yapf: disable\n",
        "                        edge_index, edge_weight, x.size(self.node_dim),\n",
        "                        self.improved, self.add_self_loops)\n",
        "                    if self.cached:\n",
        "                        self._cached_adj_t = edge_index\n",
        "                else:\n",
        "                    edge_index = cache\n",
        "\n",
        "        x = x @ self.weight\n",
        "\n",
        "        # propagate_type: (x: Tensor, edge_weight: OptTensor)\n",
        "        # out = self.propagate(edge_index, x=x, edge_weight=edge_weight,\n",
        "        #                      size=None)\n",
        "\n",
        "        out = x\n",
        "\n",
        "        if self.bias is not None:\n",
        "            out += self.bias\n",
        "\n",
        "        return out\n",
        "\n",
        "    def message(self, x_j: Tensor, edge_weight: OptTensor) -> Tensor:\n",
        "        return x_j if edge_weight is None else edge_weight.view(-1, 1) * x_j\n",
        "\n",
        "    def message_and_aggregate(self, adj_t: SparseTensor, x: Tensor) -> Tensor:\n",
        "        return matmul(adj_t, x, reduce=self.aggr)\n",
        "\n",
        "    def __repr__(self):\n",
        "        return '{}({}, {})'.format(self.__class__.__name__, self.in_channels,\n",
        "                                   self.out_channels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "F0D0mMhPyMSG",
        "outputId": "5a11bca6-8efd-4b00-f04b-8654430d070a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Downloading https://github.com/kimiyoung/planetoid/raw/master/data/ind.cora.x\n",
            "Downloading https://github.com/kimiyoung/planetoid/raw/master/data/ind.cora.tx\n",
            "Downloading https://github.com/kimiyoung/planetoid/raw/master/data/ind.cora.allx\n",
            "Downloading https://github.com/kimiyoung/planetoid/raw/master/data/ind.cora.y\n",
            "Downloading https://github.com/kimiyoung/planetoid/raw/master/data/ind.cora.ty\n",
            "Downloading https://github.com/kimiyoung/planetoid/raw/master/data/ind.cora.ally\n",
            "Downloading https://github.com/kimiyoung/planetoid/raw/master/data/ind.cora.graph\n",
            "Downloading https://github.com/kimiyoung/planetoid/raw/master/data/ind.cora.test.index\n",
            "Processing...\n",
            "Done!\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 001, Train: 0.7357, Val: 0.2880, Test: 0.2910\n",
            "Epoch: 002, Train: 0.9786, Val: 0.4560, Test: 0.4740\n",
            "Epoch: 003, Train: 1.0000, Val: 0.5360, Test: 0.5250\n",
            "Epoch: 004, Train: 0.9929, Val: 0.5360, Test: 0.5250\n",
            "Epoch: 005, Train: 0.9929, Val: 0.5360, Test: 0.5250\n",
            "Epoch: 006, Train: 0.9929, Val: 0.5360, Test: 0.5250\n",
            "Epoch: 007, Train: 1.0000, Val: 0.5360, Test: 0.5250\n",
            "Epoch: 008, Train: 1.0000, Val: 0.5360, Test: 0.5250\n",
            "Epoch: 009, Train: 1.0000, Val: 0.5720, Test: 0.5570\n",
            "Epoch: 010, Train: 1.0000, Val: 0.5860, Test: 0.5660\n",
            "Epoch: 011, Train: 1.0000, Val: 0.6000, Test: 0.5750\n",
            "Epoch: 012, Train: 1.0000, Val: 0.6000, Test: 0.5750\n",
            "Epoch: 013, Train: 1.0000, Val: 0.6000, Test: 0.5750\n",
            "Epoch: 014, Train: 1.0000, Val: 0.6000, Test: 0.5750\n",
            "Epoch: 015, Train: 1.0000, Val: 0.6000, Test: 0.5750\n",
            "Epoch: 016, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 017, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 018, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 019, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 020, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 021, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 022, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 023, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 024, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 025, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 026, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 027, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 028, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 029, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 030, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 031, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 032, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 033, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 034, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 035, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 036, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 037, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 038, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 039, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 040, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 041, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 042, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 043, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 044, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 045, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 046, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 047, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 048, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 049, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 050, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 051, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 052, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 053, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 054, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 055, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 056, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 057, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 058, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 059, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 060, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 061, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 062, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 063, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 064, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 065, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 066, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 067, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 068, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 069, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 070, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 071, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 072, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 073, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 074, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 075, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 076, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 077, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 078, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 079, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 080, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 081, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 082, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 083, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 084, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 085, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 086, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 087, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 088, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 089, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 090, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 091, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 092, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 093, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 094, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 095, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 096, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 097, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 098, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 099, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 100, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 101, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 102, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 103, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 104, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 105, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 106, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 107, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 108, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 109, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 110, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 111, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 112, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 113, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 114, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 115, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 116, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 117, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 118, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 119, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 120, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 121, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 122, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 123, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 124, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 125, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 126, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 127, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 128, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 129, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 130, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 131, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 132, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 133, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 134, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 135, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 136, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 137, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 138, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 139, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 140, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 141, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 142, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 143, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 144, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 145, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 146, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 147, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 148, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 149, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 150, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 151, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 152, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 153, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 154, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 155, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 156, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 157, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 158, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 159, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 160, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 161, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 162, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 163, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 164, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 165, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 166, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 167, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 168, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 169, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 170, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 171, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 172, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 173, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 174, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 175, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 176, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 177, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 178, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 179, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 180, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 181, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 182, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 183, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 184, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 185, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 186, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 187, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 188, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 189, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 190, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 191, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 192, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 193, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 194, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 195, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 196, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 197, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 198, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 199, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "Epoch: 200, Train: 1.0000, Val: 0.6020, Test: 0.5720\n",
            "#############\n",
            "PeerMLP                   , Train: 1.0000, Val: 0.5700, Test: 0.5920\n",
            "GCN with Weight of PeerMLP, Train: 0.9000, Val: 0.7840, Test: 0.7920\n",
            "#############\n"
          ]
        }
      ],
      "source": [
        "dataset = 'Cora'\n",
        "# dataset = 'CiteSeer'\n",
        "# dataset = 'PubMed'\n",
        "path = osp.join('./data', dataset)\n",
        "dataset = Planetoid(path, dataset, transform=T.NormalizeFeatures())\n",
        "data = dataset[0]\n",
        "\n",
        "use_gdc = True\n",
        "gdc = T.GDC(self_loop_weight=1, normalization_in='sym',\n",
        "            normalization_out='col',\n",
        "            diffusion_kwargs=dict(method='ppr', alpha=0.05),\n",
        "            sparsification_kwargs=dict(method='topk', k=128,\n",
        "                                    dim=0), exact=True)\n",
        "data = gdc(data)\n",
        "\n",
        "\n",
        "class Net_GCN(torch.nn.Module):\n",
        "    def __init__(self):\n",
        "        super(Net_GCN, self).__init__()\n",
        "        self.conv1 = GCNConv(dataset.num_features, 128, cached=True,\n",
        "                            normalize=not use_gdc)\n",
        "        self.conv2 = GCNConv(128, dataset.num_classes, cached=True,\n",
        "                            normalize=not use_gdc)\n",
        "\n",
        "        # self.conv1 = ChebConv(data.num_features, 16, K=2)\n",
        "        # self.conv2 = ChebConv(16, data.num_features, K=2)\n",
        "\n",
        "    def forward(self):\n",
        "        x, edge_index, edge_weight = data.x, data.edge_index, data.edge_attr\n",
        "        x = F.relu(self.conv1(x, edge_index, edge_weight))\n",
        "        x = F.dropout(x, training=self.training)\n",
        "        x = self.conv2(x, edge_index, edge_weight)\n",
        "        return F.log_softmax(x, dim=1)\n",
        "\n",
        "\n",
        "class Net_MLP(torch.nn.Module):\n",
        "    def __init__(self):\n",
        "        super(Net_MLP, self).__init__()\n",
        "        self.conv1 = GCNConv_MLP(dataset.num_features, 128, cached=True,\n",
        "                            normalize=not use_gdc)\n",
        "        self.conv2 = GCNConv_MLP(128, dataset.num_classes, cached=True,\n",
        "                            normalize=not use_gdc)\n",
        "\n",
        "        # self.conv1 = ChebConv(data.num_features, 16, K=2)\n",
        "        # self.conv2 = ChebConv(16, data.num_features, K=2)\n",
        "\n",
        "    def forward(self):\n",
        "        x, edge_index, edge_weight = data.x, data.edge_index, data.edge_attr\n",
        "        x = F.relu(self.conv1(x, edge_index, edge_weight))\n",
        "        x = F.dropout(x, training=self.training)\n",
        "        x = self.conv2(x, edge_index, edge_weight)\n",
        "        return F.log_softmax(x, dim=1)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "model = Net_MLP().to(device)\n",
        "data =  data.to(device)\n",
        "\n",
        "optimizer = torch.optim.Adam([\n",
        "    dict(params=model.conv1.parameters(), weight_decay=5e-4),\n",
        "    dict(params=model.conv2.parameters(), weight_decay=0)\n",
        "], lr=0.01)  # Only perform weight-decay on first convolution.\n",
        "\n",
        "\n",
        "def train():\n",
        "    model.train()\n",
        "    optimizer.zero_grad()\n",
        "    F.nll_loss(model()[data.train_mask], data.y[data.train_mask]).backward()\n",
        "    optimizer.step()\n",
        "\n",
        "\n",
        "@torch.no_grad()\n",
        "def test():\n",
        "    model.eval()\n",
        "    logits, accs = model(), []\n",
        "    for _, mask in data('train_mask', 'val_mask', 'test_mask'):\n",
        "        pred = logits[mask].max(1)[1]\n",
        "        acc = pred.eq(data.y[mask]).sum().item() / mask.sum().item()\n",
        "        accs.append(acc)\n",
        "    return accs\n",
        "\n",
        "\n",
        "best_val_acc = test_acc = 0\n",
        "for epoch in range(1, 201):\n",
        "    train()\n",
        "    train_acc, val_acc, tmp_test_acc = test()\n",
        "    if val_acc > best_val_acc:\n",
        "        best_val_acc = val_acc\n",
        "        test_acc = tmp_test_acc\n",
        "\n",
        "        weight_of_peermlp = model.state_dict()\n",
        "\n",
        "    log = 'Epoch: {:03d}, Train: {:.4f}, Val: {:.4f}, Test: {:.4f}'\n",
        "    print(log.format(epoch, train_acc, best_val_acc, test_acc))\n",
        "\n",
        "\n",
        "print( \"#############\" )\n",
        "model = Net_MLP().to(device)\n",
        "model.load_state_dict( weight_of_peermlp )\n",
        "train_acc, val_acc, test_acc = test()\n",
        "log = 'PeerMLP                   , Train: {:.4f}, Val: {:.4f}, Test: {:.4f}'\n",
        "print(log.format(train_acc, val_acc, test_acc))\n",
        "\n",
        "\n",
        "model = Net_GCN().to(device)\n",
        "model.load_state_dict( weight_of_peermlp )\n",
        "train_acc, val_acc, test_acc = test()\n",
        "log = 'GCN with Weight of PeerMLP, Train: {:.4f}, Val: {:.4f}, Test: {:.4f}'\n",
        "print(log.format(train_acc, val_acc, test_acc))\n",
        "print( \"#############\" )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0cVcNcEEyMSH"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.16"
    },
    "orig_nbformat": 4,
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}