PyTorch_Geometric.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E5T-P3Ur2TlI",
        "outputId": "b4cd8290-3483-4395-e4e8-296fa1e06eea"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2.0.1+cu118\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.2/10.2 MB\u001b[0m \u001b[31m44.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.8/4.8 MB\u001b[0m \u001b[31m20.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for torch_geometric (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ],
      "source": [
        "import os\n",
        "import torch\n",
        "os.environ['TORCH'] = torch.__version__\n",
        "print(torch.__version__)\n",
        "\n",
        "!pip install -q torch-scatter -f https://data.pyg.org/whl/torch-${TORCH}.html\n",
        "!pip install -q torch-sparse -f https://data.pyg.org/whl/torch-${TORCH}.html\n",
        "!pip install -q git+https://github.com/pyg-team/pytorch_geometric.git"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F"
      ],
      "metadata": {
        "id": "JVrQv1ne2k6s"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Structure"
      ],
      "metadata": {
        "id": "EMs8UgvF2mue"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class GATLayer(nn.Module):\n",
        "    \"\"\"\n",
        "    Simple PyTorch Implementation of the Graph Attention layer.\n",
        "    \"\"\"\n",
        "    def __init__(self):\n",
        "        super(GATLayer, self).__init__()\n",
        "      \n",
        "    def forward(self, input, adj):\n",
        "        print(\"\")"
      ],
      "metadata": {
        "id": "Hr6BLMJf2mZU"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Forward Method"
      ],
      "metadata": {
        "id": "is3mW5B92sa5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "in_features = 5\n",
        "out_features = 2\n",
        "nb_nodes = 3\n",
        "\n",
        "W = nn.Parameter(torch.zeros(size=(in_features, out_features))) #xavier paramiter inizializator\n",
        "nn.init.xavier_uniform_(W.data, gain=1.414)\n",
        "\n",
        "input = torch.rand(nb_nodes,in_features) \n",
        "\n",
        "\n",
        "# linear transformation\n",
        "h = torch.mm(input, W)\n",
        "N = h.size()[0]\n",
        "\n",
        "print(h.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MhpdM8aw2sF_",
        "outputId": "8dd9ada5-6a0e-4419-e4c8-bd5648fe6cbd"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([3, 2])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Attention Mechanism"
      ],
      "metadata": {
        "id": "RfgUztfG2y_V"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "a = nn.Parameter(torch.zeros(size=(2*out_features, 1))) #xavier paramiter inizializator\n",
        "nn.init.xavier_uniform_(a.data, gain=1.414)\n",
        "print(a.shape)\n",
        "\n",
        "leakyrelu = nn.LeakyReLU(0.2)  # LeakyReLU"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eG-6vWHF2yoU",
        "outputId": "50f6aadf-bb70-4aa9-edf9-89a28cb6c4e5"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([4, 1])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "a_input = torch.cat([h.repeat(1, N).view(N * N, -1), h.repeat(N, 1)], dim=1).view(N, -1, 2 * out_features)"
      ],
      "metadata": {
        "id": "L-IC5ZZq25dV"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "e = leakyrelu(torch.matmul(a_input, a).squeeze(2))"
      ],
      "metadata": {
        "id": "uvzJjtT12-dR"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(a_input.shape,a.shape)\n",
        "print(\"\")\n",
        "print(torch.matmul(a_input,a).shape)\n",
        "print(\"\")\n",
        "print(torch.matmul(a_input,a).squeeze(2).shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y-vX8oCI3AbB",
        "outputId": "c7d7bcdc-0533-409b-d50a-baa6457aebb5"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([3, 3, 4]) torch.Size([4, 1])\n",
            "\n",
            "torch.Size([3, 3, 1])\n",
            "\n",
            "torch.Size([3, 3])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Masked Attention"
      ],
      "metadata": {
        "id": "DovqmSQX3DW3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Masked Attention\n",
        "adj = torch.randint(2, (3, 3))\n",
        "\n",
        "zero_vec  = -9e15*torch.ones_like(e)\n",
        "print(zero_vec.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lsIpehJQ3Cnk",
        "outputId": "8a065c5d-1aa3-4448-97f5-00c68af1af2c"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([3, 3])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "attention = torch.where(adj > 0, e, zero_vec)\n",
        "print(adj,\"\\n\",e,\"\\n\",zero_vec)\n",
        "attention"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "myuFfuyM3G8r",
        "outputId": "671c85bd-2cd8-41ef-e24f-71ccb8b48604"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[0, 1, 0],\n",
            "        [0, 1, 0],\n",
            "        [0, 1, 1]]) \n",
            " tensor([[-0.3002, -0.1046, -0.2263],\n",
            "        [-0.2777, -0.0822, -0.2039],\n",
            "        [-0.2669, -0.0713, -0.1930]], grad_fn=<LeakyReluBackward0>) \n",
            " tensor([[-9.0000e+15, -9.0000e+15, -9.0000e+15],\n",
            "        [-9.0000e+15, -9.0000e+15, -9.0000e+15],\n",
            "        [-9.0000e+15, -9.0000e+15, -9.0000e+15]])\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[-9.0000e+15, -1.0461e-01, -9.0000e+15],\n",
              "        [-9.0000e+15, -8.2152e-02, -9.0000e+15],\n",
              "        [-9.0000e+15, -7.1330e-02, -1.9304e-01]], grad_fn=<WhereBackward0>)"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "attention = F.softmax(attention, dim=1)\n",
        "h_prime   = torch.matmul(attention, h)"
      ],
      "metadata": {
        "id": "JT8Il2nx3IeL"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "attention"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uEyEq06j3Kq1",
        "outputId": "1b1f5772-155b-426d-a35f-cae72c1ce2ef"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0.0000, 1.0000, 0.0000],\n",
              "        [0.0000, 1.0000, 0.0000],\n",
              "        [0.0000, 0.5304, 0.4696]], grad_fn=<SoftmaxBackward0>)"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "h_prime"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XnOZJ6ee3L9X",
        "outputId": "184e7ab6-e97a-40e1-b04c-9924baea69be"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 0.5208, -0.8006],\n",
              "        [ 0.5208, -0.8006],\n",
              "        [ 0.6788, -0.7385]], grad_fn=<MmBackward0>)"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# h_prime vs h"
      ],
      "metadata": {
        "id": "8fdRTTCx3OJU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(h_prime,\"\\n\",h)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CKs6FVc73NxL",
        "outputId": "eb07c473-8e4d-453d-eeb4-56ae87756622"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[ 0.5208, -0.8006],\n",
            "        [ 0.5208, -0.8006],\n",
            "        [ 0.6788, -0.7385]], grad_fn=<MmBackward0>) \n",
            " tensor([[ 1.2439, -0.7443],\n",
            "        [ 0.5208, -0.8006],\n",
            "        [ 0.8573, -0.6683]], grad_fn=<MmBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Building the layer"
      ],
      "metadata": {
        "id": "k5XU6Y963SGR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class GATLayer(nn.Module):\n",
        "    def __init__(self, in_features, out_features, dropout, alpha, concat=True):\n",
        "        super(GATLayer, self).__init__()\n",
        "        \n",
        "        '''\n",
        "        TODO\n",
        "        '''\n",
        "        \n",
        "    def forward(self, input, adj):\n",
        "        # Linear Transformation\n",
        "        h = torch.mm(input, self.W) # matrix multiplication\n",
        "        N = h.size()[0]\n",
        "\n",
        "        # Attention Mechanism\n",
        "        a_input = torch.cat([h.repeat(1, N).view(N * N, -1), h.repeat(N, 1)], dim=1).view(N, -1, 2 * self.out_features)\n",
        "        e       = self.leakyrelu(torch.matmul(a_input, self.a).squeeze(2))\n",
        "\n",
        "        # Masked Attention\n",
        "        zero_vec  = -9e15*torch.ones_like(e)\n",
        "        attention = torch.where(adj > 0, e, zero_vec)\n",
        "        \n",
        "        attention = F.softmax(attention, dim=1)\n",
        "        attention = F.dropout(attention, self.dropout, training=self.training)\n",
        "        h_prime   = torch.matmul(attention, h)\n",
        "\n",
        "        if self.concat:\n",
        "            return F.elu(h_prime)\n",
        "        else:\n",
        "          return h_prime"
      ],
      "metadata": {
        "id": "vUOt-DiY3Rd-"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class GATLayer(nn.Module):\n",
        "    def __init__(self, in_features, out_features, dropout, alpha, concat=True):\n",
        "        super(GATLayer, self).__init__()\n",
        "        self.dropout       = dropout        # drop prob = 0.6\n",
        "        self.in_features   = in_features    # \n",
        "        self.out_features  = out_features   # \n",
        "        self.alpha         = alpha          # LeakyReLU with negative input slope, alpha = 0.2\n",
        "        self.concat        = concat         # conacat = True for all layers except the output layer.\n",
        "\n",
        "        \n",
        "        # Xavier Initialization of Weights\n",
        "        # Alternatively use weights_init to apply weights of choice \n",
        "        self.W = nn.Parameter(torch.zeros(size=(in_features, out_features)))\n",
        "        nn.init.xavier_uniform_(self.W.data, gain=1.414)\n",
        "        \n",
        "        self.a = nn.Parameter(torch.zeros(size=(2*out_features, 1)))\n",
        "        nn.init.xavier_uniform_(self.a.data, gain=1.414)\n",
        "        \n",
        "        # LeakyReLU\n",
        "        self.leakyrelu = nn.LeakyReLU(self.alpha)\n",
        "\n",
        "    def forward(self, input, adj):\n",
        "        # Linear Transformation\n",
        "        h = torch.mm(input, self.W) # matrix multiplication\n",
        "        N = h.size()[0]\n",
        "        print(N)\n",
        "\n",
        "    \n",
        "        # Attention Mechanism\n",
        "        a_input = torch.cat([h.repeat(1, N).view(N * N, -1), h.repeat(N, 1)], dim=1).view(N, -1, 2 * self.out_features)\n",
        "        e       = self.leakyrelu(torch.matmul(a_input, self.a).squeeze(2))\n",
        "\n",
        "        # Masked Attention\n",
        "        zero_vec  = -9e15*torch.ones_like(e)\n",
        "        attention = torch.where(adj > 0, e, zero_vec)\n",
        "        \n",
        "        attention = F.softmax(attention, dim=1)\n",
        "        attention = F.dropout(attention, self.dropout, training=self.training)\n",
        "        h_prime   = torch.matmul(attention, h)\n",
        "\n",
        "        if self.concat:\n",
        "            return F.elu(h_prime)\n",
        "        else:\n",
        "            return h_prime"
      ],
      "metadata": {
        "id": "AVT_uuHv3dMp"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Using the model"
      ],
      "metadata": {
        "id": "N13PgXsG3oGc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from torch_geometric.data import Data\n",
        "from torch_geometric.nn import GATConv\n",
        "from torch_geometric.datasets import Planetoid\n",
        "import torch_geometric.transforms as T\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "name_data = 'Cora'\n",
        "dataset = Planetoid(root= '/tmp/' + name_data, name = name_data)\n",
        "dataset.transform = T.NormalizeFeatures()\n",
        "\n",
        "print(f\"Number of Classes in {name_data}:\", dataset.num_classes)\n",
        "print(f\"Number of Node Features in {name_data}:\", dataset.num_node_features)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KcfiAXw63nok",
        "outputId": "ad574e13-fbbe-4aab-ee55-e33e29da10a5"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Downloading https://github.com/kimiyoung/planetoid/raw/master/data/ind.cora.x\n",
            "Downloading https://github.com/kimiyoung/planetoid/raw/master/data/ind.cora.tx\n",
            "Downloading https://github.com/kimiyoung/planetoid/raw/master/data/ind.cora.allx\n",
            "Downloading https://github.com/kimiyoung/planetoid/raw/master/data/ind.cora.y\n",
            "Downloading https://github.com/kimiyoung/planetoid/raw/master/data/ind.cora.ty\n",
            "Downloading https://github.com/kimiyoung/planetoid/raw/master/data/ind.cora.ally\n",
            "Downloading https://github.com/kimiyoung/planetoid/raw/master/data/ind.cora.graph\n",
            "Downloading https://github.com/kimiyoung/planetoid/raw/master/data/ind.cora.test.index\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of Classes in Cora: 7\n",
            "Number of Node Features in Cora: 1433\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Processing...\n",
            "Done!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class GAT(torch.nn.Module):\n",
        "    def __init__(self):\n",
        "        super(GAT, self).__init__()\n",
        "        self.hid = 8\n",
        "        self.in_head = 8\n",
        "        self.out_head = 1\n",
        "        \n",
        "        \n",
        "        self.conv1 = GATConv(dataset.num_features, self.hid, heads=self.in_head, dropout=0.6)\n",
        "        self.conv2 = GATConv(self.hid*self.in_head, dataset.num_classes, concat=False,\n",
        "                             heads=self.out_head, dropout=0.6)\n",
        "\n",
        "    def forward(self, data):\n",
        "        x, edge_index = data.x, data.edge_index\n",
        "                \n",
        "        x = F.dropout(x, p=0.6, training=self.training)\n",
        "        x = self.conv1(x, edge_index)\n",
        "        x = F.elu(x)\n",
        "        x = F.dropout(x, p=0.6, training=self.training)\n",
        "        x = self.conv2(x, edge_index)\n",
        "        \n",
        "        return F.log_softmax(x, dim=1)\n",
        "\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "device = \"cpu\"\n",
        "\n",
        "model = GAT().to(device)\n",
        "data = dataset[0].to(device)\n",
        "\n",
        "\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=0.005, weight_decay=5e-4)\n",
        "\n",
        "model.train()\n",
        "for epoch in range(1000):\n",
        "    model.train()\n",
        "    optimizer.zero_grad()\n",
        "    out = model(data)\n",
        "    loss = F.nll_loss(out[data.train_mask], data.y[data.train_mask])\n",
        "    \n",
        "    if epoch%200 == 0:\n",
        "        print(loss)\n",
        "    \n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ervyqb8w3wcx",
        "outputId": "91340008-1151-4382-a53a-5e41d6f9c9c3"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor(1.9464, grad_fn=<NllLossBackward0>)\n",
            "tensor(0.7227, grad_fn=<NllLossBackward0>)\n",
            "tensor(0.6244, grad_fn=<NllLossBackward0>)\n",
            "tensor(0.5424, grad_fn=<NllLossBackward0>)\n",
            "tensor(0.5582, grad_fn=<NllLossBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Model Acuuracy"
      ],
      "metadata": {
        "id": "-VbLlAeZ4PHP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model.eval()\n",
        "_, pred = model(data).max(dim=1)\n",
        "correct = float(pred[data.test_mask].eq(data.y[data.test_mask]).sum().item())\n",
        "acc = correct / data.test_mask.sum().item()\n",
        "print('Accuracy: {:.4f}'.format(acc))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iL4PrcNF4OaX",
        "outputId": "b6d8c4a4-dcd4-4eed-e43a-95f02c0f1d84"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy: 0.8290\n"
          ]
        }
      ]
    }
  ]
}