GAD-EBM_weibo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "065f66a4-5f68-4e14-acc7-0aa91959aa75",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e1bcddab-f939-4f06-9009-da4a425cde65",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/roy206/.conda/envs/cent7/2020.11-py38/NWRGAE/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import scipy.sparse as sp\n",
    "import time\n",
    "import torch\n",
    "import scipy\n",
    "import random\n",
    "import pdb\n",
    "import copy\n",
    "import os\n",
    "import argparse\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "import pickle as pkl\n",
    "import networkx as nx\n",
    "import scipy.sparse as sp\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# from scipy.sparse.linalg.eigen.arpack import eigsh\n",
    "import sys\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from torch_geometric.nn import GCNConv,GINConv,SAGEConv,GATConv,PNAConv, GraphSAGE\n",
    "from torch_geometric.utils import to_dense_adj\n",
    "from pygod.utils import load_data\n",
    "from pygod.metrics import eval_roc_auc"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0fc66f2b-ba3a-46b4-9b1e-4ae84087280f",
   "metadata": {},
   "source": [
    "# Helper Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0c1b89a7-31e7-462b-b118-b961675b58f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def aug_random_edge(input_adj, perturb_percent=0.2, drop_edge = True, add_edge = True, self_loop = True):\n",
    "\n",
    "    aug_adj = copy.deepcopy(input_adj)\n",
    "    nb_nodes = input_adj.shape[0]\n",
    "    edge_index = (input_adj>0).nonzero().t()\n",
    "    \n",
    "    edge_dict = {}\n",
    "    for i in range(nb_nodes):\n",
    "        edge_dict[i] = set()\n",
    "    \n",
    "    for edge in edge_index:\n",
    "        i,j = edge[0],edge[1]\n",
    "        i = i.item()\n",
    "        j = j.item()\n",
    "        edge_dict[i].add(j)\n",
    "        edge_dict[j].add(i)\n",
    "    \n",
    "    if drop_edge: \n",
    "        for i in range(nb_nodes):\n",
    "            d = len(edge_dict[i])\n",
    "            node_list = list(edge_dict[i])\n",
    "            num_edge_to_drop = int(d * perturb_percent)\n",
    "\n",
    "            sampled_nodes = random.sample(node_list, num_edge_to_drop)\n",
    "\n",
    "            for j in sampled_nodes:\n",
    "                aug_adj[i][j] = 0\n",
    "                aug_adj[j][i] = 0\n",
    "\n",
    "            \n",
    "    node_list = [i for i in range(nb_nodes)]\n",
    "    \n",
    "    add_list = []\n",
    "    for i in range(nb_nodes):\n",
    "        d = len(edge_dict[i])\n",
    "        num_edge_to_add = int(d * perturb_percent)\n",
    "        sampled_nodes = random.sample(node_list, num_edge_to_add)\n",
    "        for j in sampled_nodes:\n",
    "            add_list.append((i,j))\n",
    "            \n",
    "    if add_edge:\n",
    "        for i in add_list:\n",
    "            aug_adj[i[0]][i[1]] = 1\n",
    "            aug_adj[i[1]][i[0]] = 1\n",
    "    \n",
    "    if self_loop: \n",
    "        for i in range(nb_nodes):\n",
    "            aug_adj[i][i] = 1\n",
    "            aug_adj[i][i] = 1\n",
    "    \n",
    "    \n",
    "    return aug_adj\n",
    "\n",
    "\n",
    "def _aug_random_edge(nb_nodes, edge_index, perturb_percent=0.2, drop_edge = True, add_edge = True, self_loop = True, use_avg_deg = True):\n",
    "\n",
    "    \n",
    "    total_edges = edge_index.shape[1]\n",
    "    avg_degree = int(total_edges/nb_nodes)\n",
    "    \n",
    "    \n",
    "    edge_dict = {}\n",
    "    for i in range(nb_nodes):\n",
    "        edge_dict[i] = set()\n",
    "    \n",
    "    for edge in edge_index:\n",
    "        i,j = edge[0],edge[1]\n",
    "        i = i.item()\n",
    "        j = j.item()\n",
    "        edge_dict[i].add(j)\n",
    "        edge_dict[j].add(i)\n",
    "    \n",
    "    if drop_edge: \n",
    "        for i in range(nb_nodes):\n",
    "            \n",
    "            d = len(edge_dict[i])\n",
    "            if use_avg_deg:\n",
    "                num_edge_to_drop = avg_degree\n",
    "            else:\n",
    "                num_edge_to_drop = int(d * perturb_percent)\n",
    "\n",
    "            node_list = list(edge_dict[i])\n",
    "            num_edge_to_drop = min(num_edge_to_drop, d)\n",
    "            sampled_nodes = random.sample(node_list, num_edge_to_drop)\n",
    "\n",
    "            for j in sampled_nodes:\n",
    "                edge_dict[i].discard(j)\n",
    "                edge_dict[j].discard(i)\n",
    "            \n",
    "    node_list = [i for i in range(nb_nodes)]\n",
    "    \n",
    "    add_list = []\n",
    "    for i in range(nb_nodes):\n",
    "        \n",
    "        if use_avg_deg:\n",
    "            num_edge_to_add =  avg_degree\n",
    "        else:\n",
    "            d = len(edge_dict[i])\n",
    "            num_edge_to_add = int(d * perturb_percent)\n",
    "        \n",
    "        sampled_nodes = random.sample(node_list, num_edge_to_add)\n",
    "        for j in sampled_nodes:\n",
    "            add_list.append((i,j))\n",
    "            \n",
    "    if add_edge:\n",
    "        for edge in add_list:\n",
    "            u = edge[0]\n",
    "            v = edge[1]\n",
    "            \n",
    "            edge_dict[u].add(v)\n",
    "            edge_dict[v].add(u)\n",
    "    \n",
    "    if self_loop: \n",
    "        for i in range(nb_nodes):\n",
    "            edge_dict[i].add(i)\n",
    "            \n",
    "    updated_edges = set()\n",
    "    for i in range(nb_nodes):\n",
    "        for j in edge_dict[i]:\n",
    "            updated_edges.add((i,j))\n",
    "            updated_edges.add((j,i))\n",
    "    \n",
    "    row = []\n",
    "    col = []\n",
    "    for edge in updated_edges:\n",
    "        u = edge[0]\n",
    "        v = edge[1]\n",
    "        row.append(u)\n",
    "        col.append(v)\n",
    "    \n",
    "    aug_edge_index = [row,col]\n",
    "    aug_edge_index = torch.tensor(aug_edge_index)\n",
    "    \n",
    "    return aug_edge_index\n",
    "\n",
    "\n",
    "def preprocess_features(features):\n",
    "    \"\"\"Row-normalize feature matrix and convert to tuple representation\"\"\"\n",
    "    features = features.squeeze()\n",
    "    rowsum = np.array(features.sum(1))\n",
    "    r_inv = np.power(rowsum, -1).flatten()\n",
    "    r_inv[np.isinf(r_inv)] = 0.\n",
    "    r_mat_inv = sp.diags(r_inv)\n",
    "    features = r_mat_inv.dot(features)\n",
    "    return features\n",
    "    # return features, sparse_to_tuple(features)\n",
    "\n",
    "def normalize_adj(adj):\n",
    "    \"\"\"Symmetrically normalize adjacency matrix.\"\"\"\n",
    "    adj = sp.coo_matrix(adj)\n",
    "    rowsum = np.array(adj.sum(1))\n",
    "    d_inv_sqrt = np.power(rowsum, -0.5).flatten()\n",
    "    d_inv_sqrt[np.isinf(d_inv_sqrt)] = 0.\n",
    "    d_mat_inv_sqrt = sp.diags(d_inv_sqrt)\n",
    "    return adj.dot(d_mat_inv_sqrt).transpose().dot(d_mat_inv_sqrt).tocoo()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea0866d6-d178-4377-93a3-6d6cc8de664a",
   "metadata": {},
   "source": [
    "# MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f4bd95b9-2f6a-4331-bb74-bd175d2a8135",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLP(nn.Module):\n",
    "    def __init__(self, num_layers, input_dim, hidden_dim, output_dim):\n",
    "        \n",
    "        super(MLP, self).__init__()\n",
    "\n",
    "        self.linear_or_not = True  # default is linear model\n",
    "        self.num_layers = num_layers\n",
    "\n",
    "        if num_layers < 1:\n",
    "            raise ValueError(\"number of layers should be positive!\")\n",
    "        elif num_layers == 1:\n",
    "            # Linear model\n",
    "            self.linear = nn.Linear(input_dim, output_dim)\n",
    "        else:\n",
    "            # Multi-layer model\n",
    "            self.linear_or_not = False\n",
    "            self.linears = torch.nn.ModuleList()\n",
    "            self.batch_norms = torch.nn.ModuleList()\n",
    "\n",
    "            self.linears.append(nn.Linear(input_dim, hidden_dim))\n",
    "            for layer in range(num_layers - 2):\n",
    "                self.linears.append(nn.Linear(hidden_dim, hidden_dim))\n",
    "            self.linears.append(nn.Linear(hidden_dim, output_dim))\n",
    "\n",
    "            for layer in range(num_layers - 1):\n",
    "                self.batch_norms.append(nn.BatchNorm1d((hidden_dim)))\n",
    "\n",
    "    def forward(self, x):\n",
    "        if self.linear_or_not:\n",
    "            # If linear model\n",
    "            return self.linear(x)\n",
    "        else:\n",
    "            # If MLP\n",
    "            h = x\n",
    "            for layer in range(self.num_layers - 1):\n",
    "                h = self.linears[layer](h)\n",
    "                \n",
    "                if len(h.shape) > 2:\n",
    "                    h = torch.transpose(h, 0, 1)\n",
    "                    h = torch.transpose(h, 1, 2)\n",
    "                    \n",
    "                # h = self.batch_norms[layer](h)\n",
    "                \n",
    "                if len(h.shape) > 2:\n",
    "                    h = torch.transpose(h, 1, 2)\n",
    "                    h = torch.transpose(h, 0, 1)\n",
    "\n",
    "                h = F.relu(h)\n",
    "                # h = F.relu(self.linears[layer](h))\n",
    "                \n",
    "            return self.linears[self.num_layers - 1](h)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64572324-7c46-4363-b541-8c9194e22e6e",
   "metadata": {},
   "source": [
    "# GNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d7d67c6-672e-428d-91d5-dd0dc6d82fe1",
   "metadata": {},
   "outputs": [],
   "source": [
    "class GNN(nn.Module):\n",
    "    def __init__(self, in_dim, out_dim, GNN_name = \"GCN\"):\n",
    "    \n",
    "        super(GNN, self).__init__()    \n",
    "        \n",
    "        self.mlp0 = MLP(3, in_dim, out_dim, out_dim)\n",
    "        \n",
    "        if GNN_name == \"GIN\":\n",
    "            self.linear1 = MLP(4, out_dim, out_dim, out_dim)\n",
    "            self.graphconv1 = GINConv(self.linear1)\n",
    "        elif GNN_name == \"GCN\":\n",
    "            self.graphconv1 = GCNConv(out_dim, out_dim, aggr='mean')\n",
    "        elif GNN_name == \"GAT\":\n",
    "            self.graphconv1 = GATConv(out_dim, out_dim, aggr='mean')\n",
    "        elif GNN_name == \"SAGE\":\n",
    "            self.graphconv1 = SAGEConv(out_dim, out_dim, aggr='mean')\n",
    "            \n",
    "        self.mlp1 = nn.Linear(out_dim,1)\n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "        self.relu = nn.ReLU()\n",
    "    \n",
    "    def forward(self, x, edge_index):\n",
    "        h0 = self.mlp0(x)\n",
    "        h1 = self.graphconv1(h0,edge_index)\n",
    "        h2 = self.mlp1(h1)\n",
    "        h2 = self.relu(h2)\n",
    "        p = torch.exp(h2)\n",
    "        return p\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7aaafc73-dc35-4c78-80eb-779cb790fa14",
   "metadata": {},
   "source": [
    "# Execution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "id": "ceafb622-a294-4cd9-b76a-dd8ae8539be1",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------------------------------------------\n",
      "Namespace(GNN_name='GCN', add_edge=False, dataset='weibo', drop_edge=True, f='/home/roy206/.local/share/jupyter/runtime/kernel-30530e72-cc9f-40c5-8783-b2fc7faa17a4.json', gpu=0, hidden_dim=64, l2_coef=100, lr=0.1, nb_epochs=200, num_neigh=1, perturb_percent=0.05, preprocess_feat=False, save_name='try.pkl', seed=42, self_loop=False, use_avg_deg=False)\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Loss:  1.1765979528427124 AUC Score:  38.76158722957526\n",
      "Loss:  0.758903980255127 AUC Score:  27.348490916394642\n",
      "Loss:  1.1790156364440918 AUC Score:  82.03960308866426\n",
      "Loss:  1.1016675233840942 AUC Score:  53.181325430487625\n",
      "Loss:  1.311104655265808 AUC Score:  73.29925822165184\n",
      "Loss:  0.9086015224456787 AUC Score:  30.98253684281966\n",
      "Loss:  0.9521691799163818 AUC Score:  37.90757455233138\n",
      "Loss:  0.9890186786651611 AUC Score:  43.27720572365271\n",
      "Loss:  1.0037858486175537 AUC Score:  50.14629058854963\n",
      "Loss:  1.0084463357925415 AUC Score:  50.93991760464045\n",
      "Loss:  1.0037215948104858 AUC Score:  50.74884792626728\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0004394054412842 AUC Score:  50.74884792626728\n",
      "Loss:  1.0145270824432373 AUC Score:  56.017120454605205\n",
      "Loss:  1.0067793130874634 AUC Score:  58.46806293254354\n",
      "Loss:  1.0009766817092896 AUC Score:  25.48734232165862\n",
      "Loss:  1.000397801399231 AUC Score:  54.87566255321673\n",
      "Loss:  0.9992377758026123 AUC Score:  93.41281169578772\n",
      "Loss:  0.9994750022888184 AUC Score:  75.45064930062384\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.000583291053772 AUC Score:  57.58423574268631\n",
      "Loss:  1.000705599784851 AUC Score:  46.49599609667575\n",
      "Loss:  1.000408411026001 AUC Score:  47.7058951568575\n",
      "Loss:  1.0001590251922607 AUC Score:  27.174281226441117\n",
      "Loss:  1.0000890493392944 AUC Score:  64.78986309628262\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  0.9999065399169922 AUC Score:  59.82404622602229\n",
      "Loss:  0.9999287128448486 AUC Score:  59.58821274339985\n",
      "Loss:  1.0000476837158203 AUC Score:  31.6216802025522\n",
      "Loss:  1.000038504600525 AUC Score:  40.90635965488842\n",
      "Loss:  1.0000028610229492 AUC Score:  60.927618831582954\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  0.9999980926513672 AUC Score:  59.636071876438756\n",
      "Loss:  1.0000028610229492 AUC Score:  40.63279373218084\n",
      "Loss:  0.9999947547912598 AUC Score:  70.20696361849896\n",
      "Loss:  1.0000028610229492 AUC Score:  43.80937299185768\n",
      "Loss:  1.00001859664917 AUC Score:  37.94865453318161\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  0.9999971389770508 AUC Score:  64.10808826991145\n",
      "Loss:  1.0000009536743164 AUC Score:  34.63700429646921\n",
      "Loss:  0.9999940395355225 AUC Score:  64.03914268716727\n",
      "Loss:  0.9999990463256836 AUC Score:  63.24488131974425\n",
      "Loss:  1.0000038146972656 AUC Score:  35.55768500589106\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0000014305114746 AUC Score:  35.787106190107295\n",
      "Loss:  1.0000014305114746 AUC Score:  38.86633927004657\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.079607270797396\n",
      "Loss:  1.0000009536743164 AUC Score:  36.88588676813434\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  0.9999995231628418 AUC Score:  60.474263984313325\n",
      "Loss:  0.9999995231628418 AUC Score:  62.02902852838439\n",
      "Loss:  0.9999995231628418 AUC Score:  62.4688403568509\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  0.9999995231628418 AUC Score:  60.446612380459165\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Loss:  1.0 AUC Score:  50.0\n",
      "Maximum AUC:  93.41281169578772\n",
      "Required Time:  2.163184642791748\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "parser = argparse.ArgumentParser(\"GAD-EBM: Graph Anomaly Detection via Energy-based Model\")\n",
    "\n",
    "parser.add_argument('-f')\n",
    "parser.add_argument('--dataset',          type=str,           default=\"weibo\",          help='dataset name')\n",
    "parser.add_argument('--perturb_percent',  type=float,         default=0.05,             help='perturb percent')\n",
    "parser.add_argument('--seed',             type=int,           default=42,               help='seed')\n",
    "parser.add_argument('--nb_epochs',        type=int,           default=200,              help='total epochs')\n",
    "parser.add_argument('--hidden_dim',       type=int,           default=64,               help='hidden dimension')\n",
    "parser.add_argument('--lr',               type=float,         default=0.1,              help='learning rate')\n",
    "parser.add_argument('--l2_coef',          type=float,         default=100,              help='regularization coefficeint')\n",
    "parser.add_argument('--gpu',              type=int,           default=0,                help='gpu')\n",
    "parser.add_argument('--save_name',        type=str,           default='try.pkl',        help='save ckpt name')\n",
    "parser.add_argument('--drop_edge',        type=bool,          default=True,             help='drop edge flag to produce state space neighbor')\n",
    "parser.add_argument('--add_edge',         type=bool,          default=False,            help='add edge flag to produce state space neighbor')\n",
    "parser.add_argument('--self_loop',        type=bool,          default=False,            help='self loop in state space neighbor')\n",
    "parser.add_argument('--use_avg_deg',      type=bool,          default=False,            help='use avg_deg to add/drop edge in state space neighbor')\n",
    "parser.add_argument('--preprocess_feat',  type=bool,          default=False,            help='preprocess feature flag')\n",
    "parser.add_argument('--GNN_name',         type=str,           default=\"GCN\",            help='gnn encoder')\n",
    "parser.add_argument('--num_neigh',        type=int,           default=1,                help='state space graph number of neighbors')\n",
    "\n",
    "args = parser.parse_args()\n",
    "\n",
    "print('-' * 100)\n",
    "print(args)\n",
    "print('-' * 100)\n",
    "\n",
    "dataset_str = args.dataset\n",
    "perturb_percent = args.perturb_percent\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = str(args.gpu) \n",
    "seed = args.seed\n",
    "# random.seed(seed)\n",
    "# np.random.seed(seed)\n",
    "# torch.manual_seed(seed)\n",
    "# torch.cuda.manual_seed(seed)\n",
    "\n",
    "nb_epochs = args.nb_epochs\n",
    "lr = args.lr\n",
    "l2_coef = args.l2_coef\n",
    "hidden_dim = args.hidden_dim\n",
    "k = args.num_neigh\n",
    "\n",
    "\n",
    "\n",
    "data = load_data(dataset_str)\n",
    "edge_index = data.edge_index\n",
    "\n",
    "\n",
    "adj = to_dense_adj(edge_index).squeeze()\n",
    "features = data.x\n",
    "labels = data.y\n",
    "y = labels.bool()\n",
    "\n",
    "anomaly_nodes = np.nonzero(y)\n",
    "\n",
    "nb_nodes = features.shape[0]  # total node\n",
    "input_dim = features.shape[1]   # total features\n",
    "\n",
    "\n",
    "\n",
    "model = GNN(input_dim, hidden_dim, args.GNN_name)\n",
    "optimiser = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=l2_coef)\n",
    "\n",
    "\n",
    "if args.preprocess_feat:\n",
    "    features = preprocess_features(features)\n",
    "\n",
    "    \n",
    "\n",
    "\n",
    "st = time.time()\n",
    "mx_auc = 0\n",
    "\n",
    "aug_edge_indexes = []\n",
    "\n",
    "for i in range(k):\n",
    "    aug_edge_index = _aug_random_edge(nb_nodes,edge_index, perturb_percent=perturb_percent,drop_edge=args.drop_edge,\n",
    "                                  add_edge=args.add_edge,self_loop=args.self_loop, use_avg_deg = args.use_avg_deg) # add/drop perturb percentage of edges\n",
    "    \n",
    "    \n",
    "    aug_edge_index = aug_edge_index.to(device)\n",
    "    \n",
    "    aug_edge_indexes.append(aug_edge_index)\n",
    "    \n",
    "    \n",
    "model = model.to(device)\n",
    "features = torch.FloatTensor(features[np.newaxis])\n",
    "features = features.to(device)\n",
    "edge_index = edge_index.to(device)\n",
    "\n",
    "losses = []\n",
    "    \n",
    "for epoch in range(nb_epochs):\n",
    "\n",
    "    model.train()\n",
    "    optimiser.zero_grad()\n",
    "    \n",
    "    p_data = model(features, edge_index)\n",
    "    loss = 0\n",
    "    \n",
    "    for i in range(k):\n",
    "        \n",
    "        aug_edge_index = aug_edge_indexes[i]\n",
    "        \n",
    "        shuf_fts = features\n",
    "        idx = np.random.permutation(nb_nodes)\n",
    "        shuf_fts = features[:, idx, :]\n",
    "        \n",
    "        \n",
    "        p_neigh = model(shuf_fts, aug_edge_index)\n",
    "    \n",
    "        c_theta_j1 = p_neigh/p_data\n",
    "        c_theta_j2 = p_data/p_neigh\n",
    "            \n",
    "        j1 = (c_theta_j1**2 + 2 * c_theta_j1).mean()\n",
    "        j2 = (2 * c_theta_j2).mean()\n",
    "        \n",
    "        \n",
    "        \n",
    "        neigh_loss = j1 - j2\n",
    "        neigh_loss = neigh_loss.mean()\n",
    "        loss = loss + neigh_loss\n",
    "    \n",
    "    loss = loss / k\n",
    "    \n",
    "    losses.append(loss)\n",
    "    \n",
    "\n",
    "    \n",
    "    logits = p_data.squeeze().detach().cpu() \n",
    "    auc_score = eval_roc_auc(y.numpy(), logits.numpy()) * 100\n",
    "    \n",
    "    print(\"Loss: \", loss.item(), \"AUC Score: \", auc_score)\n",
    "    mx_auc = max(mx_auc, auc_score)\n",
    "    \n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "\n",
    "en = time.time()\n",
    "\n",
    "\n",
    "print(\"Maximum AUC: \", mx_auc)\n",
    "print(\"Required Time: \", en-st)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73e22068-253d-4a69-aa37-a2e2b9481fa2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0710947-7fcc-4b4a-b365-315a35ff0950",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "NWRGAE",
   "language": "python",
   "name": "nwrgae"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}