nb_scratchbook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "single-efficiency",
   "metadata": {},
   "outputs": [],
   "source": [
    "import blocksci\n",
    "\n",
    "import sys, os, os.path, socket\n",
    "import numpy as np\n",
    "import numpy_indexed as npi\n",
    "import zarr\n",
    "import time\n",
    "import pandas as pd\n",
    "\n",
    "SYMBOLS = { \n",
    "            \"BTC\": \"bitcoin\",\n",
    "            \"LTC\": \"litecoin\",\n",
    "            \"DOGE\": \"dogecoin\",\n",
    "            \"BCH\": \"bitcoin_cash\",\n",
    "            \"BSV\": \"bitcoin_sv\",\n",
    "            \"LCH\": \"litecoin_cash\",\n",
    "            \"FTC\": \"feathercoin\",\n",
    "            \"MONA\": \"monacoin\"\n",
    "          } \n",
    "\n",
    "DIR_BCHAIN=\"/mnt/hdd_data/blockchain_data/\"\n",
    "DIR_PARSED=\"/mnt/hdd_data/blockchain_parsed/\"\n",
    "\n",
    "    \n",
    "class AddressMapper():\n",
    "    def __init__(self, chain):\n",
    "        self.chain = chain\n",
    "\n",
    "        self.__address_types = [blocksci.address_type.nonstandard, blocksci.address_type.pubkey,\n",
    "                                blocksci.address_type.pubkeyhash, blocksci.address_type.multisig_pubkey,\n",
    "                                blocksci.address_type.scripthash, blocksci.address_type.multisig,\n",
    "                                blocksci.address_type.nulldata, blocksci.address_type.witness_pubkeyhash,\n",
    "                                blocksci.address_type.witness_scripthash, blocksci.address_type.witness_unknown]\n",
    "\n",
    "        self.__counter_addresses = { _:self.chain.address_count(_) for _ in self.__address_types }\n",
    "\n",
    "        self.__offsets = {}\n",
    "        offset = 0\n",
    "        for _ in self.__address_types:\n",
    "            self.__offsets[_] = offset\n",
    "            offset += self.__counter_addresses[_]\n",
    "\n",
    "\n",
    "        self.total_addresses = offset\n",
    "        print(self.total_addresses)\n",
    "        print(self.__counter_addresses)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "current-burns",
   "metadata": {},
   "source": [
    "# load everything"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "tight-discharge",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cfg loading time = 0.274371862411499\n",
      "clusters loading time = 0.00021576881408691406\n",
      "address cluster map loading time = 8.235018014907837\n",
      "ground truth loading time = 17.9113986492157\n",
      "4022303272\n",
      "{address_type.nonstandard: 831942, address_type.pubkey: 857014675, address_type.pubkeyhash: 857014675, address_type.multisig_pubkey: 857014675, address_type.scripthash: 235784252, address_type.multisig: 74844649, address_type.nulldata: 46999459, address_type.witness_pubkeyhash: 857014675, address_type.witness_scripthash: 235784252, address_type.witness_unknown: 18}\n",
      "am creation time = 0.00028228759765625\n"
     ]
    }
   ],
   "source": [
    "# load the chain\n",
    "\n",
    "currency = \"bitcoin\"\n",
    "heur = \"heur_1\"\n",
    "\n",
    "cfg_file = f\"/mnt/hdd_data/blockchain_parsed/{currency}.cfg\" # later\n",
    "# cfg_file = f\"/mnt/hdd_data/blockchain-parsed/bitcoin.cfg\" # abacus\n",
    "t = time.time()\n",
    "chain = blocksci.Blockchain(cfg_file)\n",
    "print(f\"cfg loading time = {time.time()-t}\")\n",
    "\n",
    "t = time.time()\n",
    "cm = blocksci.cluster.ClusterManager(f\"{DIR_PARSED}/{currency}/{heur}/\", chain)\n",
    "print(f\"clusters loading time = {time.time()-t}\")\n",
    "\n",
    "t = time.time()\n",
    "address_cluster_map = zarr.load(f\"{DIR_PARSED}/{currency}/{heur}_data/address_cluster_map.zarr\")\n",
    "print(f\"address cluster map loading time = {time.time()-t}\")\n",
    "\n",
    "t = time.time()\n",
    "df = pd.read_csv(f\"{DIR_PARSED}/bitcoin_darknet/ground_truth_id.csv\")\n",
    "print(f\"ground truth loading time = {time.time()-t}\")\n",
    "\n",
    "t = time.time()\n",
    "am = AddressMapper(chain)\n",
    "print(f\"am creation time = {time.time()-t}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0d297e4a-9eb2-42cf-aef2-71456a2c5678",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{address_type.nonstandard: 0,\n",
       " address_type.pubkey: 831942,\n",
       " address_type.pubkeyhash: 857846617,\n",
       " address_type.multisig_pubkey: 1714861292,\n",
       " address_type.scripthash: 2571875967,\n",
       " address_type.multisig: 2807660219,\n",
       " address_type.nulldata: 2882504868,\n",
       " address_type.witness_pubkeyhash: 2929504327,\n",
       " address_type.witness_scripthash: 3786519002,\n",
       " address_type.witness_unknown: 4022303254}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "am._AddressMapper__offsets"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fb3c2bd-35b3-404b-9b53-257f5fe7929c",
   "metadata": {},
   "source": [
    "## check\n",
    "clusters are id from 1 to len(clusters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "sensitive-writer",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "591692716"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cm.clusters())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2de4ba3a-ae12-4633-8000-d61d51089ed3",
   "metadata": {},
   "source": [
    "```address_cluster_map``` goes from address(index) to cluster(cluster number - 1, or python index in clusters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "veterinary-welsh",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4022303272\n",
      "591692715\n"
     ]
    }
   ],
   "source": [
    "print(len(address_cluster_map))\n",
    "print(np.max(address_cluster_map))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3659a9d2-24b9-41f2-b32f-4b0cb15c7765",
   "metadata": {},
   "source": [
    "```df``` rows have address, controlling entity, entity sector, and personal id(ignore this, it is obsolete)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "17a6e4e4-5801-4da6-9233-2498184913bf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>address</th>\n",
       "      <th>entity</th>\n",
       "      <th>sector</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1E421ChpHWEqQ2Kcy2JqEE8eDx7nFN4AXj</td>\n",
       "      <td>777Coin.com</td>\n",
       "      <td>Gambling</td>\n",
       "      <td>34916770.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15uWGv4QFoa7Hjkfzs6YCDVGNhxZ4Yz2wW</td>\n",
       "      <td>777Coin.com</td>\n",
       "      <td>Gambling</td>\n",
       "      <td>26152409.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19s8Jzb6fQ9kLu3Gcb46aX5h9uc93PULgC</td>\n",
       "      <td>777Coin.com</td>\n",
       "      <td>Gambling</td>\n",
       "      <td>29354407.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1LfiL43VHwe2tRNjra5uhVAc4J5nFUay5X</td>\n",
       "      <td>777Coin.com</td>\n",
       "      <td>Gambling</td>\n",
       "      <td>29902318.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1Bh6y3BkJVzodfGfLh9GCuuKPqgwoyRibF</td>\n",
       "      <td>777Coin.com</td>\n",
       "      <td>Gambling</td>\n",
       "      <td>29739794.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              address       entity    sector          id\n",
       "0  1E421ChpHWEqQ2Kcy2JqEE8eDx7nFN4AXj  777Coin.com  Gambling  34916770.0\n",
       "1  15uWGv4QFoa7Hjkfzs6YCDVGNhxZ4Yz2wW  777Coin.com  Gambling  26152409.0\n",
       "2  19s8Jzb6fQ9kLu3Gcb46aX5h9uc93PULgC  777Coin.com  Gambling  29354407.0\n",
       "3  1LfiL43VHwe2tRNjra5uhVAc4J5nFUay5X  777Coin.com  Gambling  29902318.0\n",
       "4  1Bh6y3BkJVzodfGfLh9GCuuKPqgwoyRibF  777Coin.com  Gambling  29739794.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "hollywood-occurrence",
   "metadata": {},
   "source": [
    "# from adress to cluster\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ca94ecdd-8eef-43f3-9442-aacbac510947",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PubkeyAddress(1JoiKZz2QRd47ARtcYgvgxC9jhnre9aphv)\n",
      "10055\n",
      "831942\n",
      "77270814\n",
      "77270814\n"
     ]
    }
   ],
   "source": [
    "# this block load an adress from block 10000\n",
    "# identify the corresponding cluster id in address_cluster map \n",
    "# and check that the identifier is the same of the internal blocksci identifier\n",
    "\n",
    "for t in chain.blocks[10000].outputs:\n",
    "    add = t.address\n",
    "    num = add.address_num\n",
    "    typ = add.type\n",
    "    off = am._AddressMapper__offsets[typ]\n",
    "    print(add)\n",
    "    print(add.address_num)\n",
    "    print(am._AddressMapper__offsets[typ])\n",
    "    \n",
    "c_num = address_cluster_map[off + num]-1\n",
    "print(c_num)\n",
    "\n",
    "c = cm.cluster_with_address(add)\n",
    "print(c.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "751f811d-dd77-426a-b957-dc2bd72d0e38",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "TxIn(spent_tx_index=213171, address=PubkeyHashAddress(1BNwxHGaFbeUBitpjy2AsKpJ29Ybxntqvb), value=5000000000)\n",
      "TxOut(spending_tx_index=216574, address=PubkeyHashAddress(1BNwxHGaFbeUBitpjy2AsKpJ29Ybxntqvb), value=5000000000)\n",
      "\n",
      "\n",
      "TxIn(spent_tx_index=205760, address=PubkeyHashAddress(15vScfMHNrXN4QvWe54q5hwfVoYwG79CS1), value=300000000)\n",
      "TxOut(spending_tx_index=402825, address=PubkeyHashAddress(1JHJYGshG8Ds9XXHbXuTrDkf8XAXzNhi5c), value=27000000)\n",
      "TxOut(spending_tx_index=216575, address=PubkeyHashAddress(15vScfMHNrXN4QvWe54q5hwfVoYwG79CS1), value=300000000)\n",
      "\n",
      "\n",
      "TxIn(spent_tx_index=215898, address=PubkeyHashAddress(1JxDJCyWNakZ5kECKdCU9Zka6mh34mZ7B2), value=1000000)\n",
      "TxOut(spending_tx_index=216576, address=PubkeyHashAddress(1JxDJCyWNakZ5kECKdCU9Zka6mh34mZ7B2), value=1000000)\n"
     ]
    }
   ],
   "source": [
    "for t in chain.blocks[100000].txes:\n",
    "    print('\\n')\n",
    "    for i in t.inputs:\n",
    "        print(i)\n",
    "        for o in chain.tx_with_index(i.spent_tx_index).outputs:\n",
    "            print(o)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c31977c0-6911-4498-a77d-816b5744f08b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1E421ChpHWEqQ2Kcy2JqEE8eDx7nFN4AXj\n",
      "79703656\n",
      "87341931\n"
     ]
    }
   ],
   "source": [
    "# load a black address from df, identify the corresponding cluster and\n",
    "\n",
    "add_string = df.address[0]  # black address string\n",
    "add = chain.address_from_string(add_string)  # load blocksci address object\n",
    "add_typ = add.type\n",
    "add_id = add.address_num + am._AddressMapper__offsets[add_typ]  # compute address identifier in the addres_cluster_map\n",
    "\n",
    "c_num = address_cluster_map[add_id]-1\n",
    "c = cm.cluster_with_address(add)\n",
    "c_id = c.index\n",
    "\n",
    "print(add_string)\n",
    "print(c_num)\n",
    "print(c_id)\n",
    "\n",
    "# print(c.address_count())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22c79cb2-cd2d-43a8-8928-64d72e94a89c",
   "metadata": {},
   "source": [
    "```c_num``` and ```c_id``` differ and *it shouldnt happen*."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e57c1403-4a29-4e72-9189-b5dbf8c28998",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5966\n"
     ]
    }
   ],
   "source": [
    "print(c.address_count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "16cf0b26-59a0-4746-ad12-971c480230bb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "print(add in c.addresses.to_list())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "767ba503-ec91-480c-9cf3-29437d3a1c37",
   "metadata": {},
   "source": [
    "## what happens if i pass inexisting address"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "776f3a64-e5c7-40f8-a383-fd92452a6bc5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nan\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    a = chain.address_from_string('fake')\n",
    "except:\n",
    "    a = np.nan\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "97ece638-f84f-4fe0-a992-2becef707fed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[PubkeyHashAddress(1E421ChpHWEqQ2Kcy2JqEE8eDx7nFN4AXj), PubkeyHashAddress(15uWGv4QFoa7Hjkfzs6YCDVGNhxZ4Yz2wW), PubkeyHashAddress(19s8Jzb6fQ9kLu3Gcb46aX5h9uc93PULgC), PubkeyHashAddress(1LfiL43VHwe2tRNjra5uhVAc4J5nFUay5X), PubkeyHashAddress(1Bh6y3BkJVzodfGfLh9GCuuKPqgwoyRibF), PubkeyHashAddress(1DraDSB5BRNtafEsQ1WzBfQ2n84pyD5o49), PubkeyHashAddress(1FJiPTa4q81H9yvPgatfqEfwngCfvQdxZg), PubkeyHashAddress(16JGPQbLXUf8KhFybG9AbMqmoTrok2xaKC), PubkeyHashAddress(1MPpXsEffG6XBueaNhDgazzAFu2s5zGrc6), PubkeyHashAddress(1FruhAC8DEtuYj7tCyBVKb3LqgkziAEyWU)]\n"
     ]
    }
   ],
   "source": [
    "def catch(a):\n",
    "    try:\n",
    "        return chain.address_from_string(a)\n",
    "    except:\n",
    "        return np.nan\n",
    "    \n",
    "a = [catch(a) for a in df.address[:10]]\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "842e9579-6fab-43ab-af17-577cfc18d16c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "int64\n",
      "[2 3]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-45-8594c583707d>:6: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.\n",
      "  print(c[[nan_i]])\n"
     ]
    }
   ],
   "source": [
    "c = np.array([1,2,3,4])\n",
    "i = np.array([1,2, np.nan])\n",
    "#print(c[i,])\n",
    "nan_i= i[~np.isnan(i)].astype(int)\n",
    "print(nan_i.dtype)\n",
    "print(c[[nan_i]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "3053cded-de78-4d2e-980a-19b8bfd9410c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc['id', 0] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "56f4409c-b9d7-4431-91e6-a3ced6d9bd72",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1           26152409\n",
       "2           29354407\n",
       "3           29902318\n",
       "4           29739794\n",
       "5           24795829\n",
       "              ...   \n",
       "28027926    32503538\n",
       "28027927    29594299\n",
       "28027928    34033368\n",
       "28027929    26941191\n",
       "28027930    26178131\n",
       "Name: id, Length: 23369119, dtype: int64"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.id[~np.isnan(df.id)].astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ba23cd1-fa2c-4e7b-b74c-453f93f2f7ce",
   "metadata": {},
   "source": [
    "# who got a black address"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b4d9d665-7d51-4421-bb4d-1350c211dcaf",
   "metadata": {},
   "source": [
    "## has at least a black address?\n",
    "here we count if a cluster got at least one black address"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6981bf0e-4127-4034-b583-356445e08231",
   "metadata": {},
   "outputs": [],
   "source": [
    "# v1\n",
    "black_clusters = np.zeros(len(cm.clusters()), dtype=bool)\n",
    "t = time.time()\n",
    "for a in df.address:\n",
    "    add = chain.address_from_string(a)\n",
    "    add_typ = add.type\n",
    "    add_id = add.address_num + am._AddressMapper__offsets[add_typ]\n",
    "    c_index = address_cluster_map[add_id]\n",
    "    black_clusters[c_index] = True\n",
    "print(f\"time = {time.time() - t}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1570739-cf5e-4c90-bd07-ca925dc92375",
   "metadata": {},
   "outputs": [],
   "source": [
    "# v2\n",
    "black_clusters_2 = np.zeros(len(cm.clusters()), dtype=bool)\n",
    "c_indices = []\n",
    "t = time.time()\n",
    "for a in df.address:\n",
    "    add = chain.address_from_string(a)\n",
    "    add_typ = add.type\n",
    "    c_indices.append(address_cluster_map[add.address_num + am._AddressMapper__offsets[add_typ]])\n",
    "    \n",
    "black_clusters_2[c_indices] = True\n",
    "print(f\"time = {time.time() - t}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "aad26ac7-1ee2-47f1-bb02-a471336f7ee8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m----------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                    Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-9fb2225db6ab>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mc_indices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mc_indices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maddress_cluster_map\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mchain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddress_from_string\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddress_num\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mam\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_AddressMapper__offsets\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mchain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddress_from_string\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddress\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0mblack_clusters_2\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mc_indices\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"time = {time.time() - t}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-3-9fb2225db6ab>\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mc_indices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mc_indices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maddress_cluster_map\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mchain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddress_from_string\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddress_num\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mam\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_AddressMapper__offsets\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mchain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddress_from_string\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddress\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0mblack_clusters_2\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mc_indices\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"time = {time.time() - t}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# v3\n",
    "black_clusters_3 = np.zeros(len(cm.clusters()), dtype=bool)\n",
    "c_indices = []\n",
    "t = time.time()\n",
    "c_indices = np.array([address_cluster_map[chain.address_from_string(a).address_num + am._AddressMapper__offsets[chain.address_from_string(a).type]] for a in df.address])\n",
    "black_clusters_3[c_indices] = True\n",
    "print(f\"time = {time.time() - t}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05b459ce-223a-467e-9e24-968f6ccdb01c",
   "metadata": {},
   "source": [
    "## more elaborate version 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a3efa439-64ee-4453-bf47-1364a0db94cc",
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'Series' object has no attribute 'applymap'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m----------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-20-e7890ada5a10>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"test2\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddress\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapplymap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddress_from_string\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/miniconda3/envs/blocksci_env/lib/python3.8/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   5463\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_info_axis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_can_hold_identifiers_and_holds_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5464\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5465\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5466\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5467\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__setattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'Series' object has no attribute 'applymap'"
     ]
    }
   ],
   "source": [
    "df[\"test2\"] = df.address[:10].applymap(chain.address_from_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2730e18c-1c98-44c2-897f-6f7a8acdb622",
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m----------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                    Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-5d06d7d3cf75>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mbc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclusters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"num\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mchain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddress_from_string\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddress\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"time = {t - time.time()}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-6-5d06d7d3cf75>\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mbc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclusters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"num\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mchain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddress_from_string\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddress\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"time = {t - time.time()}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "bc = np.zeros(len(cm.clusters()), dtype=bool)\n",
    "t = time.time()\n",
    "df[\"num\"] = [chain.address_from_string(a) for a in df.address]\n",
    "print(f\"time = {t - time.time()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32adfb6e-3dab-4c76-b5a3-c19072476232",
   "metadata": {},
   "outputs": [],
   "source": [
    "t = time.time()\n",
    "df[\"offset\"] = [chain.address_from_string(a).type for a in df.address]\n",
    "print(f\"time = {t - time.time()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b31e1bb6-ca6f-4483-abba-242516288441",
   "metadata": {},
   "outputs": [],
   "source": [
    "t = time.time()\n",
    "df[\"c_index\"] = address_cluster_map[df.num+df.offset]\n",
    "print(f\"time = {t - time.time()}\")\n",
    "bc[c_indices] = True"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "rental-samuel",
   "metadata": {},
   "source": [
    "# Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "921bece0-0e79-4fcb-8d79-799b2ed0bef5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tx(len(txins)=1, len(txouts)=2, size_bytes=257, block_height=212504, tx_index=9980900)\n"
     ]
    }
   ],
   "source": [
    "a = chain.tx_with_index(10200851)\n",
    "\n",
    "for inp in a.inputs:\n",
    "    print(chain.tx_with_index(inp.spent_tx_index))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc2d0bc3-d581-4c70-a729-a0825cca67b2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.2 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  },
  "vscode": {
   "interpreter": {
    "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}