pyclan_examples.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mDEPRECATION: Python 2.7 reached the end of its life on January 1st, 2020. Please upgrade your Python as Python 2.7 is no longer maintained. A future version of pip will drop support for Python 2.7. More details about Python 2 support in pip, can be found at https://pip.pypa.io/en/latest/development/release-process/#python-2-support\u001b[0m\n",
      "Package         Version Location                                                  \n",
      "--------------- ------- ----------------------------------------------------------\n",
      "codecheck       1.0     /home/sarp/Desktop/SeedlingsBabyLab/.annotid/src/codecheck\n",
      "Distance        0.1.3   \n",
      "enum34          1.1.10  \n",
      "msgpack         1.0.0   \n",
      "mysql-connector 2.2.9   \n",
      "numpy           1.16.4  \n",
      "pandas          0.24.2  \n",
      "pip             20.0.2  \n",
      "pkg-resources   0.0.0   \n",
      "pudb            2019.2  \n",
      "pyclan          0.2     /home/sarp/Desktop/SeedlingsBabyLab/pyclan                \n",
      "Pygments        2.5.2   \n",
      "Pyment          0.3.3   \n",
      "python-dateutil 2.8.0   \n",
      "pytz            2019.1  \n",
      "setuptools      41.0.1  \n",
      "six             1.12.0  \n",
      "urwid           2.1.0   \n",
      "wheel           0.33.4  \n",
      "\u001b[33mWARNING: You are using pip version 20.0.2; however, version 20.1.1 is available.\n",
      "You should consider upgrading via the '/home/sarp/Desktop/SeedlingsBabyLab/.annotid/bin/python2 -m pip install --upgrade pip' command.\u001b[0m\n",
      "['', '/usr/lib/python2.7', '/usr/lib/python2.7/plat-x86_64-linux-gnu', '/usr/lib/python2.7/lib-tk', '/usr/lib/python2.7/lib-old', '/usr/lib/python2.7/lib-dynload', '/home/sarp/.local/lib/python2.7/site-packages', '/usr/local/lib/python2.7/dist-packages', '/usr/lib/python2.7/dist-packages', '/usr/lib/python2.7/dist-packages/PILcompat', '/usr/lib/python2.7/dist-packages/gtk-2.0', '/home/sarp/.local/lib/python2.7/site-packages/IPython/extensions', '/home/sarp/.ipython']\n"
     ]
    },
    {
     "ename": "ImportError",
     "evalue": "No module named codecheck",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-71a492af31b6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mpyclan\u001b[0m \u001b[0;31m# first, import the library\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;31m# ignore this it's just for pretty printing...\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sarp/Desktop/SeedlingsBabyLab/pyclan/pyclan/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mclanfile\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mClanFile\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mfilters\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0melements\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mClanBlock\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mClanLine\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mBlockGroup\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLineRange\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mAnnotation\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0merrors\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mparse\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sarp/Desktop/SeedlingsBabyLab/pyclan/pyclan/clanfile.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mre\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mpyclan\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mfilters\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mpyclan\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0melements\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mpyclan\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sarp/Desktop/SeedlingsBabyLab/pyclan/pyclan/filters.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mre\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mcodecheck\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mcollections\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mOrderedDict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mregexes\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0merrors\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mParseError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mImportError\u001b[0m: No module named codecheck"
     ]
    }
   ],
   "source": [
    "!pip list\n",
    "import sys\n",
    "print(sys.path)\n",
    "import pyclan # first, import the library\n",
    "\n",
    "# ignore this it's just for pretty printing...\n",
    "import pprint\n",
    "pp = pprint.PrettyPrinter()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# objects\n",
    "\n",
    "These are the objects that pyclan exposes. They represent progressively smaller subdivisions of a CLAN file:\n",
    "- **ClanFile**\n",
    " - this represents the whole CLAN file (.cha)\n",
    "- **BlockGroup**\n",
    " - this is a collection of ClanBlocks\n",
    "- **ClanBlock**\n",
    " - this is a single conversation block\n",
    "  - delimited by:\n",
    "   - @Bg Conversation XYZ (to begin)\n",
    "   - @Eg Conversation XYZ (to end)\n",
    "- **LineRange**\n",
    " - this is a collection of single ClanLine's\n",
    "- **ClanLine**\n",
    " - this is a single line within the CLAN file. Line's are delimited by \"\\n\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# loading a CLAN file\n",
    "\n",
    "First you need to construct a **ClanFile** object by loading a .cha file into it. Just supply a path to the file:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clan_file = pyclan.ClanFile(\"sample_data/31_14_coderSD_final.cha\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# class ClanFile\n",
    "\n",
    "A ClanFile object has a bunch of different values associated with it and methods you can call to filter/get info about the file. \n",
    "\n",
    "\n",
    "Let's print some of the basic variables that are part of every ClanFile object:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "714"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clan_file.num_full_blocks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'sample_data/31_14_coderSD_final.cha'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clan_file.clan_path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ClanFile.line_map\n",
    "\n",
    "Each ClanFile has a line_map member variable. This is a list of ClanLines. The line_map list is *the* fundamental internal representation of a CLAN file. You can loop through the line_map and print the content within each ClanLine:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "%xdb:\taverage_dB=\"-34.55\" peak_dB=\"-19.83\"\n",
      "\n",
      "*OLN:\t0 . \u001514510_16190\u0015\n",
      "\n",
      "%xdb:\taverage_dB=\"-30.45\" peak_dB=\"-20.70\"\n",
      "\n",
      "*NOF:\t0 . \u001516190_17000\u0015\n",
      "\n",
      "%xdb:\taverage_dB=\"-40.45\" peak_dB=\"-32.04\"\n",
      "\n",
      "*OLN:\t0 . \u001517000_18300\u0015\n",
      "\n",
      "%xdb:\taverage_dB=\"-27.94\" peak_dB=\"-17.41\"\n",
      "\n",
      "*NOF:\t0 . \u001518300_19100\u0015\n",
      "\n",
      "%xdb:\taverage_dB=\"-42.45\" peak_dB=\"-27.65\"\n",
      "\n",
      "*SIL:\t0 . \u001519100_20180\u0015\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# we're just looking at lines 50-60 to save space, \n",
    "# but you can loop through the entire line map if \n",
    "# you want everything in a CLAN file\n",
    "\n",
    "for line in clan_file.line_map[50:60]:\n",
    "    print line.line"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You have access to all the information in every ClanLine object in the line_map. In this next example, we loop through, check if a line is a tiered ClanLine, and if so, print just the tier:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OLN\n",
      "NOF\n",
      "OLN\n",
      "NOF\n",
      "SIL\n"
     ]
    }
   ],
   "source": [
    "# again, just looking at lines 50-60 to save space\n",
    "\n",
    "for line in clan_file.line_map[50:60]:\n",
    "    if line.is_tier_line:\n",
    "        print line.tier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# class ClanLine\n",
    "\n",
    "Here are all the variables that belong to a ClanLine object:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'content': '0 . ',\n",
      " 'conv_block_num': 0,\n",
      " 'index': 149,\n",
      " 'is_clan_comment': False,\n",
      " 'is_conv_block_delimiter': False,\n",
      " 'is_end_header': False,\n",
      " 'is_header': False,\n",
      " 'is_multi_parent': False,\n",
      " 'is_paus_block_delimiter': False,\n",
      " 'is_tier_line': True,\n",
      " 'is_tier_without_timestamp': False,\n",
      " 'is_user_comment': False,\n",
      " 'line': '*NOF:\\t0 . \\x1570240_74140\\x15\\n',\n",
      " 'multi_line_parent': None,\n",
      " 'tier': 'NOF',\n",
      " 'time_offset': 74140,\n",
      " 'time_onset': 70240,\n",
      " 'total_time': 3900,\n",
      " 'within_conv_block': False,\n",
      " 'within_paus_block': False,\n",
      " 'xdb_average': 0,\n",
      " 'xdb_line': False,\n",
      " 'xdb_peak': 0}\n"
     ]
    }
   ],
   "source": [
    "# just select a random ClanLine (happens to be 149th line in the line_map\n",
    "# in this example) and print all the member variables:\n",
    "\n",
    "random_clanline = clan_file.line_map[149]\n",
    "pp.pprint(random_clanline.__dict__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# filters for ClanFile\n",
    "\n",
    "There are a bunch of filters available to all the classes which behave roughly identically across different objects. In other words, a filter like get_user_comments() will return the same kind of result whether it's on a ClanFile, ClanBlock, or LineRange.\n",
    "\n",
    "Here are some examples of filters and their results:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ClanFile.get_tiers(*tiers)\n",
    "\n",
    "get_tiers() will return a **LineRange** filled will all the lines in a ClanFile that have the specified tier. For example, let's get all the lines that are \"FAN\" or \"MAN\" tiered:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2134717\n"
     ]
    }
   ],
   "source": [
    "# fan_or_man will be a LineRange object with just \n",
    "# the tiered lines that are \"FAN\" or \"MAN\"\n",
    "\n",
    "fan_or_man = clan_file.get_tiers(\"FAN\", \"MAN\")\n",
    "\n",
    "# a LineRange object has a \"total_time\" member. This\n",
    "# is the cumulative time in milliseconds of all the \n",
    "# ClanLines \n",
    "print fan_or_man.total_time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "or how about \"FAN\", \"MAN\", \"FAF\" and \"OLN\":\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5175587\n"
     ]
    }
   ],
   "source": [
    "fan_man_faf_oln = clan_file.get_tiers(\"FAN\", \"MAN\", \"FAF\", \"OLN\")\n",
    "\n",
    "print fan_man_faf_oln.total_time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ClanFile.get_conv_block(block_num)\n",
    "\n",
    "get_conv_block() returns a **ClanBlock** object given an integer number. For example:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "block_42 = clan_file.get_conv_block(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13290"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "block_42.total_time # in milliseconds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1179960"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "block_42.onset # in milliseconds, relative to start of CLAN file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1193250"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "block_42.offset # in milliseconds, relative to start of CLAN file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remember when we mentioned that filters are available across objects? Here's an example. Instead of calling get_tiers() on a ClanFile object, let's call it on this ClanBlock object we've pulled out:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fan_man_in_block42 = block_42.get_tiers(\"FAN\", \"MAN\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5780"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fan_man_in_block42.total_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the tier: FAN\n",
      "the timestamp: 1179960_1181080\n",
      "the raw content of the line:   *FAN:\t&=w4_74 . \u00151179960_1181080\u0015\n",
      "\n",
      "the tier: MAN\n",
      "the timestamp: 1181890_1183030\n",
      "the raw content of the line:   *MAN:\t&=w4_78 . \u00151181890_1183030\u0015\n",
      "\n",
      "the tier: MAN\n",
      "the timestamp: 1188790_1189790\n",
      "the raw content of the line:   *MAN:\t&=w0_90 . \u00151188790_1189790\u0015\n",
      "\n",
      "the tier: FAN\n",
      "the timestamp: 1190730_1193250\n",
      "the raw content of the line:   *FAN:\t&=w11_78 . \u00151190730_1193250\u0015\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for line in fan_man_in_block42.line_map:\n",
    "    print \"the tier: \" + line.tier\n",
    "    print \"the timestamp: \" + line.timestamp()\n",
    "    print \"the raw content of the line:   \" + line.line\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "fan_man_in_block42 is a LineRange object representing just the FAN and MAN tiered lines in block 42 of this CLAN file."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ClanFile.get_conv_blocks(begin=1, end=None, select=None)\n",
    "\n",
    "Instead of just picking out a single conversation block, you can filter out more than one at a time. There's two ways you can call this function. Option 1 is giving it \"begin\" and \"end\" markers. This will return all the blocks between begin and end. Option 2 is supplying a list of specific indices of blocks, and it will return just these specific blocks (in ascending order, list doesn't have to be ordered).\n",
    "\n",
    "Example of Option 1:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "356880\n"
     ]
    }
   ],
   "source": [
    "blocks_3_to_50 = clan_file.get_conv_blocks(begin=3, end=50)\n",
    "\n",
    "print blocks_3_to_50.total_time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example of Option 2:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32330\n"
     ]
    }
   ],
   "source": [
    "blocks_7_12_56_and_158 = clan_file.get_conv_blocks(select=[7, 12, 56, 158])\n",
    "\n",
    "print blocks_7_12_56_and_158.total_time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The resulting object of a get_conv_blocks() function call is a **BlockGroup**.\n",
    "\n",
    "A BlockGroup is a collection of **ClanBlock** objects, layed out into a single line_map. So you can loop through a BlockGroup's line_map just like in a ClanFile of ClanBlock or LineRange:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the tier:     MAN\n",
      "timestamp:    299340_300350\n",
      "raw content:  *MAN:\t&=w2_90 . \u0015299340_300350\u0015\n",
      "\n",
      "the tier:     CXN\n",
      "timestamp:    579360_580520\n",
      "raw content:  *CXN:\t0 . \u0015579360_580520\u0015\n",
      "\n",
      "the tier:     FAN\n",
      "timestamp:    1486680_1487840\n",
      "raw content:  *FAN:\t&=w6_54 . \u00151486680_1487840\u0015\n",
      "\n",
      "the tier:     OLN\n",
      "timestamp:    1487840_1488640\n",
      "raw content:  *OLN:\t0 . \u00151487840_1488640\u0015\n",
      "\n",
      "the tier:     FAN\n",
      "timestamp:    1488640_1490680\n",
      "raw content:  *FAN:\t&=w8_17 . \u00151488640_1490680\u0015\n",
      "\n",
      "the tier:     OLN\n",
      "timestamp:    1490680_1491660\n",
      "raw content:  *OLN:\t0 . \u00151490680_1491660\u0015\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# just looping through the first 20 lines to save space...\n",
    "for line in blocks_7_12_56_and_158.line_map[0:20]:\n",
    "    if line.is_tier_line:\n",
    "        print \"the tier:     \" + line.tier\n",
    "        print \"timestamp:    \" + line.timestamp()\n",
    "        print \"raw content:  \" + line.line"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Each block within a **BlockGroup** is also represented as a distinct **ClanBlock** in the BlockGroup.blocks member variable (so not just a list of **ClanLines** in the line_map variable). So in our current example, we should have 4 ClanBlock objects in the blocks_7_12_56_and_158 BlockGroup variable:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<pyclan.elements.ClanBlock at 0x105342e90>,\n",
       " <pyclan.elements.ClanBlock at 0x105361450>,\n",
       " <pyclan.elements.ClanBlock at 0x1053614d0>,\n",
       " <pyclan.elements.ClanBlock at 0x105361510>]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "blocks_7_12_56_and_158.blocks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looks like we do. Let's loop through them and print some info about each one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "block index:   7\n",
      "onset:         299340\n",
      "offset:        300350\n",
      "total time:    1010\n",
      "\n",
      "block index:   12\n",
      "onset:         579360\n",
      "offset:        580520\n",
      "total time:    1160\n",
      "\n",
      "block index:   56\n",
      "onset:         1486680\n",
      "offset:        1500240\n",
      "total time:    13560\n",
      "\n",
      "block index:   158\n",
      "onset:         4004000\n",
      "offset:        4020600\n",
      "total time:    16600\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for block in blocks_7_12_56_and_158.blocks:\n",
    "    print \"block index:   {}\".format(block.index)\n",
    "    print \"onset:         {}\".format(block.onset)\n",
    "    print \"offset:        {}\".format(block.offset)\n",
    "    print \"total time:    {}\".format(block.total_time)\n",
    "    print\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# More Filters\n",
    "\n",
    "Here are some more useful filters that are available:\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## get_user_comments()\n",
    "\n",
    "returns a list of comment strings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[%com:\tthis is a user comment\n",
      ",\n",
      " %xcom:\tsubregion 1 of 5  (ranked 1 of 5)  starts at 2100000 -- previous timestamp adjusted: was 2100420\n",
      ",\n",
      " %com:\tdo not know what SIS is saying here\n",
      ",\n",
      " %com:\tFAT is listing off book titles of a stack of books\n",
      ",\n",
      " %com:\trefers to a piece of clothing, not the fabric\n",
      ",\n",
      " %com:\tMOT misspeaks, means mothership, corrects herself\n",
      ",\n",
      " %xcom:\tsubregion 1 of 5  (ranked 1 of 5)  ends at 5700000 -- previous\n",
      ",\n",
      " %xcom:\tsubregion 2 of 5  (ranked 2 of 5)  starts at 8400000 -- previous timestamp adjusted: was 8402910\n",
      ",\n",
      " %com:\trefers to physical DVDs they're looking at\n",
      ",\n",
      " %com:\trefers to physical DVDs they're looking at\n",
      ",\n",
      " %com:\trefers to physical DVDs they're looking at\n",
      ",\n",
      " %com:\trefers to physical DVDs they're looking at\n",
      ",\n",
      " %com:\tthe movie, holding the DVD case\n",
      ",\n",
      " %xcom:\tthe movie, looking at the DVD case\n",
      ",\n",
      " %com:\tsuper muffled while CHI has a coat on\n",
      ",\n",
      " %com:\tcan't understand what MOT and SIS are saying here\n",
      ",\n",
      " %com:\tbegin car ride\n",
      ",\n",
      " %com:\tMOT takes the vest off here to feed CHI\n",
      ",\n",
      " %xcom:\tsubregion 2 of 5  (ranked 2 of 5)  ends at 12000000 -- previous\n",
      ",\n",
      " %xcom:\tmanually adjusted timestamp\n",
      ",\n",
      " %xcom:\tsubregion 3 of 5 (ranked 3 of 5) starts at 12000000 -- previous\n",
      ",\n",
      " %com:\tCHI runs out of the room but MOT follows with the vest shortly\n",
      ",\n",
      " %com:\tCHI runs out of the room again\n",
      ",\n",
      " %xcom:\tsilence 1 of 2 starts at 14622850.0 -- previous timestamp adjusted: was 14623340\n",
      ",\n",
      " %com:\tbegin skip\n",
      ",\n",
      " %com:\tend skip\n",
      ",\n",
      " %xcom:\tsubregion 3 of 5  (ranked 3 of 5)  ends at 15600000 -- previous\n",
      ",\n",
      " %xcom:\tsubregion 4 of 5  (ranked 5 of 5)  starts at 22200000 -- previous timestamp adjusted: was 23827750 - lowest ranked region; [contains silent region: [14622850.0, 24253786.873] ]\n",
      ",\n",
      " %xcom:\tsilence 1 of 2 ends at 24253786.873 -- previous timestamp adjusted: was 24265370\n",
      ",\n",
      " %xcom:\tsubregion 4 of 5   (ranked 5 of 5)  ends at 25800000 -- previous timestamp adjusted: was 25800620. lowest ranked region; skip unless necessary\n",
      ",\n",
      " %xcom:\tsubregion 5 of 5  (ranked 4 of 5)  starts at 35400000 -- previous timestamp adjusted: was 35400980\n",
      ",\n",
      " %com:\tMOT realizes the vest hasn't been on CHI\n",
      ",\n",
      " %com:\tvest goes back on, started make-up region here\n",
      ",\n",
      " %com:\tend make-up region\n",
      ",\n",
      " %xcom:\tsubregion 5 of 5  (ranked 4 of 5)  ends at 39000000 -- previous\n",
      ",\n",
      " %xcom:\tsilence 2 of 2 starts at 39156632.017 -- previous timestamp adjusted: was 39157630\n",
      ",\n",
      " %xcom:\tsilence 2 of 2 ends at 57600140.0 -- previous timestamp adjusted: was 57599990\n",
      "]\n"
     ]
    }
   ],
   "source": [
    "user_comments = clan_file.get_user_comments()\n",
    "\n",
    "pp.pprint(user_comments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n"
     ]
    }
   ],
   "source": [
    "comments_in_block = block_42.get_user_comments()\n",
    "\n",
    "# no comments in this block\n",
    "print comments_in_block"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# get_within_time(begin=0, end=None)\n",
    "\n",
    "This returns a LineRange with all the lines between the specified time range. If \"begin\" is left out, it'll start from the very beggining, until end. If \"end\" is left out, it will start from \"begin\" until the very end. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "the lines within 10000ms and 20000ms: \n",
      "\n",
      "[*NOF:\t0 . \u001511890_14510\u0015\n",
      ", %xdb:\taverage_dB=\"-34.55\" peak_dB=\"-19.83\"\n",
      ", *OLN:\t0 . \u001514510_16190\u0015\n",
      ", %xdb:\taverage_dB=\"-30.45\" peak_dB=\"-20.70\"\n",
      ", *NOF:\t0 . \u001516190_17000\u0015\n",
      ", %xdb:\taverage_dB=\"-40.45\" peak_dB=\"-32.04\"\n",
      ", *OLN:\t0 . \u001517000_18300\u0015\n",
      ", %xdb:\taverage_dB=\"-27.94\" peak_dB=\"-17.41\"\n",
      ", *NOF:\t0 . \u001518300_19100\u0015\n",
      ", %xdb:\taverage_dB=\"-42.45\" peak_dB=\"-27.65\"\n",
      ", *SIL:\t0 . \u001519100_20180\u0015\n",
      ", %xdb:\taverage_dB=\"-46.96\" peak_dB=\"-31.70\"\n",
      ", @Eg:\tPause 1\n",
      ", @Bg:\tConversation 1\n",
      "]\n",
      "\n",
      "\n",
      "total time of LineRange: 8290\n"
     ]
    }
   ],
   "source": [
    "lines_within_10000_and_20000 = clan_file.get_within_time(begin=10000, end=20000)\n",
    "\n",
    "print \"\\nthe lines within 10000ms and 20000ms: \\n\\n{}\\n\\n\".format(lines_within_10000_and_20000.line_map)\n",
    " \n",
    "\n",
    "print \"total time of LineRange: {}\".format(lines_within_10000_and_20000.total_time)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ClanFile.new_file_from_blocks(path, blocks=[], rewrite_timestamps=False, begin=1, end=None)\n",
    "\n",
    "This function allows you to create a brand new .cha file containing only the blocks you specify as argument. You can either specify a range with begin= and end= or select specific blocks by passing in a list of indices with the blocks=[] argument. For example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clan_file.new_file_from_blocks(\"new_CLAN_file_blocks10-45.cha\",begin=10, end=45)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "or"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clan_file.new_file_from_blocks(\"new_CLAN_file_blocks_3_7_13_29_147.cha\", blocks=[3, 7, 13, 29, 147])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "the rewrite_timestamps argument hasn't been implemented yet, but once it is, you'll be able to have all the timestamps in the new .cha file start at 0 and be contiguous with each other."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# More examples coming soon...."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}