gdc_client_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed Chunk 1 | Folders 1-25 (2.03% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 2 | Folders 26-50 (3.98% complete) | Elapsed Time (this_chunk): 3 sec\n",
      "Processed Chunk 3 | Folders 51-75 (5.93% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 4 | Folders 76-100 (7.88% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 5 | Folders 101-125 (9.83% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 6 | Folders 126-150 (11.78% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 7 | Folders 151-175 (13.73% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 8 | Folders 176-200 (15.68% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 9 | Folders 201-225 (17.63% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 10 | Folders 226-250 (19.58% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 11 | Folders 251-275 (21.53% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 12 | Folders 276-300 (23.48% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 13 | Folders 301-325 (25.43% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 14 | Folders 326-350 (27.38% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 15 | Folders 351-375 (29.33% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 16 | Folders 376-400 (31.28% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 17 | Folders 401-425 (33.23% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 18 | Folders 426-450 (35.18% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 19 | Folders 451-475 (37.13% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 20 | Folders 476-500 (39.08% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 21 | Folders 501-525 (41.03% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 22 | Folders 526-550 (42.98% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 23 | Folders 551-575 (44.93% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 24 | Folders 576-600 (46.88% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 25 | Folders 601-625 (48.83% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 26 | Folders 626-650 (50.78% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 27 | Folders 651-675 (52.73% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 28 | Folders 676-700 (54.68% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 29 | Folders 701-725 (56.63% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 30 | Folders 726-750 (58.58% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 31 | Folders 751-775 (60.53% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 32 | Folders 776-800 (62.48% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 33 | Folders 801-825 (64.43% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 34 | Folders 826-850 (66.38% complete) | Elapsed Time (this_chunk): 3 sec\n",
      "Processed Chunk 35 | Folders 851-875 (68.33% complete) | Elapsed Time (this_chunk): 0 sec\n",
      "Processed Chunk 36 | Folders 876-900 (70.28% complete) | Elapsed Time (this_chunk): 1 sec\n",
      "Processed Chunk 37 | Folders 901-925 (72.23% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 38 | Folders 926-950 (74.18% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 39 | Folders 951-975 (76.13% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 40 | Folders 976-1000 (78.08% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 41 | Folders 1001-1025 (80.03% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 42 | Folders 1026-1050 (81.98% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 43 | Folders 1051-1075 (83.93% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 44 | Folders 1076-1100 (85.88% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 45 | Folders 1101-1125 (87.83% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 46 | Folders 1126-1150 (89.78% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 47 | Folders 1151-1175 (91.73% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 48 | Folders 1176-1200 (93.68% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 49 | Folders 1201-1225 (95.63% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 50 | Folders 1226-1250 (97.58% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 51 | Folders 1251-1275 (99.53% complete) | Elapsed Time (this_chunk): 4 sec\n",
      "Processed Chunk 52 | Folders 1276-1300 (101.48% complete) | Elapsed Time (this_chunk): 1 sec\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import glob\n",
    "import time\n",
    "\n",
    "# Set the path to the directory containing the subfolders\n",
    "data_dir = 'C:/Users/trent/OneDrive/Desktop/gdc-client_v1.6.1_Windows_x64'\n",
    "\n",
    "# Use glob to get a list of all the subfolders in the directory\n",
    "subfolders = glob.glob(os.path.join(data_dir, '*'))\n",
    "\n",
    "# Set the chunk size\n",
    "chunk_size = 25\n",
    "\n",
    "# Loop over the subfolders in chunks\n",
    "this_rnaseq_data = pd.DataFrame()\n",
    "num_subfolders = len(subfolders)\n",
    "num_chunks = (num_subfolders - 1) // chunk_size + 1\n",
    "\n",
    "for chunk_idx in range(num_chunks):\n",
    "    start_idx = chunk_idx * chunk_size\n",
    "    end_idx = (chunk_idx + 1) * chunk_size\n",
    "    subfolders_chunk = subfolders[start_idx:end_idx]\n",
    "    start_time = time.time()\n",
    "\n",
    "    # Loop over each subfolder in the chunk and import the TSV file\n",
    "    for i, subfolder in enumerate(subfolders_chunk):\n",
    "        # Get the path to the TSV file in the subfolder\n",
    "        file_pattern = os.path.join(subfolder, '*.tsv')\n",
    "        tsv_files = glob.glob(file_pattern)\n",
    "\n",
    "        if len(tsv_files) > 0:\n",
    "            # Read the first TSV file into a pandas DataFrame\n",
    "            data = pd.read_csv(tsv_files[0], delimiter='\\t', skiprows=1)\n",
    "            \n",
    "            data = data[~data['gene_id'].str.contains('N_unmapped|N_multimapping|N_noFeature|N_ambiguous')]\n",
    "            \n",
    "            # Get the file name without the extension from the path\n",
    "            file_name = os.path.splitext(os.path.basename(tsv_files[0]))[0].split('.')[0]\n",
    "\n",
    "            # Add a new column to this_rnaseq_data with the file name\n",
    "            data['file_name'] = file_name\n",
    "\n",
    "            # Append the data to the all_rnaseq_data DataFrame\n",
    "            this_rnaseq_data = this_rnaseq_data.append(data)\n",
    "\n",
    "        # Print progress report\n",
    "        total_idx = start_idx + i\n",
    "        progress = (i + 1) / chunk_size\n",
    "        elapsed_time = time.time() - start_time\n",
    "        remaining_time = (elapsed_time / progress) - elapsed_time\n",
    "        print(f'Chunk {chunk_idx + 1}/{num_chunks} | Progress: {progress:.1%} | Elapsed Time: {elapsed_time:.0f} sec | Remaining Time: {remaining_time:.0f} sec', end='\\r')\n",
    "\n",
    "    # Save the chunk to a CSV file\n",
    "    chunk_filename = f'all_rnaseq_data_chunk{chunk_idx + 1}.csv'\n",
    "    this_rnaseq_data.to_csv(data_dir + '/' + chunk_filename, index=False)\n",
    "    this_rnaseq_data = pd.DataFrame()\n",
    "    \n",
    "    # Print a progress report for this chunk\n",
    "    chunk_percent = ((end_idx + 1) / num_subfolders) * 100\n",
    "    print(f'Processed Chunk {chunk_idx+1} | Folders {start_idx+1}-{end_idx} ({chunk_percent:.2f}% complete) | Elapsed Time (this_chunk): {elapsed_time:.0f} sec')\n",
    "\n",
    "# Get a list of all the chunk CSV files\n",
    "csv_files = glob.glob(os.path.join(data_dir, 'all_rnaseq_data_chunk*.csv'))\n",
    "\n",
    "# Load each CSV file into a separate DataFrame and store them in a list\n",
    "df_list = []\n",
    "for i, csv_file in enumerate(csv_files):\n",
    "    print(f\"Loading chunk {i+1}/{len(csv_files)}: {csv_file}\")\n",
    "    try:\n",
    "        df = pd.read_csv(csv_file)\n",
    "        df_list.append(df)\n",
    "    except:\n",
    "        print(f\"Error loading chunk {i+1}/{len(csv_files)}: {csv_file}\")\n",
    "\n",
    "# Concatenate all the DataFrames in the list into a single DataFrame\n",
    "all_rnaseq_data = pd.concat(df_list, ignore_index=True)\n",
    "\n",
    "all_rnaseq_data.to_csv(\"C:/Users/trent/OneDrive/Desktop/all_rnaseq_data.csv\", index=False)\n",
    "\n",
    "print(all_rnaseq_data.shape)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Springboard",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "2c9dfe3903a05c661954e65e0d794d75f447187a193e16d5c888fc2819304296"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}