qwen.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "plaintext"
    }
   },
   "outputs": [],
   "source": [
    "# Install required libraries\n",
    "!pip install transformers accelerate datasets torch torchvision pillow\n",
    "\n",
    "# Import necessary libraries\n",
    "from transformers import Qwen2VLForConditionalGeneration, AutoProcessor, TrainingArguments, Trainer\n",
    "from datasets import load_dataset\n",
    "from PIL import Image\n",
    "import torch\n",
    "\n",
    "# Load the pretrained model and processor\n",
    "model = Qwen2VLForConditionalGeneration.from_pretrained(\n",
    "    \"Qwen/Qwen2-VL-2B-Instruct\",\n",
    "    torch_dtype=\"auto\",\n",
    "    device_map=\"auto\",\n",
    ")\n",
    "processor = AutoProcessor.from_pretrained(\"Qwen/Qwen2-VL-2B-Instruct\")\n",
    "\n",
    "# Load the dataset\n",
    "ds = load_dataset(\"mdwiratathya/ROCO-radiology\")\n",
    "\n",
    "# Debugging function to inspect dataset integrity\n",
    "def check_data_integrity(dataset):\n",
    "    for idx, example in enumerate(dataset):\n",
    "        try:\n",
    "            assert isinstance(example[\"image\"], Image.Image), f\"Invalid image format at index {idx}\"\n",
    "            assert isinstance(example[\"caption\"], str), f\"Invalid caption format at index {idx}\"\n",
    "        except AssertionError as e:\n",
    "            print(f\"Dataset integrity check failed: {e}\")\n",
    "            return False\n",
    "    print(\"All dataset entries are valid.\")\n",
    "    return True\n",
    "\n",
    "# Verify dataset integrity\n",
    "if not check_data_integrity(ds[\"train\"]):\n",
    "    raise ValueError(\"Dataset contains invalid entries.\")\n",
    "\n",
    "# Preprocessing function with robust error handling\n",
    "def preprocess_data(example):\n",
    "    try:\n",
    "        # Resize image\n",
    "        image = example[\"image\"].resize((128, 128))\n",
    "        caption = example[\"caption\"]\n",
    "\n",
    "        # Prepare text prompt\n",
    "        text_prompt = processor.apply_chat_template(\n",
    "            [\n",
    "                {\n",
    "                    \"role\": \"user\",\n",
    "                    \"content\": [\n",
    "                        {\"type\": \"image\"},\n",
    "                        {\"type\": \"text\", \"text\": caption},\n",
    "                    ],\n",
    "                }\n",
    "            ],\n",
    "            add_generation_prompt=True,\n",
    "        )\n",
    "\n",
    "        # Tokenize and prepare inputs\n",
    "        inputs = processor(\n",
    "            text=[text_prompt],\n",
    "            images=[image],\n",
    "            padding=True,\n",
    "            truncation=True,\n",
    "            max_length=512,\n",
    "            return_tensors=\"pt\",\n",
    "        )\n",
    "        inputs[\"labels\"] = inputs[\"input_ids\"]\n",
    "\n",
    "        return {\n",
    "            \"input_ids\": inputs[\"input_ids\"].squeeze(),\n",
    "            \"attention_mask\": inputs[\"attention_mask\"].squeeze(),\n",
    "            \"pixel_values\": inputs[\"pixel_values\"].squeeze(),\n",
    "            \"labels\": inputs[\"labels\"].squeeze(),\n",
    "        }\n",
    "    except Exception as e:\n",
    "        print(f\"Error processing example: {example}, {e}\")\n",
    "        raise\n",
    "\n",
    "# Preprocess dataset without multiprocessing\n",
    "try:\n",
    "    print(\"Preprocessing train dataset...\")\n",
    "    train_data = ds[\"train\"].map(preprocess_data, batched=False)\n",
    "    print(\"Preprocessing validation dataset...\")\n",
    "    val_data = ds[\"validation\"].map(preprocess_data, batched=False)\n",
    "    print(\"Dataset preprocessing completed.\")\n",
    "except Exception as e:\n",
    "    print(f\"Error during dataset preprocessing: {e}\")\n",
    "    raise\n",
    "\n",
    "# Training arguments\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=\"./results\",\n",
    "    per_device_train_batch_size=4,\n",
    "    gradient_accumulation_steps=4,\n",
    "    per_device_eval_batch_size=4,\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    save_strategy=\"epoch\",\n",
    "    learning_rate=5e-5,\n",
    "    num_train_epochs=3,\n",
    "    logging_dir=\"./logs\",\n",
    "    logging_steps=500,\n",
    "    save_total_limit=2,\n",
    "    load_best_model_at_end=True,\n",
    "    fp16=True,\n",
    ")\n",
    "\n",
    "# Trainer setup\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_data,\n",
    "    eval_dataset=val_data,\n",
    "    tokenizer=processor,\n",
    ")\n",
    "\n",
    "# Train the model\n",
    "try:\n",
    "    print(\"Training model...\")\n",
    "    trainer.train()\n",
    "    print(\"Model training completed.\")\n",
    "except Exception as e:\n",
    "    print(f\"Error during training: {e}\")\n",
    "    raise\n",
    "\n",
    "# Save the fine-tuned model and processor\n",
    "model.save_pretrained(\"./fine_tuned_qwen2vl\")\n",
    "processor.save_pretrained(\"./fine_tuned_qwen2vl\")\n",
    "\n",
    "# Test the fine-tuned model\n",
    "test_example = ds[\"test\"][0]\n",
    "test_image = test_example[\"image\"].resize((128, 128))\n",
    "test_caption = test_example[\"caption\"]\n",
    "\n",
    "# Prepare inputs for testing\n",
    "messages = [\n",
    "    {\n",
    "        \"role\": \"user\",\n",
    "        \"content\": [\n",
    "            {\"type\": \"image\"},\n",
    "            {\"type\": \"text\", \"text\": test_caption},\n",
    "        ],\n",
    "    }\n",
    "]\n",
    "text_prompt = processor.apply_chat_template(messages, add_generation_prompt=True)\n",
    "inputs = processor(\n",
    "    text=[text_prompt],\n",
    "    images=[test_image],\n",
    "    padding=True,\n",
    "    truncation=True,\n",
    "    max_length=512,\n",
    "    return_tensors=\"pt\",\n",
    ").to(\"cuda\")\n",
    "\n",
    "# Generate output\n",
    "output_ids = model.generate(**inputs, max_new_tokens=1024)\n",
    "output_text = processor.batch_decode(\n",
    "    output_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True\n",
    ")\n",
    "\n",
    "# Print generated response\n",
    "print(\"Generated Response:\", output_text[0])"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}