test_clipvip.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "529cddeb-dc94-47a5-80f2-0535cee76d16",
   "metadata": {},
   "outputs": [],
   "source": [
    "import av\n",
    "import torch\n",
    "from torch.nn import functional as F\n",
    "import numpy as np\n",
    "from easydict import EasyDict as edict\n",
    "\n",
    "from transformers.models.clip.configuration_clip import CLIPConfig, CLIPTextConfig, CLIPVisionConfig\n",
    "from transformers import CLIPProcessor, CLIPTokenizerFast\n",
    "from clipvip.CLIP_VIP import CLIPModel, clip_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "b792cfc7-b3e3-456d-93e0-597dd6e3f4fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extraCfg = edict({\n",
    "    \"type\": \"ViP\",\n",
    "    \"temporal_size\": 12,\n",
    "    \"if_use_temporal_embed\": 1,\n",
    "    \"logit_scale_init_value\": 4.60,\n",
    "    \"add_cls_num\": 3\n",
    "})\n",
    "\n",
    "clipconfig = CLIPConfig.from_pretrained(\"openai/clip-vit-base-patch32\")\n",
    "clipconfig.vision_additional_config = extraCfg\n",
    "\n",
    "checkpoint = torch.load(\"/Users/teli/www/test/视频向量提取/XPretrain-视频获取/CLIP-ViP/pretrain_clipvip_base_32.pt\")\n",
    "cleanDict = { key.replace(\"clipmodel.\", \"\") : value for key, value in checkpoint.items() }\n",
    "model =  CLIPModel(config=clipconfig)\n",
    "model.load_state_dict(cleanDict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "847d7c4f-4c8b-43cf-842a-2ae7b85442ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "!export https_proxy=http://127.0.0.1:7890 http_proxy=http://127.0.0.1:7890 all_proxy=socks5://127.0.0.1:7890"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "6df5d6b1-d711-4412-8cf3-ddb0a40847e0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 512])\n"
     ]
    }
   ],
   "source": [
    "tokenizer = CLIPTokenizerFast.from_pretrained(\"openai/clip-vit-base-patch32\")\n",
    "tokens = tokenizer([\"in the forest\"], padding=True, return_tensors=\"pt\")\n",
    "textOutput = model.get_text_features(**tokens)\n",
    "print(textOutput.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "47d9fc8b-40a5-447e-b159-1d610d83ac5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_video_pyav(container, indices):\n",
    "    frames = []\n",
    "    container.seek(0)\n",
    "    start_index = indices[0]\n",
    "    end_index = indices[-1]\n",
    "    for i, frame in enumerate(container.decode(video=0)):\n",
    "        if i > end_index:\n",
    "            break\n",
    "        if i >= start_index and i in indices:\n",
    "            frames.append(frame)\n",
    "    return np.stack([x.to_ndarray(format=\"rgb24\") for x in frames])\n",
    "\n",
    "\n",
    "def sample_frame_indices(clip_len, frame_sample_rate, seg_len):\n",
    "    '''\n",
    "    Sample a given number of frame indices from the video.\n",
    "    Args:\n",
    "        clip_len (`int`): Total number of frames to sample.\n",
    "        frame_sample_rate (`int`): Sample every n-th frame.\n",
    "        seg_len (`int`): Maximum allowed index of sample's last frame.\n",
    "    Returns:\n",
    "        indices (`List[int]`): List of sampled frame indices\n",
    "    '''\n",
    "    converted_len = int(clip_len * frame_sample_rate)\n",
    "    end_idx = np.random.randint(converted_len, seg_len)\n",
    "    start_idx = end_idx - converted_len\n",
    "    indices = np.linspace(start_idx, end_idx, num=clip_len)\n",
    "    indices = np.clip(indices, start_idx, end_idx - 1).astype(np.int64)\n",
    "    return indices\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "1eeb02e8-05bc-461b-9a9b-5c8ae7d4ffd6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoProcessor\n",
    "processor = AutoProcessor.from_pretrained(\"microsoft/xclip-base-patch16\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "55b6b5c1-e561-461c-94e0-4b6854c2234c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No accelerated colorspace conversion found from yuv420p to rgb24.\n",
      " (repeated 11 more times)\n",
      "No accelerated colorspace conversion found from yuv420p to rgb24.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "video.shape torch.Size([1, 12, 3, 224, 224])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/teli/miniforge3/envs/clip4clip/lib/python3.10/site-packages/transformers/feature_extraction_utils.py:141: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at /Users/runner/work/pytorch/pytorch/pytorch/torch/csrc/utils/tensor_new.cpp:264.)\n",
      "  return torch.tensor(value)\n"
     ]
    }
   ],
   "source": [
    "container = av.open(\"/Volumes/Samsung_T3/_download/test_cut/py_053.mp4\")\n",
    "\n",
    "clip_len = 12\n",
    "fcount = container.streams.video[0].frames\n",
    "# sample 16 frames\n",
    "indices = sample_frame_indices(clip_len=clip_len, frame_sample_rate=fcount//clip_len, seg_len=fcount)\n",
    "video = read_video_pyav(container, indices)\n",
    "pixel_values = processor(videos=list(video), return_tensors=\"pt\").pixel_values\n",
    "\n",
    "print(\"video.shape\", pixel_values.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b8129ab6-95ed-436b-af28-e83f6a0030b5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([12, 3, 224, 224])\n"
     ]
    }
   ],
   "source": [
    "B, N, C, H, W = pixel_values.shape\n",
    "vv = pixel_values.reshape(-1, C, H, W)\n",
    "print(vv.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "9387b9f8-5b77-479c-8110-6f18cbe01056",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 4.2968e-02,  4.5826e-03, -2.0510e-02,  2.1235e-02,  2.9924e-02,\n",
      "         -2.1660e-02,  1.0331e-02, -5.7496e-03,  1.9632e-03, -1.6750e-02,\n",
      "          2.7891e-02,  1.9469e-03,  1.7285e-02,  3.7726e-02, -2.0460e-03,\n",
      "         -2.4150e-02, -1.2243e-01,  2.5370e-02, -3.1692e-02, -2.2621e-02,\n",
      "         -5.9742e-02,  5.1138e-02, -1.8714e-02, -3.2980e-02, -1.4633e-02,\n",
      "         -6.1462e-02, -4.4542e-03, -7.4782e-03,  8.8724e-03,  1.6593e-02,\n",
      "         -5.3571e-02, -1.2733e-02, -7.3391e-03,  5.5302e-03,  8.7937e-02,\n",
      "         -1.3672e-02, -4.0799e-02, -2.3102e-02,  1.0312e-02,  2.3676e-02,\n",
      "         -5.8805e-03,  2.3927e-02,  1.3316e-02, -1.1890e-02,  1.8357e-02,\n",
      "          7.2390e-02, -8.1439e-02, -3.8800e-04,  5.2957e-02,  1.6394e-02,\n",
      "          2.0144e-02, -4.0233e-02, -2.2974e-02, -2.2099e-02,  6.2945e-04,\n",
      "         -1.3704e-02,  8.3659e-02, -1.9435e-02, -4.7705e-03, -5.0799e-02,\n",
      "          6.5837e-02, -5.7450e-03, -4.7846e-02,  1.4080e-02,  5.2817e-03,\n",
      "         -7.3937e-03,  4.3109e-02, -2.9816e-02,  1.7472e-02, -1.8528e-02,\n",
      "          1.9086e-02,  6.7099e-02,  1.9946e-02,  4.1341e-03, -5.2120e-02,\n",
      "          5.5661e-03, -2.7610e-02, -1.9233e-02, -5.6992e-03,  5.9818e-03,\n",
      "          3.8132e-02, -6.5703e-02,  1.0403e-02,  1.0997e-01,  2.8007e-02,\n",
      "         -3.8057e-02, -2.1491e-02, -3.8106e-02,  1.9293e-02, -3.1739e-02,\n",
      "          2.6116e-04, -5.0076e-03, -6.5500e-01,  6.2071e-02,  5.4428e-03,\n",
      "         -2.1553e-02, -3.6007e-04, -2.6902e-02, -1.3170e-02,  4.0234e-03,\n",
      "          2.0144e-02, -3.3486e-02, -3.3633e-02,  2.7245e-02,  1.0242e-01,\n",
      "          1.6802e-02,  1.1291e-02,  9.0297e-03,  6.1747e-03, -4.2028e-02,\n",
      "         -2.6287e-03, -6.5109e-02, -6.7764e-03,  3.0512e-02,  3.9734e-02,\n",
      "         -2.7887e-02, -7.0855e-03, -5.1161e-02, -1.1537e-02, -4.2414e-02,\n",
      "         -2.3436e-02,  3.6816e-02, -1.1365e-02, -3.8221e-02,  2.2610e-02,\n",
      "         -3.8196e-03,  1.7846e-02, -8.4271e-04,  1.9180e-02,  1.3780e-02,\n",
      "          2.7928e-02,  2.1226e-02,  1.8370e-02,  5.9320e-02,  6.2814e-02,\n",
      "         -1.1267e-02, -2.4189e-03, -3.0987e-03,  7.0230e-03,  1.2333e-02,\n",
      "          1.6076e-02,  3.2892e-02,  2.1314e-02, -4.0406e-02,  2.6669e-02,\n",
      "          5.7420e-02, -1.9034e-02,  1.1101e-02, -2.6642e-02,  2.5240e-02,\n",
      "          1.0685e-05,  4.1675e-03, -2.7294e-02, -4.4130e-02, -1.7845e-02,\n",
      "          8.4131e-03, -6.0968e-03,  5.2139e-02,  2.0509e-02,  1.3212e-02,\n",
      "         -2.9735e-02,  2.7683e-02, -2.4663e-02,  3.7026e-02,  3.0479e-02,\n",
      "          2.3106e-02,  1.1439e-02,  4.0724e-02, -1.2998e-03, -5.5241e-02,\n",
      "         -1.4615e-04, -9.9703e-03, -6.1029e-02, -1.4451e-02,  7.2916e-02,\n",
      "          2.6504e-02,  1.1316e-02,  5.2677e-02,  1.7144e-02, -2.4691e-03,\n",
      "         -1.0159e-02, -2.2523e-02,  1.7184e-02, -3.6072e-02, -1.2492e-02,\n",
      "         -6.1511e-03, -1.6099e-02,  8.9060e-02, -3.0123e-03,  1.3579e-02,\n",
      "         -2.2558e-02, -1.8502e-02, -4.1432e-02, -1.8303e-02,  5.8325e-03,\n",
      "          1.8963e-02,  1.7020e-02,  4.0559e-03, -4.4809e-03,  3.7295e-02,\n",
      "         -2.6635e-02, -3.2418e-02,  2.2509e-02, -5.3819e-03, -2.3422e-02,\n",
      "          1.9642e-02,  1.8439e-02, -1.2174e-02, -1.5289e-02,  1.6680e-02,\n",
      "          1.0148e-01, -2.5153e-02,  2.6855e-02, -3.3411e-02,  3.7513e-03,\n",
      "          9.3260e-03, -1.7159e-02,  6.7766e-03, -5.2018e-02,  4.2834e-02,\n",
      "         -1.3716e-02, -2.5716e-02,  2.0884e-02,  1.9624e-03, -4.4343e-02,\n",
      "          2.7317e-02,  3.6148e-03, -1.1862e-02, -2.2078e-02, -2.3516e-03,\n",
      "          6.2758e-02, -2.1241e-03,  5.9680e-02,  1.4305e-03,  1.4381e-02,\n",
      "          3.0480e-02, -4.2336e-03, -1.3922e-02, -1.0374e-04,  2.2014e-02,\n",
      "         -7.9395e-03, -1.6462e-02, -2.5140e-02,  5.8786e-04,  3.3255e-03,\n",
      "         -1.2003e-02,  5.4495e-03, -1.1778e-02, -6.3402e-02,  2.0175e-02,\n",
      "         -5.9290e-02,  2.1701e-02, -4.7258e-02, -1.1713e-03, -2.0250e-02,\n",
      "          2.4158e-02, -5.5648e-02,  6.7346e-03,  7.4486e-02,  4.2978e-03,\n",
      "          3.4981e-02,  1.6257e-02,  2.7912e-02,  1.4554e-02, -6.5946e-03,\n",
      "         -8.3685e-03, -1.8854e-02, -2.1801e-02, -4.8561e-03,  6.2470e-02,\n",
      "          1.2036e-02, -1.0247e-02, -3.4446e-02,  1.7223e-02, -1.4127e-02,\n",
      "          3.6349e-02,  2.7774e-02, -8.1842e-03, -1.5056e-02,  2.7007e-02,\n",
      "         -7.7338e-03,  4.4293e-02, -2.3387e-02,  4.2511e-02, -2.1707e-02,\n",
      "          3.1061e-02, -4.1112e-02,  4.5382e-02, -6.2382e-02,  3.7081e-02,\n",
      "         -6.0284e-03,  3.5389e-02,  3.5211e-03, -5.1012e-02, -1.8673e-03,\n",
      "          1.8666e-02,  2.1113e-02, -4.7202e-03,  2.0139e-02,  4.2537e-02,\n",
      "          1.7140e-02,  6.5737e-02,  8.1025e-03, -5.4703e-02, -1.6271e-04,\n",
      "          8.1339e-02, -6.9049e-02,  8.6994e-03, -1.8571e-03,  1.9991e-02,\n",
      "         -1.3114e-02, -6.6045e-02,  5.9215e-02, -4.5379e-03, -3.7451e-02,\n",
      "          2.7781e-02, -4.9740e-02, -6.0898e-04,  1.5412e-02,  8.0735e-02,\n",
      "          5.8027e-02,  1.2375e-01, -1.5039e-02,  3.7410e-03, -4.4371e-03,\n",
      "         -4.1035e-03, -2.9830e-03, -1.3198e-02,  3.0166e-02, -1.9643e-02,\n",
      "          1.2636e-02, -6.4493e-03,  9.5025e-03,  5.4073e-02, -3.0690e-02,\n",
      "         -3.7296e-02,  2.9789e-02, -3.0540e-02, -3.7531e-02, -8.4522e-03,\n",
      "          3.1646e-02, -9.7781e-03, -6.2846e-04, -2.5140e-02, -3.3236e-02,\n",
      "          1.4656e-02,  4.0819e-03, -1.0609e-02, -1.8958e-02,  2.5472e-02,\n",
      "         -9.8710e-02, -1.4845e-03, -3.6661e-02,  1.2374e-02,  2.4442e-03,\n",
      "          3.2446e-02,  6.0752e-02,  1.1728e-01, -4.8197e-03, -4.6901e-02,\n",
      "         -8.8136e-03,  6.8979e-02, -2.1797e-02,  3.6428e-02, -1.2214e-02,\n",
      "          3.4161e-02, -1.8054e-02, -3.0755e-02,  9.8140e-03,  6.6398e-03,\n",
      "          2.3985e-03, -4.1698e-02,  2.6191e-03,  3.6676e-03,  3.9917e-02,\n",
      "          1.5721e-02,  3.7684e-02,  7.2189e-03,  1.0760e-02,  4.0086e-03,\n",
      "         -5.3105e-02,  2.5856e-02, -1.4477e-02,  9.5476e-03, -9.2241e-04,\n",
      "          9.6127e-05, -4.2725e-03, -2.4245e-02,  2.4564e-02,  1.3147e-01,\n",
      "         -5.5457e-02, -3.2671e-02, -4.5018e-02,  2.9718e-02, -1.5660e-02,\n",
      "         -1.1105e-02, -4.0532e-02, -7.9137e-03,  2.1402e-02, -2.9454e-02,\n",
      "          4.2345e-03,  1.6338e-02,  2.8209e-02,  3.4367e-02,  2.4788e-02,\n",
      "         -1.2727e-02,  4.0670e-02,  1.6304e-02, -7.8697e-02,  1.5201e-02,\n",
      "         -5.6143e-02,  2.0924e-03,  1.6329e-02,  1.6121e-02, -1.5415e-02,\n",
      "         -1.5435e-02,  2.1507e-02, -1.5103e-02, -5.7801e-03, -1.2467e-02,\n",
      "         -2.6345e-02, -1.8070e-04, -2.8159e-02,  3.8796e-03,  4.6005e-03,\n",
      "          4.3740e-02,  3.8130e-02, -3.2589e-02,  2.6568e-02,  2.0343e-02,\n",
      "         -1.2432e-02, -2.7639e-02,  5.2539e-03, -3.8884e-02, -1.4502e-02,\n",
      "          3.0370e-03, -2.7531e-02,  2.3094e-02,  2.0890e-02,  1.7514e-02,\n",
      "         -3.1102e-02,  4.1807e-02, -2.6835e-02,  2.1054e-02,  1.6964e-02,\n",
      "         -2.7275e-02,  3.5884e-02, -4.0429e-02,  3.3694e-02, -1.3544e-03,\n",
      "          3.8153e-02, -4.1632e-02, -7.2119e-03,  2.8649e-02, -1.8378e-02,\n",
      "         -3.1784e-02,  3.5598e-02,  6.4888e-03, -4.8388e-02, -2.5603e-02,\n",
      "         -5.0692e-02, -3.8733e-02, -1.6021e-03, -3.5409e-03,  6.4011e-02,\n",
      "         -1.8287e-02,  1.3539e-02,  1.5179e-02, -1.7251e-02, -2.2048e-02,\n",
      "         -1.9591e-02, -1.7675e-02, -3.0663e-02,  3.9875e-02, -3.3814e-02,\n",
      "         -9.6362e-03, -1.3783e-02,  8.3028e-03, -3.0769e-02, -4.5723e-02,\n",
      "         -2.2787e-02, -1.4473e-02, -1.4861e-02,  8.7507e-03, -8.7079e-03,\n",
      "          4.5076e-02,  1.1242e-02,  3.4124e-02,  2.3239e-03,  6.9016e-02,\n",
      "          3.9873e-02,  3.4173e-02,  9.8947e-03, -1.9386e-02, -1.4594e-02,\n",
      "          5.9373e-02,  1.3484e-02, -2.0977e-02,  6.9884e-03,  2.9694e-03,\n",
      "         -2.2417e-02,  1.1562e-02,  4.8560e-02,  2.2448e-03,  9.5930e-02,\n",
      "          2.2815e-02,  2.7926e-02, -2.2285e-02,  3.3694e-03, -5.9484e-02,\n",
      "          4.6517e-02,  2.6544e-02]])\n"
     ]
    }
   ],
   "source": [
    "inputs = {\n",
    "        \"if_norm\": True,\n",
    "        \"pixel_values\": pixel_values}\n",
    "with torch.no_grad():\n",
    "    video_features = model.get_image_features(**inputs)\n",
    "print(video_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "c29d7a90-38cd-4598-9a96-437f51646acb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.1142], grad_fn=<SumBackward1>)\n"
     ]
    }
   ],
   "source": [
    "sim = F.cosine_similarity(textOutput, video_features, dim=1)\n",
    "print(sim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8706652-2883-4e74-b762-d878cbb23f0b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "37632765-1f4a-426e-a85f-7bdff1093d9e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "51dcc73d-5221-4d95-a55b-cee8a625ccfb",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "36bfe50e-1e7f-4dcf-b4f6-1bf448aab07f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "07a44f96-c548-4a73-ac6a-98ede5c904af",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 4.2968e-02,  4.5826e-03, -2.0510e-02,  2.1235e-02,  2.9924e-02,\n",
      "         -2.1660e-02,  1.0331e-02, -5.7496e-03,  1.9632e-03, -1.6750e-02,\n",
      "          2.7891e-02,  1.9469e-03,  1.7285e-02,  3.7726e-02, -2.0460e-03,\n",
      "         -2.4150e-02, -1.2243e-01,  2.5370e-02, -3.1692e-02, -2.2621e-02,\n",
      "         -5.9742e-02,  5.1138e-02, -1.8714e-02, -3.2980e-02, -1.4633e-02,\n",
      "         -6.1462e-02, -4.4542e-03, -7.4782e-03,  8.8724e-03,  1.6593e-02,\n",
      "         -5.3571e-02, -1.2733e-02, -7.3391e-03,  5.5302e-03,  8.7937e-02,\n",
      "         -1.3672e-02, -4.0799e-02, -2.3102e-02,  1.0312e-02,  2.3676e-02,\n",
      "         -5.8805e-03,  2.3927e-02,  1.3316e-02, -1.1890e-02,  1.8357e-02,\n",
      "          7.2390e-02, -8.1439e-02, -3.8800e-04,  5.2957e-02,  1.6394e-02,\n",
      "          2.0144e-02, -4.0233e-02, -2.2974e-02, -2.2099e-02,  6.2945e-04,\n",
      "         -1.3704e-02,  8.3659e-02, -1.9435e-02, -4.7705e-03, -5.0799e-02,\n",
      "          6.5837e-02, -5.7450e-03, -4.7846e-02,  1.4080e-02,  5.2817e-03,\n",
      "         -7.3937e-03,  4.3109e-02, -2.9816e-02,  1.7472e-02, -1.8528e-02,\n",
      "          1.9086e-02,  6.7099e-02,  1.9946e-02,  4.1341e-03, -5.2120e-02,\n",
      "          5.5661e-03, -2.7610e-02, -1.9233e-02, -5.6992e-03,  5.9818e-03,\n",
      "          3.8132e-02, -6.5703e-02,  1.0403e-02,  1.0997e-01,  2.8007e-02,\n",
      "         -3.8057e-02, -2.1491e-02, -3.8106e-02,  1.9293e-02, -3.1739e-02,\n",
      "          2.6116e-04, -5.0076e-03, -6.5500e-01,  6.2071e-02,  5.4428e-03,\n",
      "         -2.1553e-02, -3.6007e-04, -2.6902e-02, -1.3170e-02,  4.0234e-03,\n",
      "          2.0144e-02, -3.3486e-02, -3.3633e-02,  2.7245e-02,  1.0242e-01,\n",
      "          1.6802e-02,  1.1291e-02,  9.0297e-03,  6.1747e-03, -4.2028e-02,\n",
      "         -2.6287e-03, -6.5109e-02, -6.7764e-03,  3.0512e-02,  3.9734e-02,\n",
      "         -2.7887e-02, -7.0855e-03, -5.1161e-02, -1.1537e-02, -4.2414e-02,\n",
      "         -2.3436e-02,  3.6816e-02, -1.1365e-02, -3.8221e-02,  2.2610e-02,\n",
      "         -3.8196e-03,  1.7846e-02, -8.4271e-04,  1.9180e-02,  1.3780e-02,\n",
      "          2.7928e-02,  2.1226e-02,  1.8370e-02,  5.9320e-02,  6.2814e-02,\n",
      "         -1.1267e-02, -2.4189e-03, -3.0987e-03,  7.0230e-03,  1.2333e-02,\n",
      "          1.6076e-02,  3.2892e-02,  2.1314e-02, -4.0406e-02,  2.6669e-02,\n",
      "          5.7420e-02, -1.9034e-02,  1.1101e-02, -2.6642e-02,  2.5240e-02,\n",
      "          1.0685e-05,  4.1675e-03, -2.7294e-02, -4.4130e-02, -1.7845e-02,\n",
      "          8.4131e-03, -6.0968e-03,  5.2139e-02,  2.0509e-02,  1.3212e-02,\n",
      "         -2.9735e-02,  2.7683e-02, -2.4663e-02,  3.7026e-02,  3.0479e-02,\n",
      "          2.3106e-02,  1.1439e-02,  4.0724e-02, -1.2998e-03, -5.5241e-02,\n",
      "         -1.4615e-04, -9.9703e-03, -6.1029e-02, -1.4451e-02,  7.2916e-02,\n",
      "          2.6504e-02,  1.1316e-02,  5.2677e-02,  1.7144e-02, -2.4691e-03,\n",
      "         -1.0159e-02, -2.2523e-02,  1.7184e-02, -3.6072e-02, -1.2492e-02,\n",
      "         -6.1511e-03, -1.6099e-02,  8.9060e-02, -3.0123e-03,  1.3579e-02,\n",
      "         -2.2558e-02, -1.8502e-02, -4.1432e-02, -1.8303e-02,  5.8325e-03,\n",
      "          1.8963e-02,  1.7020e-02,  4.0559e-03, -4.4809e-03,  3.7295e-02,\n",
      "         -2.6635e-02, -3.2418e-02,  2.2509e-02, -5.3819e-03, -2.3422e-02,\n",
      "          1.9642e-02,  1.8439e-02, -1.2174e-02, -1.5289e-02,  1.6680e-02,\n",
      "          1.0148e-01, -2.5153e-02,  2.6855e-02, -3.3411e-02,  3.7513e-03,\n",
      "          9.3260e-03, -1.7159e-02,  6.7766e-03, -5.2018e-02,  4.2834e-02,\n",
      "         -1.3716e-02, -2.5716e-02,  2.0884e-02,  1.9624e-03, -4.4343e-02,\n",
      "          2.7317e-02,  3.6148e-03, -1.1862e-02, -2.2078e-02, -2.3516e-03,\n",
      "          6.2758e-02, -2.1241e-03,  5.9680e-02,  1.4305e-03,  1.4381e-02,\n",
      "          3.0480e-02, -4.2336e-03, -1.3922e-02, -1.0374e-04,  2.2014e-02,\n",
      "         -7.9395e-03, -1.6462e-02, -2.5140e-02,  5.8786e-04,  3.3255e-03,\n",
      "         -1.2003e-02,  5.4495e-03, -1.1778e-02, -6.3402e-02,  2.0175e-02,\n",
      "         -5.9290e-02,  2.1701e-02, -4.7258e-02, -1.1713e-03, -2.0250e-02,\n",
      "          2.4158e-02, -5.5648e-02,  6.7346e-03,  7.4486e-02,  4.2978e-03,\n",
      "          3.4981e-02,  1.6257e-02,  2.7912e-02,  1.4554e-02, -6.5946e-03,\n",
      "         -8.3685e-03, -1.8854e-02, -2.1801e-02, -4.8561e-03,  6.2470e-02,\n",
      "          1.2036e-02, -1.0247e-02, -3.4446e-02,  1.7223e-02, -1.4127e-02,\n",
      "          3.6349e-02,  2.7774e-02, -8.1842e-03, -1.5056e-02,  2.7007e-02,\n",
      "         -7.7338e-03,  4.4293e-02, -2.3387e-02,  4.2511e-02, -2.1707e-02,\n",
      "          3.1061e-02, -4.1112e-02,  4.5382e-02, -6.2382e-02,  3.7081e-02,\n",
      "         -6.0284e-03,  3.5389e-02,  3.5211e-03, -5.1012e-02, -1.8673e-03,\n",
      "          1.8666e-02,  2.1113e-02, -4.7202e-03,  2.0139e-02,  4.2537e-02,\n",
      "          1.7140e-02,  6.5737e-02,  8.1025e-03, -5.4703e-02, -1.6271e-04,\n",
      "          8.1339e-02, -6.9049e-02,  8.6994e-03, -1.8571e-03,  1.9991e-02,\n",
      "         -1.3114e-02, -6.6045e-02,  5.9215e-02, -4.5379e-03, -3.7451e-02,\n",
      "          2.7781e-02, -4.9740e-02, -6.0898e-04,  1.5412e-02,  8.0735e-02,\n",
      "          5.8027e-02,  1.2375e-01, -1.5039e-02,  3.7410e-03, -4.4371e-03,\n",
      "         -4.1035e-03, -2.9830e-03, -1.3198e-02,  3.0166e-02, -1.9643e-02,\n",
      "          1.2636e-02, -6.4493e-03,  9.5025e-03,  5.4073e-02, -3.0690e-02,\n",
      "         -3.7296e-02,  2.9789e-02, -3.0540e-02, -3.7531e-02, -8.4522e-03,\n",
      "          3.1646e-02, -9.7781e-03, -6.2846e-04, -2.5140e-02, -3.3236e-02,\n",
      "          1.4656e-02,  4.0819e-03, -1.0609e-02, -1.8958e-02,  2.5472e-02,\n",
      "         -9.8710e-02, -1.4845e-03, -3.6661e-02,  1.2374e-02,  2.4442e-03,\n",
      "          3.2446e-02,  6.0752e-02,  1.1728e-01, -4.8197e-03, -4.6901e-02,\n",
      "         -8.8136e-03,  6.8979e-02, -2.1797e-02,  3.6428e-02, -1.2214e-02,\n",
      "          3.4161e-02, -1.8054e-02, -3.0755e-02,  9.8140e-03,  6.6398e-03,\n",
      "          2.3985e-03, -4.1698e-02,  2.6191e-03,  3.6676e-03,  3.9917e-02,\n",
      "          1.5721e-02,  3.7684e-02,  7.2189e-03,  1.0760e-02,  4.0086e-03,\n",
      "         -5.3105e-02,  2.5856e-02, -1.4477e-02,  9.5476e-03, -9.2241e-04,\n",
      "          9.6127e-05, -4.2725e-03, -2.4245e-02,  2.4564e-02,  1.3147e-01,\n",
      "         -5.5457e-02, -3.2671e-02, -4.5018e-02,  2.9718e-02, -1.5660e-02,\n",
      "         -1.1105e-02, -4.0532e-02, -7.9137e-03,  2.1402e-02, -2.9454e-02,\n",
      "          4.2345e-03,  1.6338e-02,  2.8209e-02,  3.4367e-02,  2.4788e-02,\n",
      "         -1.2727e-02,  4.0670e-02,  1.6304e-02, -7.8697e-02,  1.5201e-02,\n",
      "         -5.6143e-02,  2.0924e-03,  1.6329e-02,  1.6121e-02, -1.5415e-02,\n",
      "         -1.5435e-02,  2.1507e-02, -1.5103e-02, -5.7801e-03, -1.2467e-02,\n",
      "         -2.6345e-02, -1.8070e-04, -2.8159e-02,  3.8796e-03,  4.6005e-03,\n",
      "          4.3740e-02,  3.8130e-02, -3.2589e-02,  2.6568e-02,  2.0343e-02,\n",
      "         -1.2432e-02, -2.7639e-02,  5.2539e-03, -3.8884e-02, -1.4502e-02,\n",
      "          3.0370e-03, -2.7531e-02,  2.3094e-02,  2.0890e-02,  1.7514e-02,\n",
      "         -3.1102e-02,  4.1807e-02, -2.6835e-02,  2.1054e-02,  1.6964e-02,\n",
      "         -2.7275e-02,  3.5884e-02, -4.0429e-02,  3.3694e-02, -1.3544e-03,\n",
      "          3.8153e-02, -4.1632e-02, -7.2119e-03,  2.8649e-02, -1.8378e-02,\n",
      "         -3.1784e-02,  3.5598e-02,  6.4888e-03, -4.8388e-02, -2.5603e-02,\n",
      "         -5.0692e-02, -3.8733e-02, -1.6021e-03, -3.5409e-03,  6.4011e-02,\n",
      "         -1.8287e-02,  1.3539e-02,  1.5179e-02, -1.7251e-02, -2.2048e-02,\n",
      "         -1.9591e-02, -1.7675e-02, -3.0663e-02,  3.9875e-02, -3.3814e-02,\n",
      "         -9.6362e-03, -1.3783e-02,  8.3028e-03, -3.0769e-02, -4.5723e-02,\n",
      "         -2.2787e-02, -1.4473e-02, -1.4861e-02,  8.7507e-03, -8.7079e-03,\n",
      "          4.5076e-02,  1.1242e-02,  3.4124e-02,  2.3239e-03,  6.9016e-02,\n",
      "          3.9873e-02,  3.4173e-02,  9.8947e-03, -1.9386e-02, -1.4594e-02,\n",
      "          5.9373e-02,  1.3484e-02, -2.0977e-02,  6.9884e-03,  2.9694e-03,\n",
      "         -2.2417e-02,  1.1562e-02,  4.8560e-02,  2.2448e-03,  9.5930e-02,\n",
      "          2.2815e-02,  2.7926e-02, -2.2285e-02,  3.3694e-03, -5.9484e-02,\n",
      "          4.6517e-02,  2.6544e-02]])\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c21aa86f-6c02-4d88-b310-4ee9c502e7a2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "2abb59bd-076c-4e30-837e-a8b950ccf153",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[ 0.0205,  0.0028, -0.0089,  ...,  0.0097,  0.0270,  0.0024],\n",
       "        [ 0.0115,  0.0075, -0.0120,  ...,  0.0050,  0.0048, -0.0432],\n",
       "        [ 0.0141,  0.0140,  0.0021,  ...,  0.0164,  0.0107, -0.0082],\n",
       "        ...,\n",
       "        [-0.0023,  0.0027,  0.0039,  ..., -0.0287,  0.0108,  0.0049],\n",
       "        [ 0.0486, -0.0194, -0.0194,  ..., -0.0122,  0.0148,  0.0281],\n",
       "        [-0.0008,  0.0048, -0.0132,  ...,  0.0080,  0.0052, -0.0132]],\n",
       "       requires_grad=True)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "918bb4be-8780-4865-a40c-a93caf053662",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}