feature_extrach.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pororo import Pororo\n",
    "from pororo.tasks.utils.download_utils import download_or_load\n",
    "from pororo.tasks import PororoAsrFactory\n",
    "import torch\n",
    "import fairseq\n",
    "from fairseq.tasks.audio_pretraining import AudioPretrainingTask\n",
    "import soundfile as sf\n",
    "import torch.nn.functional as F\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Namespace(activation_dropout=0.0, activation_fn='gelu', adam_betas='(0.9,0.98)', adam_eps=1e-06, all_gather_list_size=16384, arch='wav2vec2', attention_dropout=0.1, best_checkpoint_metric='loss', bf16=False, bpe=None, broadcast_buffers=False, bucket_cap_mb=25, checkpoint_suffix='', clip_norm=0.0, codebook_negatives=0, conv_bias=False, conv_feature_layers='[(512, 10, 5)] + [(512, 3, 2)] * 4 + [(512,2,2)] * 2', conv_pos=128, conv_pos_groups=16, cpu=False, criterion='wav2vec', cross_sample_negatives=0, curriculum=0, data='./manifest/', data_buffer_size=10, dataset_impl=None, ddp_backend='c10d', device_id=0, disable_validation=False, distributed_backend='nccl', distributed_init_method='env://', distributed_no_spawn=True, distributed_port=-1, distributed_rank=0, distributed_world_size=64, distributed_wrapper='DDP', dropout=0.0, dropout_features=0.1, dropout_input=0.1, empty_cache_freq=0, enable_padding=False, encoder_attention_heads=16, encoder_embed_dim=1024, encoder_ffn_embed_dim=4096, encoder_layerdrop=0.0, encoder_layers=24, end_learning_rate=0.0, extractor_mode='default', fast_stat_sync=False, feature_grad_mult=0.03, final_dim=768, find_unused_parameters=False, finetune_from_model=None, fix_batches_to_gpus=False, fixed_validation_seed=None, force_anneal=None, fp16=True, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, infonce=True, keep_best_checkpoints=-1, keep_interval_updates=-1, keep_last_epochs=-1, labels=None, latent_dim=0, latent_groups=2, latent_temp='(2.0,0.1,0.999995)', latent_vars=320, layer_norm_first=False, localsgd_frequency=3, log_format=None, log_interval=100, log_keys='[\"prob_perplexity\",\"code_perplexity\",\"temp\"]', logit_temp=0.1, loss_weights='[0.1, 10]', lr=[0.0003], lr_scheduler='polynomial_decay', mask_channel_length=10, mask_channel_min_space=1, mask_channel_other=0, mask_channel_prob=0, mask_channel_selection='static', mask_length=10, mask_min_space=1, mask_other=0.0, mask_prob=0.65, mask_selection='static', max_epoch=0, max_sample_size=320000, max_sentences=None, max_sentences_valid=None, max_tokens=1200000, max_tokens_valid=1200000, max_update=600000, maximize_best_checkpoint_metric=False, memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, min_lr=-1, min_sample_size=32000, model_parallel_size=1, negatives_from_everywhere=False, no_epoch_checkpoints=False, no_last_checkpoints=False, no_mask_channel_overlap=False, no_mask_overlap=False, no_progress_bar=False, no_save=False, no_save_optimizer_state=False, no_seed_provided=True, normalize=False, nprocs_per_node=4, num_negatives=100, num_workers=6, optimizer='adam', optimizer_overrides='{}', patience=-1, power=1.0, profile=False, quantization_config_path=None, quantize_input=False, quantize_targets=True, required_batch_size_multiple=8, reset_dataloader=False, reset_lr_scheduler=False, reset_meters=False, reset_optimizer=False, restore_file='checkpoint_last.pt', same_quantizer=False, sample_rate=16000, save_dir='/data/project/rw/kaki/model/wav2vec-kspon-pretrain8/', save_interval=1, save_interval_updates=0, scoring='bleu', seed=1, sentence_avg=False, skip_invalid_size_inputs_valid_test=True, slowmo_algorithm='LocalSGD', slowmo_momentum=None, stop_time_hours=0, target_glu=False, task='audio_pretraining', tensorboard_logdir='/data/project/rw/kaki/tensorboard/wav2vec-kspon-pretrain8/', threshold_loss_scale=None, tokenizer=None, total_num_update=600000, tpu=False, train_subset='train', update_freq=[2], use_bmuf=False, use_old_adam=False, user_dir=None, valid_subset='valid', validate_after_updates=0, validate_interval=1, validate_interval_updates=0, warmup_updates=32000, weight_decay=0.01)\n"
     ]
    }
   ],
   "source": [
    "model_path = 'misc/wav2vec.ko.pt'\n",
    "\n",
    "model = torch.load(model_path)\n",
    "w2v_args = model['pretrain_args']\n",
    "task = AudioPretrainingTask.setup_task(w2v_args)\n",
    "w2v_model = task.build_model(w2v_args).cuda()\n",
    "print(w2v_args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "225360\n",
      "expected feature len: 14.085\n",
      "torch.Size([1, 704, 1024])\n"
     ]
    }
   ],
   "source": [
    "audio_file = '/mnt/data_input/speechDATA_eng/train-clean-100/103/1240/103-1240-0000.flac'\n",
    "x, sr = sf.read(audio_file)\n",
    "\n",
    "print(len(x))\n",
    "source = torch.from_numpy(x).float().cuda()\n",
    "# normalize\n",
    "source = F.layer_norm(source, source.shape)\n",
    "\n",
    "# (len(data),) -> (1,-1)\n",
    "source = source.view(1, -1)\n",
    "\n",
    "\n",
    "input_args = {\n",
    "    \"source\": source,\n",
    "    \"padding_mask\": None,\n",
    "    \"mask\": False,\n",
    "}\n",
    "w2v_feat, _ = w2v_model.extract_features(**input_args)\n",
    "print(f\"expected feature len: {(len(x)//16000) * 50}\")\n",
    "print(w2v_feat.shape)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "d4d1e4263499bec80672ea0156c357c1ee493ec2b1c70f0acce89fc37c4a6abe"
  },
  "kernelspec": {
   "display_name": "Python 3.7.7 64-bit ('base': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}