From 99d8f7a1eaa703470cb54d033e26d233b445f0b0 Mon Sep 17 00:00:00 2001 From: Phillip Rust Date: Tue, 15 Oct 2024 01:37:30 +0200 Subject: [PATCH 1/6] Fix SignHiera loading from CLIP checkpoint --- src/ssvp_slt/modeling/sign_hiera.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/ssvp_slt/modeling/sign_hiera.py b/src/ssvp_slt/modeling/sign_hiera.py index 324a38c..73af93e 100644 --- a/src/ssvp_slt/modeling/sign_hiera.py +++ b/src/ssvp_slt/modeling/sign_hiera.py @@ -600,28 +600,27 @@ def from_clip_model(cls, model_id: str, clip_model_path: str) -> nn.Module: from ssvp_slt.modeling.clip import CLIP, CLIPTextCfg, CLIPVisionCfg checkpoint = torch.load(clip_model_path) - args = checkpoint["args"] + clip_model_cfg = checkpoint["cfg"]["model"] model_params = checkpoint["clip"] vision_cfg = CLIPVisionCfg( - model_id=args.model, - proj="mlp", + model_id=clip_model_cfg["vision_model_name"], + proj=clip_model_cfg["vision_model_proj"], ) - # FIXME: might cause errors if proj and pooler are not `mlp` and `mean_pooler` text_cfg = CLIPTextCfg( - hf_model_name=args.text_model_name_or_path, - proj="mlp", - pooler_type="mean_pooler", + hf_model_name=clip_model_cfg["text_model_name_or_path"], + proj=clip_model_cfg["text_model_proj"], + pooler_type=clip_model_cfg["text_model_pooler"], ) - clip = CLIP(embed_dim=768, vision_cfg=vision_cfg, text_cfg=text_cfg, output_dict=True) + clip = CLIP(embed_dim=clip_model_cfg.pop("embed_dim"), vision_cfg=vision_cfg, text_cfg=text_cfg, output_dict=True) print(f"Loading CLIP weights from {clip_model_path}") msg = clip.load_state_dict(model_params) print(msg) print("Loading SignHiera weights from CLIP vision tower") - model = sys.modules[__name__].__dict__[model_id](**args.__dict__) + model = sys.modules[__name__].__dict__[model_id](**clip_model_cfg) msg = model.load_state_dict(clip.visual.transformer.state_dict(), strict=False) print(msg) From 885cf4a200e71d807d1fdb594af0bfd03143f24f Mon Sep 17 00:00:00 2001 From: Joozef315 Date: Wed, 23 Oct 2024 21:11:01 +0300 Subject: [PATCH 2/6] edit INSTALL.md & add script to test the environment Signed-off-by: Joozef315 --- INSTALL.md | 65 ++++++++++++++++++++++++++++++++++++++++++++++++----- test_env.py | 26 +++++++++++++++++++++ 2 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 test_env.py diff --git a/INSTALL.md b/INSTALL.md index f3888fd..273cfdd 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,24 +1,31 @@ ## Installation ### Requirements + - Linux, CUDA >= 11.7, Python >= 3.8, PyTorch >= 2.0.0 (our setup below is based on CUDA 11.8, Python 3.10, PyTorch 2.0.1; more recent versions should work too, but no guarantees) - Conda (anaconda / miniconda work well) +- visual studio visual studio (2022/2019/2017) build tools ### Setup #### 1. Create conda environment + ```bash -conda create --name ssvp_slt python=3.10 +conda create --name ssvp_slt python=3.10 cmake ipython conda activate ssvp_slt conda install pytorch==2.0.1 torchvision==0.15.2 pytorch-cuda=11.8 -c pytorch -c nvidia conda install av -c conda-forge +python -m pip install torch==2.2.0 torchvision==0.17.0 torchaudio==2.2.0 --index-url https://download.pytorch.org/whl/cu118 ``` #### 2. (Optional) Video dataloader GPU decoding backend support + If you want to use our video dataloader with GPU decoding backend, you need to reinstall torchvision by building it from scratch following the steps at [https://github.com/pytorch/vision/tree/main/torchvision/csrc/io/decoder/gpu](https://github.com/pytorch/vision/tree/main/torchvision/csrc/io/decoder/gpu). We found that this does not work with ffmpeg 6.1, so we recommend running `conda install 'ffmpeg<6.0'`. If you get a warning that torchvision is built without GPU decoding support due to `bsf.h` missing, we recommend manually downloading `bsf.h` from the `ffmpeg` source code ([https://github.com/FFmpeg/FFmpeg/blob/master/libavcodec/bsf.h](https://github.com/FFmpeg/FFmpeg/blob/master/libavcodec/bsf.h), **make sure it matches your ffmpeg version!**) and placing it under `$(path-to-your-conda)/ssvp_slt/include/libavcodec`. #### 3. Pip Installs + Install the remaining dependencies and an egg of our ssvp-slt package: + ```bash pip install git+https://github.com/facebookresearch/stopes.git pip install -r requirements.txt @@ -32,17 +39,65 @@ cd .. pip install -e . ``` -#### 4. BLEURT +#### 4. Install dlib + +- Install CUDA and cuDNN with + +```bash +conda install cuda cudnn -c nvidia +``` + +- Install dlib from the source (write your VS versions) + +```bash +git clone https://github.com/davisking/dlib.git +cd dlib +mkdir build +cd build +cmake .. -DDLIB_USE_CUDA=1 -DUSE_AVX_INSTRUCTIONS=1 -DCUDAToolkit_ROOT=/path/to/your/conda/envs/dlib/bin/ -G "Visual Studio 17 2022" -A x64 --verbose +cmake --build . +cd .. +python setup.py install --set DLIB_USE_CUDA=1 +``` + +#### 5. BLEURT + If you want to compute BLEURT scores as part of the translation evals (via `common.compute_bleurt=true`), you need to install BLEURT: + ```bash pip install git+https://github.com/google-research/bleurt.git ``` -You will also likely need to set some environment variables before running the translation code: +#### 6. Set environment variables + +You will also likely need to set some environment variables before running the translation code: + ```bash export XLA_FLAGS=--xla_gpu_cuda_data_dir=$(path-to-your-cuda-11.8) export LD_LIBRARY_PATH=$(path-to-your-cuda-11.8)/lib64:${LD_LIBRARY_PATH} ``` -#### 5. Weights and Biases -If you want to use [Weights and Biases](https://wandb.ai) to track your training runs (via `cfg.wandb.enabled=true`), you need to ensure your `WAND_API_KEY` environment variable is set correctly. \ No newline at end of file +#### 7. Weights and Biases + +If you want to use [Weights and Biases](https://wandb.ai) to track your training runs (via `cfg.wandb.enabled=true`), you need to ensure your `WAND_API_KEY` environment variable is set correctly. + +#### 7. Test your env + +you can excute `test_env.py` python script to check if the dependencies are installed + +```bash +python test_env.py +``` + +The output (values could be quite different) + +```bash +CUDA is available! +2.2.0+cu118 +11.8 +Device count: 1 +Current device: 0 +Device name: NVIDIA A100-SXM4-40GB +dlib CUDA is available! +Number of dlib CUDA devices: 1 +``` diff --git a/test_env.py b/test_env.py new file mode 100644 index 0000000..b26c052 --- /dev/null +++ b/test_env.py @@ -0,0 +1,26 @@ +import torch +import dlib +import os + +# Check if CUDA is available and if it's using a GPU +if torch.cuda.is_available(): + print("CUDA is available!") + print(torch.__version__) + print(torch.version.cuda) + print(f"Device count: {torch.cuda.device_count()}") + print(f"Current device: {torch.cuda.current_device()}") + print(f"Device name: {torch.cuda.get_device_name(0)}") + +else: + print("CUDA is not available.") + # Consider using a runtime with GPU acceleration in Colab. + print("Consider using a runtime with GPU acceleration in Colab.") + + +# If dlib CUDA is available, print details +if dlib.DLIB_USE_CUDA: + print("dlib CUDA is available!") + print(f"Number of dlib CUDA devices: {dlib.cuda.get_num_devices()}") +else: + print("dlib CUDA is not available.") + # Consider alternative libraries or configurations if needed. \ No newline at end of file From 14172b2f52b68820e57e4af159fb44a794ec72e6 Mon Sep 17 00:00:00 2001 From: Joozef315 Date: Mon, 28 Oct 2024 13:07:11 +0300 Subject: [PATCH 3/6] edit intall.md file Signed-off-by: Joozef315 --- INSTALL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index 273cfdd..3d53675 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -11,7 +11,7 @@ #### 1. Create conda environment ```bash -conda create --name ssvp_slt python=3.10 cmake ipython +conda create --name ssvp_slt python=3.10 cmake conda activate ssvp_slt conda install pytorch==2.0.1 torchvision==0.15.2 pytorch-cuda=11.8 -c pytorch -c nvidia conda install av -c conda-forge From ded5040d72ee3899e255066ecf63fdf3e3f14ea5 Mon Sep 17 00:00:00 2001 From: Joozef315 Date: Mon, 4 Nov 2024 20:59:53 +0200 Subject: [PATCH 4/6] remove typos & add LICENSE Signed-off-by: Joozef315 --- .gitignore | 5 ++++- INSTALL.md | 11 +++++------ test_env.py => tests/test_env.py | 14 +++++++++++--- 3 files changed, 20 insertions(+), 10 deletions(-) rename test_env.py => tests/test_env.py (61%) diff --git a/.gitignore b/.gitignore index 93690b6..dee27b3 100644 --- a/.gitignore +++ b/.gitignore @@ -94,4 +94,7 @@ dmypy.json # misc *.mp4 sweep*/ -core* \ No newline at end of file +core* + +features_outputs +*.pth \ No newline at end of file diff --git a/INSTALL.md b/INSTALL.md index 3d53675..5362435 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -4,7 +4,7 @@ - Linux, CUDA >= 11.7, Python >= 3.8, PyTorch >= 2.0.0 (our setup below is based on CUDA 11.8, Python 3.10, PyTorch 2.0.1; more recent versions should work too, but no guarantees) - Conda (anaconda / miniconda work well) -- visual studio visual studio (2022/2019/2017) build tools +- visual studio (2019) build tools (Just for windows) ### Setup @@ -13,9 +13,8 @@ ```bash conda create --name ssvp_slt python=3.10 cmake conda activate ssvp_slt -conda install pytorch==2.0.1 torchvision==0.15.2 pytorch-cuda=11.8 -c pytorch -c nvidia -conda install av -c conda-forge python -m pip install torch==2.2.0 torchvision==0.17.0 torchaudio==2.2.0 --index-url https://download.pytorch.org/whl/cu118 +conda install av -c conda-forge ``` #### 2. (Optional) Video dataloader GPU decoding backend support @@ -81,12 +80,12 @@ export LD_LIBRARY_PATH=$(path-to-your-cuda-11.8)/lib64:${LD_LIBRARY_PATH} If you want to use [Weights and Biases](https://wandb.ai) to track your training runs (via `cfg.wandb.enabled=true`), you need to ensure your `WAND_API_KEY` environment variable is set correctly. -#### 7. Test your env +#### 8. Test your env -you can excute `test_env.py` python script to check if the dependencies are installed +you can execute `test_env.py` python script to check if the dependencies are installed ```bash -python test_env.py +python tests/test_env.py ``` The output (values could be quite different) diff --git a/test_env.py b/tests/test_env.py similarity index 61% rename from test_env.py rename to tests/test_env.py index b26c052..0fe91f2 100644 --- a/test_env.py +++ b/tests/test_env.py @@ -1,6 +1,16 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +# -------------------------------------------------------- +# References: +# DeiT: https://github.com/facebookresearch/deit +# BEiT: https://github.com/microsoft/unilm/tree/master/beit +# -------------------------------------------------------- + import torch import dlib -import os # Check if CUDA is available and if it's using a GPU if torch.cuda.is_available(): @@ -13,7 +23,6 @@ else: print("CUDA is not available.") - # Consider using a runtime with GPU acceleration in Colab. print("Consider using a runtime with GPU acceleration in Colab.") @@ -23,4 +32,3 @@ print(f"Number of dlib CUDA devices: {dlib.cuda.get_num_devices()}") else: print("dlib CUDA is not available.") - # Consider alternative libraries or configurations if needed. \ No newline at end of file From 17040e60b366967ff79a4ee40a9fc51a70e0a521 Mon Sep 17 00:00:00 2001 From: Joozef315 Date: Wed, 6 Nov 2024 14:49:51 +0200 Subject: [PATCH 5/6] removing L6-10 in the header --- tests/test_env.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/test_env.py b/tests/test_env.py index 0fe91f2..155329f 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -1,13 +1,7 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. - -# This source code is licensed under the license found in the +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -# -------------------------------------------------------- -# References: -# DeiT: https://github.com/facebookresearch/deit -# BEiT: https://github.com/microsoft/unilm/tree/master/beit -# -------------------------------------------------------- import torch import dlib From bdfddd20d92eb2899591169fcc4380235d6b3c8c Mon Sep 17 00:00:00 2001 From: Joozef315 Date: Wed, 6 Nov 2024 15:22:18 +0200 Subject: [PATCH 6/6] update the header --- tests/test_env.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_env.py b/tests/test_env.py index 155329f..60f89f0 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -1,6 +1,7 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. import torch