Skip to content

Commit

Permalink
Wrap train-taskcluster.sh in train_taskcluster.py
Browse files Browse the repository at this point in the history
  • Loading branch information
bhearsum committed Apr 30, 2024
1 parent 658d6c2 commit 2eaca54
Show file tree
Hide file tree
Showing 8 changed files with 66 additions and 5 deletions.
2 changes: 1 addition & 1 deletion Taskfile.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ tasks:
task test -- tests/test_alignments.py
deps: [poetry-install-tests]
cmds:
- PYTHONPATH=$(pwd) poetry run pytest -vv {{.CLI_ARGS}}
- PYTHONPATH="$(pwd):$(pwd)/taskcluster/scripts/pipeline" poetry run pytest -vv {{.CLI_ARGS}}

test-docker:
desc: Run the unit tests in the docker image. Some tests require the pre-built Linux executables.
Expand Down
4 changes: 3 additions & 1 deletion taskcluster/kinds/finetune-student/kind.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ tasks:
- pipeline/train/configs/opustrainer/student.yml
- pipeline/train/configs/training/student.train.yml
- pipeline/train/train.sh
- taskcluster/scripts/pipeline/train_taskcluster.py
- taskcluster/scripts/pipeline/train-taskcluster.sh
from-parameters:
marian_args: training_config.marian-args.training-student-finetuned

Expand Down Expand Up @@ -88,7 +90,7 @@ tasks:
pip3 install $VCS_PATH/tracking &&
export PATH="$HOME/.local/bin:$PATH" &&
export MARIAN=$MOZ_FETCHES_DIR &&
$VCS_PATH/taskcluster/scripts/pipeline/train-taskcluster.sh
$VCS_PATH/taskcluster/scripts/pipeline/train_taskcluster.py
student
finetune
{src_locale}
Expand Down
3 changes: 2 additions & 1 deletion taskcluster/kinds/train-backwards/kind.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ tasks:
type: train-backwards
resources:
- pipeline/train/train.sh
- taskcluster/scripts/pipeline/train_taskcluster.py
- taskcluster/scripts/pipeline/train-taskcluster.sh
- pipeline/train/configs/model/backward.yml
- pipeline/train/configs/opustrainer/backward.yml
Expand Down Expand Up @@ -98,7 +99,7 @@ tasks:
pip3 install $VCS_PATH/tracking &&
export PATH="$HOME/.local/bin:$PATH" &&
export MARIAN=$MOZ_FETCHES_DIR &&
$VCS_PATH/taskcluster/scripts/pipeline/train-taskcluster.sh
$VCS_PATH/taskcluster/scripts/pipeline/train_taskcluster.py
backward
train
{trg_locale}
Expand Down
4 changes: 3 additions & 1 deletion taskcluster/kinds/train-student/kind.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ tasks:
- pipeline/train/configs/opustrainer/student.yml
- pipeline/train/configs/training/student.train.yml
- pipeline/train/train.sh
- taskcluster/scripts/pipeline/train_taskcluster.py
- taskcluster/scripts/pipeline/train-taskcluster.sh
from-parameters:
marian_args: training_config.marian-args.training-student
worker-type:
Expand Down Expand Up @@ -94,7 +96,7 @@ tasks:
pip3 install $VCS_PATH/tracking &&
export PATH="$HOME/.local/bin:$PATH" &&
export MARIAN=$MOZ_FETCHES_DIR &&
$VCS_PATH/taskcluster/scripts/pipeline/train-taskcluster.sh
$VCS_PATH/taskcluster/scripts/pipeline/train_taskcluster.py
student
train
{src_locale}
Expand Down
3 changes: 2 additions & 1 deletion taskcluster/kinds/train-teacher/kind.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ tasks:
- pipeline/train/configs/opustrainer/teacher.yml
- pipeline/train/configs/training/teacher.train.yml
- pipeline/train/train.sh
- taskcluster/scripts/pipeline/train_taskcluster.py
- taskcluster/scripts/pipeline/train-taskcluster.sh
from-parameters:
marian_args: training_config.marian-args.training-teacher
Expand Down Expand Up @@ -119,7 +120,7 @@ tasks:
pip3 install $VCS_PATH/tracking &&
export PATH="$HOME/.local/bin:$PATH" &&
export MARIAN=$MOZ_FETCHES_DIR &&
$VCS_PATH/taskcluster/scripts/pipeline/train-taskcluster.sh
$VCS_PATH/taskcluster/scripts/pipeline/train_taskcluster.py
teacher
train
{src_locale}
Expand Down
Empty file.
17 changes: 17 additions & 0 deletions taskcluster/scripts/pipeline/train_taskcluster.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/usr/bin/env python3

import json
import os.path
import requests
import subprocess
import sys

TRAINING_SCRIPT = os.path.join(os.path.dirname(__file__), "train-taskcluster.sh")


def main(args):
subprocess.run([TRAINING_SCRIPT, *args], check=True)


if __name__ == "__main__":
main(sys.argv[1:])
38 changes: 38 additions & 0 deletions tests/test_train_taskcluster.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
import pytest
from unittest import mock

import train_taskcluster

@pytest.mark.parametrize("args",
(
(
"foo",
"bar",
"blah",
),
(
"foo",
"bar",
"blah",
"and",
"many",
"more",
"arguments",
"way",
"way",
"way",
"way",
"way",
"way",
"more",
"than",
"we",
"currently",
"use",
),
),
)
def test_all_args_forwarded(args):
with mock.patch("train_taskcluster.subprocess") as mocked_subprocess:
train_taskcluster.main(args)
assert mocked_subprocess.run.call_args[0][0][1:] == list(args)

0 comments on commit 2eaca54

Please sign in to comment.