From a5beec9947e2a995648f488127252e7354016137 Mon Sep 17 00:00:00 2001 From: Hongbo Miao <3375461+hongbo-miao@users.noreply.github.com> Date: Mon, 6 Jan 2025 01:38:13 -0800 Subject: [PATCH] feat(ruff): enable Pylint --- .ruff.toml | 8 +- api-python/routers/seed.py | 25 ----- api-python/routers/seed_test.py | 21 ---- api-rust/scripts/download_model.py | 2 +- .../pytorch-mnist/src/models/net.py | 2 +- .../pytorch-mnist/src/utils/test.py | 8 +- .../pytorch-mnist/src/utils/train.py | 14 +-- .../iads-data-reader/src/main.py | 2 +- .../src/model/data_loader.py | 2 +- .../src/model/net.py | 2 +- machine-learning/dali/src/main.py | 4 +- .../graph-neural-network/src/main.py | 99 ++++++++++--------- .../pipelines/classify-mnist/src/main.py | 4 +- .../experiments/classify-mnist/src/main.py | 4 +- .../set_up/workspace/export_onnx.py | 2 +- .../set_up/workspace/export_pt.py | 2 +- 16 files changed, 80 insertions(+), 121 deletions(-) delete mode 100644 api-python/routers/seed.py delete mode 100644 api-python/routers/seed_test.py diff --git a/.ruff.toml b/.ruff.toml index cb7283cca5..bf5da66d66 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -175,6 +175,12 @@ select = [ "F", # Pyflakes "I", # isort "PGH", # pygrep-hooks + "PL", # Pylint "UP", # pyupgrade ] -ignore = ["E501"] +ignore = [ + "E501", + "PLR0913", + "PLR0915", + "PLR2004", +] diff --git a/api-python/routers/seed.py b/api-python/routers/seed.py deleted file mode 100644 index 065bdec545..0000000000 --- a/api-python/routers/seed.py +++ /dev/null @@ -1,25 +0,0 @@ -from fastapi import APIRouter -from pydantic import BaseModel - -router = APIRouter(tags=["seed"]) - -seed_number = 42 - - -class SeedUpdate(BaseModel): - seedNumber: int - - class Config: - json_schema_extra = {"example": {"seedNumber": 100}} - - -@router.get("/seed") -async def get_seed() -> dict[str, int]: - return {"seedNumber": seed_number} - - -@router.post("/update-seed") -async def update_seed(seed_data: SeedUpdate) -> dict[str, int]: - global seed_number - seed_number = seed_data.seedNumber - return {"seedNumber": seed_number} diff --git a/api-python/routers/seed_test.py b/api-python/routers/seed_test.py deleted file mode 100644 index 0a21a61a54..0000000000 --- a/api-python/routers/seed_test.py +++ /dev/null @@ -1,21 +0,0 @@ -from fastapi.testclient import TestClient -from main import app - -client = TestClient(app) - - -def test_get_seed(): - response = client.get("/seed") - assert response.status_code == 200 - assert response.json() == {"seedNumber": 42} - - -def test_update_seed(): - new_seed = 100 - response = client.post("/update-seed", json={"seedNumber": new_seed}) - assert response.status_code == 200 - assert response.json() == {"seedNumber": new_seed} - - response = client.get("/seed") - assert response.status_code == 200 - assert response.json() == {"seedNumber": new_seed} diff --git a/api-rust/scripts/download_model.py b/api-rust/scripts/download_model.py index 15184e5303..3a56843029 100644 --- a/api-rust/scripts/download_model.py +++ b/api-rust/scripts/download_model.py @@ -3,7 +3,7 @@ import httpx import torch -import torchvision.models as models +from torchvision import models logger = logging.getLogger(__name__) diff --git a/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/models/net.py b/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/models/net.py index ca9a3a9cb2..7f625ff1e4 100644 --- a/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/models/net.py +++ b/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/models/net.py @@ -1,5 +1,5 @@ -import torch.nn as nn import torch.nn.functional as F +from torch import nn class Net(nn.Module): diff --git a/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/test.py b/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/test.py index b10e6aeb8c..829f51d3a0 100644 --- a/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/test.py +++ b/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/test.py @@ -14,13 +14,13 @@ def test(model, test_loader, device): correct = 0 with torch.no_grad(): for data, target in test_loader: - data, target = data.to(device), target.to(device) - output = model(data) + device_data, device_target = data.to(device), target.to(device) + output = model(device_data) # sum up batch loss - test_loss += F.nll_loss(output, target, size_average=False).item() + test_loss += F.nll_loss(output, device_target, size_average=False).item() # get the index of the max log-probability pred = output.max(1, keepdim=True)[1] - correct += pred.eq(target.view_as(pred)).sum().item() + correct += pred.eq(device_target.view_as(pred)).sum().item() test_loss /= len(test_loader.dataset) logger.info( diff --git a/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/train.py b/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/train.py index fd95cb3873..823bfbb180 100644 --- a/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/train.py +++ b/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/train.py @@ -4,10 +4,10 @@ import torch import torch.distributed as dist import torch.nn.functional as F -import torch.optim as optim import torch.utils.data import torch.utils.data.distributed from models.net import Net +from torch import optim from utils.average_gradients import average_gradients from utils.get_test_data_loader import get_test_data_loader from utils.get_train_data_loader import get_train_data_loader @@ -19,10 +19,10 @@ def train(args): is_distributed = len(args.hosts) > 1 and args.backend is not None - logger.info("Distributed training:", is_distributed) + logger.info(f"Distributed training: {is_distributed}") use_cuda = args.num_gpus > 0 - logger.info("Number of gpus available:", args.num_gpus) + logger.info(f"Number of gpus available: {args.num_gpus}") kwargs = {"num_workers": 1, "pin_memory": True} if use_cuda else {} device = torch.device("cuda" if use_cuda else "cpu") @@ -71,10 +71,10 @@ def train(args): for epoch in range(1, args.epochs + 1): model.train() for batch_idx, (data, target) in enumerate(train_loader, 1): - data, target = data.to(device), target.to(device) + device_data, device_target = data.to(device), target.to(device) optimizer.zero_grad() - output = model(data) - loss = F.nll_loss(output, target) + output = model(device_data) + loss = F.nll_loss(output, device_target) loss.backward() if is_distributed and not use_cuda: # average gradients manually for a multi-machine cpu case only @@ -82,7 +82,7 @@ def train(args): optimizer.step() if batch_idx % args.log_interval == 0: logger.info( - f"Train Epoch: {epoch} [{batch_idx * len(data)}/{len(train_loader.sampler)} ({100.0 * batch_idx / len(train_loader):.0f}%)] Loss: {loss.item():.6f}" + f"Train Epoch: {epoch} [{batch_idx * len(device_data)}/{len(train_loader.sampler)} ({100.0 * batch_idx / len(train_loader):.0f}%)] Loss: {loss.item():.6f}" ) test(model, test_loader, device) save_model(model, args.model_dir) diff --git a/data-visualization/iads/iads-data-manager/iads-data-reader/src/main.py b/data-visualization/iads/iads-data-manager/iads-data-reader/src/main.py index 842c83625d..8908c61026 100644 --- a/data-visualization/iads/iads-data-manager/iads-data-reader/src/main.py +++ b/data-visualization/iads/iads-data-manager/iads-data-reader/src/main.py @@ -101,7 +101,7 @@ def export_to_parquet( str(parquet_file_path), ] logger.info(f"Executing command: {' '.join(cmd)}") - result = subprocess.run(cmd, capture_output=True, text=True) + result = subprocess.run(cmd, capture_output=True, text=True, check=False) if result.returncode == 0: return parquet_file_path else: diff --git a/machine-learning/convolutional-neural-network/src/model/data_loader.py b/machine-learning/convolutional-neural-network/src/model/data_loader.py index 71b74becde..bfc1dccdd9 100644 --- a/machine-learning/convolutional-neural-network/src/model/data_loader.py +++ b/machine-learning/convolutional-neural-network/src/model/data_loader.py @@ -1,7 +1,7 @@ import torch import torchvision -import torchvision.transforms as transforms from args import get_args +from torchvision import transforms args = get_args() transform = transforms.Compose( diff --git a/machine-learning/convolutional-neural-network/src/model/net.py b/machine-learning/convolutional-neural-network/src/model/net.py index 2ad6da8119..775b6194ee 100644 --- a/machine-learning/convolutional-neural-network/src/model/net.py +++ b/machine-learning/convolutional-neural-network/src/model/net.py @@ -1,6 +1,6 @@ import torch -import torch.nn as nn import torch.nn.functional as F +from torch import nn class Net(nn.Module): diff --git a/machine-learning/dali/src/main.py b/machine-learning/dali/src/main.py index 84050d575b..bd975bf177 100644 --- a/machine-learning/dali/src/main.py +++ b/machine-learning/dali/src/main.py @@ -2,10 +2,8 @@ import urllib.request from pathlib import Path -import nvidia.dali.fn as fn -import nvidia.dali.types as types import torch -from nvidia.dali import pipeline_def +from nvidia.dali import fn, pipeline_def, types from nvidia.dali.plugin.pytorch import DALIGenericIterator logger = logging.getLogger(__name__) diff --git a/machine-learning/graph-neural-network/src/main.py b/machine-learning/graph-neural-network/src/main.py index 4bc54ad5be..89a1de60a8 100644 --- a/machine-learning/graph-neural-network/src/main.py +++ b/machine-learning/graph-neural-network/src/main.py @@ -1,11 +1,11 @@ import numpy as np import torch -import torch.optim as optim import wandb from args import get_args from model.data_loader import fetch_dataset, get_dataloaders from model.gnn import GNN from ogb.graphproppred import Evaluator +from torch import optim from tqdm import tqdm cls_criterion = torch.nn.BCEWithLogitsLoss() @@ -17,24 +17,24 @@ def train(model, device, loader, optimizer, task_type): total_loss = 0 for step, batch in enumerate(tqdm(loader, desc="Iteration")): - batch = batch.to(device) + device_batch = batch.to(device) - if batch.x.shape[0] == 1 or batch.batch[-1] == 0: + if device_batch.x.shape[0] == 1 or device_batch.batch[-1] == 0: pass else: - pred = model(batch) + pred = model(device_batch) optimizer.zero_grad() # ignore nan targets (unlabeled) when computing training loss. - is_labeled = batch.y == batch.y + is_labeled = device_batch.y == device_batch.y if "classification" in task_type: loss = cls_criterion( pred.to(torch.float32)[is_labeled], - batch.y.to(torch.float32)[is_labeled], + device_batch.y.to(torch.float32)[is_labeled], ) else: loss = reg_criterion( pred.to(torch.float32)[is_labeled], - batch.y.to(torch.float32)[is_labeled], + device_batch.y.to(torch.float32)[is_labeled], ) loss.backward() total_loss += loss.item() @@ -49,15 +49,15 @@ def evaluate(model, device, loader, evaluator): y_pred = [] for step, batch in enumerate(tqdm(loader, desc="Iteration")): - batch = batch.to(device) + device_batch = batch.to(device) - if batch.x.shape[0] == 1: + if device_batch.x.shape[0] == 1: pass else: with torch.no_grad(): - pred = model(batch) + pred = model(device_batch) - y_true.append(batch.y.view(pred.shape).detach().cpu()) + y_true.append(device_batch.y.view(pred.shape).detach().cpu()) y_pred.append(pred.detach().cpu()) y_true = torch.cat(y_true, dim=0).numpy() @@ -93,44 +93,45 @@ def main(): val_loader = dataloaders["val"] test_loader = dataloaders["test"] - if config.gnn == "gin": - model = GNN( - gnn_type="gin", - num_tasks=dataset.num_tasks, - num_layer=config.num_layer, - emb_dim=config.emb_dim, - drop_ratio=config.drop_ratio, - virtual_node=False, - ).to(device) - elif config.gnn == "gin-virtual": - model = GNN( - gnn_type="gin", - num_tasks=dataset.num_tasks, - num_layer=config.num_layer, - emb_dim=config.emb_dim, - drop_ratio=config.drop_ratio, - virtual_node=True, - ).to(device) - elif config.gnn == "gcn": - model = GNN( - gnn_type="gcn", - num_tasks=dataset.num_tasks, - num_layer=config.num_layer, - emb_dim=config.emb_dim, - drop_ratio=config.drop_ratio, - virtual_node=False, - ).to(device) - elif config.gnn == "gcn-virtual": - model = GNN( - gnn_type="gcn", - num_tasks=dataset.num_tasks, - num_layer=config.num_layer, - emb_dim=config.emb_dim, - drop_ratio=config.drop_ratio, - virtual_node=True, - ).to(device) - else: - raise ValueError("Invalid GNN type") + match config.gnn: + case "gin": + model = GNN( + gnn_type="gin", + num_tasks=dataset.num_tasks, + num_layer=config.num_layer, + emb_dim=config.emb_dim, + drop_ratio=config.drop_ratio, + virtual_node=False, + ).to(device) + case "gin-virtual": + model = GNN( + gnn_type="gin", + num_tasks=dataset.num_tasks, + num_layer=config.num_layer, + emb_dim=config.emb_dim, + drop_ratio=config.drop_ratio, + virtual_node=True, + ).to(device) + case "gcn": + model = GNN( + gnn_type="gcn", + num_tasks=dataset.num_tasks, + num_layer=config.num_layer, + emb_dim=config.emb_dim, + drop_ratio=config.drop_ratio, + virtual_node=False, + ).to(device) + case "gcn-virtual": + model = GNN( + gnn_type="gcn", + num_tasks=dataset.num_tasks, + num_layer=config.num_layer, + emb_dim=config.emb_dim, + drop_ratio=config.drop_ratio, + virtual_node=True, + ).to(device) + case _: + raise ValueError("Invalid GNN type") wb.watch(model) diff --git a/machine-learning/hm-kubeflow/pipelines/classify-mnist/src/main.py b/machine-learning/hm-kubeflow/pipelines/classify-mnist/src/main.py index 0e247931b8..dc2ec922f9 100644 --- a/machine-learning/hm-kubeflow/pipelines/classify-mnist/src/main.py +++ b/machine-learning/hm-kubeflow/pipelines/classify-mnist/src/main.py @@ -8,10 +8,10 @@ def train(): import lightning as L import torch - import torch.nn as nn import torch.nn.functional as F - import torch.utils.data as data import torchvision + from torch import nn + from torch.utils import data class LitAutoEncoder(L.LightningModule): def __init__(self): diff --git a/machine-learning/hm-mlflow/experiments/classify-mnist/src/main.py b/machine-learning/hm-mlflow/experiments/classify-mnist/src/main.py index 7178c54004..22bc27ec32 100644 --- a/machine-learning/hm-mlflow/experiments/classify-mnist/src/main.py +++ b/machine-learning/hm-mlflow/experiments/classify-mnist/src/main.py @@ -1,12 +1,12 @@ import lightning as L import mlflow import torch -import torch.nn as nn import torch.nn.functional as F -import torch.utils.data as data import torchvision from args import get_args from lightning.pytorch.loggers.wandb import WandbLogger +from torch import nn +from torch.utils import data class LitAutoEncoder(L.LightningModule): diff --git a/machine-learning/triton-inference-server/amazon-sagemaker-triton-resnet-50/set_up/workspace/export_onnx.py b/machine-learning/triton-inference-server/amazon-sagemaker-triton-resnet-50/set_up/workspace/export_onnx.py index 77af59ce1a..d3470c0721 100644 --- a/machine-learning/triton-inference-server/amazon-sagemaker-triton-resnet-50/set_up/workspace/export_onnx.py +++ b/machine-learning/triton-inference-server/amazon-sagemaker-triton-resnet-50/set_up/workspace/export_onnx.py @@ -1,5 +1,5 @@ import torch -import torchvision.models as models +from torchvision import models def main() -> None: diff --git a/machine-learning/triton-inference-server/amazon-sagemaker-triton-resnet-50/set_up/workspace/export_pt.py b/machine-learning/triton-inference-server/amazon-sagemaker-triton-resnet-50/set_up/workspace/export_pt.py index c70cffc851..874a31b54a 100644 --- a/machine-learning/triton-inference-server/amazon-sagemaker-triton-resnet-50/set_up/workspace/export_pt.py +++ b/machine-learning/triton-inference-server/amazon-sagemaker-triton-resnet-50/set_up/workspace/export_pt.py @@ -1,5 +1,5 @@ import torch -import torchvision.models as models +from torchvision import models def main() -> None: