Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Initial Job Sizing Infrastructure #488

Merged
merged 19 commits into from
Jan 30, 2025
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Switch Job* from data classes to pydantic
* Switched `JobSize`, `JobResources` from data classes to pydantic's
  `BaseModel`.
* Remove tests that already cover features provided by pydantic`.
* Updated examples for those classes.
TimothyWillard committed Jan 29, 2025

Verified

This commit was signed with the committer’s verified signature.
brucebolt Bruce Bolt
commit 6033c38da417226d9896ea734e0db7a9dfdf64ca
73 changes: 37 additions & 36 deletions flepimop/gempyor_pkg/src/gempyor/batch.py
Original file line number Diff line number Diff line change
@@ -20,7 +20,6 @@


from abc import ABC, abstractmethod
from dataclasses import dataclass
from datetime import timedelta
from enum import Enum, auto
import json
@@ -30,6 +29,8 @@
from typing import Any, Literal, overload
import warnings

from pydantic import BaseModel, PositiveInt

from .utils import _git_head


@@ -42,8 +43,7 @@
_batch_systems = []


@dataclass(frozen=True, slots=True)
class JobResources:
class JobResources(BaseModel):
"""
A batch submission job resources request.

@@ -70,25 +70,26 @@ class JobResources:
... JobResources(nodes=0, cpus=1, memory=1024)
... except Exception as e:
... print(e)
The 'nodes' attribute must be greater than 0, but instead was given '0'.
1 validation error for JobResources
nodes
Input should be greater than 0 [type=greater_than, input_value=0, input_type=int]
For further information visit https://errors.pydantic.dev/2.10/v/greater_than
>>> try:
... JobResources(nodes=2, cpus=4.5, memory=1024)
... except Exception as e:
... print(e)
1 validation error for JobResources
cpus
Input should be a valid integer, got a number with a fractional part [type=int_from_float, input_value=4.5, input_type=float]
For further information visit https://errors.pydantic.dev/2.10/v/int_from_float
"""

nodes: int
cpus: int
memory: int

def __post_init__(self) -> None:
for p in self.__slots__:
if (val := getattr(self, p)) < 1:
raise ValueError(
(
f"The '{p}' attribute must be greater than 0, "
f"but instead was given '{val}'."
)
)
nodes: PositiveInt
cpus: PositiveInt
memory: PositiveInt

@property
def total_cpus(self) -> int:
def total_cpus(self) -> PositiveInt:
"""
Calculate the total number of CPUs.

@@ -98,7 +99,7 @@ def total_cpus(self) -> int:
return self.nodes * self.cpus

@property
def total_memory(self) -> int:
def total_memory(self) -> PositiveInt:
"""
Calculate the total amount of memory.

@@ -107,7 +108,7 @@ def total_memory(self) -> int:
"""
return self.nodes * self.memory

def total_resources(self) -> tuple[int, int, int]:
def total_resources(self) -> tuple[PositiveInt, PositiveInt, PositiveInt]:
"""
Calculate the total resources.

@@ -118,8 +119,7 @@ def total_resources(self) -> tuple[int, int, int]:
return (self.nodes, self.total_cpus, self.total_memory)


@dataclass(frozen=True, slots=True)
class JobSize:
class JobSize(BaseModel):
"""
A batch submission job size.

@@ -140,22 +140,23 @@ class JobSize:
... JobSize(jobs=10, simulations=200, blocks=0)
... except Exception as e:
... print(e)
The 'blocks' attribute must be greater than 0, but instead was given '0'.
1 validation error for JobSize
blocks
Input should be greater than 0 [type=greater_than, input_value=0, input_type=int]
For further information visit https://errors.pydantic.dev/2.10/v/greater_than
>>> try:
... JobSize(jobs=10, simulations=200.25, blocks=5)
... except Exception as e:
... print(e)
1 validation error for JobSize
simulations
Input should be a valid integer, got a number with a fractional part [type=int_from_float, input_value=200.25, input_type=float]
For further information visit https://errors.pydantic.dev/2.10/v/int_from_float
"""

jobs: int
simulations: int
blocks: int

def __post_init__(self) -> None:
for p in self.__slots__:
if (val := getattr(self, p)) < 1:
raise ValueError(
(
f"The '{p}' attribute must be greater than 0, "
f"but instead was given '{val}'."
)
)
jobs: PositiveInt
simulations: PositiveInt
blocks: PositiveInt


class BatchSystem(ABC):
26 changes: 0 additions & 26 deletions flepimop/gempyor_pkg/tests/batch/test_job_resources_class.py
Original file line number Diff line number Diff line change
@@ -5,32 +5,6 @@
from gempyor.batch import BatchSystem, JobResources, JobSize


@pytest.mark.parametrize(
"kwargs",
(
{"nodes": 0, "cpus": 1, "memory": 1},
{"nodes": 1, "cpus": 0, "memory": 1},
{"nodes": 1, "cpus": 1, "memory": 0},
{"nodes": 0, "cpus": 0, "memory": 1},
{"nodes": 1, "cpus": 0, "memory": 0},
{"nodes": 0, "cpus": 1, "memory": 0},
{"nodes": 0, "cpus": 0, "memory": 0},
),
)
def test_less_than_one_value_error(
kwargs: dict[Literal["nodes", "cpus", "memory"], int]
) -> None:
param = next(k for k, v in kwargs.items() if v < 1)
with pytest.raises(
ValueError,
match=(
f"^The '{param}' attribute must be greater than 0, "
f"but instead was given '{kwargs.get(param)}'.$"
),
):
JobResources(**kwargs)


@pytest.mark.parametrize("nodes", (1, 2, 4, 8))
@pytest.mark.parametrize("cpus", (1, 2, 4, 8))
@pytest.mark.parametrize("memory", (1024, 2 * 1024, 4 * 1024, 8 * 1024))
32 changes: 0 additions & 32 deletions flepimop/gempyor_pkg/tests/batch/test_job_size_class.py

This file was deleted.