Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: unify part of the repository interfaces #1231

Merged
merged 7 commits into from
Jan 30, 2025
Prev Previous commit
Next Next commit
fix: adapt the aggregation repository
NiklasKoehneckeAA committed Jan 29, 2025
commit c5a8a9e3762481bc645109ea64e2414d11fbadaf
Original file line number Diff line number Diff line change
@@ -23,7 +23,21 @@ def store_aggregation_overview(
def aggregation_overview(
self, aggregation_id: str, aggregation_type: type[AggregatedEvaluation]
) -> Optional[AggregationOverview[AggregatedEvaluation]]:
return self._aggregation_overviews.get(aggregation_id, None)
overview = self._aggregation_overviews.get(aggregation_id, None)
if overview is None or type(overview.statistics) is aggregation_type:
return overview
return AggregationOverview[AggregatedEvaluation](
evaluation_overviews=overview.evaluation_overviews,
id=overview.id,
start=overview.start,
end=overview.end,
successful_evaluation_count=overview.successful_evaluation_count,
crashed_during_evaluation_count=overview.crashed_during_evaluation_count,
description=overview.description,
statistics=aggregation_type.model_validate(overview.statistics),
labels=overview.labels,
metadata=overview.metadata,
)

def aggregation_overview_ids(self) -> Sequence[str]:
return sorted(list(self._aggregation_overviews.keys()))
14 changes: 10 additions & 4 deletions tests/evaluation/aggregation/test_aggregation_repository.py
Original file line number Diff line number Diff line change
@@ -5,7 +5,7 @@
import pytest
from _pytest.fixtures import FixtureRequest
from fsspec.implementations.memory import MemoryFileSystem # type: ignore
from pydantic import ValidationError
from pydantic import BaseModel, ValidationError
from pytest import fixture, mark

from intelligence_layer.core import utc_now
@@ -92,23 +92,29 @@ def test_aggregation_repository_stores_and_returns_an_aggregation_overview(
assert stored_aggregation_overview == aggregation_overview


@mark.skip("TODO: fix this for consistency between repositories")
@mark.parametrize(
"repository_fixture",
test_repository_fixtures,
)
def test_aggregation_repository_aggregation_overview_does_not_work_with_incorrect_types(
def test_aggregation_overview_does_not_work_with_incorrect_types(
repository_fixture: str,
request: FixtureRequest,
aggregation_overview: AggregationOverview[DummyAggregatedEvaluation],
) -> None:
class InvalidClass(BaseModel):
data: str

aggregation_repository: AggregationRepository = request.getfixturevalue(
repository_fixture
)

aggregation_repository.store_aggregation_overview(aggregation_overview)
with pytest.raises(ValidationError):
aggregation_repository.aggregation_overview(aggregation_overview.id, str) # type: ignore
aggregation_repository.aggregation_overview(
aggregation_overview.id, InvalidClass
)
with pytest.raises(ValidationError):
next(iter(aggregation_repository.aggregation_overviews(InvalidClass)))


@mark.parametrize(