Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DM-40002: Try to support pydantic v1 and v2 #866

Merged
merged 18 commits into from
Jul 19, 2023
Merged

DM-40002: Try to support pydantic v1 and v2 #866

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
6eeb67c
Fix some tests that require optional dependencies
timj Jul 14, 2023
0bd4a79
Fix types in test to match model
timj Jul 14, 2023
2e913d3
Initial attempt to support pydantic v2
timj Jul 14, 2023
afe26c4
For storage class conversion test use usable storage classes
timj Jul 17, 2023
7e19163
Remove fastapi pin
timj Jul 17, 2023
56d2a1e
Use model_construct for pydantic v2
timj Jul 17, 2023
d3eee16
Use mambaforge for build action
timj Jul 18, 2023
37afb3f
Use python 3.11 for doc build
timj Jul 18, 2023
2e129be
Add python 3.10 definition of Self
timj Jul 18, 2023
e0894c6
Pydantic/fastAPI no longer handle generic Mapping for Config classes
timj Jul 18, 2023
e96015d
Fix mypy with pydantic
timj Jul 18, 2023
92a5351
Fix mypy for pydantic v1 and v2
timj Jul 18, 2023
45d95e6
Add model_field and model_rebuild to v1 pydantic compat layer
timj Jul 18, 2023
eba136b
Remove the --- from start of action yaml
timj Jul 18, 2023
594b379
Fix fallback model_fields class property
timj Jul 19, 2023
f6cfafa
Use model_construct in direct methods
timj Jul 19, 2023
50673f4
Add note about use of BaseModel.construct
timj Jul 19, 2023
1ce5be8
Add news fragment
timj Jul 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions tests/test_butler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,14 @@
try:
import boto3
import botocore
from lsst.resources.s3utils import setAwsEnvCredentials, unsetAwsEnvCredentials
from moto import mock_s3 # type: ignore[import]
except ImportError:
boto3 = None

def mock_s3(cls): # type: ignore[no-untyped-def]
def mock_s3(*args: Any, **kwargs: Any) -> Any: # type: ignore[no-untyped-def]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd expect you can drop type: ignore here?

"""No-op decorator in case moto mock_s3 can not be imported."""
return cls
return None


try:
Expand Down Expand Up @@ -93,7 +94,6 @@ def mock_s3(cls): # type: ignore[no-untyped-def]
from lsst.daf.butler.tests import MetricsExample, MultiDetectorFormatter
from lsst.daf.butler.tests.utils import TestCaseMixin, makeTestTempDir, removeTestTempDir, safeTestTempDir
from lsst.resources import ResourcePath
from lsst.resources.s3utils import setAwsEnvCredentials, unsetAwsEnvCredentials
from lsst.utils import doImportType
from lsst.utils.introspection import get_full_type_name

Expand Down
73 changes: 47 additions & 26 deletions tests/test_parquet.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
try:
import pandas as pd
except ImportError:
np = None
pd = None

from lsst.daf.butler import (
Butler,
Expand All @@ -54,31 +54,52 @@
StorageClassConfig,
StorageClassFactory,
)
from lsst.daf.butler.delegates.arrowastropy import ArrowAstropyDelegate
from lsst.daf.butler.delegates.arrownumpy import ArrowNumpyDelegate
from lsst.daf.butler.delegates.arrowtable import ArrowTableDelegate
from lsst.daf.butler.delegates.dataframe import DataFrameDelegate
from lsst.daf.butler.formatters.parquet import (
ArrowAstropySchema,
ArrowNumpySchema,
DataFrameSchema,
ParquetFormatter,
_append_numpy_multidim_metadata,
_astropy_to_numpy_dict,
_numpy_dict_to_numpy,
_numpy_dtype_to_arrow_types,
_numpy_style_arrays_to_arrow_arrays,
_numpy_to_numpy_dict,
arrow_to_astropy,
arrow_to_numpy,
arrow_to_numpy_dict,
arrow_to_pandas,
astropy_to_arrow,
compute_row_group_size,
numpy_dict_to_arrow,
numpy_to_arrow,
pandas_to_arrow,
)

try:
from lsst.daf.butler.delegates.arrowastropy import ArrowAstropyDelegate
except ImportError:
atable = None
pa = None
try:
from lsst.daf.butler.delegates.arrownumpy import ArrowNumpyDelegate
except ImportError:
np = None
pa = None
try:
from lsst.daf.butler.delegates.arrowtable import ArrowTableDelegate
except ImportError:
pa = None
try:
from lsst.daf.butler.delegates.dataframe import DataFrameDelegate
except ImportError:
pd = None
try:
from lsst.daf.butler.formatters.parquet import (
ArrowAstropySchema,
ArrowNumpySchema,
DataFrameSchema,
ParquetFormatter,
_append_numpy_multidim_metadata,
_astropy_to_numpy_dict,
_numpy_dict_to_numpy,
_numpy_dtype_to_arrow_types,
_numpy_style_arrays_to_arrow_arrays,
_numpy_to_numpy_dict,
arrow_to_astropy,
arrow_to_numpy,
arrow_to_numpy_dict,
arrow_to_pandas,
astropy_to_arrow,
compute_row_group_size,
numpy_dict_to_arrow,
numpy_to_arrow,
pandas_to_arrow,
)
except ImportError:
pa = None
pd = None
atable = None
np = None
from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir

TESTDIR = os.path.abspath(os.path.dirname(__file__))
Expand Down
8 changes: 7 additions & 1 deletion tests/test_postgresql.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

from __future__ import annotations

import gc
import itertools
import os
Expand All @@ -41,7 +43,11 @@
import sqlalchemy
from lsst.daf.butler import Timespan, ddl
from lsst.daf.butler.registry import _ButlerRegistry, _RegistryFactory
from lsst.daf.butler.registry.databases.postgresql import PostgresqlDatabase, _RangeTimespanType

try:
from lsst.daf.butler.registry.databases.postgresql import PostgresqlDatabase, _RangeTimespanType
except ImportError:
testing = None
from lsst.daf.butler.registry.tests import DatabaseTests, RegistryTests
from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir

Expand Down