Datapipes update

pvnet/data/utils.py

@@ -1,8 +1,7 @@
 """Utils common between Wind and PV datamodules"""
 import numpy as np
 import torch
-from ocf_datapipes.batch import unstack_np_batch_into_examples
-from ocf_datapipes.utils.consts import BatchKey
+from ocf_datapipes.batch import BatchKey, unstack_np_batch_into_examples
 from torch.utils.data import IterDataPipe, functional_datapipe
 
 

pvnet/models/base_model.py

@@ -17,7 +17,7 @@
 from huggingface_hub.file_download import hf_hub_download
 from huggingface_hub.hf_api import HfApi
 from huggingface_hub.utils._deprecation import _deprecate_positional_args
-from ocf_datapipes.utils.consts import BatchKey
+from ocf_datapipes.batch import BatchKey
 from ocf_ml_metrics.evaluation.evaluation import evaluation
 from ocf_ml_metrics.metrics.errors import common_metrics
 

pvnet/models/baseline/last_value.py

@@ -1,6 +1,6 @@
 """Persistence model"""
 
-from ocf_datapipes.utils.consts import BatchKey
+from ocf_datapipes.batch import BatchKey
 
 import pvnet
 from pvnet.models.base_model import BaseModel

pvnet/models/baseline/single_value.py

@@ -1,6 +1,6 @@
 """Average value model"""
 import torch
-from ocf_datapipes.utils.consts import BatchKey
+from ocf_datapipes.batch import BatchKey
 from torch import nn
 
 import pvnet

pvnet/models/multimodal/deep_supervision.py

@@ -5,7 +5,7 @@
 
 import torch
 import torch.nn.functional as F
-from ocf_datapipes.utils.consts import BatchKey
+from ocf_datapipes.batch import BatchKey
 from torch import nn
 
 import pvnet

pvnet/models/multimodal/multimodal.py

@@ -4,7 +4,7 @@
 from typing import Optional
 
 import torch
-from ocf_datapipes.utils.consts import BatchKey, NWPBatchKey
+from ocf_datapipes.batch import BatchKey, NWPBatchKey
 from torch import nn
 
 import pvnet

pvnet/models/multimodal/nwp_weighting.py

@@ -2,7 +2,7 @@
 from typing import Optional
 
 import torch
-from ocf_datapipes.utils.consts import BatchKey
+from ocf_datapipes.batch import BatchKey
 from torch import nn
 
 import pvnet

pvnet/models/multimodal/site_encoders/encoders.py

@@ -3,7 +3,7 @@
 """
 
 import torch
-from ocf_datapipes.utils.consts import BatchKey
+from ocf_datapipes.batch import BatchKey
 from torch import nn
 
 from pvnet.models.multimodal.linear_networks.networks import ResFCNet2

pvnet/models/multimodal/weather_residual.py

@@ -4,7 +4,7 @@
 
 import torch
 import torch.nn.functional as F
-from ocf_datapipes.utils.consts import BatchKey
+from ocf_datapipes.batch import BatchKey
 from torch import nn
 
 import pvnet

pvnet/models/utils.py

@@ -6,7 +6,7 @@
 
 import numpy as np
 import torch
-from ocf_datapipes.utils.consts import BatchKey
+from ocf_datapipes.batch import BatchKey
 
 logger = logging.getLogger(__name__)
 

pvnet/utils.py

@@ -15,7 +15,8 @@
 import yaml
 from lightning.pytorch.loggers import Logger
 from lightning.pytorch.utilities import rank_zero_only
-from ocf_datapipes.utils.consts import BatchKey, Location
+from ocf_datapipes.batch import BatchKey
+from ocf_datapipes.utils import Location
 from ocf_datapipes.utils.geospatial import osgb_to_lon_lat
 from omegaconf import DictConfig, OmegaConf
 

requirements.txt

@@ -1,4 +1,4 @@
-ocf_datapipes>=2.3.0
+ocf_datapipes>=3.0.0
 ocf_ml_metrics
 numpy
 pandas

scripts/hindcast.py

@@ -10,9 +10,9 @@
 import pandas as pd
 import torch
 import xarray as xr
+from ocf_datapipes.batch import BatchKey
 from ocf_datapipes.load import OpenGSP
 from ocf_datapipes.training.pvnet import construct_sliced_data_pipeline
-from ocf_datapipes.utils.consts import BatchKey
 from ocf_datapipes.utils.utils import stack_np_examples_into_batch
 from torchdata.dataloader2 import DataLoader2, MultiProcessingReadingService
 from torchdata.datapipes.iter import IterableWrapper

scripts/save_concurrent_batches.py

@@ -23,12 +23,11 @@
 import hydra
 import numpy as np
 import torch
-from ocf_datapipes.batch import stack_np_examples_into_batch
+from ocf_datapipes.batch import BatchKey, stack_np_examples_into_batch
 from ocf_datapipes.training.common import (
     open_and_return_datapipes,
 )
 from ocf_datapipes.training.pvnet import construct_loctime_pipelines, construct_sliced_data_pipeline
-from ocf_datapipes.utils.consts import BatchKey
 from omegaconf import DictConfig, OmegaConf
 from sqlalchemy import exc as sa_exc
 from torch.utils.data import DataLoader

tests/conftest.py

@@ -7,7 +7,7 @@
 import torch
 import hydra
 
-from ocf_datapipes.utils.consts import BatchKey
+from ocf_datapipes.batch import BatchKey
 from datetime import timedelta
 
 import pvnet

tests/models/multimodal/site_encoders/test_encoders.py

@@ -1,5 +1,5 @@
 import torch
-from ocf_datapipes.utils.consts import BatchKey
+from ocf_datapipes.batch import BatchKey
 from torch import nn
 
 from pvnet.models.multimodal.site_encoders.encoders import (