diff --git a/src/mx_bluesky/hyperion/external_interaction/callbacks/ispyb_callback_base.py b/src/mx_bluesky/hyperion/external_interaction/callbacks/ispyb_callback_base.py index fcd4a822e..f8bc99b6f 100644 --- a/src/mx_bluesky/hyperion/external_interaction/callbacks/ispyb_callback_base.py +++ b/src/mx_bluesky/hyperion/external_interaction/callbacks/ispyb_callback_base.py @@ -110,9 +110,9 @@ def _handle_ispyb_hardware_read(self, doc) -> Sequence[ScanDataInfo]: slitgap_vertical=doc["data"]["s4_slit_gaps_ygap"], ) hwscan_position_info = DataCollectionPositionInfo( - pos_x=doc["data"]["smargon-x"], - pos_y=doc["data"]["smargon-y"], - pos_z=doc["data"]["smargon-z"], + pos_x=float(doc["data"]["smargon-x"]), + pos_y=float(doc["data"]["smargon-y"]), + pos_z=float(doc["data"]["smargon-z"]), ) scan_data_infos = self.populate_info_for_update( hwscan_data_collection_info, hwscan_position_info, self.params diff --git a/src/mx_bluesky/hyperion/parameters/components.py b/src/mx_bluesky/hyperion/parameters/components.py index 890d3b1bc..e0b092a64 100644 --- a/src/mx_bluesky/hyperion/parameters/components.py +++ b/src/mx_bluesky/hyperion/parameters/components.py @@ -149,7 +149,6 @@ class WithVisit(BaseModel): visit: str = Field(min_length=1) zocalo_environment: str = Field(default=CONST.ZOCALO_ENV) beamline: str = Field(default=CONST.I03.BEAMLINE, pattern=r"BL\d{2}[BIJS]") - storage_directory: str det_dist_to_beam_converter_path: str = Field( default=CONST.PARAM.DETECTOR.BEAM_XY_LUT_PATH ) @@ -172,6 +171,7 @@ class DiffractionExperiment( selected_aperture: ApertureValue | None = Field(default=None) transmission_frac: float = Field(default=0.1) ispyb_experiment_type: IspybExperimentType + storage_directory: str @model_validator(mode="before") @classmethod diff --git a/src/mx_bluesky/hyperion/parameters/load_centre_collect.py b/src/mx_bluesky/hyperion/parameters/load_centre_collect.py index afc259238..f47132655 100644 --- a/src/mx_bluesky/hyperion/parameters/load_centre_collect.py +++ b/src/mx_bluesky/hyperion/parameters/load_centre_collect.py @@ -30,7 +30,17 @@ class LoadCentreCollect(HyperionParameters, WithVisit, WithSample): @model_validator(mode="before") @classmethod - def robot_load_params(cls, values): + def validate_model(cls, values): + allowed_keys = ( + LoadCentreCollect.model_fields.keys() + | RobotLoadThenCentre.model_fields.keys() + | MultiRotationScan.model_fields.keys() + ) + disallowed_keys = values.keys() - allowed_keys + assert ( + disallowed_keys == set() + ), f"Unexpected fields found in LoadCentreCollect {disallowed_keys}" + values["robot_load_then_centre"] = construct_from_values( values, "robot_load_then_centre", RobotLoadThenCentre ) diff --git a/tests/conftest.py b/tests/conftest.py index 6e91bd6b6..bed497a4e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -281,6 +281,10 @@ def smargon(RE: RunEngine) -> Generator[Smargon, None, None]: set_mock_value(smargon.z.user_readback, 0.0) set_mock_value(smargon.x.high_limit_travel, 2) set_mock_value(smargon.x.low_limit_travel, -2) + set_mock_value(smargon.y.high_limit_travel, 2) + set_mock_value(smargon.y.low_limit_travel, -2) + set_mock_value(smargon.z.high_limit_travel, 2) + set_mock_value(smargon.z.low_limit_travel, -2) with ( patch_async_motor(smargon.omega), @@ -405,7 +409,12 @@ def dcm(RE): dcm = i03.dcm(fake_with_ophyd_sim=True) set_mock_value(dcm.energy_in_kev.user_readback, 12.7) set_mock_value(dcm.pitch_in_mrad.user_readback, 1) - return dcm + with ( + oa_patch_motor(dcm.roll_in_mrad), + oa_patch_motor(dcm.pitch_in_mrad), + oa_patch_motor(dcm.offset_in_mm), + ): + yield dcm @pytest.fixture @@ -594,7 +603,7 @@ def fake_create_rotation_devices( xbpm_feedback: XBPMFeedback, ): set_mock_value(smargon.omega.max_velocity, 131) - oav.zoom_controller.onst.sim_put("1.0x") # type: ignore + oav.zoom_controller.zrst.sim_put("1.0x") # type: ignore oav.zoom_controller.fvst.sim_put("5.0x") # type: ignore return RotationScanComposite( diff --git a/tests/system_tests/conftest.py b/tests/system_tests/conftest.py new file mode 100644 index 000000000..9c3c67886 --- /dev/null +++ b/tests/system_tests/conftest.py @@ -0,0 +1,84 @@ +from unittest.mock import MagicMock, patch + +import pytest +from dodal.beamlines import i03 +from dodal.devices.oav.oav_parameters import OAVConfigParams +from ophyd_async.core import AsyncStatus, set_mock_value +from requests import Response + + +@pytest.fixture +def undulator_for_system_test(undulator): + set_mock_value(undulator.current_gap, 1.11) + return undulator + + +@pytest.fixture +def oav_for_system_test(test_config_files): + parameters = OAVConfigParams( + test_config_files["zoom_params_file"], test_config_files["display_config"] + ) + oav = i03.oav(fake_with_ophyd_sim=True, params=parameters) + oav.zoom_controller.zrst.set("1.0x") + oav.zoom_controller.onst.set("7.5x") + oav.cam.array_size.array_size_x.sim_put(1024) + oav.cam.array_size.array_size_y.sim_put(768) + + unpatched_method = oav.parameters.load_microns_per_pixel + + def patch_lmpp(zoom, xsize, ysize): + unpatched_method(zoom, 1024, 768) + + # Grid snapshots + oav.grid_snapshot.x_size.sim_put(1024) + oav.grid_snapshot.y_size.sim_put(768) + oav.grid_snapshot.top_left_x.set(50) + oav.grid_snapshot.top_left_y.set(100) + oav.grid_snapshot.box_width.set(0.1 * 1000 / 1.25) # size in pixels + unpatched_snapshot_trigger = oav.grid_snapshot.trigger + + def mock_grid_snapshot_trigger(): + oav.grid_snapshot.last_path_full_overlay.set("test_1_y") + oav.grid_snapshot.last_path_outer.set("test_2_y") + oav.grid_snapshot.last_saved_path.set("test_3_y") + return unpatched_snapshot_trigger() + + # Plain snapshots + def next_snapshot(): + next_snapshot_idx = 1 + while True: + yield f"/tmp/snapshot{next_snapshot_idx}.png" + next_snapshot_idx += 1 + + empty_response = MagicMock(spec=Response) + empty_response.content = b"" + with ( + patch( + "dodal.devices.areadetector.plugins.MJPG.requests.get", + return_value=empty_response, + ), + patch("dodal.devices.areadetector.plugins.MJPG.Image.open"), + patch.object(oav.grid_snapshot, "post_processing"), + patch.object( + oav.grid_snapshot, "trigger", side_effect=mock_grid_snapshot_trigger + ), + patch.object( + oav.parameters, + "load_microns_per_pixel", + new=MagicMock(side_effect=patch_lmpp), + ), + patch.object(oav.snapshot.last_saved_path, "get") as mock_last_saved_path, + ): + it_next_snapshot = next_snapshot() + + @AsyncStatus.wrap + async def mock_rotation_snapshot_trigger(): + mock_last_saved_path.side_effect = lambda: next(it_next_snapshot) + + with patch.object( + oav.snapshot, + "trigger", + side_effect=mock_rotation_snapshot_trigger, + ): + oav.parameters.load_microns_per_pixel(1.0, 1024, 768) + yield oav diff --git a/tests/system_tests/hyperion/external_interaction/conftest.py b/tests/system_tests/hyperion/external_interaction/conftest.py index d1a99d397..49a42d000 100644 --- a/tests/system_tests/hyperion/external_interaction/conftest.py +++ b/tests/system_tests/hyperion/external_interaction/conftest.py @@ -1,19 +1,46 @@ import os -from collections.abc import Callable +from collections.abc import Callable, Sequence from functools import partial from typing import Any +from unittest.mock import patch import ispyb.sqlalchemy +import numpy import pytest +from dodal.devices.aperturescatterguard import ApertureScatterguard +from dodal.devices.attenuator import Attenuator +from dodal.devices.backlight import Backlight +from dodal.devices.dcm import DCM +from dodal.devices.detector.detector_motion import DetectorMotion +from dodal.devices.eiger import EigerDetector +from dodal.devices.flux import Flux +from dodal.devices.oav.oav_detector import OAV +from dodal.devices.robot import BartRobot +from dodal.devices.s4_slit_gaps import S4SlitGaps +from dodal.devices.smargon import Smargon +from dodal.devices.synchrotron import Synchrotron, SynchrotronMode +from dodal.devices.undulator import Undulator +from dodal.devices.xbpm_feedback import XBPMFeedback +from dodal.devices.zebra import Zebra +from dodal.devices.zebra_controlled_shutter import ZebraShutter from ispyb.sqlalchemy import DataCollection, DataCollectionGroup, GridInfo, Position +from ophyd.sim import NullStatus +from ophyd_async.core import AsyncStatus, set_mock_value from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker +from mx_bluesky.hyperion.experiment_plans.grid_detect_then_xray_centre_plan import ( + GridDetectThenXRayCentreComposite, +) +from mx_bluesky.hyperion.experiment_plans.rotation_scan_plan import ( + RotationScanComposite, +) from mx_bluesky.hyperion.external_interaction.ispyb.ispyb_store import StoreInIspyb from mx_bluesky.hyperion.parameters.constants import CONST from mx_bluesky.hyperion.parameters.gridscan import ThreeDGridScan +from mx_bluesky.hyperion.utils.utils import convert_angstrom_to_eV -from ....conftest import raw_params_from_file +from ....conftest import fake_read, pin_tip_edge_data, raw_params_from_file TEST_RESULT_LARGE = [ { @@ -62,6 +89,14 @@ def get_current_datacollection_comment(Session: Callable, dcid: int) -> str: return current_comment +def get_datacollections(Session: Callable, dcg_id: int) -> Sequence[int]: + with Session.begin() as session: + query = session.query(DataCollection.dataCollectionId).filter( + DataCollection.dataCollectionGroupId == dcg_id + ) + return [row[0] for row in query.all()] + + def get_current_datacollection_attribute( Session: Callable, dcid: int, attr: str ) -> str: @@ -105,7 +140,7 @@ def get_current_datacollectiongroup_attribute( ): with Session() as session: query = session.query(DataCollectionGroup).filter( - DataCollection.dataCollectionGroupId == dcg_id + DataCollectionGroup.dataCollectionGroupId == dcg_id ) first_result = query.first() return getattr(first_result, attr) @@ -123,6 +158,13 @@ def fetch_comment(sqlalchemy_sessionmaker) -> Callable: return partial(get_current_datacollection_comment, sqlalchemy_sessionmaker) +@pytest.fixture +def fetch_datacollection_ids_for_group_id( + sqlalchemy_sessionmaker, +) -> Callable[[int], Sequence]: + return partial(get_datacollections, sqlalchemy_sessionmaker) + + @pytest.fixture def fetch_datacollection_attribute(sqlalchemy_sessionmaker) -> Callable: return partial(get_current_datacollection_attribute, sqlalchemy_sessionmaker) @@ -167,3 +209,158 @@ def dummy_ispyb_3d(dummy_params) -> StoreInIspyb: @pytest.fixture def zocalo_env(): os.environ["ZOCALO_CONFIG"] = "/dls_sw/apps/zocalo/live/configuration.yaml" + + +# noinspection PyUnreachableCode +@pytest.fixture +def grid_detect_then_xray_centre_composite( + fast_grid_scan, + backlight, + smargon, + undulator_for_system_test, + synchrotron, + s4_slit_gaps, + attenuator, + xbpm_feedback, + detector_motion, + zocalo, + aperture_scatterguard, + zebra, + eiger, + robot, + oav_for_system_test, + dcm, + flux, + ophyd_pin_tip_detection, + sample_shutter, +): + composite = GridDetectThenXRayCentreComposite( + zebra_fast_grid_scan=fast_grid_scan, + pin_tip_detection=ophyd_pin_tip_detection, + backlight=backlight, + panda_fast_grid_scan=None, # type: ignore + smargon=smargon, + undulator=undulator_for_system_test, + synchrotron=synchrotron, + s4_slit_gaps=s4_slit_gaps, + attenuator=attenuator, + xbpm_feedback=xbpm_feedback, + detector_motion=detector_motion, + zocalo=zocalo, + aperture_scatterguard=aperture_scatterguard, + zebra=zebra, + eiger=eiger, + panda=None, # type: ignore + robot=robot, + oav=oav_for_system_test, + dcm=dcm, + flux=flux, + sample_shutter=sample_shutter, + ) + + @AsyncStatus.wrap + async def mock_pin_tip_detect(): + tip_x_px, tip_y_px, top_edge_array, bottom_edge_array = pin_tip_edge_data() + set_mock_value( + ophyd_pin_tip_detection.triggered_top_edge, + top_edge_array, + ) + + set_mock_value( + ophyd_pin_tip_detection.triggered_bottom_edge, + bottom_edge_array, + ) + set_mock_value( + zocalo.bbox_sizes, numpy.array([[10, 10, 10]], dtype=numpy.uint64) + ) + set_mock_value(ophyd_pin_tip_detection.triggered_tip, (tip_x_px, tip_y_px)) + + @AsyncStatus.wrap + async def mock_complete_status(): + pass + + @AsyncStatus.wrap + async def mock_zocalo_complete(): + await zocalo._put_results(TEST_RESULT_MEDIUM, {"dcid": 0, "dcgid": 0}) + + with ( + patch.object(eiger, "wait_on_arming_if_started"), + # xsize, ysize will always be wrong since computed as 0 before we get here + # patch up load_microns_per_pixel connect to receive non-zero values + patch.object( + ophyd_pin_tip_detection, "trigger", side_effect=mock_pin_tip_detect + ), + patch.object(fast_grid_scan, "kickoff", return_value=NullStatus()), + patch.object(fast_grid_scan, "complete", return_value=NullStatus()), + patch.object(zocalo, "trigger", side_effect=mock_zocalo_complete), + ): + yield composite + + +@pytest.fixture +def composite_for_rotation_scan( + eiger: EigerDetector, + smargon: Smargon, + zebra: Zebra, + detector_motion: DetectorMotion, + backlight: Backlight, + attenuator: Attenuator, + flux: Flux, + undulator_for_system_test: Undulator, + aperture_scatterguard: ApertureScatterguard, + synchrotron: Synchrotron, + s4_slit_gaps: S4SlitGaps, + dcm: DCM, + robot: BartRobot, + oav_for_system_test: OAV, + sample_shutter: ZebraShutter, + xbpm_feedback: XBPMFeedback, +): + set_mock_value(smargon.omega.max_velocity, 131) + oav_for_system_test.zoom_controller.zrst.sim_put("1.0x") # type: ignore + oav_for_system_test.zoom_controller.fvst.sim_put("5.0x") # type: ignore + + fake_create_rotation_devices = RotationScanComposite( + attenuator=attenuator, + backlight=backlight, + dcm=dcm, + detector_motion=detector_motion, + eiger=eiger, + flux=flux, + smargon=smargon, + undulator=undulator_for_system_test, + aperture_scatterguard=aperture_scatterguard, + synchrotron=synchrotron, + s4_slit_gaps=s4_slit_gaps, + zebra=zebra, + robot=robot, + oav=oav_for_system_test, + sample_shutter=sample_shutter, + xbpm_feedback=xbpm_feedback, + ) + + energy_ev = convert_angstrom_to_eV(0.71) + set_mock_value( + fake_create_rotation_devices.dcm.energy_in_kev.user_readback, + energy_ev / 1000, # pyright: ignore + ) + set_mock_value( + fake_create_rotation_devices.synchrotron.synchrotron_mode, + SynchrotronMode.USER, + ) + set_mock_value( + fake_create_rotation_devices.synchrotron.top_up_start_countdown, # pyright: ignore + -1, + ) + fake_create_rotation_devices.s4_slit_gaps.xgap.user_readback.sim_put( # pyright: ignore + 0.123 + ) + fake_create_rotation_devices.s4_slit_gaps.ygap.user_readback.sim_put( # pyright: ignore + 0.234 + ) + + with ( + patch("bluesky.preprocessors.__read_and_stash_a_motor", fake_read), + patch("bluesky.plan_stubs.wait"), + ): + yield fake_create_rotation_devices diff --git a/tests/system_tests/hyperion/external_interaction/test_ispyb_dev_connection.py b/tests/system_tests/hyperion/external_interaction/test_ispyb_dev_connection.py index 11da80c3a..d2e6d2461 100644 --- a/tests/system_tests/hyperion/external_interaction/test_ispyb_dev_connection.py +++ b/tests/system_tests/hyperion/external_interaction/test_ispyb_dev_connection.py @@ -6,18 +6,12 @@ from copy import deepcopy from decimal import Decimal from typing import Any, Literal -from unittest.mock import MagicMock, patch -import numpy import pytest from bluesky.run_engine import RunEngine -from dodal.devices.oav.oav_detector import OAV from dodal.devices.oav.oav_parameters import OAVParameters from dodal.devices.synchrotron import SynchrotronMode -from ophyd.sim import NullStatus -from ophyd_async.core import AsyncStatus, set_mock_value -from mx_bluesky.hyperion.experiment_plans import oav_grid_detection_plan from mx_bluesky.hyperion.experiment_plans.grid_detect_then_xray_centre_plan import ( GridDetectThenXRayCentreComposite, grid_detect_then_xray_centre, @@ -57,9 +51,7 @@ ThreeDGridScan, ) from mx_bluesky.hyperion.parameters.rotation import RotationScan -from mx_bluesky.hyperion.utils.utils import convert_angstrom_to_eV -from ....conftest import fake_read, pin_tip_edge_data from .conftest import raw_params_from_file EXPECTED_DATACOLLECTION_FOR_ROTATION = { @@ -67,7 +59,7 @@ "beamSizeAtSampleX": 0.02, "beamSizeAtSampleY": 0.02, "exposureTime": 0.023, - "undulatorGap1": 1.12, + "undulatorGap1": 1.11, "synchrotronMode": SynchrotronMode.USER.value, "slitGapHorizontal": 0.123, "slitGapVertical": 0.234, @@ -238,123 +230,6 @@ def grid_detect_then_xray_centre_parameters(): return GridScanWithEdgeDetect(**json_dict) -# noinspection PyUnreachableCode -@pytest.fixture -def grid_detect_then_xray_centre_composite( - fast_grid_scan, - backlight, - smargon, - undulator, - synchrotron, - s4_slit_gaps, - attenuator, - xbpm_feedback, - detector_motion, - zocalo, - aperture_scatterguard, - zebra, - eiger, - robot, - oav: OAV, - dcm, - flux, - ophyd_pin_tip_detection, - sample_shutter, - done_status, -): - composite = GridDetectThenXRayCentreComposite( - zebra_fast_grid_scan=fast_grid_scan, - pin_tip_detection=ophyd_pin_tip_detection, - backlight=backlight, - panda_fast_grid_scan=None, # type: ignore - smargon=smargon, - undulator=undulator, - synchrotron=synchrotron, - s4_slit_gaps=s4_slit_gaps, - attenuator=attenuator, - xbpm_feedback=xbpm_feedback, - detector_motion=detector_motion, - zocalo=zocalo, - aperture_scatterguard=aperture_scatterguard, - zebra=zebra, - eiger=eiger, - panda=None, # type: ignore - robot=robot, - oav=oav, - dcm=dcm, - flux=flux, - sample_shutter=sample_shutter, - ) - oav.zoom_controller.zrst.set("1.0x") - oav.cam.array_size.array_size_x.sim_put(1024) # type: ignore - oav.cam.array_size.array_size_y.sim_put(768) # type: ignore - oav.grid_snapshot.x_size.sim_put(1024) # type: ignore - oav.grid_snapshot.y_size.sim_put(768) # type: ignore - oav.grid_snapshot.top_left_x.set(50) - oav.grid_snapshot.top_left_y.set(100) - oav.grid_snapshot.box_width.set(0.1 * 1000 / 1.25) # size in pixels - set_mock_value(undulator.current_gap, 1.11) - - unpatched_method = oav.parameters.load_microns_per_pixel - - unpatched_snapshot_trigger = oav.grid_snapshot.trigger - - def mock_snapshot_trigger(): - oav.grid_snapshot.last_path_full_overlay.set("test_1_y") - oav.grid_snapshot.last_path_outer.set("test_2_y") - oav.grid_snapshot.last_saved_path.set("test_3_y") - return unpatched_snapshot_trigger() - - def patch_lmpp(zoom, xsize, ysize): - unpatched_method(zoom, 1024, 768) - - def mock_pin_tip_detect(_): - tip_x_px, tip_y_px, top_edge_array, bottom_edge_array = pin_tip_edge_data() - set_mock_value( - ophyd_pin_tip_detection.triggered_top_edge, - top_edge_array, - ) - - set_mock_value( - ophyd_pin_tip_detection.triggered_bottom_edge, - bottom_edge_array, - ) - set_mock_value( - zocalo.bbox_sizes, numpy.array([[10, 10, 10]], dtype=numpy.uint64) - ) - - yield from [] - return tip_x_px, tip_y_px - - @AsyncStatus.wrap - async def mock_complete_status(): - pass - - with ( - patch.object(eiger, "wait_on_arming_if_started"), - # xsize, ysize will always be wrong since computed as 0 before we get here - # patch up load_microns_per_pixel connect to receive non-zero values - patch.object( - oav.parameters, - "load_microns_per_pixel", - new=MagicMock(side_effect=patch_lmpp), - ), - patch.object( - oav_grid_detection_plan, - "wait_for_tip_to_be_found", - side_effect=mock_pin_tip_detect, - ), - patch("dodal.devices.areadetector.plugins.MJPG.requests.get"), - patch("dodal.devices.areadetector.plugins.MJPG.Image.open"), - patch.object(oav.grid_snapshot, "post_processing"), - patch.object(oav.grid_snapshot, "trigger", side_effect=mock_snapshot_trigger), - patch.object(fast_grid_scan, "kickoff", return_value=NullStatus()), - patch.object(fast_grid_scan, "complete", return_value=NullStatus()), - patch.object(zocalo, "trigger", return_value=NullStatus()), - ): - yield composite - - def scan_xy_data_info_for_update( data_collection_group_id, dummy_params: ThreeDGridScan, scan_data_info_for_begin ): @@ -418,57 +293,6 @@ def scan_data_infos_for_update_3d( return [scan_xy_data_info_for_update, scan_xz_data_info_for_update] -@pytest.fixture -def composite_for_rotation_scan(fake_create_rotation_devices: RotationScanComposite): - energy_ev = convert_angstrom_to_eV(0.71) - set_mock_value( - fake_create_rotation_devices.dcm.energy_in_kev.user_readback, - energy_ev / 1000, # pyright: ignore - ) - set_mock_value(fake_create_rotation_devices.undulator.current_gap, 1.12) # pyright: ignore - set_mock_value( - fake_create_rotation_devices.synchrotron.synchrotron_mode, - SynchrotronMode.USER, - ) - set_mock_value( - fake_create_rotation_devices.synchrotron.top_up_start_countdown, # pyright: ignore - -1, - ) - fake_create_rotation_devices.s4_slit_gaps.xgap.user_readback.sim_put( # pyright: ignore - 0.123 - ) - fake_create_rotation_devices.s4_slit_gaps.ygap.user_readback.sim_put( # pyright: ignore - 0.234 - ) - it_snapshot_filenames = iter( - [ - "/tmp/snapshot1.png", - "/tmp/snapshot2.png", - "/tmp/snapshot3.png", - "/tmp/snapshot4.png", - ] - ) - - with ( - patch("bluesky.preprocessors.__read_and_stash_a_motor", fake_read), - patch.object( - fake_create_rotation_devices.oav.snapshot.last_saved_path, "get" - ) as mock_last_saved_path, - patch("bluesky.plan_stubs.wait"), - ): - - @AsyncStatus.wrap - async def apply_snapshot_filename(): - mock_last_saved_path.return_value = next(it_snapshot_filenames) - - with patch.object( - fake_create_rotation_devices.oav.snapshot, - "trigger", - side_effect=apply_snapshot_filename, - ): - yield fake_create_rotation_devices - - @pytest.fixture def params_for_rotation_scan(test_rotation_params: RotationScan): test_rotation_params.rotation_increment_deg = 0.27 @@ -745,7 +569,6 @@ def test_ispyb_deposition_in_rotation_plan( RE: RunEngine, fetch_comment: Callable[..., Any], fetch_datacollection_attribute: Callable[..., Any], - fetch_datacollectiongroup_attribute: Callable[..., Any], fetch_datacollection_position_attribute: Callable[..., Any], ): os.environ["ISPYB_CONFIG_PATH"] = CONST.SIM.DEV_ISPYB_DATABASE_CFG diff --git a/tests/system_tests/hyperion/external_interaction/test_load_centre_collect_full_plan.py b/tests/system_tests/hyperion/external_interaction/test_load_centre_collect_full_plan.py new file mode 100644 index 000000000..02399f8d6 --- /dev/null +++ b/tests/system_tests/hyperion/external_interaction/test_load_centre_collect_full_plan.py @@ -0,0 +1,305 @@ +from __future__ import annotations + +import os +import re +from collections.abc import Callable +from contextlib import ExitStack +from decimal import Decimal +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from bluesky.run_engine import RunEngine +from dodal.devices.oav.oav_parameters import OAVParameters +from dodal.devices.synchrotron import SynchrotronMode +from dodal.devices.util.test_utils import patch_motor +from ophyd_async.core import set_mock_value + +from mx_bluesky.hyperion.experiment_plans.load_centre_collect_full_plan import ( + LoadCentreCollectComposite, + load_centre_collect_full_plan, +) +from mx_bluesky.hyperion.external_interaction.callbacks.robot_load.ispyb_callback import ( + RobotLoadISPyBCallback, +) +from mx_bluesky.hyperion.external_interaction.callbacks.rotation.ispyb_callback import ( + RotationISPyBCallback, +) +from mx_bluesky.hyperion.external_interaction.callbacks.xray_centre.ispyb_callback import ( + GridscanISPyBCallback, +) +from mx_bluesky.hyperion.parameters.constants import CONST +from mx_bluesky.hyperion.parameters.load_centre_collect import LoadCentreCollect + +from .conftest import raw_params_from_file +from .test_ispyb_dev_connection import DATA_COLLECTION_COLUMN_MAP + + +@pytest.fixture +def load_centre_collect_params(): + json_dict = raw_params_from_file( + "tests/test_data/parameter_json_files/example_load_centre_collect_params.json" + ) + return LoadCentreCollect(**json_dict) + + +@pytest.fixture +def load_centre_collect_composite( + grid_detect_then_xray_centre_composite, + composite_for_rotation_scan, + thawer, + vfm, + vfm_mirror_voltages, + undulator_dcm, + webcam, + lower_gonio, +): + composite = LoadCentreCollectComposite( + aperture_scatterguard=composite_for_rotation_scan.aperture_scatterguard, + attenuator=composite_for_rotation_scan.attenuator, + backlight=composite_for_rotation_scan.backlight, + dcm=composite_for_rotation_scan.dcm, + detector_motion=composite_for_rotation_scan.detector_motion, + eiger=grid_detect_then_xray_centre_composite.eiger, + flux=composite_for_rotation_scan.flux, + robot=composite_for_rotation_scan.robot, + smargon=composite_for_rotation_scan.smargon, + undulator=composite_for_rotation_scan.undulator, + synchrotron=composite_for_rotation_scan.synchrotron, + s4_slit_gaps=composite_for_rotation_scan.s4_slit_gaps, + sample_shutter=composite_for_rotation_scan.sample_shutter, + zebra=grid_detect_then_xray_centre_composite.zebra, + oav=grid_detect_then_xray_centre_composite.oav, + xbpm_feedback=composite_for_rotation_scan.xbpm_feedback, + zebra_fast_grid_scan=grid_detect_then_xray_centre_composite.zebra_fast_grid_scan, + pin_tip_detection=grid_detect_then_xray_centre_composite.pin_tip_detection, + zocalo=grid_detect_then_xray_centre_composite.zocalo, + panda=grid_detect_then_xray_centre_composite.panda, + panda_fast_grid_scan=grid_detect_then_xray_centre_composite.panda_fast_grid_scan, + thawer=thawer, + vfm=vfm, + vfm_mirror_voltages=vfm_mirror_voltages, + undulator_dcm=undulator_dcm, + webcam=webcam, + lower_gonio=lower_gonio, + ) + + set_mock_value(composite.dcm.bragg_in_degrees.user_readback, 5) + + with ExitStack() as stack: + [ + stack.enter_context(context_mgr) + for context_mgr in [ + patch.object(vc, "set") + for vc in vfm_mirror_voltages.voltage_channels.values() + ] + + [patch_motor(vfm.x_mm)] + ] + + yield composite + + +GRID_DC_1_EXPECTED_VALUES = { + "BLSAMPLEID": 5461074, + "detectorid": 78, + "axisstart": 0.0, + "axisrange": 0, + "axisend": 0, + "focalspotsizeatsamplex": 0.02, + "focalspotsizeatsampley": 0.02, + "slitgapvertical": 0.234, + "slitgaphorizontal": 0.123, + "beamsizeatsamplex": 0.02, + "beamsizeatsampley": 0.02, + "transmission": 100, + "datacollectionnumber": 1, + "detectordistance": 255.0, + "exposuretime": 0.002, + "imagedirectory": "/tmp/dls/i03/data/2024/cm31105-4/auto/123457/xraycentring/", + "imageprefix": "robot_load_centring_file", + "imagesuffix": "h5", + "numberofpasses": 1, + "overlap": 0, + "omegastart": 0, + "startimagenumber": 1, + "wavelength": 0.71, + "xbeam": 75.6027, + "ybeam": 79.4935, + "xtalsnapshotfullpath1": "test_1_y", + "xtalsnapshotfullpath2": "test_2_y", + "xtalsnapshotfullpath3": "test_3_y", + "synchrotronmode": "User", + "undulatorgap1": 1.11, + "filetemplate": "robot_load_centring_file_1_master.h5", + "numberofimages": 120, +} + +GRID_DC_2_EXPECTED_VALUES = GRID_DC_1_EXPECTED_VALUES | { + "axisstart": 90, + "axisend": 90, + "omegastart": 90, + "datacollectionnumber": 2, + "filetemplate": "robot_load_centring_file_2_master.h5", + "numberofimages": 90, +} + +ROTATION_DC_EXPECTED_VALUES = { + "axisStart": 10, + "axisEnd": 370, + # "chiStart": 0, mx-bluesky 325 + "wavelength": 0.71, + "beamSizeAtSampleX": 0.1, + "beamSizeAtSampleY": 0.02, + "exposureTime": 0.004, + "undulatorGap1": 1.11, + "synchrotronMode": SynchrotronMode.USER.value, + "slitGapHorizontal": 0.123, + "slitGapVertical": 0.234, + "xtalSnapshotFullPath1": "/tmp/snapshot2.png", + "xtalSnapshotFullPath2": "/tmp/snapshot3.png", + "xtalSnapshotFullPath3": "/tmp/snapshot4.png", + "xtalSnapshotFullPath4": "/tmp/snapshot5.png", +} + +ROTATION_DC_2_EXPECTED_VALUES = { + "axisStart": 10, + "axisEnd": 370, + # "chiStart": 30, mx-bluesky 325 + "wavelength": 0.71, + "beamSizeAtSampleX": 0.1, + "beamSizeAtSampleY": 0.02, + "exposureTime": 0.004, + "undulatorGap1": 1.11, + "synchrotronMode": SynchrotronMode.USER.value, + "slitGapHorizontal": 0.123, + "slitGapVertical": 0.234, + "xtalSnapshotFullPath1": "/tmp/snapshot6.png", + "xtalSnapshotFullPath2": "/tmp/snapshot7.png", + "xtalSnapshotFullPath3": "/tmp/snapshot8.png", + "xtalSnapshotFullPath4": "/tmp/snapshot9.png", +} + + +@pytest.mark.s03 +def test_execute_load_centre_collect_full_plan( + load_centre_collect_composite: LoadCentreCollectComposite, + load_centre_collect_params: LoadCentreCollect, + oav_parameters_for_rotation: OAVParameters, + RE: RunEngine, + fetch_datacollection_attribute: Callable[..., Any], + fetch_datacollectiongroup_attribute: Callable[..., Any], + fetch_datacollection_ids_for_group_id: Callable[..., Any], +): + os.environ["ISPYB_CONFIG_PATH"] = CONST.SIM.DEV_ISPYB_DATABASE_CFG + ispyb_gridscan_cb = GridscanISPyBCallback() + ispyb_rotation_cb = RotationISPyBCallback() + robot_load_cb = RobotLoadISPyBCallback() + robot_load_cb.expeye = MagicMock() + robot_load_cb.expeye.start_load.return_value = 1234 + RE.subscribe(ispyb_gridscan_cb) + RE.subscribe(ispyb_rotation_cb) + RE.subscribe(robot_load_cb) + RE( + load_centre_collect_full_plan( + load_centre_collect_composite, + load_centre_collect_params, + oav_parameters_for_rotation, + ) + ) + + assert robot_load_cb.expeye.start_load.called_once_with("cm37235", 4, 5461074, 2, 6) + assert robot_load_cb.expeye.update_barcode_and_snapshots( + 1234, + "BARCODE", + "/tmp/dls/i03/data/2024/cm31105-4/auto/123457/xraycentring/snapshots/160705_webcam_after_load.png", + "/tmp/snapshot1.png", + ) + assert robot_load_cb.expeye.end_load(1234, "success", "OK") + + # Compare gridscan collection + compare_actual_and_expected( + ispyb_gridscan_cb.ispyb_ids.data_collection_group_id, + {"experimentType": "Mesh3D", "blSampleId": 5461074}, + fetch_datacollectiongroup_attribute, + ) + compare_actual_and_expected( + ispyb_gridscan_cb.ispyb_ids.data_collection_ids[0], + GRID_DC_1_EXPECTED_VALUES, + fetch_datacollection_attribute, + DATA_COLLECTION_COLUMN_MAP, + ) + compare_actual_and_expected( + ispyb_gridscan_cb.ispyb_ids.data_collection_ids[1], + GRID_DC_2_EXPECTED_VALUES, + fetch_datacollection_attribute, + DATA_COLLECTION_COLUMN_MAP, + ) + + compare_comment( + fetch_datacollection_attribute, + ispyb_gridscan_cb.ispyb_ids.data_collection_ids[0], + "Hyperion: Xray centring - Diffraction grid scan of 30 by 4 " + "images in 20.0 um by 20.0 um steps. Top left (px): [100,152], " + "bottom right (px): [844,251]. Aperture: ApertureValue.SMALL. ", + ) + compare_comment( + fetch_datacollection_attribute, + ispyb_gridscan_cb.ispyb_ids.data_collection_ids[1], + "Hyperion: Xray centring - Diffraction grid scan of 30 by 3 " + "images in 20.0 um by 20.0 um steps. Top left (px): [100,165], " + "bottom right (px): [844,239]. Aperture: ApertureValue.SMALL. ", + ) + + rotation_dcg_id = ispyb_rotation_cb.ispyb_ids.data_collection_group_id + rotation_dc_ids = fetch_datacollection_ids_for_group_id(rotation_dcg_id) + compare_actual_and_expected( + rotation_dcg_id, + {"experimentType": "SAD", "blSampleId": 5461074}, + fetch_datacollectiongroup_attribute, + ) + compare_actual_and_expected( + rotation_dc_ids[0], + ROTATION_DC_EXPECTED_VALUES, + fetch_datacollection_attribute, + ) + compare_actual_and_expected( + rotation_dc_ids[1], + ROTATION_DC_2_EXPECTED_VALUES, + fetch_datacollection_attribute, + ) + + compare_comment( + fetch_datacollection_attribute, + ispyb_rotation_cb.ispyb_ids.data_collection_ids[0], + "Sample position (µm): (675, 737, -381) Hyperion Rotation Scan - Aperture: ApertureValue.LARGE. ", + ) + + +def compare_actual_and_expected( + id, expected_values, fetch_datacollection_attribute, column_map: dict | None = None +): + results = "\n" + for k, v in expected_values.items(): + actual = fetch_datacollection_attribute( + id, column_map[k.lower()] if column_map else k + ) + if isinstance(actual, Decimal): + actual = float(actual) + if isinstance(v, float): + actual_v = actual == pytest.approx(v) + else: + actual_v = actual == v + if not actual_v: + results += f"expected {k} {v} == {actual}\n" + assert results == "\n", results + + +def compare_comment( + fetch_datacollection_attribute, data_collection_id, expected_comment +): + actual_comment = fetch_datacollection_attribute( + data_collection_id, DATA_COLLECTION_COLUMN_MAP["comments"] + ) + match = re.search(" Zocalo processing took", actual_comment) + truncated_comment = actual_comment[: match.start()] if match else actual_comment + assert truncated_comment == expected_comment diff --git a/tests/test_data/parameter_json_files/example_load_centre_collect_params.json b/tests/test_data/parameter_json_files/example_load_centre_collect_params.json new file mode 100644 index 000000000..538b47549 --- /dev/null +++ b/tests/test_data/parameter_json_files/example_load_centre_collect_params.json @@ -0,0 +1,53 @@ +{ + "parameter_model_version": "5.0.0", + "visit": "cm37235-4", + "detector_distance_mm": 255, + "sample_id": 5461074, + "sample_puck": 2, + "sample_pin": 6, + "robot_load_then_centre": { + "storage_directory": "/tmp/dls/i03/data/2024/cm31105-4/auto/123457/xraycentring", + "file_name": "robot_load_centring_file", + "exposure_time_s": 0.002, + "use_roi_mode": true, + "demand_energy_ev": 11100, + "transmission_frac": 1.0, + "omega_start_deg": 0, + "chi_start_deg": 30, + "use_panda": false + }, + "multi_rotation_scan": { + "comment": "Hyperion Rotation Scan - ", + "file_name": "protk", + "storage_directory": "/tmp/dls/i03/data/2024/cm31105-4/auto/123457/", + "demand_energy_ev": 11200, + "exposure_time_s": 0.004, + "rotation_increment_deg": 0.1, + "snapshot_omegas_deg": [ + 0, + 90, + 180, + 270 + ], + "rotation_scans": [ + { + "omega_start_deg": 10, + "chi_start_deg": 0, + "rotation_axis": "omega", + "rotation_direction": "Negative", + "scan_width_deg": 360, + "transmission_frac": 0.05, + "ispyb_experiment_type": "SAD" + }, + { + "omega_start_deg": 10, + "chi_start_deg": 30, + "rotation_axis": "omega", + "rotation_direction": "Negative", + "scan_width_deg": 360, + "transmission_frac": 0.05, + "ispyb_experiment_type": "SAD" + } + ] + } +} diff --git a/tests/test_data/parameter_json_files/good_test_load_centre_collect_params.json b/tests/test_data/parameter_json_files/good_test_load_centre_collect_params.json index 9fcefddd2..a0785f033 100644 --- a/tests/test_data/parameter_json_files/good_test_load_centre_collect_params.json +++ b/tests/test_data/parameter_json_files/good_test_load_centre_collect_params.json @@ -2,7 +2,6 @@ "parameter_model_version": "5.0.0", "zocalo_environment": "dev_artemis", "beamline": "BL03S", - "storage_directory": "/tmp/dls/i03/data/2024/cm31105-4/auto/123456/", "det_dist_to_beam_converter_path": "tests/test_data/test_lookup_table.txt", "insertion_prefix": "SR03S", "visit": "cm31105-4", @@ -11,6 +10,7 @@ "sample_puck": 40, "sample_pin": 3, "robot_load_then_centre": { + "storage_directory": "/tmp/dls/i03/data/2024/cm31105-4/auto/123458/xraycentring", "file_name": "robot_load_centring_file", "comment": "Robot load and centre", "exposure_time_s": 0.004, @@ -20,6 +20,7 @@ }, "multi_rotation_scan": { "comment": "Rotation", + "storage_directory": "/tmp/dls/i03/data/2024/cm31105-4/auto/123458/", "file_name": "file_name", "exposure_time_s": 0.004, "selected_aperture": "SMALL_APERTURE", diff --git a/tests/unit_tests/hyperion/experiment_plans/conftest.py b/tests/unit_tests/hyperion/experiment_plans/conftest.py index b6837bff4..2a07c75da 100644 --- a/tests/unit_tests/hyperion/experiment_plans/conftest.py +++ b/tests/unit_tests/hyperion/experiment_plans/conftest.py @@ -261,6 +261,7 @@ def robot_load_composite( panda=panda, panda_fast_grid_scan=panda_fast_grid_scan, thawer=thawer, + sample_shutter=sample_shutter, vfm=vfm, vfm_mirror_voltages=vfm_mirror_voltages, dcm=dcm, @@ -268,7 +269,6 @@ def robot_load_composite( robot=robot, webcam=webcam, lower_gonio=lower_gonio, - sample_shutter=sample_shutter )