diff --git a/moto/core/utils.py b/moto/core/utils.py index ffea7a197bf7..921ccfd3e4db 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -337,6 +337,8 @@ def merge_dicts( merge_dicts(dict1[key], dict2[key], remove_nulls) else: dict1[key] = dict2[key] + if isinstance(dict1[key], dict): + remove_null_from_dict(dict1) if dict1[key] == {} and remove_nulls: dict1.pop(key) else: @@ -345,6 +347,14 @@ def merge_dicts( dict1.pop(key) +def remove_null_from_dict(dct: Dict[str, Any]) -> None: + for key in list(dct.keys()): + if dct[key] is None: + dct.pop(key) + elif isinstance(dct[key], dict): + remove_null_from_dict(dct[key]) + + def aws_api_matches(pattern: str, string: Any) -> bool: """ AWS API can match a value based on a glob, or an exact match diff --git a/moto/iotdata/models.py b/moto/iotdata/models.py index 88046653827b..b8f3e78faac3 100644 --- a/moto/iotdata/models.py +++ b/moto/iotdata/models.py @@ -72,12 +72,17 @@ def create_from_previous_version( # type: ignore[misc] @classmethod def parse_payload(cls, desired: Optional[str], reported: Optional[str]) -> Any: # type: ignore[misc] - if desired is None: - delta = reported - elif reported is None: + if not desired and not reported: + delta = None + elif reported is None and desired: delta = desired + elif desired and reported: + delta = jsondiff.diff(reported, desired) + delta.pop(jsondiff.add, None) # type: ignore + delta.pop(jsondiff.delete, None) # type: ignore + delta.pop(jsondiff.replace, None) # type: ignore else: - delta = jsondiff.diff(desired, reported) + delta = None return delta def _create_metadata_from_state(self, state: Any, ts: Any) -> Any: @@ -129,11 +134,11 @@ def to_dict(self, include_delta: bool = True) -> Dict[str, Any]: return {"timestamp": self.timestamp, "version": self.version} delta = self.parse_payload(self.desired, self.reported) payload = {} - if self.desired is not None: + if self.desired: payload["desired"] = self.desired - if self.reported is not None: + if self.reported: payload["reported"] = self.reported - if include_delta and (delta is not None and len(delta.keys()) != 0): + if include_delta and delta: payload["delta"] = delta metadata = {} @@ -214,11 +219,9 @@ def delete_thing_shadow( def publish(self, topic: str, payload: bytes) -> None: self.published_payloads.append((topic, payload)) - def list_named_shadows_for_thing(self, thing_name: str) -> List[FakeShadow]: + def list_named_shadows_for_thing(self, thing_name: str) -> List[str]: thing = self.iot_backend.describe_thing(thing_name) - return [ - shadow for name, shadow in thing.thing_shadows.items() if name is not None - ] + return [name for name in thing.thing_shadows.keys() if name is not None] iotdata_backends = BackendDict(IoTDataPlaneBackend, "iot-data") diff --git a/moto/iotdata/responses.py b/moto/iotdata/responses.py index 4db68514e9df..2a4e72255cd9 100644 --- a/moto/iotdata/responses.py +++ b/moto/iotdata/responses.py @@ -63,4 +63,4 @@ def publish(self) -> str: def list_named_shadows_for_thing(self) -> str: thing_name = self._get_param("thingName") shadows = self.iotdata_backend.list_named_shadows_for_thing(thing_name) - return json.dumps({"results": [shadow.to_dict() for shadow in shadows]}) + return json.dumps({"results": shadows}) diff --git a/tests/test_iotdata/__init__.py b/tests/test_iotdata/__init__.py index 08a1c1568c9c..6d54825e0421 100644 --- a/tests/test_iotdata/__init__.py +++ b/tests/test_iotdata/__init__.py @@ -1 +1,56 @@ -# This file is intentionally left blank. +import os +from functools import wraps +from typing import TYPE_CHECKING, Callable, TypeVar +from uuid import uuid4 + +import boto3 + +from moto import mock_aws + +if TYPE_CHECKING: + from typing_extensions import ParamSpec + + P = ParamSpec("P") + +T = TypeVar("T") + + +def iot_aws_verified() -> "Callable[[Callable[P, T]], Callable[P, T]]": + """ + Function that is verified to work against AWS. + Can be run against AWS at any time by setting: + MOTO_TEST_ALLOW_AWS_REQUEST=true + + If this environment variable is not set, the function runs in a `mock_aws` context. + """ + + def inner(func: "Callable[P, T]") -> "Callable[P, T]": + @wraps(func) + def pagination_wrapper(*args: "P.args", **kwargs: "P.kwargs") -> T: + allow_aws_request = ( + os.environ.get("MOTO_TEST_ALLOW_AWS_REQUEST", "false").lower() == "true" + ) + + if allow_aws_request: + return _create_thing_and_execute_test(func, *args, **kwargs) + else: + with mock_aws(): + return _create_thing_and_execute_test(func, *args, **kwargs) + + return pagination_wrapper + + return inner + + +def _create_thing_and_execute_test( + func: "Callable[P, T]", *args: "P.args", **kwargs: "P.kwargs" +) -> T: + iot_client = boto3.client("iot", region_name="ap-northeast-1") + name = str(uuid4()) + + iot_client.create_thing(thingName=name) + + try: + return func(*args, **kwargs, name=name) # type: ignore + finally: + iot_client.delete_thing(thingName=name) diff --git a/tests/test_iotdata/test_iotdata.py b/tests/test_iotdata/test_iotdata.py index 8f369d4d0db0..b2185859ee42 100644 --- a/tests/test_iotdata/test_iotdata.py +++ b/tests/test_iotdata/test_iotdata.py @@ -1,5 +1,6 @@ import json import sys +from typing import Dict, Optional from unittest import SkipTest import boto3 @@ -11,16 +12,17 @@ from moto.core import DEFAULT_ACCOUNT_ID as ACCOUNT_ID from moto.utilities.distutils_version import LooseVersion +from . import iot_aws_verified + boto3_version = sys.modules["botocore"].__version__ -@mock_aws -def test_basic() -> None: - iot_client = boto3.client("iot", region_name="ap-northeast-1") +@iot_aws_verified() +@pytest.mark.aws_verified +def test_basic(name: Optional[str] = None) -> None: client = boto3.client("iot-data", region_name="ap-northeast-1") - name = "my-thing" + raw_payload = b'{"state": {"desired": {"led": "on"}}}' - iot_client.create_thing(thingName=name) with pytest.raises(ClientError): client.get_thing_shadow(thingName=name) @@ -47,13 +49,11 @@ def test_basic() -> None: client.get_thing_shadow(thingName=name) -@mock_aws -def test_update() -> None: - iot_client = boto3.client("iot", region_name="ap-northeast-1") +@iot_aws_verified() +@pytest.mark.aws_verified +def test_update(name: Optional[str] = None) -> None: client = boto3.client("iot-data", region_name="ap-northeast-1") - name = "my-thing" raw_payload = b'{"state": {"desired": {"led": "on"}}}' - iot_client.create_thing(thingName=name) # first update res = client.update_thing_shadow(thingName=name, payload=raw_payload) @@ -97,14 +97,13 @@ def test_update() -> None: assert ex.value.response["Error"]["Message"] == "Version conflict" -@mock_aws -def test_create_named_shadows() -> None: +@iot_aws_verified() +@pytest.mark.aws_verified +def test_create_named_shadows(name: Optional[str] = None) -> None: if LooseVersion(boto3_version) < LooseVersion("1.29.0"): raise SkipTest("Parameter only available in newer versions") - iot_client = boto3.client("iot", region_name="ap-northeast-1") client = boto3.client("iot-data", region_name="ap-northeast-1") - thing_name = "my-thing" - iot_client.create_thing(thingName=thing_name) + thing_name = name # default shadow default_payload = json.dumps({"state": {"desired": {"name": "default"}}}) @@ -128,23 +127,16 @@ def test_create_named_shadows() -> None: # List named shadows shadows = client.list_named_shadows_for_thing(thingName=thing_name)["results"] assert len(shadows) == 2 - - for shadow in shadows: - shadow.pop("metadata") - shadow.pop("timestamp") - shadow.pop("version") - - # Verify both named shadows are present - for name in ["shadow1", "shadow2"]: - assert { - "state": {"reported": {"name": name}, "delta": {"name": name}} - } in shadows + assert "shadow1" in shadows + assert "shadow2" in shadows # Verify we can delete a named shadow client.delete_thing_shadow(thingName=thing_name, shadowName="shadow2") - with pytest.raises(ClientError): - client.get_thing_shadow(thingName="shadow1") + with pytest.raises(ClientError) as exc: + client.get_thing_shadow(thingName=thing_name, shadowName="shadow2") + err = exc.value.response["Error"] + assert err["Code"] == "ResourceNotFoundException" # The default and other named shadow are still there assert "payload" in client.get_thing_shadow(thingName=thing_name) @@ -171,38 +163,86 @@ def test_publish() -> None: assert ("test/topic4", b"string") in mock_backend.published_payloads -@mock_aws -def test_delete_field_from_device_shadow() -> None: - test_thing_name = "TestThing" - - iot_raw_client = boto3.client("iot", region_name="eu-central-1") - iot_raw_client.create_thing(thingName=test_thing_name) - iot = boto3.client("iot-data", region_name="eu-central-1") +@iot_aws_verified() +@pytest.mark.aws_verified +def test_delete_field_from_device_shadow(name: Optional[str] = None) -> None: + iot = boto3.client("iot-data", region_name="ap-northeast-1") iot.update_thing_shadow( - thingName=test_thing_name, + thingName=name, payload=json.dumps({"state": {"desired": {"state1": 1, "state2": 2}}}), ) - response = json.loads( - iot.get_thing_shadow(thingName=test_thing_name)["payload"].read() - ) + response = json.loads(iot.get_thing_shadow(thingName=name)["payload"].read()) assert len(response["state"]["desired"]) == 2 iot.update_thing_shadow( - thingName=test_thing_name, + thingName=name, payload=json.dumps({"state": {"desired": {"state1": None}}}), ) - response = json.loads( - iot.get_thing_shadow(thingName=test_thing_name)["payload"].read() - ) + response = json.loads(iot.get_thing_shadow(thingName=name)["payload"].read()) assert len(response["state"]["desired"]) == 1 assert "state2" in response["state"]["desired"] iot.update_thing_shadow( - thingName=test_thing_name, + thingName=name, payload=json.dumps({"state": {"desired": {"state2": None}}}), ) - response = json.loads( - iot.get_thing_shadow(thingName=test_thing_name)["payload"].read() - ) + response = json.loads(iot.get_thing_shadow(thingName=name)["payload"].read()) assert "desired" not in response["state"] + + +@iot_aws_verified() +@pytest.mark.aws_verified +@pytest.mark.parametrize( + "desired,initial_delta,reported,delta_after_report", + [ + ( + {"desired": {"online": True}}, + {"desired": {"online": True}, "delta": {"online": True}}, + {"reported": {"online": False}}, + { + "desired": {"online": True}, + "reported": {"online": False}, + "delta": {"online": True}, + }, + ), + ( + {"desired": {"enabled": True}}, + {"desired": {"enabled": True}, "delta": {"enabled": True}}, + {"reported": {"online": False, "enabled": True}}, + { + "desired": {"enabled": True}, + "reported": {"online": False, "enabled": True}, + }, + ), + ({}, {}, {"reported": {"online": False}}, {"reported": {"online": False}}), + ({}, {}, {"reported": {"online": None}}, {}), + ( + {"desired": {}}, + {}, + {"reported": {"online": False}}, + {"reported": {"online": False}}, + ), + ], +) +def test_delta_calculation( + desired: Dict[str, Dict[str, Optional[bool]]], + initial_delta: Dict[str, Dict[str, Optional[bool]]], + reported: Dict[str, Dict[str, Optional[bool]]], + delta_after_report: Dict[str, Dict[str, Optional[bool]]], + name: Optional[str] = None, +) -> None: + client = boto3.client("iot-data", region_name="ap-northeast-1") + desired_payload = json.dumps({"state": desired}).encode("utf-8") + client.update_thing_shadow(thingName=name, payload=desired_payload) + + res = client.get_thing_shadow(thingName=name) + payload = json.loads(res["payload"].read()) + assert payload["state"] == initial_delta + + reported_payload = json.dumps({"state": reported}).encode("utf-8") + client.update_thing_shadow(thingName=name, payload=reported_payload) + + res = client.get_thing_shadow(thingName=name) + payload = json.loads(res["payload"].read()) + assert payload["state"] == delta_after_report