Skip to content

[Core] Add retry for failed upserts and handle circular dependencies

Sign in for the full log view
GitHub Actions / JUnit Test Report failed Dec 23, 2024 in 0s

48 tests run, 47 passed, 0 skipped, 1 failed.

Annotations

Check failure on line 379 in port_ocean/tests/core/handlers/mixins/test_sync_raw.py

See this annotation in the file changed.

@github-actions github-actions / JUnit Test Report

test_sync_raw.test_sync_raw_mixin_dependency

AssertionError: Expected one failed entity callback due to retry logic
assert 7 == 5
 +  where 7 = len([Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_2'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'}), Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_4'}), Entity(identifier='entity_3', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': ''}), Entity(identifier='entity_4', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), ...])
 +    where [Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_2'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'}), Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_4'}), Entity(identifier='entity_3', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': ''}), Entity(identifier='entity_4', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), ...] = EntityTopologicalSorter(entities=[Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_2'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'}), Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_4'}), Entity(identifier='entity_3', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': ''}), Entity(identifier='entity_4', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_5', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'})]).entities
 +      where EntityTopologicalSorter(entities=[Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_2'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'}), Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_4'}), Entity(identifier='entity_3', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': ''}), Entity(identifier='entity_4', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_5', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'})]) = EventContext(event_type='resync', trigger_type='machine', attributes={}, _aborted=False, _port_app_config=PortAppConfig(enable_merge_entity=True, delete_dependent_entities=True, create_missing_related_entities=False, resources=[ResourceConfig(kind='project', selector=Selector(query='true'), port=PortResourceConfig(entity=MappingsConfig(mappings=EntityMapping(identifier='.id | tostring', title='.name', blueprint='"service"', team=None, properties={'url': '.web_url'}, relations={})), items_to_parse=None))]), _parent_event=None, _event_id='89b5a205-a447-4902-8964-0c47388dfd4e', _on_abort_callbacks=[<function SyncRawMixin.sync_raw_all.<locals>.<lambda> at 0x7f58e5433740>]).entity_topological_sorter
Raw output
mock_sync_raw_mixin = <port_ocean.core.integrations.mixins.sync_raw.SyncRawMixin object at 0x7f58e53357c0>
mock_ocean = <port_ocean.ocean.Ocean object at 0x7f58e534b980>

    @pytest.mark.asyncio
    async def test_sync_raw_mixin_dependency(
        mock_sync_raw_mixin: SyncRawMixin, mock_ocean: Ocean
    ) -> None:
        entities_params = [
            ("entity_1", "service", {"service": "entity_3"}, True),
            ("entity_2", "service", {"service": "entity_4"}, True),
            ("entity_3", "service", {"service": ""}, True),
            ("entity_4", "service", {"service": "entity_3"}, True),
            ("entity_5", "service", {"service": "entity_1"}, True),
        ]
        entities = [create_entity(*entity_param) for entity_param in entities_params]
    
        calc_result_mock = MagicMock()
        calc_result_mock.entity_selector_diff.passed = entities
        calc_result_mock.errors = []
    
        mock_sync_raw_mixin.entity_processor.parse_items = AsyncMock(return_value=calc_result_mock)  # type: ignore
    
        mock_order_by_entities_dependencies = MagicMock(
            side_effect=EntityTopologicalSorter.order_by_entities_dependencies
        )
        async with event_context(EventType.RESYNC, trigger_type="machine") as event:
            app_config = (
                await mock_sync_raw_mixin.port_app_config_handler.get_port_app_config(
                    use_cache=False
                )
            )
            event.port_app_config = app_config
            org = event.entity_topological_sorter.register_entity
    
            def mock_register_entity(*args: Any, **kwargs: Any) -> None:
                entity = args[0]
                entity.properties["mock_is_to_fail"] = False
                return org(*args, **kwargs)
    
            event.entity_topological_sorter.register_entity = MagicMock(side_effect=mock_register_entity)  # type: ignore
            raiesed_error_handle_failed = []
            org_event_get_entities = event.entity_topological_sorter.get_entities
    
            def get_entities_wrapper(*args: Any, **kwargs: Any) -> Any:
                try:
                    return org_event_get_entities(*args, **kwargs)
                except Exception as e:
                    raiesed_error_handle_failed.append(e)
                    raise e
    
            event.entity_topological_sorter.get_entities = MagicMock(side_effect=lambda *args, **kwargs: get_entities_wrapper(*args, **kwargs))  # type: ignore
    
            with patch(
                "port_ocean.core.integrations.mixins.sync_raw.event_context",
                lambda *args, **kwargs: no_op_event_context(event),
            ):
                with patch(
                    "port_ocean.core.utils.entity_topological_sorter.EntityTopologicalSorter.order_by_entities_dependencies",
                    mock_order_by_entities_dependencies,
                ):
    
                    await mock_sync_raw_mixin.sync_raw_all(
                        trigger_type="machine", user_agent_type=UserAgentType.exporter
                    )
    
                    assert event.entity_topological_sorter.register_entity.call_count == 5
>                   assert (
                        len(event.entity_topological_sorter.entities) == 5
                    ), "Expected one failed entity callback due to retry logic"
E                   AssertionError: Expected one failed entity callback due to retry logic
E                   assert 7 == 5
E                    +  where 7 = len([Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_2'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'}), Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_4'}), Entity(identifier='entity_3', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': ''}), Entity(identifier='entity_4', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), ...])
E                    +    where [Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_2'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'}), Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_4'}), Entity(identifier='entity_3', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': ''}), Entity(identifier='entity_4', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), ...] = EntityTopologicalSorter(entities=[Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_2'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'}), Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_4'}), Entity(identifier='entity_3', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': ''}), Entity(identifier='entity_4', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_5', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'})]).entities
E                    +      where EntityTopologicalSorter(entities=[Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_2'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'}), Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_4'}), Entity(identifier='entity_3', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': ''}), Entity(identifier='entity_4', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_5', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'})]) = EventContext(event_type='resync', trigger_type='machine', attributes={}, _aborted=False, _port_app_config=PortAppConfig(enable_merge_entity=True, delete_dependent_entities=True, create_missing_related_entities=False, resources=[ResourceConfig(kind='project', selector=Selector(query='true'), port=PortResourceConfig(entity=MappingsConfig(mappings=EntityMapping(identifier='.id | tostring', title='.name', blueprint='"service"', team=None, properties={'url': '.web_url'}, relations={})), items_to_parse=None))]), _parent_event=None, _event_id='89b5a205-a447-4902-8964-0c47388dfd4e', _on_abort_callbacks=[<function SyncRawMixin.sync_raw_all.<locals>.<lambda> at 0x7f58e5433740>]).entity_topological_sorter

port_ocean/tests/core/handlers/mixins/test_sync_raw.py:379: AssertionError