Skip to content

Commit

Permalink
[Core] Add retry for failed upserts and handle circular dependencies
Browse files Browse the repository at this point in the history 
1. Fix lint
2. change order of assert in test
  • Loading branch information
Ivan Kalinovski authored and Ivan Kalinovski committed Dec 23, 2024
1 parent f029a8f commit 79054b2
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 12 deletions.
22 changes: 11 additions & 11 deletions integrations/sonarqube/examples/project.entity.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,17 @@
"title": "Port API",
"team": [],
"properties": {
"organization": "port-labs",
"link": "https://sonarcloud.io/project/overview?id=port-labs_Port_port-api",
"qualityGateStatus": "ERROR",
"numberOfBugs": 20,
"numberOfCodeSmells": 262,
"numberOfVulnerabilities": 0,
"numberOfHotSpots": 3,
"numberOfDuplications": 18,
"coverage": 0,
"mainBranch": "main"
"organization": "port-labs",
"link": "https://sonarcloud.io/project/overview?id=port-labs_Port_port-api",
"qualityGateStatus": "ERROR",
"numberOfBugs": 20,
"numberOfCodeSmells": 262,
"numberOfVulnerabilities": 0,
"numberOfHotSpots": 3,
"numberOfDuplications": 18,
"coverage": 0,
"mainBranch": "main"
},
"relations": {},
"icon": "sonarqube"
}
}
2 changes: 1 addition & 1 deletion port_ocean/tests/core/handlers/mixins/test_sync_raw.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -375,10 +375,10 @@ def get_entities_wrapper(*args: Any, **kwargs: Any) -> Any:
trigger_type="machine", user_agent_type=UserAgentType.exporter
)

assert event.entity_topological_sorter.register_entity.call_count == 5
assert (

Check failure on line 379 in port_ocean/tests/core/handlers/mixins/test_sync_raw.py

View workflow job for this annotation

GitHub Actions / JUnit Test Report

test_sync_raw.test_sync_raw_mixin_dependency 

AssertionError: Expected one failed entity callback due to retry logic
assert 7 == 5
 +  where 7 = len([Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_2'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'}), Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_4'}), Entity(identifier='entity_3', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': ''}), Entity(identifier='entity_4', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), ...])
 +    where [Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_2'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'}), Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_4'}), Entity(identifier='entity_3', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': ''}), Entity(identifier='entity_4', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), ...] = EntityTopologicalSorter(entities=[Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_2'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'}), Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_4'}), Entity(identifier='entity_3', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': ''}), Entity(identifier='entity_4', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_5', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'})]).entities
 +      where EntityTopologicalSorter(entities=[Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_2'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'}), Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_4'}), Entity(identifier='entity_3', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': ''}), Entity(identifier='entity_4', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_5', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'})]) = EventContext(event_type='resync', trigger_type='machine', attributes={}, _aborted=False, _port_app_config=PortAppConfig(enable_merge_entity=True, delete_dependent_entities=True, create_missing_related_entities=False, resources=[ResourceConfig(kind='project', selector=Selector(query='true'), port=PortResourceConfig(entity=MappingsConfig(mappings=EntityMapping(identifier='.id | tostring', title='.name', blueprint='"service"', team=None, properties={'url': '.web_url'}, relations={})), items_to_parse=None))]), _parent_event=None, _event_id='89b5a205-a447-4902-8964-0c47388dfd4e', _on_abort_callbacks=[<function SyncRawMixin.sync_raw_all.<locals>.<lambda> at 0x7f58e5433740>]).entity_topological_sorter
Raw output 
mock_sync_raw_mixin = <port_ocean.core.integrations.mixins.sync_raw.SyncRawMixin object at 0x7f58e53357c0>
mock_ocean = <port_ocean.ocean.Ocean object at 0x7f58e534b980>

    @pytest.mark.asyncio
    async def test_sync_raw_mixin_dependency(
        mock_sync_raw_mixin: SyncRawMixin, mock_ocean: Ocean
    ) -> None:
        entities_params = [
            ("entity_1", "service", {"service": "entity_3"}, True),
            ("entity_2", "service", {"service": "entity_4"}, True),
            ("entity_3", "service", {"service": ""}, True),
            ("entity_4", "service", {"service": "entity_3"}, True),
            ("entity_5", "service", {"service": "entity_1"}, True),
        ]
        entities = [create_entity(*entity_param) for entity_param in entities_params]
    
        calc_result_mock = MagicMock()
        calc_result_mock.entity_selector_diff.passed = entities
        calc_result_mock.errors = []
    
        mock_sync_raw_mixin.entity_processor.parse_items = AsyncMock(return_value=calc_result_mock)  # type: ignore
    
        mock_order_by_entities_dependencies = MagicMock(
            side_effect=EntityTopologicalSorter.order_by_entities_dependencies
        )
        async with event_context(EventType.RESYNC, trigger_type="machine") as event:
            app_config = (
                await mock_sync_raw_mixin.port_app_config_handler.get_port_app_config(
                    use_cache=False
                )
            )
            event.port_app_config = app_config
            org = event.entity_topological_sorter.register_entity
    
            def mock_register_entity(*args: Any, **kwargs: Any) -> None:
                entity = args[0]
                entity.properties["mock_is_to_fail"] = False
                return org(*args, **kwargs)
    
            event.entity_topological_sorter.register_entity = MagicMock(side_effect=mock_register_entity)  # type: ignore
            raiesed_error_handle_failed = []
            org_event_get_entities = event.entity_topological_sorter.get_entities
    
            def get_entities_wrapper(*args: Any, **kwargs: Any) -> Any:
                try:
                    return org_event_get_entities(*args, **kwargs)
                except Exception as e:
                    raiesed_error_handle_failed.append(e)
                    raise e
    
            event.entity_topological_sorter.get_entities = MagicMock(side_effect=lambda *args, **kwargs: get_entities_wrapper(*args, **kwargs))  # type: ignore
    
            with patch(
                "port_ocean.core.integrations.mixins.sync_raw.event_context",
                lambda *args, **kwargs: no_op_event_context(event),
            ):
                with patch(
                    "port_ocean.core.utils.entity_topological_sorter.EntityTopologicalSorter.order_by_entities_dependencies",
                    mock_order_by_entities_dependencies,
                ):
    
                    await mock_sync_raw_mixin.sync_raw_all(
                        trigger_type="machine", user_agent_type=UserAgentType.exporter
                    )
    
                    assert event.entity_topological_sorter.register_entity.call_count == 5
>                   assert (
                        len(event.entity_topological_sorter.entities) == 5
                    ), "Expected one failed entity callback due to retry logic"
E                   AssertionError: Expected one failed entity callback due to retry logic
E                   assert 7 == 5
E                    +  where 7 = len([Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_2'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'}), Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_4'}), Entity(identifier='entity_3', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': ''}), Entity(identifier='entity_4', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), ...])
E                    +    where [Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_2'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'}), Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_4'}), Entity(identifier='entity_3', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': ''}), Entity(identifier='entity_4', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), ...] = EntityTopologicalSorter(entities=[Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_2'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'}), Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_4'}), Entity(identifier='entity_3', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': ''}), Entity(identifier='entity_4', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_5', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'})]).entities
E                    +      where EntityTopologicalSorter(entities=[Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_2'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'}), Entity(identifier='entity_1', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_2', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_4'}), Entity(identifier='entity_3', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': ''}), Entity(identifier='entity_4', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_3'}), Entity(identifier='entity_5', blueprint='service', title=None, team=[], properties={'mock_is_to_fail': False}, relations={'service': 'entity_1'})]) = EventContext(event_type='resync', trigger_type='machine', attributes={}, _aborted=False, _port_app_config=PortAppConfig(enable_merge_entity=True, delete_dependent_entities=True, create_missing_related_entities=False, resources=[ResourceConfig(kind='project', selector=Selector(query='true'), port=PortResourceConfig(entity=MappingsConfig(mappings=EntityMapping(identifier='.id | tostring', title='.name', blueprint='"service"', team=None, properties={'url': '.web_url'}, relations={})), items_to_parse=None))]), _parent_event=None, _event_id='89b5a205-a447-4902-8964-0c47388dfd4e', _on_abort_callbacks=[<function SyncRawMixin.sync_raw_all.<locals>.<lambda> at 0x7f58e5433740>]).entity_topological_sorter

port_ocean/tests/core/handlers/mixins/test_sync_raw.py:379: AssertionError
len(event.entity_topological_sorter.entities) == 5
), "Expected one failed entity callback due to retry logic"
assert event.entity_topological_sorter.register_entity.call_count == 5
assert event.entity_topological_sorter.get_entities.call_count == 1
assert len(raiesed_error_handle_failed) == 0
assert mock_ocean.port_client.client.post.call_count == 10 # type: ignore
Expand Down

0 comments on commit 79054b2

Please sign in to comment.