From 5a2047b282ef350b1f6f3dc8a5db9001b9058a7b Mon Sep 17 00:00:00 2001 From: driesdeprest Date: Fri, 5 May 2023 14:24:00 +0200 Subject: [PATCH 01/10] Foul & card changes for Wyscout and shot result & counter attack changes for Opta --- .../serializers/event/opta/deserializer.py | 20 ++++++++++++ .../event/wyscout/deserializer_v3.py | 31 ++++++++++++++----- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/kloppy/infra/serializers/event/opta/deserializer.py b/kloppy/infra/serializers/event/opta/deserializer.py index 2789a609..a6c27c29 100644 --- a/kloppy/infra/serializers/event/opta/deserializer.py +++ b/kloppy/infra/serializers/event/opta/deserializer.py @@ -40,6 +40,7 @@ BodyPart, PassType, PassQualifier, + CounterAttackQualifier ) from kloppy.exceptions import DeserializationError from kloppy.infra.serializers.event.deserializer import EventDataDeserializer @@ -105,6 +106,8 @@ EVENT_QUALIFIER_SECOND_YELLOW_CARD = 32 EVENT_QUALIFIER_RED_CARD = 33 +EVENT_QUALIFIER_COUNTER_ATTACK = 23 + EVENT_QUALIFIER_TEAM_FORMATION = 130 event_type_names = { @@ -301,6 +304,14 @@ def _parse_shot( # timestamp = else: result = ShotResult.GOAL + elif 82 in raw_qualifiers: + result = ShotResult.BLOCKED + elif type_id == EVENT_TYPE_SHOT_MISS: + result = ShotResult.OFF_TARGET + elif type_id == EVENT_TYPE_SHOT_POST: + result = ShotResult.OFF_TARGET + elif type_id == EVENT_TYPE_SHOT_SAVED: + result = ShotResult.SAVED else: result = None @@ -380,6 +391,7 @@ def _get_event_qualifiers(raw_qualifiers: List) -> List[Qualifier]: qualifiers.extend(_get_event_bodypart_qualifiers(raw_qualifiers)) qualifiers.extend(_get_event_pass_qualifiers(raw_qualifiers)) qualifiers.extend(_get_event_card_qualifiers(raw_qualifiers)) + qualifiers.extend(_get_event_counter_attack_qualifiers(raw_qualifiers)) return qualifiers @@ -451,6 +463,14 @@ def _get_event_card_qualifiers(raw_qualifiers: List) -> List[Qualifier]: return qualifiers +def _get_event_counter_attack_qualifiers(raw_qualifiers: List) -> List[Qualifier]: + qualifiers = [] + if EVENT_QUALIFIER_COUNTER_ATTACK in raw_qualifiers: + qualifiers.append(CounterAttackQualifier(True)) + + return qualifiers + + def _get_event_type_name(type_id: int) -> str: return event_type_names.get(type_id, "unknown") diff --git a/kloppy/infra/serializers/event/wyscout/deserializer_v3.py b/kloppy/infra/serializers/event/wyscout/deserializer_v3.py index 413fa35a..56c58364 100644 --- a/kloppy/infra/serializers/event/wyscout/deserializer_v3.py +++ b/kloppy/infra/serializers/event/wyscout/deserializer_v3.py @@ -39,7 +39,6 @@ ) from kloppy.utils import performance_logging -from . import wyscout_tags from ..deserializer import EventDataDeserializer from .deserializer_v2 import WyscoutInputs @@ -184,12 +183,10 @@ def _parse_foul(raw_event: Dict) -> Dict: def _parse_card(raw_event: Dict) -> Dict: qualifiers = _generic_qualifiers(raw_event) card_type = None - if _has_tag(raw_event, wyscout_tags.RED_CARD): - card_type = CardType.RED - elif _has_tag(raw_event, wyscout_tags.YELLOW_CARD): + if _check_secondary_event_types(raw_event, ["yellow_card"]): card_type = CardType.FIRST_YELLOW - elif _has_tag(raw_event, wyscout_tags.SECOND_YELLOW_CARD): - card_type = CardType.SECOND_YELLOW + elif _check_secondary_event_types(raw_event, ["red_card"]): + card_type = CardType.RED return {"result": None, "qualifiers": qualifiers, "card_type": card_type} @@ -347,7 +344,9 @@ def deserialize(self, inputs: WyscoutInputs) -> EventDataset: "period": periods[-1], "timestamp": float( raw_event["second"] + raw_event["minute"] * 60 - ), + ) + if period_id == 1 + else float(raw_event["second"] + (raw_event["minute"] * 60) - (60 * 45)), } primary_event_type = raw_event["type"]["primary"] @@ -401,6 +400,24 @@ def deserialize(self, inputs: WyscoutInputs) -> EventDataset: event = self.event_factory.build_shot( **set_piece_event_args, **generic_event_args ) + elif primary_event_type == "infraction": + if "foul" in secondary_event_types: + foul_event_args = _parse_foul(raw_event) + event = self.event_factory.build_foul_committed( + **foul_event_args, **generic_event_args + ) + # We already append event to events + # as we potentially have a card and foul event for one raw event + if event and self.should_include_event(event): + events.append(transformer.transform_event(event)) + if "yellow_card" in secondary_event_types: + card_event_args = _parse_card(raw_event) + event = self.event_factory.build_card( + **card_event_args, **generic_event_args + ) + if event and self.should_include_event(event): + events.append(transformer.transform_event(event)) + continue else: event = self.event_factory.build_generic( From 3611c99d3b6988ff67892618ceb58b6514517e0c Mon Sep 17 00:00:00 2001 From: driesdeprest Date: Thu, 15 Jun 2023 13:42:29 +0200 Subject: [PATCH 02/10] Opta - fix for missing pass end coordinate qualifiers --- kloppy/infra/serializers/event/opta/deserializer.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/kloppy/infra/serializers/event/opta/deserializer.py b/kloppy/infra/serializers/event/opta/deserializer.py index a6c27c29..0f5ff647 100644 --- a/kloppy/infra/serializers/event/opta/deserializer.py +++ b/kloppy/infra/serializers/event/opta/deserializer.py @@ -228,16 +228,13 @@ def _parse_f24_datetime(dt_str: str) -> float: def _parse_pass(raw_qualifiers: Dict[int, str], outcome: int) -> Dict: if outcome: - receiver_coordinates = Point( - x=float(raw_qualifiers[140]), y=float(raw_qualifiers[141]) - ) result = PassResult.COMPLETE else: result = PassResult.INCOMPLETE - # receiver_coordinates = None - receiver_coordinates = Point( - x=float(raw_qualifiers[140]), y=float(raw_qualifiers[141]) - ) + receiver_coordinates = Point( + x=float(raw_qualifiers[140]) if 140 in raw_qualifiers else 0, + y=float(raw_qualifiers[141]) if 141 in raw_qualifiers else 0 + ) qualifiers = _get_event_qualifiers(raw_qualifiers) From 8e096f075989aa7f514e3058ebad823fd88a3512 Mon Sep 17 00:00:00 2001 From: driesdeprest Date: Mon, 19 Jun 2023 11:36:50 +0200 Subject: [PATCH 03/10] Write tests & fix infraction handling of Wyscout --- .../event/wyscout/deserializer_v3.py | 1 + kloppy/tests/files/wyscout_events_v3.json | 77 +++++++++++++++++++ kloppy/tests/test_opta.py | 4 + kloppy/tests/test_wyscout.py | 18 ++--- 4 files changed, 90 insertions(+), 10 deletions(-) diff --git a/kloppy/infra/serializers/event/wyscout/deserializer_v3.py b/kloppy/infra/serializers/event/wyscout/deserializer_v3.py index 56c58364..35389a3a 100644 --- a/kloppy/infra/serializers/event/wyscout/deserializer_v3.py +++ b/kloppy/infra/serializers/event/wyscout/deserializer_v3.py @@ -410,6 +410,7 @@ def deserialize(self, inputs: WyscoutInputs) -> EventDataset: # as we potentially have a card and foul event for one raw event if event and self.should_include_event(event): events.append(transformer.transform_event(event)) + continue if "yellow_card" in secondary_event_types: card_event_args = _parse_card(raw_event) event = self.event_factory.build_card( diff --git a/kloppy/tests/files/wyscout_events_v3.json b/kloppy/tests/files/wyscout_events_v3.json index 2f012ddd..cd64ceba 100644 --- a/kloppy/tests/files/wyscout_events_v3.json +++ b/kloppy/tests/files/wyscout_events_v3.json @@ -434,6 +434,83 @@ "name": "Bologna" }, "videoTimestamp": "8.148438" + },{ + "id": 1343788476, + "matchId": 2852835, + "matchPeriod": "1H", + "minute": 0, + "second": 52, + "matchTimestamp": "00:00:52.511", + "videoTimestamp": "53.511357", + "type": { + "primary": "infraction", + "secondary": [ + "foul" + ] + }, + "location": { + "x": 65, + "y": 11 + }, + "team": { + "formation": "4-2-3-1", + "id": 3166, + "name": "Bologna" + }, + "opponentTeam": { + "formation": "3-4-3", + "id": 3185, + "name": "Torino" + }, + "player": { + "id": 20623, + "name": "R. Soriano", + "position": "AMF" + }, + "pass": null, + "shot": null, + "groundDuel": null, + "aerialDuel": null, + "infraction": { + "yellowCard": false, + "redCard": false, + "type": "regular_foul", + "opponent": { + "id": 20583, + "name": "Danilo", + "position": "RCB" + } + }, + "carry": null, + "possession": { + "attack": { + "flank": "right", + "withGoal": false, + "withShot": false, + "withShotOnGoal": false, + "xg": 0 + }, + "duration": "28.0201355", + "endLocation": { + "x": 83, + "y": 0 + }, + "eventIndex": 2, + "eventsNumber": 14, + "id": 663291836, + "startLocation": { + "x": 40, + "y": 68 + }, + "team": { + "formation": "4-2-3-1", + "id": 3166, + "name": "Bologna" + }, + "types": [ + "attack" + ] + } } ], "formations": { diff --git a/kloppy/tests/test_opta.py b/kloppy/tests/test_opta.py index 6d1a6347..f481d419 100644 --- a/kloppy/tests/test_opta.py +++ b/kloppy/tests/test_opta.py @@ -13,6 +13,7 @@ DatasetType, CardType, FormationType, + CounterAttackQualifier ) from kloppy import opta @@ -107,6 +108,9 @@ def test_correct_deserialization(self, f7_data: str, f24_data: str): # Check OFFSIDE pass has end_coordinates assert dataset.events[20].receiver_coordinates.x == 89.3 # 2360555167 + # Check counterattack + assert CounterAttackQualifier(value=True) in dataset.events[17].qualifiers # 2318695229 + def test_correct_normalized_deserialization( self, f7_data: str, f24_data: str ): diff --git a/kloppy/tests/test_wyscout.py b/kloppy/tests/test_wyscout.py index fab413aa..0cef0827 100644 --- a/kloppy/tests/test_wyscout.py +++ b/kloppy/tests/test_wyscout.py @@ -1,7 +1,7 @@ from pathlib import Path import pytest -from kloppy.domain import Point, SetPieceType, SetPieceQualifier +from kloppy.domain import Point, SetPieceType, SetPieceQualifier, EventType from kloppy import wyscout @@ -23,7 +23,12 @@ def test_correct_v3_deserialization(self, event_v3_data: Path): coordinates="wyscout", data_version="V3", ) - assert dataset.records[2].coordinates == Point(36.0, 78.0) + assert dataset.events[2].coordinates == Point(36.0, 78.0) + assert ( + dataset.events[4].get_qualifier_value(SetPieceQualifier) + == SetPieceType.CORNER_KICK + ) + assert dataset.events[5].event_type == EventType.FOUL_COMMITTED def test_correct_normalized_v3_deserialization(self, event_v3_data: Path): dataset = wyscout.load(event_data=event_v3_data, data_version="V3") @@ -41,11 +46,4 @@ def test_correct_auto_recognize_deserialization(self, event_v2_data: Path): dataset = wyscout.load(event_data=event_v2_data, coordinates="wyscout") assert dataset.records[2].coordinates == Point(29.0, 6.0) - def test_correct_handling_of_corner_shot(self, event_v3_data: Path): - dataset = wyscout.load( - event_data=event_v3_data, data_version="V3", event_types=["shot"] - ) - assert ( - dataset.events[0].get_qualifier_value(SetPieceQualifier) - == SetPieceType.CORNER_KICK - ) + From 1d14d354014dd3aae7abe0680d6cc9677f7c48d5 Mon Sep 17 00:00:00 2001 From: driesdeprest Date: Mon, 19 Jun 2023 11:38:11 +0200 Subject: [PATCH 04/10] Black formatting --- kloppy/infra/serializers/event/opta/deserializer.py | 8 +++++--- kloppy/infra/serializers/event/wyscout/deserializer_v3.py | 6 +++++- kloppy/tests/test_opta.py | 6 ++++-- kloppy/tests/test_wyscout.py | 6 ++---- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/kloppy/infra/serializers/event/opta/deserializer.py b/kloppy/infra/serializers/event/opta/deserializer.py index 6dc1d991..b463319e 100644 --- a/kloppy/infra/serializers/event/opta/deserializer.py +++ b/kloppy/infra/serializers/event/opta/deserializer.py @@ -40,7 +40,7 @@ BodyPart, PassType, PassQualifier, - CounterAttackQualifier + CounterAttackQualifier, ) from kloppy.exceptions import DeserializationError from kloppy.infra.serializers.event.deserializer import EventDataDeserializer @@ -234,7 +234,7 @@ def _parse_pass(raw_qualifiers: Dict[int, str], outcome: int) -> Dict: result = PassResult.INCOMPLETE receiver_coordinates = Point( x=float(raw_qualifiers[140]) if 140 in raw_qualifiers else 0, - y=float(raw_qualifiers[141]) if 141 in raw_qualifiers else 0 + y=float(raw_qualifiers[141]) if 141 in raw_qualifiers else 0, ) qualifiers = _get_event_qualifiers(raw_qualifiers) @@ -461,7 +461,9 @@ def _get_event_card_qualifiers(raw_qualifiers: List) -> List[Qualifier]: return qualifiers -def _get_event_counter_attack_qualifiers(raw_qualifiers: List) -> List[Qualifier]: +def _get_event_counter_attack_qualifiers( + raw_qualifiers: List, +) -> List[Qualifier]: qualifiers = [] if EVENT_QUALIFIER_COUNTER_ATTACK in raw_qualifiers: qualifiers.append(CounterAttackQualifier(True)) diff --git a/kloppy/infra/serializers/event/wyscout/deserializer_v3.py b/kloppy/infra/serializers/event/wyscout/deserializer_v3.py index 35389a3a..5e72cb75 100644 --- a/kloppy/infra/serializers/event/wyscout/deserializer_v3.py +++ b/kloppy/infra/serializers/event/wyscout/deserializer_v3.py @@ -346,7 +346,11 @@ def deserialize(self, inputs: WyscoutInputs) -> EventDataset: raw_event["second"] + raw_event["minute"] * 60 ) if period_id == 1 - else float(raw_event["second"] + (raw_event["minute"] * 60) - (60 * 45)), + else float( + raw_event["second"] + + (raw_event["minute"] * 60) + - (60 * 45) + ), } primary_event_type = raw_event["type"]["primary"] diff --git a/kloppy/tests/test_opta.py b/kloppy/tests/test_opta.py index f481d419..1fe4e317 100644 --- a/kloppy/tests/test_opta.py +++ b/kloppy/tests/test_opta.py @@ -13,7 +13,7 @@ DatasetType, CardType, FormationType, - CounterAttackQualifier + CounterAttackQualifier, ) from kloppy import opta @@ -109,7 +109,9 @@ def test_correct_deserialization(self, f7_data: str, f24_data: str): assert dataset.events[20].receiver_coordinates.x == 89.3 # 2360555167 # Check counterattack - assert CounterAttackQualifier(value=True) in dataset.events[17].qualifiers # 2318695229 + assert ( + CounterAttackQualifier(value=True) in dataset.events[17].qualifiers + ) # 2318695229 def test_correct_normalized_deserialization( self, f7_data: str, f24_data: str diff --git a/kloppy/tests/test_wyscout.py b/kloppy/tests/test_wyscout.py index 0cef0827..f56ca9d6 100644 --- a/kloppy/tests/test_wyscout.py +++ b/kloppy/tests/test_wyscout.py @@ -25,8 +25,8 @@ def test_correct_v3_deserialization(self, event_v3_data: Path): ) assert dataset.events[2].coordinates == Point(36.0, 78.0) assert ( - dataset.events[4].get_qualifier_value(SetPieceQualifier) - == SetPieceType.CORNER_KICK + dataset.events[4].get_qualifier_value(SetPieceQualifier) + == SetPieceType.CORNER_KICK ) assert dataset.events[5].event_type == EventType.FOUL_COMMITTED @@ -45,5 +45,3 @@ def test_correct_v2_deserialization(self, event_v2_data: Path): def test_correct_auto_recognize_deserialization(self, event_v2_data: Path): dataset = wyscout.load(event_data=event_v2_data, coordinates="wyscout") assert dataset.records[2].coordinates == Point(29.0, 6.0) - - From 4d49d2174a801f1be01fdeed6fe63ea41f104e32 Mon Sep 17 00:00:00 2001 From: driesdeprest Date: Tue, 20 Jun 2023 11:56:31 +0200 Subject: [PATCH 05/10] Recognize yellow AND red cards --- kloppy/infra/serializers/event/wyscout/deserializer_v3.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kloppy/infra/serializers/event/wyscout/deserializer_v3.py b/kloppy/infra/serializers/event/wyscout/deserializer_v3.py index 5e72cb75..de8832ab 100644 --- a/kloppy/infra/serializers/event/wyscout/deserializer_v3.py +++ b/kloppy/infra/serializers/event/wyscout/deserializer_v3.py @@ -415,7 +415,10 @@ def deserialize(self, inputs: WyscoutInputs) -> EventDataset: if event and self.should_include_event(event): events.append(transformer.transform_event(event)) continue - if "yellow_card" in secondary_event_types: + if ( + "yellow_card" in secondary_event_types + or "red_card" in secondary_event_types + ): card_event_args = _parse_card(raw_event) event = self.event_factory.build_card( **card_event_args, **generic_event_args From dca2794dec9589cf5445d245554c74e3cd9e6eed Mon Sep 17 00:00:00 2001 From: driesdeprest Date: Wed, 21 Jun 2023 11:09:12 +0200 Subject: [PATCH 06/10] Pin Python version 3.7.16 for GitHub workflows --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5c138c57..9a4c3b15 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] + python-version: [3.7.16, 3.8, 3.9, "3.10", "3.11"] steps: - uses: actions/checkout@v2 From 5e8e3becdb894fcaf9fb9e203c7839e07be8ce8b Mon Sep 17 00:00:00 2001 From: driesdeprest Date: Wed, 21 Jun 2023 11:12:20 +0200 Subject: [PATCH 07/10] Pin Python version 3.7.16 for GitHub workflows try #2 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9a4c3b15..7aa5256a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [3.7.16, 3.8, 3.9, "3.10", "3.11"] + python-version: ["3.7.16", 3.8, 3.9, "3.10", "3.11"] steps: - uses: actions/checkout@v2 From 979f93959d232703ca92dbf4dff7c5f19517d7ae Mon Sep 17 00:00:00 2001 From: driesdeprest Date: Fri, 23 Jun 2023 15:28:43 +0200 Subject: [PATCH 08/10] Fix: Make it so that we don't have FOUL_COMMITTED events for events describing a player was fouled. Opta description of foul event: "Indicates a foul has been committed. The event comes in pairs, with one for the team committing the foul (has outcome = 0) and another for the team fouled (outcome = 1)." --- kloppy/infra/serializers/event/opta/deserializer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kloppy/infra/serializers/event/opta/deserializer.py b/kloppy/infra/serializers/event/opta/deserializer.py index b463319e..59e39cb9 100644 --- a/kloppy/infra/serializers/event/opta/deserializer.py +++ b/kloppy/infra/serializers/event/opta/deserializer.py @@ -663,7 +663,7 @@ def deserialize(self, inputs: OptaInputs) -> EventDataset: **generic_event_kwargs, ) - elif type_id == EVENT_TYPE_FOUL_COMMITTED: + elif (type_id == EVENT_TYPE_FOUL_COMMITTED) and (outcome == 0): event = self.event_factory.build_foul_committed( result=None, qualifiers=None, From b7e888731606d7ec0f18d1d578e5a3dd5be7e4cf Mon Sep 17 00:00:00 2001 From: driesdeprest Date: Fri, 23 Jun 2023 15:41:06 +0200 Subject: [PATCH 09/10] black format --- kloppy/infra/serializers/event/opta/deserializer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kloppy/infra/serializers/event/opta/deserializer.py b/kloppy/infra/serializers/event/opta/deserializer.py index 59e39cb9..f7e9df86 100644 --- a/kloppy/infra/serializers/event/opta/deserializer.py +++ b/kloppy/infra/serializers/event/opta/deserializer.py @@ -663,7 +663,9 @@ def deserialize(self, inputs: OptaInputs) -> EventDataset: **generic_event_kwargs, ) - elif (type_id == EVENT_TYPE_FOUL_COMMITTED) and (outcome == 0): + elif (type_id == EVENT_TYPE_FOUL_COMMITTED) and ( + outcome == 0 + ): event = self.event_factory.build_foul_committed( result=None, qualifiers=None, From e8ff8cad0a72ec79d1fb4ba8cb83799c24cda9ad Mon Sep 17 00:00:00 2001 From: driesdeprest Date: Fri, 7 Jul 2023 08:54:04 +0200 Subject: [PATCH 10/10] Black formatting --- kloppy/infra/serializers/event/opta/deserializer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kloppy/infra/serializers/event/opta/deserializer.py b/kloppy/infra/serializers/event/opta/deserializer.py index 65d8165a..b78937f1 100644 --- a/kloppy/infra/serializers/event/opta/deserializer.py +++ b/kloppy/infra/serializers/event/opta/deserializer.py @@ -667,8 +667,8 @@ def deserialize(self, inputs: OptaInputs) -> EventDataset: **generic_event_kwargs, ) elif (type_id == EVENT_TYPE_FOUL_COMMITTED) and ( - outcome == 0 - ): + outcome == 0 + ): event = self.event_factory.build_foul_committed( result=None, qualifiers=None,