From dce0ec3326a52724822d0f1468d2039bd04d562b Mon Sep 17 00:00:00 2001 From: Garance Gourdel Date: Thu, 22 Aug 2024 11:48:15 +0200 Subject: [PATCH] feat(incidents): implement models and client for /incidents/secrets/ endpoint --- ...optimize_files_to_scan_on_merge_commits.md | 3 + pygitguardian/client.py | 25 +++ pygitguardian/models.py | 178 +++++++++++++++- tests/test_client.py | 82 ++++++++ tests/test_models.py | 197 ++++++++++++++++++ 5 files changed, 484 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20240823_170420_fnareoh_scrt_4759_optimize_files_to_scan_on_merge_commits.md diff --git a/changelog.d/20240823_170420_fnareoh_scrt_4759_optimize_files_to_scan_on_merge_commits.md b/changelog.d/20240823_170420_fnareoh_scrt_4759_optimize_files_to_scan_on_merge_commits.md new file mode 100644 index 00000000..22da1e60 --- /dev/null +++ b/changelog.d/20240823_170420_fnareoh_scrt_4759_optimize_files_to_scan_on_merge_commits.md @@ -0,0 +1,3 @@ +### Added + +- Added `GGClient.retrieve_secret_incident()` to retrieve the incident associated with a secret (see https://api.gitguardian.com/docs#tag/Secret-Incidents/operation/retrieve-incidents) diff --git a/pygitguardian/client.py b/pygitguardian/client.py index 5275c6cb..c6162fd8 100644 --- a/pygitguardian/client.py +++ b/pygitguardian/client.py @@ -37,6 +37,7 @@ QuotaResponse, RemediationMessages, ScanResult, + SecretIncident, SecretScanPreferences, ServerMetadata, ) @@ -454,6 +455,30 @@ def multi_content_scan( return obj + def retrieve_secret_incident( + self, incident_id: int, with_occurrences: int = 20 + ) -> Union[Detail, SecretIncident]: + """ + retrieve_secret_incident handles the /incidents/secret/{incident_id} endpoint of the API + + :param incident_id: incident id + :param with_occurrences: number of occurrences of the incident to retrieve (default 20) + """ + + resp = self.get( + endpoint=f"incidents/secrets/{incident_id}", + params={"with_occurrences": with_occurrences}, + ) + + obj: Union[Detail, SecretIncident] + if is_ok(resp): + obj = SecretIncident.from_dict(resp.json()) + else: + obj = load_detail(resp) + + obj.status_code = resp.status_code + return obj + def quota_overview( self, extra_headers: Optional[Dict[str, str]] = None, diff --git a/pygitguardian/models.py b/pygitguardian/models.py index 3f8c69d7..0d8b4196 100644 --- a/pygitguardian/models.py +++ b/pygitguardian/models.py @@ -4,7 +4,7 @@ from dataclasses import dataclass, field from datetime import date, datetime from enum import Enum -from typing import Any, ClassVar, Dict, List, Optional, cast +from typing import Any, ClassVar, Dict, List, Literal, Optional, Type, cast from uuid import UUID import marshmallow_dataclass @@ -794,3 +794,179 @@ class JWTService(Enum): """Enum for the different services GIM can generate a JWT for.""" HMSL = "hmsl" + + +@dataclass +class Detector(Base, FromDictMixin): + name: str + display_name: str + nature: str + family: str + detector_group_name: str + detector_group_display_name: str + + +Severity = Literal["low", "medium", "high", "critical", "unknown"] +ValidityStatus = Literal["valid", "invalid", "failed_to_check", "no_checker", "unknown"] +IncidentStatus = Literal["IGNORED", "TRIGGERED", "RESOLVED", "ASSIGNED"] +Tag = Literal[ + "DEFAULT_BRANCH", + "FROM_HISTORICAL_SCAN", + "CHECK_RUN_SKIP_FALSE_POSITIVE", + "CHECK_RUN_SKIP_LOW_RISK", + "CHECK_RUN_SKIP_TEST_CRED", + "IGNORED_IN_CHECK_RUN", + "FALSE_POSITIVE", + "PUBLICLY_EXPOSED", + "PUBLICLY_LEAKED", + "REGRESSION", + "SENSITIVE_FILE", + "TEST_FILE", +] +IgnoreReason = Literal["test_credential", "false_positive", "low_risk"] +OccurrenceKind = Literal["realtime", "historical"] +OccurrencePresence = Literal["present", "removed"] +Visibility = Literal["private", "internal", "public"] + + +@dataclass +class SecretPresence(Base, FromDictMixin): + files_requiring_code_fix: int + files_pending_merge: int + files_fixed: int + outside_vcs: int + removed_outside_vcs: int + in_vcs: int + removed_in_vcs: int + + +@dataclass +class Answer(Base, FromDictMixin): + type: str + field_ref: str + field_label: str + boolean: Optional[bool] = None + text: Optional[str] = None + + +@dataclass +class Feedback(Base, FromDictMixin): + created_at: datetime + updated_at: datetime + member_id: int + email: str + answers: List[Answer] + + +@dataclass +class Source(Base, FromDictMixin): + id: int + url: str + type: str + full_name: str + health: Literal["safe", "unknown", "at_risk"] + default_branch: Optional[str] + default_branch_head: Optional[str] + open_incidents_count: int + closed_incidents_count: int + secret_incidents_breakdown: Dict[str, Any] # TODO: add SecretIncidentsBreakdown + visibility: Visibility + external_id: str + source_criticality: str + last_scan: Optional[Dict[str, Any]] # TODO: add LastScan + monitored: bool + + +@dataclass +class OccurrenceMatch(Base, FromDictMixin): + """ + Describes the match of an occurrence, different from the Match return as part of a PolicyBreak. + + name: type of the match such as "api_key", "password", "client_id", "client_secret"... + indice_start: start index of the match in the document (0-based) + indice_end: end index of the match in the document (0-based, strictly greater than indice_start) + pre_line_start: Optional start line number (1-based) of the match in the document (before the git patch) + pre_line_end: Optional end line number (1-based) of the match in the document (before the git patch) + post_line_start: Optional start line number (1-based) of the match in the document (after the git patch) + post_line_end: Optional end line number (1-based) of the match in the document (after the git patch) + """ + + name: str + indice_start: int + indice_end: int + pre_line_start: Optional[int] + pre_line_end: Optional[int] + post_line_start: Optional[int] + post_line_end: Optional[int] + + +@dataclass +class SecretOccurrence(Base, FromDictMixin): + id: int + incident_id: int + kind: OccurrenceKind + source: Source + author_name: str + author_info: str + date: datetime # Publish date + url: str + matches: List[OccurrenceMatch] + tags: List[str] + sha: Optional[str] # Commit sha + presence: OccurrencePresence + filepath: Optional[str] + + +SecretOccurrenceSchema = cast( + Type[BaseSchema], + marshmallow_dataclass.class_schema(SecretOccurrence, base_schema=BaseSchema), +) +SecretOccurrence.SCHEMA = SecretOccurrenceSchema() + + +@dataclass(repr=False) # the default repr would be too long +class SecretIncident(Base, FromDictMixin): + """ + Secret Incident describes a leaked secret incident. + """ + + id: int + date: datetime + detector: Detector + secret_hash: str + hmsl_hash: str + gitguardian_url: str + regression: bool + status: IncidentStatus + assignee_id: Optional[int] + assignee_email: Optional[str] + occurrences_count: int + secret_presence: SecretPresence + ignore_reason: Optional[IgnoreReason] + triggered_at: Optional[datetime] + ignored_at: Optional[datetime] + ignorer_id: Optional[int] + ignorer_api_token_id: Optional[UUID] + resolver_id: Optional[int] + resolver_api_token_id: Optional[UUID] + secret_revoked: bool + severity: Severity + validity: ValidityStatus + resolved_at: Optional[datetime] + share_url: Optional[str] + tags: List[Tag] + feedback_list: List[Feedback] + occurrences: Optional[List[SecretOccurrence]] + + def __repr__(self) -> str: + return ( + f"id:{self.id}, detector_name:{self.detector.name}," + f" url:{self.gitguardian_url}" + ) + + +SecretIncidentSchema = cast( + Type[BaseSchema], + marshmallow_dataclass.class_schema(SecretIncident, base_schema=BaseSchema), +) +SecretIncident.SCHEMA = SecretIncidentSchema() diff --git a/tests/test_client.py b/tests/test_client.py index 3dbf410c..071ad61f 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -616,6 +616,88 @@ def test_multiscan_parameters( assert mock_response.call_count == 1 +@responses.activate +def test_retrieve_secret_incident(client: GGClient): + """ + GIVEN a ggclient + WHEN calling retrieve_secret_incident with parameters + THEN the parameters are passed in the request + """ + + mock_response = responses.get( + url=client._url_from_endpoint("incidents/secrets/3759", "v1"), + status=200, + match=[matchers.query_param_matcher({"with_occurrences": 20})], + json={ + "id": 3759, + "date": "2019-08-22T14:15:22Z", + "detector": { + "name": "slack_bot_token", + "display_name": "Slack Bot Token", + "nature": "specific", + "family": "apikey", + "detector_group_name": "slackbot_token", + "detector_group_display_name": "Slack Bot Token", + }, + "secret_hash": "Ri9FjVgdOlPnBmujoxP4XPJcbe82BhJXB/SAngijw/juCISuOMgPzYhV28m6OG24", + "hmsl_hash": "05975add34ddc9a38a0fb57c7d3e676ffed57080516fc16bf8d8f14308fedb86", + "gitguardian_url": "https://dashboard.gitguardian.com/workspace/1/incidents/3899", + "regression": False, + "status": "IGNORED", + "assignee_id": 309, + "assignee_email": "eric@gitguardian.com", + "occurrences_count": 4, + "secret_presence": { + "files_requiring_code_fix": 1, + "files_pending_merge": 1, + "files_fixed": 1, + "outside_vcs": 1, + "removed_outside_vcs": 0, + "in_vcs": 3, + "removed_in_vcs": 0, + }, + "ignore_reason": "test_credential", + "triggered_at": "2019-05-12T09:37:49Z", + "ignored_at": "2019-08-24T14:15:22Z", + "ignorer_id": 309, + "ignorer_api_token_id": "fdf075f9-1662-4cf1-9171-af50568158a8", + "resolver_id": 395, + "resolver_api_token_id": "fdf075f9-1662-4cf1-9171-af50568158a8", + "secret_revoked": False, + "severity": "high", + "validity": "valid", + "resolved_at": None, + "share_url": "https://dashboard.gitguardian.com/share/incidents/11111111-1111-1111-1111-111111111111", + "tags": ["FROM_HISTORICAL_SCAN", "SENSITIVE_FILE"], + "feedback_list": [ + { + "created_at": "2021-05-20T12:40:55.662949Z", + "updated_at": "2021-05-20T12:40:55.662949Z", + "member_id": 42, + "email": "eric@gitguardian.com", + "answers": [ + { + "type": "boolean", + "field_ref": "actual_secret_yes_no", + "field_label": "Is it an actual secret?", + "boolean": True, + } + ], + } + ], + "occurrences": None, + }, + ) + + result = client.retrieve_secret_incident(3759) + + assert mock_response.call_count == 1 + assert result.id == 3759 + assert result.detector.name == "slack_bot_token" + assert result.ignore_reason == "test_credential" + assert result.secret_revoked is False + + @responses.activate def test_rate_limit(): """ diff --git a/tests/test_models.py b/tests/test_models.py index 483217fc..1c1acca9 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -24,6 +24,10 @@ QuotaSchema, ScanResult, ScanResultSchema, + SecretIncident, + SecretIncidentSchema, + SecretOccurrence, + SecretOccurrenceSchema, ) @@ -177,6 +181,199 @@ def test_document_handle_surrogates(self): "gitguardian_url": "https://dashboard.gitguardian.com/workspace/1/honeytokens/d45a123f-b15d-4fea-abf6-ff2a8479de5b", # noqa: E501 }, ), + ( + SecretIncidentSchema, + SecretIncident, + { + "id": 3759, + "date": "2019-08-22T14:15:22Z", + "detector": { + "name": "slack_bot_token", + "display_name": "Slack Bot Token", + "nature": "specific", + "family": "apikey", + "detector_group_name": "slackbot_token", + "detector_group_display_name": "Slack Bot Token", + }, + "secret_hash": "Ri9FjVgdOlPnBmujoxP4XPJcbe82BhJXB/SAngijw/juCISuOMgPzYhV28m6OG24", + "hmsl_hash": "05975add34ddc9a38a0fb57c7d3e676ffed57080516fc16bf8d8f14308fedb86", + "gitguardian_url": "https://dashboard.gitguardian.com/workspace/1/incidents/3899", + "regression": False, + "status": "IGNORED", + "assignee_id": 309, + "assignee_email": "eric@gitguardian.com", + "occurrences_count": 4, + "secret_presence": { + "files_requiring_code_fix": 1, + "files_pending_merge": 1, + "files_fixed": 1, + "outside_vcs": 1, + "removed_outside_vcs": 0, + "in_vcs": 3, + "removed_in_vcs": 0, + }, + "ignore_reason": "test_credential", + "triggered_at": "2019-05-12T09:37:49Z", + "ignored_at": "2019-08-24T14:15:22Z", + "ignorer_id": 309, + "ignorer_api_token_id": "fdf075f9-1662-4cf1-9171-af50568158a8", + "resolver_id": 395, + "resolver_api_token_id": "fdf075f9-1662-4cf1-9171-af50568158a8", + "secret_revoked": False, + "severity": "high", + "validity": "valid", + "resolved_at": None, + "share_url": "https://dashboard.gitguardian.com/share/incidents/11111111-11111", + "tags": ["FROM_HISTORICAL_SCAN", "SENSITIVE_FILE"], + "feedback_list": [ + { + "created_at": "2021-05-20T12:40:55.662949Z", + "updated_at": "2021-05-20T12:40:55.662949Z", + "member_id": 42, + "email": "eric@gitguardian.com", + "answers": [ + { + "type": "boolean", + "field_ref": "actual_secret_yes_no", + "field_label": "Is it an actual secret?", + "boolean": True, + } + ], + } + ], + "occurrences": None, + }, + ), + ( + SecretOccurrenceSchema, + SecretOccurrence, + { + "id": 16424242, + "incident_id": 133424242, + "author_info": "toto@gitguardian.com", + "author_name": "toto@gitguardian.com", + "date": "2024-01-17T16:05:43Z", + "filepath": ".pre-commit-config.yaml", + "kind": "historical", + "sha": "ee95f89e211831f07f07e07fde478", + "presence": "present", + "url": "https://github.com/GitGuardian/py-gitguardian/commit/ee95f89e211831f07f07e07fde478", + "matches": [ + { + "name": "connection_uri", + "indice_start": 62, + "indice_end": 131, + "pre_line_start": None, + "pre_line_end": None, + "post_line_start": 3, + "post_line_end": 3, + }, + { + "name": "scheme", + "indice_start": 62, + "indice_end": 70, + "pre_line_start": None, + "pre_line_end": None, + "post_line_start": 3, + "post_line_end": 3, + }, + { + "name": "username", + "indice_start": 73, + "indice_end": 81, + "pre_line_start": None, + "pre_line_end": None, + "post_line_start": 3, + "post_line_end": 3, + }, + { + "name": "password", + "indice_start": 82, + "indice_end": 99, + "pre_line_start": None, + "pre_line_end": None, + "post_line_start": 3, + "post_line_end": 3, + }, + { + "name": "host", + "indice_start": 100, + "indice_end": 112, + "pre_line_start": None, + "pre_line_end": None, + "post_line_start": 3, + "post_line_end": 3, + }, + { + "name": "port", + "indice_start": 113, + "indice_end": 117, + "pre_line_start": None, + "pre_line_end": None, + "post_line_start": 3, + "post_line_end": 3, + }, + { + "name": "database", + "indice_start": 118, + "indice_end": 131, + "pre_line_start": None, + "pre_line_end": None, + "post_line_start": 3, + "post_line_end": 3, + }, + ], + "source": { + "id": 16218989, + "type": "github", + "full_name": "py-gitguardian", + "health": "at_risk", + "source_criticality": "unknown", + "default_branch": "main", + "default_branch_head": None, + "open_incidents_count": 19, + "closed_incidents_count": 0, + "last_scan": { + "date": "2024-08-07T14:15:33.829070Z", + "status": "finished", + "failing_reason": "", + "commits_scanned": 49, + "duration": "0.0", + "branches_scanned": 14, + "progress": 100, + }, + "monitored": True, + "visibility": "internal", + "external_id": "139", + "secret_incidents_breakdown": { + "open_secret_incidents": { + "total": 19, + "severity_breakdown": { + "critical": 0, + "high": 7, + "medium": 0, + "low": 0, + "info": 0, + "unknown": 12, + }, + }, + "closed_secret_incidents": { + "total": 0, + "severity_breakdown": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "info": 0, + "unknown": 0, + }, + }, + }, + "url": "https://github.com/GitGuardian/py-gitguardian", + }, + "tags": ["FROM_HISTORICAL_SCAN"], + }, + ), ], ) def test_schema_loads(self, schema_klass, expected_klass, instance_data):