Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Integration][Datadog] Datadog Teams and Users #1256

Open
wants to merge 21 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
ade687c
- Add `get_teams()`, `get_users()`, and `get_paginated_team_members()…
shariff-6 Dec 19, 2024
f1aa4d8
Add team and user resync handlers, enrich teams with members
shariff-6 Dec 19, 2024
54f718b
Add TeamSelector and TeamResourceConfig for team member inclusion
shariff-6 Dec 19, 2024
0ab265f
Add Datadog team and user blueprints and resource configuration
shariff-6 Dec 19, 2024
d336640
Refactor Datadog client to simplify team member fetching
shariff-6 Dec 19, 2024
770d9e7
Clean up Datadog blueprints and resource mappings
shariff-6 Dec 19, 2024
b2e1f07
Fixes lint
shariff-6 Dec 19, 2024
78c73ba
Updates version and Changelog
shariff-6 Dec 19, 2024
37545ed
Merge branch 'main' into PORT-12104-Datadog-Enhancements-Users-and-Teams
PeyGis Dec 20, 2024
1ac08e5
Update integrations/datadog/client.py
shariff-6 Dec 23, 2024
ecf456d
refactor(datadog): enhance logging and team enrichment flow
shariff-6 Dec 23, 2024
4b7664d
Merge branch 'PORT-12104-Datadog-Enhancements-Users-and-Teams' of htt…
shariff-6 Dec 23, 2024
ca57481
Merge branch 'main' of https://github.com/port-labs/ocean into PORT-1…
shariff-6 Dec 23, 2024
d9e2f5c
Updates changelog
shariff-6 Dec 23, 2024
3d050be
Removes unused enrich function, function is now in client
shariff-6 Dec 23, 2024
f52ba59
Remove unused imports
shariff-6 Dec 23, 2024
0c942cf
Removes redundant log
shariff-6 Dec 23, 2024
d1a4fee
Limits team seletor to team kind
shariff-6 Dec 23, 2024
d18958f
Reorder resources, users and teams first
shariff-6 Dec 23, 2024
5223cff
Merge branch 'main' into PORT-12104-Datadog-Enhancements-Users-and-Teams
phalbert Dec 24, 2024
deadb11
Update integrations/datadog/CHANGELOG.md
phalbert Dec 24, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 105 additions & 15 deletions integrations/datadog/.port/resources/blueprints.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,100 @@
[
{
"identifier": "datadogUser",
"description": "This blueprint represents a Datadog user account. Users can be assigned to teams, granted specific permissions, and can interact with various Datadog features based on their access levels.",
"title": "Datadog User",
"icon": "Datadog",
"schema": {
"properties": {
"email": {
"type": "string",
"format": "email",
"title": "Email",
"description": "The email address associated with the user account"
},
"handle": {
"type": "string",
"title": "Handle",
"description": "The unique handle identifier for the user within Datadog"
},
"status": {
"type": "string",
"title": "Status",
"description": "The current status of the user account (e.g., active, pending, disabled)"
},
"disabled": {
"type": "boolean",
"title": "Disabled",
"description": "Indicates whether the user account is currently disabled"
},
"verified": {
"type": "boolean",
"title": "Verified",
"description": "Indicates whether the user's email address has been verified"
},
"createdAt": {
"type": "string",
"format": "date-time",
"title": "Created At",
"description": "The timestamp when the user account was created"
}
},
"required": []
},
"mirrorProperties": {},
"calculationProperties": {},
"aggregationProperties": {},
"relations": {}
},
{
"identifier": "datadogTeam",
"description": "This blueprint represents a Datadog team",
"title": "Datadog Team",
"icon": "Datadog",
"schema": {
"properties": {
"description": {
"type": "string",
"title": "Description",
"description": "A description of the team's purpose and responsibilities"
},
"handle": {
"type": "string",
"title": "Handle",
"description": "The unique handle identifier for the team within Datadog"
},
"userCount": {
"type": "number",
"title": "User Count",
"description": "The total number of users that are members of this team"
},
"summary": {
"type": "string",
"title": "Summary",
"description": "A brief summary of the team's purpose or main responsibilities"
},
"createdAt": {
"type": "string",
"format": "date-time",
"title": "Created At",
"description": "The timestamp when the team was created"
}
},
"required": []
},
"mirrorProperties": {},
"calculationProperties": {},
"aggregationProperties": {},
"relations": {
"members": {
"target": "datadogUser",
"title": "Members",
"description": "Users who are members of this team",
"many": true,
"required": false
}
}
},
{
"identifier": "datadogHost",
"description": "This blueprint represents a datadog host",
Expand Down Expand Up @@ -202,7 +298,14 @@
"mirrorProperties": {},
"calculationProperties": {},
"aggregationProperties": {},
"relations": {}
"relations": {
"team": {
"target": "datadogTeam",
"title": "Team",
"many": false,
"required": false
}
}
},
{
"identifier": "datadogSlo",
Expand Down Expand Up @@ -252,20 +355,7 @@
},
"mirrorProperties": {},
"calculationProperties": {},
"aggregationProperties": {
phalbert marked this conversation as resolved.
Show resolved Hide resolved
"sli_average": {
"title": "SLI Average",
"type": "number",
"target": "datadogSloHistory",
"calculationSpec": {
"func": "average",
"averageOf": "total",
"property": "sliValue",
"measureTimeBy": "$createdAt",
"calculationBy": "property"
}
}
},
"aggregationProperties": {},
"relations": {
"monitors": {
"title": "SLO Monitors",
Expand Down
42 changes: 42 additions & 0 deletions integrations/datadog/.port/resources/port-app-config.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,41 @@
deleteDependentEntities: true
createMissingRelatedEntities: true
resources:
- kind: user
selector:
query: 'true'
port:
entity:
mappings:
identifier: .id | tostring
title: .attributes.name
blueprint: '"datadogUser"'
properties:
email: .attributes.email
handle: .attributes.handle
status: .attributes.status
disabled: .attributes.disabled
verified: .attributes.verified
createdAt: .attributes.created_at | todate
- kind: team
selector:
query: 'true'
includeMembers: 'true'
port:
entity:
mappings:
identifier: .id | tostring
title: .attributes.name
blueprint: '"datadogTeam"'
properties:
description: .attributes.description
handle: .attributes.handle
userCount: .attributes.user_count
summary: .attributes.summary
createdAt: .attributes.created_at | todate
relations:
members: if .__members then .__members[].id else [] end

- kind: host
selector:
query: "true"
Expand Down Expand Up @@ -58,6 +93,13 @@ resources:
owners: >-
[.attributes.schema.contacts[] | select(.type == "email") |
.contact]
relations:
team:
combinator: '"and"'
rules:
- property: '"handle"'
operator: '"="'
value: .attributes.schema.team
- kind: slo
selector:
query: "true"
Expand Down
4 changes: 4 additions & 0 deletions integrations/datadog/.port/spec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,16 @@ features:
- type: exporter
section: APM & Alerting
resources:
- kind: user
- kind: team
- kind: host
- kind: monitor
- kind: service
- kind: slo
- kind: sloHistory
- kind: serviceMetric


configurations:
- name: datadogBaseUrl
description: Datadog Base URL (e.g., <a target="_blank" href="https://api.datadoghq.com">https://api.datadoghq.com</a> or <a target="_blank" href= "https://api.datadoghq.eu")>https://api.datadoghq.eu</a>. To identify your base URL, see the <a target="_blank" href="https://docs.datadoghq.com/getting_started/site/#:~:text=within%20their%20environments.-,Access%20the%20Datadog%20site,-You%20can%20identify">Datadog documentation</a>.
Expand Down
8 changes: 8 additions & 0 deletions integrations/datadog/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

<!-- towncrier release notes start -->

## 0.1.68 (2024-12-24)


### Improvements

- Added Datadog Users and Teams


## 0.1.67 (2024-12-22)


Expand Down
87 changes: 86 additions & 1 deletion integrations/datadog/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import http
import json
import time
from typing import Any, AsyncGenerator, Optional
from typing import Any, AsyncGenerator, Optional, List, Dict
from urllib.parse import urlparse, urlunparse

import httpx
Expand Down Expand Up @@ -158,6 +158,91 @@ async def _fetch_with_rate_limit_handling(
raise
return response.json()

async def get_team_members(
self, team_id: str, page_size: int = MAX_PAGE_SIZE
) -> AsyncGenerator[List[Dict[str, Dict[str, Any]]], None]:
logger.info(f"Enriching team {team_id} with members information")

page = 0

phalbert marked this conversation as resolved.
Show resolved Hide resolved
while True:
url = f"{self.api_url}/api/v2/team/{team_id}/memberships"
result = await self._send_api_request(
url,
params={
"page[size]": page_size,
"page[number]": page,
},
)

users = result.get("included", [])

if not users:
break

yield users
page += 1

async def get_teams(self) -> AsyncGenerator[List[Dict[str, Dict[str, Any]]], None]:
page = 0
page_size = MAX_PAGE_SIZE

while True:
url = f"{self.api_url}/api/v2/team"
result = await self._send_api_request(
url,
params={
"page[size]": page_size,
"page[number]": page,
},
)

teams = result.get("data", [])
if not teams:
break

yield teams
page += 1

async def get_users(self) -> AsyncGenerator[list[dict[str, Any]], None]:
page = 0
page_size = MAX_PAGE_SIZE

while True:
url = f"{self.api_url}/api/v2/users"
result = await self._send_api_request(
url,
params={
"page[number]": page,
"page[size]": page_size,
"schema_version": "v2.2",
},
)

users = result.get("data", [])
if not users:
break

yield users
page += 1

async def enrich_teams_with_members(
self, teams: List[Dict[str, Any]]
) -> List[Dict[str, Any]]:
async def fetch_team_members(team: Dict[str, Any]) -> List[Dict[str, Any]]:
members = []
async for batch in self.get_team_members(team["id"]):
members.extend(batch)
return members

team_tasks = [fetch_team_members(team) for team in teams]
results = await asyncio.gather(*team_tasks)

for team, members in zip(teams, results):
team["__members"] = members

return teams

async def get_hosts(self) -> AsyncGenerator[list[dict[str, Any]], None]:
start = 0
count = MAX_PAGE_SIZE
Expand Down
33 changes: 31 additions & 2 deletions integrations/datadog/main.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,16 @@
import typing
from enum import StrEnum
from typing import Any
from typing import Any, cast

from loguru import logger

from client import DatadogClient
from overrides import SLOHistoryResourceConfig, DatadogResourceConfig, DatadogSelector
from overrides import (
SLOHistoryResourceConfig,
DatadogResourceConfig,
DatadogSelector,
TeamResourceConfig,
)
from port_ocean.context.event import event
from port_ocean.context.ocean import ocean
from port_ocean.core.ocean_types import ASYNC_GENERATOR_RESYNC_TYPE
Expand All @@ -18,6 +23,8 @@ class ObjectKind(StrEnum):
SERVICE = "service"
SLO_HISTORY = "sloHistory"
SERVICE_METRIC = "serviceMetric"
TEAM = "team"
USER = "user"


def init_client() -> DatadogClient:
Expand All @@ -28,6 +35,28 @@ def init_client() -> DatadogClient:
)


@ocean.on_resync(ObjectKind.TEAM)
async def on_resync_teams(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
dd_client = init_client()

selector = cast(TeamResourceConfig, event.resource_config).selector

async for teams in dd_client.get_teams():
logger.info(f"Received teams batch with {len(teams)} teams")
phalbert marked this conversation as resolved.
Show resolved Hide resolved
if selector.include_members:
teams = await dd_client.enrich_teams_with_members(teams)
yield teams
Comment on lines +46 to +48
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the enrich_teams_with_members should be just get_team_members and the logic of the merge will be in the main file

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay, makes sense. 👍 .



@ocean.on_resync(ObjectKind.USER)
async def on_resync_users(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
dd_client = init_client()

async for users in dd_client.get_users():
logger.info(f"Received batch with {len(users)} users")
yield users


@ocean.on_resync(ObjectKind.HOST)
async def on_resync_hosts(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
dd_client = init_client()
Expand Down
Loading
Loading