diff --git a/docs-v2/guides/custom.mdx b/docs-v2/guides/custom.mdx index 50eadefed2a..3466f49d7e2 100644 --- a/docs-v2/guides/custom.mdx +++ b/docs-v2/guides/custom.mdx @@ -6,7 +6,7 @@ description: 'Extend & create your own custom syncs & actions.' # Step 1: Setup the Nango CLI & nango-integrations folder -Install the Nango CLI globally: +Install the Nango CLI globally: ```bash npm install -g nango ``` @@ -66,7 +66,7 @@ integrations: type: action returns: SlackAlertResponse # Optional data model (defined below) as returned by your action script -models: +models: AsanaTask: # Data model referenced above id: string # Required unique ID project_id: string @@ -113,12 +113,12 @@ This lets you easily **create your own unified API** with standard data models t ### Write your _sync_ -Modify the configuration of `nango.yaml` as you need and run (in `./nango-integrations`): +Modify the configuration of `nango.yaml` as you need and run (in `./nango-integrations`): ``` nango generate ``` -This will generate the scaffold for your _sync_ script(s). Open any _sync_ script (named `[sync-name].ts`) which contains the following template (for the Asana example above): +This will generate the scaffold for your _sync_ script(s). Open any _sync_ script (named `[sync-name].ts`) which contains the following template (for the Asana example above): ```typescript asana-tasks.ts import { NangoSync, AsanaTask } from './models'; @@ -127,7 +127,7 @@ export default async function fetchData(nango: NangoSync): Promise { // Integration code goes here. } ``` -_Sync_ scripts mostly do 2 things. They: +_Sync_ scripts mostly do 2 things. They: - incrementally fetch data from external APIs (with HTTP requests) - transform the external data into the models that you defined in `nango.yaml` @@ -143,7 +143,7 @@ To develop syncs locally and test them run the following within `./nango-integra ```bash nango dev # Continuously watches integration files for changes. ``` -Nango now watches your `nango-integrations` folder for changes and compiles the sync scripts & data models as needed. If there are any compilation errors (e.g. due to type issues), you can see them in the terminal where `nango dev` runs. +Nango now watches your `nango-integrations` folder for changes and compiles the sync scripts & data models as needed. If there are any compilation errors (e.g. due to type issues), you can see them in the terminal where `nango dev` runs. Fill in the `fetchData` method with your integration code (in the example here, we fetch tasks from Asana): @@ -243,17 +243,17 @@ Use `await nango.log()` to log data from within integration scripts. ### Write your _action_ -Modify the configuration of `nango.yaml` as you need and run (in `./nango-integrations`): +Modify the configuration of `nango.yaml` as you need and run (in `./nango-integrations`): ``` nango generate ``` -This will generate the scaffold for your _action_ script(s). Open any _action_ script (named `[action-name].ts`) which contains the following template (for the Slack example above): +This will generate the scaffold for your _action_ script(s). Open any _action_ script (named `[action-name].ts`) which contains the following template (for the Slack example above): ```typescript slack-alert.ts -import { NangoSync, SlackAlertResponse } from './models'; +import { NangoAction, SlackAlertResponse } from './models'; -export default async function runAction(nango: NangoSync, input: any): Promise { +export default async function runAction(nango: NangoAction, input: any): Promise { // Integration code goes here. } ``` @@ -269,18 +269,18 @@ To develop _actions_ locally and test them, run the following within `./nango-in ```bash nango dev # Continuously watches integration files for changes. ``` -Nango now watches your `nango-integrations` folder for changes and compiles the _action_ scripts & data models as needed. If there are any compilation errors (e.g. due to type issues), you can see them in the terminal where `nango dev` runs. +Nango now watches your `nango-integrations` folder for changes and compiles the _action_ scripts & data models as needed. If there are any compilation errors (e.g. due to type issues), you can see them in the terminal where `nango dev` runs. Fill in the `runAction` method with your integration code: ```ts slack-alert.ts -import { NangoSync, SlackAlertResponse } from './models'; +import { NangoAction, SlackAlertResponse } from './models'; interface SlackAlertParams { channel: string } -export default async function runAction(nango: NangoSync, input: SlackAlertParams): Promise { +export default async function runAction(nango: NangoAction, input: SlackAlertParams): Promise { const res = await nango.post({ endpoint: '/chat.postMessage', params: { @@ -324,7 +324,7 @@ Because this is a dry run, the fetched data will not be stored in Nango. Instead By default, the _connection_ ID is fetched from your `Dev` environment. You can fetch _connections_ from your `Prod` environment with the `-e prod` flag. -To test incremental _sync_ runs, add the `-l` flag (which will populate the `nango.lastSyncDate` value in your script): +To test incremental _sync_ runs, add the `-l` flag (which will populate the `nango.lastSyncDate` value in your script): ```bash nango dryrun asana-tasks -l "2023-06-20T10:00:00.000Z" ``` @@ -333,7 +333,7 @@ nango dryrun asana-tasks -l "2023-06-20T10:00:00.000Z" # Step 3: Deploy a _sync/action_ **1. Deploy to the `Dev` environment** - + When your _sync_ script is ready, you can deploy it to your `Dev` environment in Nango: ```bash @@ -344,16 +344,16 @@ nango dryrun asana-tasks -l "2023-06-20T10:00:00.000Z" When you deploy your _sync_, Nango automatically adds it to all the existing _connections_ of the _integration_, and starts syncing their data. - + It will also add the _sync_ to any new _connection_ that is created (OAuth flow completes) for the _integration_. - + You can see all _syncs_ (and their status) for a _connection_ in the dashboard: ![View syncs in Dashboard](/images/connection-syncs.png) **2. Deploy to the `Prod` environment** - + Once you are ready to deploy to production, run: ```bash @@ -364,15 +364,15 @@ nango dryrun asana-tasks -l "2023-06-20T10:00:00.000Z" ### Handling API rate-limits -Nango has currently two approaches to handle rate limits, a generic/naive one and an API-specific one. +Nango has currently two approaches to handle rate limits, a generic/naive one and an API-specific one. The **generic & naive approach** is based on retries & exponential-backoff. When you make network requests with the _proxy_ in a _sync_ with a high number of retries, exponential back-off will increase the delay between retries, augmenting the chances to go back under the rate-limit. But this "blind" approach is inefficient both in terms of optimising the time between requests and avoiding complex rate-limits. -The **API-specific approach** is based on reading the rate-limit headers returned by the external APIs. Nango observes these headers and pauses the _sync_ job until the rate-limit is passed. This approach has the benefit of being more efficient both for minimizing _sync_ durations and avoid failures due to rate-limiting. +The **API-specific approach** is based on reading the rate-limit headers returned by the external APIs. Nango observes these headers and pauses the _sync_ job until the rate-limit is passed. This approach has the benefit of being more efficient both for minimizing _sync_ durations and avoid failures due to rate-limiting. This second approach requires to edit Nango's [providers.yaml](https://nango.dev/providers.yaml) file to indicate the rate-limit header name for a specific API (in the `retry` entry, under `at` or `after` fields): -Github example: +Github example: ```yaml github: auth_mode: OAUTH2 @@ -385,7 +385,7 @@ github: docs: https://docs.github.com/en/rest ``` -Discord example: +Discord example: ```yaml discord: auth_mode: OAUTH2 diff --git a/integration-templates/hubspot/hubspot-contacts.ts b/integration-templates/hubspot/hubspot-contacts.ts index 3b1dc94c1ba..5c059ac442b 100644 --- a/integration-templates/hubspot/hubspot-contacts.ts +++ b/integration-templates/hubspot/hubspot-contacts.ts @@ -1,12 +1,36 @@ import type { NangoSync, HubspotContact } from './models'; export default async function fetchData(nango: NangoSync) { - const query = `properties=firstname,lastname,email`; + //handle query properties well + const query = `firstname,lastname`; - for await (const records of nango.paginate({ endpoint: '/crm/v3/objects/contacts', params: { query } })) { - const mappedRecords = mapHubspotContacts(records); + let totalRecords = 0; - await nango.batchSave(mappedRecords, 'HubspotContact'); + try { + const endpoint = '/crm/v3/objects/contacts'; + const config = { + params: { + properties: query + }, + paginate: { + type: 'cursor', + cursor_path_in_response: 'paging.next.after', + limit_name_in_request: 'limit', + cursor_name_in_request: 'after', + response_path: 'results', + limit: 100 + } + }; + for await (const contact of nango.paginate({ ...config, endpoint })) { + const mappedContact = mapHubspotContacts(contact); + + const batchSize: number = mappedContact.length; + totalRecords += batchSize; + await nango.log(`Saving batch of ${batchSize} owners (total owners: ${totalRecords})`); + await nango.batchSave(mappedContact, 'HubspotContact'); + } + } catch (error: any) { + throw new Error(`Error in fetchData: ${error.message}`); } } diff --git a/integration-templates/hubspot/hubspot-owner.ts b/integration-templates/hubspot/hubspot-owner.ts deleted file mode 100644 index 31ff578b4ee..00000000000 --- a/integration-templates/hubspot/hubspot-owner.ts +++ /dev/null @@ -1,52 +0,0 @@ -import type { NangoSync, HubspotOwner } from './models'; - -interface Params { - limit: string; - [key: string]: any; // Allows additional properties -} - -export default async function fetchData(nango: NangoSync) { - const MAX_PAGE = 100; - - let afterLink = null; - - while (true) { - let payload = { - endpoint: '/crm/v3/owners', - params: { - limit: `${MAX_PAGE}` - } as Params - }; - - if (afterLink) { - // If there is no afterLink, then we are on the first page. - payload.params['after'] = afterLink; - } - - const response = await nango.get(payload); - - let pageData = response.data.results; - - const mappedOwners: HubspotOwner[] = pageData.map((owner: HubspotOwner) => ({ - id: owner.id, - email: owner.email, - firstName: owner.firstName, - lastName: owner.lastName, - userId: owner.userId, - createdAt: owner.createdAt, - updatedAt: owner.updatedAt, - archived: owner.archived - })); - - if (mappedOwners.length > 0) { - await nango.batchSave(mappedOwners, 'HubspotOwner'); - await nango.log(`Sent ${mappedOwners.length} owners`); - } - - if (response.data.paging?.next?.after) { - afterLink = response.data.paging.next.after; - } else { - break; - } - } -} diff --git a/integration-templates/hubspot/hubspot-owners.ts b/integration-templates/hubspot/hubspot-owners.ts new file mode 100644 index 00000000000..b9a5a7504f2 --- /dev/null +++ b/integration-templates/hubspot/hubspot-owners.ts @@ -0,0 +1,42 @@ +import type { HubspotOwner, NangoSync } from './models'; + +export default async function fetchData(nango: NangoSync) { + let totalRecords = 0; + + try { + const endpoint = '/crm/v3/owners'; + const config = { + paginate: { + type: 'cursor', + cursor_path_in_response: 'paging.next.after', + limit_name_in_request: 'limit', + cursor_name_in_request: 'after', + response_path: 'results', + limit: 100 + } + }; + for await (const owner of nango.paginate({ ...config, endpoint })) { + const mappedOwner: HubspotOwner[] = owner.map(mapOwner) || []; + + const batchSize: number = mappedOwner.length; + totalRecords += batchSize; + await nango.log(`Saving batch of ${batchSize} owners (total owners: ${totalRecords})`); + await nango.batchSave(mappedOwner, 'HubspotOwner'); + } + } catch (error: any) { + throw new Error(`Error in fetchData: ${error.message}`); + } +} + +function mapOwner(owner: any): HubspotOwner { + return { + id: owner.id, + email: owner.email, + firstName: owner.firstName, + lastName: owner.lastName, + userId: owner.userId, + createdAt: owner.createdAt, + updatedAt: owner.updatedAt, + archived: owner.archived + }; +} diff --git a/integration-templates/hubspot/hubspot-service-tickets.ts b/integration-templates/hubspot/hubspot-service-tickets.ts index 1a2ff7883d6..a5be27a79ec 100644 --- a/integration-templates/hubspot/hubspot-service-tickets.ts +++ b/integration-templates/hubspot/hubspot-service-tickets.ts @@ -1,91 +1,84 @@ import type { NangoSync, HubspotServiceTicket } from './models'; +interface PayloadData { + properties: string[]; + limit: string; + after?: string | null; + sorts: Array<{ + propertyName: string; + direction: string; + }>; + filterGroups: { + filters: Array<{ + propertyName: string; + operator: string; + value: any; + }>; + }[]; +} + +interface Payload { + endpoint: string; + data: PayloadData; +} + export default async function fetchData(nango: NangoSync) { const MAX_PAGE = 100; + const TICKET_PROPERTIES = ['hubspot_owner_id', 'hs_pipeline', 'hs_pipeline_stage', 'hs_ticket_priority', 'hs_ticket_category', 'subject', 'content']; let afterLink = null; - - let lastSyncDate = nango.lastSyncDate?.toISOString().slice(0, -8).replace('T', ' '); - let queryDate = Date.now() - 86400000; - - if (lastSyncDate) { - queryDate = Date.parse(lastSyncDate); - } + const lastSyncDate = nango.lastSyncDate?.toISOString().slice(0, -8).replace('T', ' '); + const queryDate = lastSyncDate ? Date.parse(lastSyncDate) : Date.now() - 86400000; while (true) { - let payload = { + const payload: Payload = { endpoint: '/crm/v3/objects/tickets/search', - params: { - limit: `${MAX_PAGE}` - } as Params, data: { - sorts: [ - { - propertyName: 'hs_lastmodifieddate', - direction: 'DESCENDING' - } - ], - properties: [ - // Define a list of these properties otherwise Hubspot won't return the Owner ID. - 'hubspot_owner_id', - 'hs_pipeline', - 'hs_pipeline_stage', - 'hs_ticket_priority', - 'hs_ticket_category', - 'subject', - 'content' - ], - filterGroups: [ - { - filters: [ - { - propertyName: 'hs_lastmodifieddate', - operator: 'GT', - value: queryDate - } - ] - } - ] + sorts: [{ propertyName: 'hs_lastmodifieddate', direction: 'DESCENDING' }], + properties: TICKET_PROPERTIES, + filterGroups: [{ filters: [{ propertyName: 'hs_lastmodifieddate', operator: 'GT', value: queryDate }] }], + limit: `${MAX_PAGE}`, + after: afterLink } }; - if (afterLink) { - payload.params['after'] = afterLink; - } - - const response = await nango.post(payload); + try { + const response = await nango.post(payload); + const pageData = response.data.results; - let pageData = response.data.results; + const mappedTickets: HubspotServiceTicket[] = pageData.map((ticket: any) => { + const { id, createdAt, archived } = ticket; + const { subject, content, hs_object_id, hubspot_owner_id, hs_pipeline, hs_pipeline_stage, hs_ticket_category, hs_ticket_priority } = + ticket.properties; - const mappedTickets: HubspotServiceTicket[] = pageData.map((ticket: any) => ({ - id: ticket.id, - createdAt: ticket.createdAt, - updatedAt: ticket.properties.hs_lastmodifieddate, - archived: ticket.archived, - subject: ticket.properties.subject, - content: ticket.properties.content, - objectId: ticket.properties.hs_object_id, - ownerId: ticket.properties.hubspot_owner_id, - pipelineName: ticket.properties.hs_pipeline, - pipelineStage: ticket.properties.hs_pipeline_stage, - category: ticket.properties.hs_ticket_category, - priority: ticket.properties.hs_ticket_priority - })); + return { + id, + createdAt, + updatedAt: ticket.properties.hs_lastmodifieddate, + archived, + subject, + content, + objectId: hs_object_id, + ownerId: hubspot_owner_id, + pipelineName: hs_pipeline, + pipelineStage: hs_pipeline_stage, + category: hs_ticket_category, + priority: hs_ticket_priority + }; + }); - if (mappedTickets.length > 0) { - await nango.batchSave(mappedTickets, 'HubspotServiceTicket'); - await nango.log(`Sent ${mappedTickets.length}`); - } + if (mappedTickets.length > 0) { + await nango.batchSave(mappedTickets, 'HubspotServiceTicket'); + await nango.log(`Sent ${mappedTickets.length}`); + } - if (response.data.paging?.next?.after) { - afterLink = response.data.paging.next.after; - } else { - break; + if (response.data.paging?.next?.after) { + afterLink = response.data.paging.next.after; + } else { + break; + } + } catch (error: any) { + throw new Error(`Error in fetchData: ${error.message}`); } } } - -interface Params { - limit: string; - [key: string]: any; // Allows additional properties -} diff --git a/integration-templates/hubspot/hubspot-user.ts b/integration-templates/hubspot/hubspot-user.ts deleted file mode 100644 index ace7d75cc56..00000000000 --- a/integration-templates/hubspot/hubspot-user.ts +++ /dev/null @@ -1,49 +0,0 @@ -import type { NangoSync, HubspotUser } from './models'; - -interface Params { - limit: string; - [key: string]: any; // Allows additional properties such as the 'after' property -} - -export default async function fetchData(nango: NangoSync) { - const MAX_PAGE = 100; - - let afterLink = null; - - while (true) { - let payload = { - endpoint: '/settings/v3/users', - params: { - limit: `${MAX_PAGE}` - } as Params - }; - - if (afterLink) { - // If there is no afterLink, then we are on the first page. - payload.params['after'] = afterLink; - } - - const response = await nango.get(payload); - - let pageData = response.data.results; - - const mappedUsers: HubspotUser[] = pageData.map((owner: HubspotUser) => ({ - id: owner.id, - email: owner.email, - roleId: owner.roleId, - primaryTeamId: owner.primaryTeamId, - superAdmin: owner.superAdmin - })); - - if (mappedUsers.length > 0) { - await nango.batchSave(mappedUsers, 'HubspotUser'); - await nango.log(`Sent ${mappedUsers.length} users`); - } - - if (response.data.paging?.next?.after) { - afterLink = response.data.paging.next.after; - } else { - break; - } - } -} diff --git a/integration-templates/hubspot/hubspot-users.ts b/integration-templates/hubspot/hubspot-users.ts new file mode 100644 index 00000000000..eb08e6fd3c8 --- /dev/null +++ b/integration-templates/hubspot/hubspot-users.ts @@ -0,0 +1,39 @@ +import type { HubspotUser, NangoSync } from './models'; + +export default async function fetchData(nango: NangoSync) { + let totalRecords = 0; + + try { + const endpoint = '/settings/v3/users'; + const config = { + paginate: { + type: 'cursor', + cursor_path_in_response: 'paging.next.after', + limit_name_in_request: 'limit', + cursor_name_in_request: 'after', + response_path: 'results', + limit: 100 + } + }; + for await (const user of nango.paginate({ ...config, endpoint })) { + const mappedUser: HubspotUser[] = user.map(mapUser) || []; + + const batchSize: number = mappedUser.length; + totalRecords += batchSize; + await nango.log(`Saving batch of ${batchSize} users (total users: ${totalRecords})`); + await nango.batchSave(mappedUser, 'HubspotUser'); + } + } catch (error: any) { + throw new Error(`Error in fetchData: ${error.message}`); + } +} + +function mapUser(user: any): HubspotUser { + return { + id: user.id, + email: user.email, + roleId: user.roleId, + primaryTeamId: user.primaryTeamId, + superAdmin: user.superAdmin + }; +} diff --git a/integration-templates/hubspot/nango.yaml b/integration-templates/hubspot/nango.yaml index b245ec7b317..66b8ae667cc 100644 --- a/integration-templates/hubspot/nango.yaml +++ b/integration-templates/hubspot/nango.yaml @@ -4,23 +4,43 @@ integrations: runs: every half hour returns: - HubspotServiceTicket + description: | + Retrieves a list of service tickets. + Details: incremental sync, doesn't track deletes, metadata is not required. + Scope(s): tickets hubspot-contacts: runs: every day returns: - HubspotContact - hubspot-owner: + description: | + Retrieves a list of contacts from an account. + Details: full sync, doesn't track deletes, metadata is not required. + Scope(s): crm.objects.contacts.read + hubspot-owners: runs: every day returns: - HubspotOwner - hubspot-user: + description: | + Retrieves a list of owners from an account who can assign CRM objects to + specific people in your organization. + Details: full sync, doesn't track deletes, metadata is not required. + Scope(s): crm.objects.owners.read + hubspot-users: runs: every day returns: - HubspotUser + description: | + Retrieves a list of users from an account. + Details: full sync, doesn't track deletes, metadata is not required. + Scope(s): settings.users.read hubspot-knowledge-base: runs: every day returns: - HubspotKnowledgeBase - + description: | + Retrieves a list of content related to a search term from your HubSpot hosted site. + Details: full sync, doesn't track deletes, metadata is not required. + Scope(s): content models: HubspotServiceTicket: id: integer diff --git a/packages/shared/flows.yaml b/packages/shared/flows.yaml index 553ea335d25..047b505af66 100644 --- a/packages/shared/flows.yaml +++ b/packages/shared/flows.yaml @@ -760,22 +760,43 @@ integrations: runs: every half hour returns: - HubspotServiceTicket + description: | + Retrieves a list of service tickets. + Details: incremental sync, doesn't track deletes, metadata is not required. + Scope(s): tickets hubspot-contacts: runs: every day returns: - HubspotContact + description: | + Retrieves a list of contacts from an account. + Details: full sync, doesn't track deletes, metadata is not required. + Scope(s): crm.objects.contacts.read hubspot-owner: runs: every day returns: - HubspotOwner + description: | + Retrieves a list of owners from an account who can assign CRM objects to + specific people in your organization. + Details: full sync, doesn't track deletes, metadata is not required. + Scope(s): crm.objects.owners.read hubspot-user: runs: every day returns: - HubspotUser + description: | + Retrieves a list of users from an account. + Details: full sync, doesn't track deletes, metadata is not required. + Scope(s): settings.users.read hubspot-knowledge-base: runs: every day returns: - HubspotKnowledgeBase + description: | + Retrieves a list of content related to a search term from your HubSpot hosted site. + Details: full sync, doesn't track deletes, metadata is not required. + Scope(s): content models: HubspotServiceTicket: id: integer