From ce9850b2380360c019bbb57fab1451b3f6a204c4 Mon Sep 17 00:00:00 2001 From: Erin Cochran Date: Fri, 20 Mar 2020 15:41:55 -0400 Subject: [PATCH 01/17] Fix typo --- _config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_config.yml b/_config.yml index c7eb5f7d6..5bb014844 100755 --- a/_config.yml +++ b/_config.yml @@ -50,7 +50,7 @@ sass: sass_dir: _sass -## The syntax highligher used for rendering code blocks +## The syntax highlighter used for rendering code blocks highlighter: rouge ## Configs for the Markdown processor From f785587422198b0b4b536deb1f849b9e320b2e9a Mon Sep 17 00:00:00 2001 From: Erin Cochran Date: Fri, 20 Mar 2020 15:42:07 -0400 Subject: [PATCH 02/17] Correct frequency_in_minutes to not required --- _data/connect/common/all-sources.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/connect/common/all-sources.yml b/_data/connect/common/all-sources.yml index 3bbabf7e8..39e5f15a3 100644 --- a/_data/connect/common/all-sources.yml +++ b/_data/connect/common/all-sources.yml @@ -38,7 +38,7 @@ fields: - name: *frequency type: "string" - required: true + required: false internal: false category: "interval-scheduling, anchor-scheduling" order: 1 From 51c20965afc7aea1c691cc36fbd5ae9320b9cd7c Mon Sep 17 00:00:00 2001 From: Erin Cochran Date: Fri, 20 Mar 2020 15:42:20 -0400 Subject: [PATCH 03/17] Fixing some things in the Recurly docs --- _data/taps/versions/recurly.yml | 2 +- _saas-integrations/recurly/v1/recurly-v1.md | 6 ++++-- .../recurly/v14-09-2016/recurly-v14-09-2016.md | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/_data/taps/versions/recurly.yml b/_data/taps/versions/recurly.yml index df2a3d358..7d339ba01 100644 --- a/_data/taps/versions/recurly.yml +++ b/_data/taps/versions/recurly.yml @@ -16,5 +16,5 @@ released-versions: status: "deprecated" date-released: "September 14, 2016" date-last-connection: "August 21, 2019" - deprecation-date: "" + deprecation-date: "November 21, 2019" sunset-date: "" \ No newline at end of file diff --git a/_saas-integrations/recurly/v1/recurly-v1.md b/_saas-integrations/recurly/v1/recurly-v1.md index f187952d6..2cda2c6bb 100644 --- a/_saas-integrations/recurly/v1/recurly-v1.md +++ b/_saas-integrations/recurly/v1/recurly-v1.md @@ -13,10 +13,10 @@ # -------------------------- # title: Recurly (v1) -permalink: /integrations/saas/recurly/v1 +permalink: /integrations/saas/recurly +redirect_from: /integrations/saas/recurly/v1 keywords: recurly, integration, schema, etl recurly, recurly etl, recurly schema layout: singer -input: false key: "recurly-setup" @@ -47,6 +47,8 @@ frequency: "1 hour" tier: "Standard" status-url: "https://status.recurly.com/" +api-type: "platform.recurly" + anchor-scheduling: true cron-scheduling: true diff --git a/_saas-integrations/recurly/v14-09-2016/recurly-v14-09-2016.md b/_saas-integrations/recurly/v14-09-2016/recurly-v14-09-2016.md index a8b049ba4..c35de8667 100644 --- a/_saas-integrations/recurly/v14-09-2016/recurly-v14-09-2016.md +++ b/_saas-integrations/recurly/v14-09-2016/recurly-v14-09-2016.md @@ -1,6 +1,6 @@ --- title: Recurly -permalink: /integrations/saas/recurly +permalink: /integrations/saas/recurly/v14-09-2016 keywords: recurly, integration, schema, etl recurly, recurly etl, recurly schema summary: "Connection instructions and schema details for Stitch's Recurly integration." layout: singer @@ -8,6 +8,7 @@ format: ## controls formatting options in template schema-list: true table-desc: true list: expand +input: false key: "recurly-setup" From 765f45a27380512aa18470fcfea08213d7283fac Mon Sep 17 00:00:00 2001 From: Erin Cochran Date: Fri, 20 Mar 2020 15:42:38 -0400 Subject: [PATCH 04/17] Making comments play nice in some includes --- .../api-form-property-fields-logic.html | 24 +++++----- .../api-form-property-source-saas-fields.html | 46 ++++++++++--------- 2 files changed, 38 insertions(+), 32 deletions(-) diff --git a/_includes/developers/api-form-property-fields-logic.html b/_includes/developers/api-form-property-fields-logic.html index 5182ddd21..de37589c8 100644 --- a/_includes/developers/api-form-property-fields-logic.html +++ b/_includes/developers/api-form-property-fields-logic.html @@ -1,18 +1,20 @@ - +{% comment %} +This file contains the logic that's used get a form property's attributes +and display them in a table and in the example object section. - +Destination and source form properties are a little different, +so we need to run logic specific to each type +{% endcomment %} {% case include.content %} - + {% when 'source' %} - - {% if form-property.uses-start-date == true %} @@ -21,20 +23,20 @@ {% assign start-date = null | compact %} {% endif %} - + {% assign all-source-attributes = site.data.connect.common.all-sources.fields | concat: start-date %} {% case form-property.source-type %} - + {% when 'database' %} {% include developers/api-form-property-source-database-fields.html %} - + {% when 'saas' %} {% include developers/api-form-property-source-saas-fields.html %} {% endcase %} - + {% when 'destination' %} {% include developers/api-form-property-destination-fields.html %} {% endcase %} \ No newline at end of file diff --git a/_includes/developers/api-form-property-source-saas-fields.html b/_includes/developers/api-form-property-source-saas-fields.html index f1e8a9e6a..31484b841 100644 --- a/_includes/developers/api-form-property-source-saas-fields.html +++ b/_includes/developers/api-form-property-source-saas-fields.html @@ -1,8 +1,12 @@ - +{% comment %} +This file contains the logic used to pull and display applicable fields + for SaaS source form properties. + +If the form property has its own specific attributes, get them & +concat them to the common array +{% endcomment %} + - {% if form-property.object-attributes %} {% assign form-object-attributes = form-property.object-attributes %} @@ -16,20 +20,20 @@ {% assign all-saas-docs = site.saas-integrations | where:"name",form-property.docs-name %} {% assign this-saas = all-saas-docs | where:"input",true %} - {% for saas in this-saas %} - {% assign doc-link = saas.url | prepend: site.baseurl %} - {% assign replication-frequency-value = saas.frequency | remove: " " | remove: "minutes" | remove: "hours" %} - - - {% if replication-frequency-value == "30" %} - {% assign replication-frequency = replication-frequency-value %} - {% else %} - {% assign replication-frequency = replication-frequency-value | times: 60 %} - {% endif %} - - {% if saas.cron-scheduling == true %} - {% assign cron-scheduling = true %} - {% else %} - {% assign cron-scheduling = false %} - {% endif %} - {% endfor %} \ No newline at end of file +{% for saas in this-saas %} + {% assign doc-link = saas.url | prepend: site.baseurl %} + {% assign replication-frequency-value = saas.frequency | remove: " " | remove: "minutes" | remove: "hours" %} + + + {% if replication-frequency-value == "30" %} + {% assign replication-frequency = replication-frequency-value %} + {% else %} + {% assign replication-frequency = replication-frequency-value | times: 60 %} + {% endif %} + + {% if saas.cron-scheduling == true %} + {% assign cron-scheduling = true %} + {% else %} + {% assign cron-scheduling = false %} + {% endif %} +{% endfor %} \ No newline at end of file From d70907bbfe1fe8508b74ac7e1ee25737f35814e5 Mon Sep 17 00:00:00 2001 From: Erin Cochran Date: Fri, 20 Mar 2020 15:43:01 -0400 Subject: [PATCH 05/17] Change terms for streams/fields, add info about oauth properties --- .../api/objects/connection-steps/connection-step-object.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/_developer-files/connect/api/objects/connection-steps/connection-step-object.md b/_developer-files/connect/api/objects/connection-steps/connection-step-object.md index 8b42b17d7..fef873855 100644 --- a/_developer-files/connect/api/objects/connection-steps/connection-step-object.md +++ b/_developer-files/connect/api/objects/connection-steps/connection-step-object.md @@ -27,10 +27,10 @@ object-attributes: The type of step. Possible values are: - `form` - The first step in every source's creation. - - `oauth` - If required, the OAuth step for the source's creation. + - `oauth` - If required, the OAuth step for the source's creation. **Note**: OAuth properties may be provided alongside `form` properties in a single `POST` or `PUT` request. A separate request isn't necessary. - `profile` - If required, the profile selection step. For example: Selecting a Facebook Ads profile. - - `discover_schema` - The step in which Stitch performs a [structure sync]({{ api.terms }}) to detect the tables and attributes available in the source. - - `field_selection` - The step in which tables and columns are selected for replication. + - `discover_schema` - The step in which Stitch performs a [structure sync]({{ api.terms }}) to detect the streams and fields available in the source. + - `field_selection` - The step in which streams and fields are selected for replication. - `fully_configured` - Achieved when the source has a successful connection and `field_selection` is complete. - name: "properties" From ab13ee1459f2f4d2644b38c4c87f5c4b739f31c7 Mon Sep 17 00:00:00 2001 From: Erin Cochran Date: Fri, 20 Mar 2020 15:43:08 -0400 Subject: [PATCH 06/17] Fix example request --- .../connect/api/objects/sources/retrieve-a-source.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_developer-files/connect/api/objects/sources/retrieve-a-source.md b/_developer-files/connect/api/objects/sources/retrieve-a-source.md index 05a851ca9..0179ab91d 100644 --- a/_developer-files/connect/api/objects/sources/retrieve-a-source.md +++ b/_developer-files/connect/api/objects/sources/retrieve-a-source.md @@ -54,8 +54,8 @@ examples: language: "json" code: | {% assign right-bracket = "}" %} - curl -X {{ endpoint.method | upcase }} {{ endpoint.full-url | flatify | replace: "{source_id","120643" | remove: right-bracket | strip_newlines }} - -H "Authorization: Bearer " + curl {{ endpoint.method | upcase }} {{ endpoint.full-url | flatify | replace: "{source_id","120643" | remove: right-bracket | strip_newlines }} \ + -H "Authorization: Bearer " \ -H "Content-Type: application/json" - type: "Response" language: "json" From c1828bf32423c069715087142c17c32b53dcf775 Mon Sep 17 00:00:00 2001 From: Erin Cochran Date: Fri, 20 Mar 2020 15:43:15 -0400 Subject: [PATCH 07/17] First pass at configure a source guide --- .../api/create-data-source-connect-api.md | 641 ++++++++++++++++++ 1 file changed, 641 insertions(+) create mode 100644 _developer-content/connect/guides/api/create-data-source-connect-api.md diff --git a/_developer-content/connect/guides/api/create-data-source-connect-api.md b/_developer-content/connect/guides/api/create-data-source-connect-api.md new file mode 100644 index 000000000..03d69747d --- /dev/null +++ b/_developer-content/connect/guides/api/create-data-source-connect-api.md @@ -0,0 +1,641 @@ +--- +# -------------------------- # +# PAGE INFO # +# -------------------------- # + +title: Create a Data Source with the Connect API +permalink: /developers/stitch-connect/guides/create-data-source-with-stitch-connect +summary: "Create a data source using the Stitch Connect API." + +product-type: "connect" +content-type: "guide" +content-id: "create-data-source" +topics: "sources, connect api" + +key: "create-source-connect-api" + +layout: tutorial + + +# -------------------------- # +# GUIDE PAGE INFO # +# -------------------------- # + +## This is used only on the /stitch-connect/guides page. +doc-type: "tutorial" +icon: source +order: 4 + +description: "Create and configure a data source using the Connect API." + + +# -------------------------- # +# RELATED SIDEBAR LINKS # +# -------------------------- # + +related: + - title: "Destination and source API availability" + link: "{{ link.connect.guides.connection-reference | prepend: site.baseurl }}" + + - title: "Replication scheduling for sources using the Connect API" + link: "{{ link.connect.guides.replication-scheduling-for-sources | prepend: site.baseurl }}" + + - title: "Select streams and fields with the Connect API" + link: "{{ link.connect.guides.select-streams-and-fields | prepend: site.baseurl }}" + + - title: "Field selection and compatibility rules" + link: "{{ link.connect.guides.field-selection-compatibility-rules | prepend: site.baseurl }}" + + - title: "Connect API reference" + link: "{{ link.connect.api | prepend: site.baseurl }}" + +# -------------------------- # +# GUIDE INTRO # +# -------------------------- # + +intro: | + {% include misc/data-files.html %} + + {{ page.summary }} + + +# -------------------------- # +# GUIDE REQUIREMENTS # +# -------------------------- # + +requirements: + - item: | + **Access to Stitch Connect.** Connect access is a Stitch Enterprise feature. Refer to the [Connect API reference]({{ link.connect.api | flatify | prepend: site.baseurl }}#authentication) for more info on obtaining API credentials. + + +# -------------------------- # +# TUTORIAL STEPS # +# -------------------------- # + +steps: + - title: "Get the source's API type" + anchor: "get-source-api-type" + content: | + To get started, you'll need to identify the API type of the data source you want to create. Every data source available in the Connect API has a `type`, and is typically similar to `platform.`. + + For example: The API type for a Recurly source is `platform.recurly`. + + Refer to the [Destination and Source API Availability Reference]({{ link.connect.guides.connection-reference | prepend: site.baseurl | append: "#sources-api-availability" }}) to locate the API type for your data source. + + - title: "Get the source's report card" + anchor: "get-source-report-card" + content: | + {% assign source-types = site.data.connect.core-objects.source-types %} + + When preparing for source creation, the next step is to get the report card for the source you want to create. The report card contains information about the steps required to fully configure a source. + + Use the [{{ source-types.get.method | upcase }} {{ source-types.get.name | flatify }} endpoint]({{ link.connect.api | append: source-types.get.anchor | prepend: site.baseurl }}) to get the report card for the source. In this example, we're retrieving the report card for a `platform.recurly` source: + + {% assign right-bracket = "}" %} + + ```json + curl {{ site.data.connect.api.base-url | strip_newlines }}{{ source-types.get.name | flatify | remove: right-bracket | replace:"{source_type","platform.recurly" | strip_newlines }} \ + -H 'Content-Type: application/json' \ + -H 'Authorization: Bearer ' + ``` + + The response will be a [Source object]({{ link.connect.api | prepend: site.baseurl | append: site.data.connect.core-objects.sources.object }}) with a [Connection step object]({{ link.connect.api | append: site.data.connect.data-structures.connection-steps.section | prepend: site.baseurl }}): + + ```json + { + "type": "platform.recurly", + "current_step": 1, + "current_step_type": "form", + "steps": [ + { + "type": "form", + "properties": [ + { + "name": "anchor_time", + "is_required": false, + "is_credential": false, + "system_provided": false, + "property_type": "user_provided", + "json_schema": { + "type": "string", + "format": "date-time" + }, + "provided": false, + "tap_mutable": false + }, + { + "name": "cron_expression", + "is_required": false, + "is_credential": false, + "system_provided": false, + "property_type": "user_provided", + "json_schema": null, + "provided": false, + "tap_mutable": false + }, + { + "name": "frequency_in_minutes", + "is_required": false, + "is_credential": false, + "system_provided": false, + "property_type": "user_provided", + "json_schema": { + "type": "string", + "pattern": "^1$|^30$|^60$|^360$|^720$|^1440$" + }, + "provided": false, + "tap_mutable": false + }, + { + "name": "image_version", + "is_required": true, + "is_credential": false, + "system_provided": true, + "property_type": "read_only", + "json_schema": null, + "provided": false, + "tap_mutable": false + }, + { + "name": "start_date", + "is_required": true, + "is_credential": false, + "system_provided": false, + "property_type": "user_provided", + "json_schema": { + "type": "string", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T00:00:00Z$" + }, + "provided": false, + "tap_mutable": false + }, + { + "name": "api_key", + "is_required": true, + "is_credential": true, + "system_provided": false, + "property_type": "user_provided", + "json_schema": { + "type": "string" + }, + "provided": false, + "tap_mutable": false + }, + { + "name": "subdomain", + "is_required": true, + "is_credential": false, + "system_provided": false, + "property_type": "user_provided", + "json_schema": { + "type": "string" + }, + "provided": false, + "tap_mutable": false + }, + { + "name": "quota_limit", + "is_required": true, + "is_credential": false, + "system_provided": false, + "property_type": "user_provided", + "json_schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string", + "pattern": "^\\d+" + } + ] + }, + "provided": false, + "tap_mutable": false + } + ] + }, + { + "type": "discover_schema", + "properties": [] + }, + { + "type": "field_selection", + "properties": [] + }, + { + "type": "fully_configured", + "properties": [] + } + ], + "details": { + "pricing_tier": "premium", + "pipeline_state": "released", + "default_start_date": "-1 year", + "default_scheduling_interval": 60, + "protocol": "platform.recurly", + "access": true + } + } + ``` + + **Note**: To create the source in your account, the `details.access` property must be `true`. This indicates that the plan your Stitch account is using has access to the source. + + For Recurly sources, the following steps are required to fully configure the source: + + 1. **The `form` step**. Provide values for all required user-provided properties. These properties will have a `is_required: true` value and a `property_type: user_provided` value. Refer to the [{{ form-property.title }} documentation]({{ | append: "#" | append: form-property.key }}) for more info about these properties. + + 2. **The `discover_schema` step.** Stitch runs a [connection check]({{ site.data.connect.core-objects.connection-checks.section | prepend: link.connect.api | prepend: site.baseurl }}) to test the provided `form` properties and detects the streams and fields available in the source. If all `form` properties are valid, including credentials, Stitch will automatically advance to the next step without any action required on your part. + + If the connection check fails, the source will remain on this step until a successful connection check completes. + + 3. **The `field_selection` step**. Select the streams and fields you want to replicate. + + - title: "Create the source and complete the form step" + anchor: "create-source-complete-form-step" + content: | + {% assign sources = site.data.connect.core-objects.sources %} + + Use the [{{ sources.create.method | upcase }} {{ sources.create.name | flatify }} endpoint]({{ link.connect.api | prepend: site.baseurl | append: sources.create.anchor }}) to create the Recurly source. The request body must include the following properties: + + {% assign form-property = site.developer-files | where:"key","source-form-properties-recurly-object" | first %} + + {% include developers/api-form-property-fields-logic.html content="source" %} + + - `type`: The API type of the source. In this example, this value will be `platform.recurly`. + - `display_name`: {{ site.data.connect.general.common.attributes.display-name }} + + For example: A display name of `Recurly` will create a destination schema named `recurly`. + - `properties`: A [Properties object]({{ site.data.connect.data-structures.properties.section | prepend: link.connect.api | prepend: site.baseurl }}) containing the properties required to configure the source. Refer to the [Source form property documentation]({{ site.data.connect.data-structures.source-form-properties.section | prepend: link.connect.api | prepend: site.baseurl }}) for your source for more info about the required properties. + + For `platform.recurly`, the properties are: + + {% for attribute in all-form-attributes %} + - `{{ attribute.name }}`{% if attribute.required == false %}*{% endif %} + {% endfor %} + + * While these properties have a `is_required: false` value, you must provide a replication schedule for the source. Refer to the [Replication Scheduling for Sources Using the Connect API guide]({{ link.connect.guides.replication-scheduling-for-sources | prepend: site.baseurl }}) for more info and examples. + + This request will complete the `form` step outlined in the source's report card, which you retrieved in [Step 2](#get-source-report-card): + + ```json + curl -X POST {{ site.data.connect.api.base-url | strip_newlines }}{{ site.data.connect.core-objects.sources.create.name | flatify | strip_newlines }} \ + -H 'Content-Type: application/json' \ + -H 'Authorization: Bearer ' \ + -d $'{ + "type": "platform.recurly", + "display_name": "Recurly", + "properties": { + "start_date": "2018-01-10T00:00:00Z", + "api_key": "", + "frequency_in_minutes": "60", + "quota_limit": "30", + "subdomain": "stitchdata" + } + }' + ``` + + The response will be a [Source object]({{ link.connect.api | prepend: site.baseurl | append: site.data.connect.core-objects.sources.object }}) containing the source's ID, [report card]({{ link.connect.api | prepend: site.baseurl | append: site.data.connect.data-structures.report-cards.source.section }}), and current configuration status (`report_card.current_step_type`): + + ```json + { + "properties": { + "frequency_in_minutes": "60", + "image_version": "1.latest", + "quota_limit": "30", + "start_date": "2018-01-10T00:00:00Z", + "subdomain": "stitchdata" + }, + "updated_at": "2020-03-20T16:06:19Z", + "schedule": null, + "check_job_name": "116078.233312.check.bc876980-6ac4-11ea-9197-0ee6b2399f9b", + "name": "recurly", + "type": "platform.recurly", + "deleted_at": null, + "system_paused_at": null, + "stitch_client_id": 116078, + "paused_at": null, + "id": 233312, /* Source ID */ + "display_name": "Recurly", + "created_at": "2020-03-20T16:03:16Z", + "report_card": { + "type": "platform.recurly", + "current_step": 3, + "current_step_type": "field_selection", /* Configuration status */ + "steps": [ + { + "type": "form", + "properties": [ + { + "name": "anchor_time", + "is_required": false, + "is_credential": false, + "system_provided": false, + "property_type": "user_provided", + "json_schema": { + "type": "string", + "format": "date-time" + }, + "provided": false, + "tap_mutable": false + }, + { + "name": "cron_expression", + "is_required": false, + "is_credential": false, + "system_provided": false, + "property_type": "user_provided", + "json_schema": null, + "provided": false, + "tap_mutable": false + }, + { + "name": "frequency_in_minutes", + "is_required": false, + "is_credential": false, + "system_provided": false, + "property_type": "user_provided", + "json_schema": { + "type": "string", + "pattern": "^1$|^30$|^60$|^360$|^720$|^1440$" + }, + "provided": true, + "tap_mutable": false + }, + { + "name": "image_version", + "is_required": true, + "is_credential": false, + "system_provided": true, + "property_type": "read_only", + "json_schema": null, + "provided": true, + "tap_mutable": false + }, + { + "name": "start_date", + "is_required": true, + "is_credential": false, + "system_provided": false, + "property_type": "user_provided", + "json_schema": { + "type": "string", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T00:00:00Z$" + }, + "provided": true, + "tap_mutable": false + }, + { + "name": "api_key", + "is_required": true, + "is_credential": true, + "system_provided": false, + "property_type": "user_provided", + "json_schema": { + "type": "string" + }, + "provided": true, + "tap_mutable": false + }, + { + "name": "subdomain", + "is_required": true, + "is_credential": false, + "system_provided": false, + "property_type": "user_provided", + "json_schema": { + "type": "string" + }, + "provided": true, + "tap_mutable": false + }, + { + "name": "quota_limit", + "is_required": true, + "is_credential": false, + "system_provided": false, + "property_type": "user_provided", + "json_schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string", + "pattern": "^\\d+" + } + ] + }, + "provided": true, + "tap_mutable": false + } + ] + }, + { + "type": "discover_schema", + "properties": [] + }, + { + "type": "field_selection", + "properties": [] + }, + { + "type": "fully_configured", + "properties": [] + } + ] + } + } + ``` + + - title: "Complete the field selection step" + anchor: "complete-field-selection-step" + content: | + Next, you'll select the streams and fields you want to replicate from the source. Locate the source's ID in the source's report card - in this example, it's `233312` - and follow the steps in the [Select Streams and Fields with the Connect API guide]({{ link.connect.guides.select-streams-and-fields | prepend: site.baseurl }}). + + To complete field selection, at least one stream and one field in the stream must be selected. This includes fields that are automatically selected. For example: If a stream uses an `id` field as a Primary Key, this field will be automatically included when the stream is selected. + + - title: "Check the source's configuration status" + anchor: "check-source-configuration-status" + content: | + After field selection, the source's configuration status should be `fully_configured`. When `fully_configured`, Stitch can begin replication for the source using the schedule and stream/field selection data you provided. + + You can verify the source's configuarion status by sending a request to [{{ sources.retrieve.method | upcase }} {{ sources.retrieve.name | flatify }}]({{ link.connect.api | prepend: site.baseurl | append: sources.retrieve.anchor }}), replacing `{source_id}` with the source's ID: + + ```json + curl {{ site.data.connect.api.base-url | strip_newlines }}{{ sources.retrieve.name | flatify | remove: right-bracket | replace:"{source_id","233312" | strip_newlines }} \ + -H 'Content-Type: application/json' \ + -H 'Authorization: Bearer ' + ``` + + The response will be a [Source object]({{ link.connect.api | prepend: site.baseurl | append: site.data.connect.core-objects.sources.object }}) containing the source's current configuration status (`report_card.current_step_type`): + + ```json + { + "properties": { + "anchor_time": "2020-03-20T19:09:38.509Z", + "frequency_in_minutes": "60", + "image_version": "1.latest", + "quota_limit": "30", + "start_date": "2018-01-10T00:00:00Z", + "subdomain": "stitchdata" + }, + "updated_at": "2020-03-20T19:09:03Z", + "schedule": { + "type": "interval", + "unit": "minute", + "interval": 60.0, + "next_fire_time": "2020-03-20T19:09:38Z" + }, + "name": "recurly", + "type": "platform.recurly", + "deleted_at": null, + "system_paused_at": null, + "stitch_client_id": 116078, + "paused_at": null, + "id": 233312, + "display_name": "Recurly", + "created_at": "2020-03-20T16:03:16Z", + "report_card": { + "type": "platform.recurly", + "current_step": 4, + "current_step_type": "fully_configured", /* Configuration status */ + "steps": [ + { + "type": "form", + "properties": [ + { + "name": "anchor_time", + "is_required": false, + "is_credential": false, + "system_provided": false, + "property_type": "user_provided", + "json_schema": { + "type": "string", + "format": "date-time" + }, + "provided": true, + "tap_mutable": false + }, + { + "name": "cron_expression", + "is_required": false, + "is_credential": false, + "system_provided": false, + "property_type": "user_provided", + "json_schema": null, + "provided": false, + "tap_mutable": false + }, + { + "name": "frequency_in_minutes", + "is_required": false, + "is_credential": false, + "system_provided": false, + "property_type": "user_provided", + "json_schema": { + "type": "string", + "pattern": "^1$|^30$|^60$|^360$|^720$|^1440$" + }, + "provided": true, + "tap_mutable": false + }, + { + "name": "image_version", + "is_required": true, + "is_credential": false, + "system_provided": true, + "property_type": "read_only", + "json_schema": null, + "provided": true, + "tap_mutable": false + }, + { + "name": "start_date", + "is_required": true, + "is_credential": false, + "system_provided": false, + "property_type": "user_provided", + "json_schema": { + "type": "string", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T00:00:00Z$" + }, + "provided": true, + "tap_mutable": false + }, + { + "name": "api_key", + "is_required": true, + "is_credential": true, + "system_provided": false, + "property_type": "user_provided", + "json_schema": { + "type": "string" + }, + "provided": true, + "tap_mutable": false + }, + { + "name": "subdomain", + "is_required": true, + "is_credential": false, + "system_provided": false, + "property_type": "user_provided", + "json_schema": { + "type": "string" + }, + "provided": true, + "tap_mutable": false + }, + { + "name": "quota_limit", + "is_required": true, + "is_credential": false, + "system_provided": false, + "property_type": "user_provided", + "json_schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string", + "pattern": "^\\d+" + } + ] + }, + "provided": true, + "tap_mutable": false + } + ] + }, + { + "type": "discover_schema", + "properties": [] + }, + { + "type": "field_selection", + "properties": [] + }, + { + "type": "fully_configured", + "properties": [] + } + ] + } + } + ``` + + TODO- FINISH THIS SECTION + + + +# -------------------------- # +# NEXT STEPS # +# -------------------------- # + +next-steps: | + Congratulations on configuring a data source using the Connect API! + + TODO- Add links to other Connect stuff +--- \ No newline at end of file From 858fa1a29151f5fcc6388d251fff6215f7b4b969 Mon Sep 17 00:00:00 2001 From: Erin Cochran Date: Wed, 29 Apr 2020 11:56:18 -0400 Subject: [PATCH 08/17] Update urls for source guide --- _data/urls.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/urls.yaml b/_data/urls.yaml index 5fd125d16..4dfbaffd9 100755 --- a/_data/urls.yaml +++ b/_data/urls.yaml @@ -268,7 +268,7 @@ connect: category: /developers/stitch-connect/guides api-credentials: /developers/stitch-connect/guides/connect-api-credentials - create-configure-a-source: /developers/stitch-connect/api#quick-start + create-configure-a-source: /developers/stitch-connect/guides/create-data-source-with-stitch-connect create-import-api-source: /developers/stitch-connect/guides/create-import-api-integration-with-stitch-connect manage-import-api-access-tokens: /developers/stitch-connect/guides/manage-import-api-access-tokens field-selection-compatibility-rules: /developers/stitch-connect/guides/field-selection-compatibility-rules From bb3e28d9c443e001e64f25c41fe52851c264b072 Mon Sep 17 00:00:00 2001 From: Erin Date: Thu, 30 Apr 2020 14:06:47 -0400 Subject: [PATCH 09/17] Keep it DRY --- _data/connect/core-objects/replication-jobs.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/_data/connect/core-objects/replication-jobs.yml b/_data/connect/core-objects/replication-jobs.yml index 2a8eef4d9..900ca0ded 100644 --- a/_data/connect/core-objects/replication-jobs.yml +++ b/_data/connect/core-objects/replication-jobs.yml @@ -5,11 +5,11 @@ title: "Replication jobs" section: "#replication-jobs" object: "#replication-job--object" -base: "/v4/sources/{source_id}/sync" +base: &base "/v4/sources/{source_id}/sync" description: "An object representing a replication job. A replication job consists of three phases: Extraction, Preparation, and Loading." post: - name: "{{ site.data.connect.api.core-objects.replication-jobs.base }}" + name: *base title: "Start a replication job" method: "post" anchor: "#start-a-job" @@ -17,7 +17,7 @@ post: short: *post-description delete: - name: "{{ site.data.connect.api.core-objects.replication-jobs.base }}" + name: *base title: "Stop a replication job" method: "delete" anchor: "#stop-a-job" From 25bd0d069064cebfefda08a09aa2fd92cc9edf86 Mon Sep 17 00:00:00 2001 From: Erin Date: Thu, 30 Apr 2020 14:06:55 -0400 Subject: [PATCH 10/17] Change guide order --- .../connect/guides/api/select-tables-and-fields.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_developer-content/connect/guides/api/select-tables-and-fields.md b/_developer-content/connect/guides/api/select-tables-and-fields.md index d5da4c6a4..de1286af0 100644 --- a/_developer-content/connect/guides/api/select-tables-and-fields.md +++ b/_developer-content/connect/guides/api/select-tables-and-fields.md @@ -25,7 +25,7 @@ layout: tutorial ## This is used only on the /stitch-connect/guides page. doc-type: "tutorial" icon: table-selection -order: 3 +order: 4 description: "Select streams and fields for replication." From d3ce2ca268696fc2b57ca09bc525adf1d7b5a97c Mon Sep 17 00:00:00 2001 From: Erin Date: Thu, 30 Apr 2020 14:07:12 -0400 Subject: [PATCH 11/17] Correct connection check example --- .../retrieve-sources-last-connection-check-v4.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_developer-files/connect/api/objects/connection-checks/retrieve-sources-last-connection-check-v4.md b/_developer-files/connect/api/objects/connection-checks/retrieve-sources-last-connection-check-v4.md index b217ac3ce..48a48e9d1 100644 --- a/_developer-files/connect/api/objects/connection-checks/retrieve-sources-last-connection-check-v4.md +++ b/_developer-files/connect/api/objects/connection-checks/retrieve-sources-last-connection-check-v4.md @@ -68,5 +68,5 @@ examples: code: "{{ site.data.connect.code-examples.connection-checks.successful | lstrip }}" - title: "A failed connection check" - code: "{{ site.data.connect.code-examples.connection-checks.in-progress | lstrip }}" + code: "{{ site.data.connect.code-examples.connection-checks.failed | lstrip }}" --- \ No newline at end of file From d5f2429f93a8f0b290e7981fb95719fd630e868d Mon Sep 17 00:00:00 2001 From: Erin Date: Thu, 30 Apr 2020 14:07:24 -0400 Subject: [PATCH 12/17] Correct header for replication job example --- .../api/objects/replication-jobs/start-replication-job.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_developer-files/connect/api/objects/replication-jobs/start-replication-job.md b/_developer-files/connect/api/objects/replication-jobs/start-replication-job.md index cd962b7da..46c0c8da5 100644 --- a/_developer-files/connect/api/objects/replication-jobs/start-replication-job.md +++ b/_developer-files/connect/api/objects/replication-jobs/start-replication-job.md @@ -58,7 +58,7 @@ examples: - type: "Request" request-url: | {% assign right-bracket = "}" %}{{ endpoint.short-url | flatify | replace: "{source_id","120643" | remove: right-bracket | strip_newlines }} - header: "{{ site.data.connect.request-headers.put.without-body | flatify }}" + header: "{{ site.data.connect.request-headers.post.without-body | flatify }}" code: "" - type: "Response" From 2588e8d0e1f15eee755e37920dee35e766735c30 Mon Sep 17 00:00:00 2001 From: Erin Date: Thu, 30 Apr 2020 14:07:47 -0400 Subject: [PATCH 13/17] Another pass at the guide --- .../api/create-data-source-connect-api.md | 159 ++++++++++++++---- 1 file changed, 125 insertions(+), 34 deletions(-) diff --git a/_developer-content/connect/guides/api/create-data-source-connect-api.md b/_developer-content/connect/guides/api/create-data-source-connect-api.md index 6612f4a1d..672ac2c4c 100644 --- a/_developer-content/connect/guides/api/create-data-source-connect-api.md +++ b/_developer-content/connect/guides/api/create-data-source-connect-api.md @@ -102,7 +102,7 @@ steps: The response will be a [Source object]({{ link.connect.api | prepend: site.baseurl | append: site.data.connect.core-objects.sources.object }}) with a [Connection step object]({{ link.connect.api | append: site.data.connect.data-structures.connection-steps.section | prepend: site.baseurl }}): - ```json + {% capture code %} { "type": "platform.recurly", "current_step": 1, @@ -238,7 +238,11 @@ steps: "access": true } } - ``` + {% endcapture %} + + {% assign description = "Response for GET " | append: request-url %} + + {% include layout/code-snippet.html code-description=description code=code %} **Note**: To create the source in your account, the `details.access` property must be `true`. This indicates that the plan your Stitch account is using has access to the source. @@ -279,26 +283,25 @@ steps: This request will complete the `form` step outlined in the source's report card, which you retrieved in [Step 2](#get-source-report-card): - ```json - curl -X POST {{ site.data.connect.api.base-url | strip_newlines }}{{ site.data.connect.core-objects.sources.create.name | flatify | strip_newlines }} \ - -H 'Content-Type: application/json' \ - -H 'Authorization: Bearer ' \ - -d $'{ - "type": "platform.recurly", - "display_name": "Recurly", - "properties": { - "start_date": "2018-01-10T00:00:00Z", - "api_key": "", - "frequency_in_minutes": "60", - "quota_limit": "30", - "subdomain": "stitchdata" - } - }' - ``` - + {% assign request-url = sources.create.name %} + {% assign description = "POST " | append: request-url %} + {% capture code %}'{ + "type":"platform.recurly", + "display_name":"Recurly", + "properties":{ + "start_date":"2018-01-10T00:00:00Z", + "api_key":"", + "frequency_in_minutes":"60", + "quota_limit":"30", + "subdomain":"stitchdata" + } + }' + {% endcapture %} + {% include developers/api-request-examples.html code-description=description header=site.data.connect.request-headers.post.with-body request-url=request-url code=code %} + The response will be a [Source object]({{ link.connect.api | prepend: site.baseurl | append: site.data.connect.core-objects.sources.object }}) containing the source's ID, [report card]({{ link.connect.api | prepend: site.baseurl | append: site.data.connect.data-structures.report-cards.source.section }}), and current configuration status (`report_card.current_step_type`): - ```json + {% capture code %} { "properties": { "frequency_in_minutes": "60", @@ -447,15 +450,39 @@ steps: ] } } - ``` + {% endcapture %} + {% assign description = "Response for POST " | append: request-url %} + {% include layout/code-snippet.html code-description=description code=code request-url=request-url %} + + # - title: "Wait for a successful connection check" + # anchor: "wait-successful-connection-check" + # content: | + # If all required `form` properties for the source have been provided, the source will automatically progress to `discover_schema` step. During this step, Stitch will perform a connection check using the `form` properties provided for the source. + + # After a successful connection check completes, the source will automatically progress to [the `field_selection` step](#complete-field-selection-step). + + # If you notice that the source hasn't progressed past `discover_schema`, use the source's last connection check results to pinpoint the problem. Retrieve this info using the [GET {{ site.data.connect.core-objects.connection-checks.get-source.name | flatify }} endpoint](), replacing `{source_id}` with the source's ID: + + # {% assign example-url = site.data.connect.core-objects.connection-checks.get-source.name %} + # {% assign request-url = example-url | flatify | remove: right-bracket | replace:"{source_id","233312" | strip_newlines %} + # {% assign description = "GET " | append: example-url %} + + # {% include developers/api-request-examples.html code-description=description header=site.data.connect.request-headers.get.without-body request-url=request-url %} + + # {{ site.data.connect.code-examples.connection-checks.failed | lstrip }} + - title: "Complete the field selection step" anchor: "complete-field-selection-step" content: | - Next, you'll select the streams and fields you want to replicate from the source. Locate the source's ID in the source's report card - in this example, it's `233312` - and follow the steps in the [Select Streams and Fields with the Connect API guide]({{ link.connect.guides.select-streams-and-fields | prepend: site.baseurl }}). + Next, you'll select the streams and fields you want to replicate from the source. The source will automatically progress from `discover_schema` to `field_selection` after a successful connection check completes. + + Locate the source's ID in the source's report card - in this example, it's `233312` - and follow the steps in the [Select Streams and Fields with the Connect API guide]({{ link.connect.guides.select-streams-and-fields | prepend: site.baseurl }}). To complete field selection, at least one stream and one field in the stream must be selected. This includes fields that are automatically selected. For example: If a stream uses an `id` field as a Primary Key, this field will be automatically included when the stream is selected. + **Note**: Stream and field selection may occur any time when a source's `current_step` is `field_selection` or `fully_configured`, as long as the source's report card has a `field_selection` step. + - title: "Check the source's configuration status" anchor: "check-source-configuration-status" content: | @@ -463,18 +490,17 @@ steps: You can verify the source's configuration status by sending a request to [{{ sources.retrieve.method | upcase }} {{ sources.retrieve.name | flatify }}]({{ link.connect.api | prepend: site.baseurl | append: sources.retrieve.anchor }}), replacing `{source_id}` with the source's ID: - ```json - curl {{ site.data.connect.api.base-url | strip_newlines }}{{ sources.retrieve.name | flatify | remove: right-bracket | replace:"{source_id","233312" | strip_newlines }} \ - -H 'Content-Type: application/json' \ - -H 'Authorization: Bearer ' - ``` + {% assign example-url = sources.retrieve.name %} + {% assign request-url = example-url | flatify | remove: right-bracket | replace:"{source_id","233312" | strip_newlines %} + {% assign description = "GET " | append: example-url %} + + {% include developers/api-request-examples.html code-description=description header=site.data.connect.request-headers.get.without-body request-url=request-url %} The response will be a [Source object]({{ link.connect.api | prepend: site.baseurl | append: site.data.connect.core-objects.sources.object }}) containing the source's current configuration status (`report_card.current_step_type`): - ```json + {% capture code %} { "properties": { - "anchor_time": "2020-03-20T19:09:38.509Z", "frequency_in_minutes": "60", "image_version": "1.latest", "quota_limit": "30", @@ -625,18 +651,83 @@ steps: ] } } - ``` + {% endcapture %} + + {% assign description = "Response for GET " | append: example-url %} + {% include layout/code-snippet.html code-description=description code=code request-url=request-url %} + + - title: "Start a replication job" + anchor: "start-a-replication-job" + content: | + {% include note.html type="single-line" content="**Note**: This step is optional." %} + + {% assign replication-jobs = site.data.connect.core-objects.replication-jobs %} + + Now that the source is `fully_configured`, you can start extracting data. + + Stitch will automatically schedule a replication job based on the [schedule you set](##create-source-complete-form-step). To see when the next replication job will begin, check the `schedule.next_fire_time` value in the Source object: + + {% capture code %} + { + "properties": {...}, + "updated_at": "2020-03-20T19:09:03Z", + "schedule":{ + "type":"interval", + "unit":"minute", + "interval":60.0, + "next_fire_time":"2020-03-20T19:09:38Z" + }, + [...] + } + {% endcapture %} - TODO- FINISH THIS SECTION + {% assign description = "Example schedule in a Source object" %} + {% include layout/code-snippet.html code-description=description code=code %} + If you want to start a replication job sooner than the `next_fire_time`, you can send a request to [POST {{ replication-jobs.post.name | flatify }}]({{ link.connect.api | prepend: site.baseurl | append: replication-jobs.post.anchor }}), replacing `{source_id}` with the source's ID: + {% assign example-url = replication-jobs.post.name %} + {% assign request-url = example-url | flatify | remove: right-bracket | replace:"{source_id","233312" | strip_newlines %} + {% assign description = "POST " | append: example-url %} + {% include developers/api-request-examples.html code-description=description header=site.data.connect.request-headers.post.without-body request-url=request-url %} + + **Note**: Stitch allows only one replication job to run at a time. The response to the above request may be either of the following: + + - **If the job started successfully**, the response will be a single [Replication Job object]({{ link.connect.api | prepend: site.baseurl | append: replication-jobs.object }}): + + {% capture code %} + { + "job_name":"116078.233312.sync.c12fb0a7-7e4a-11e9-abdc-0edc2c318fba" + } + {% endcapture %} + {% assign description = "Response for POST " | append: example-url | append: " indicating a job was started" %} + {% include layout/code-snippet.html use-code-block=false language="json" code-description=description code=code %} + + ```json + {{ code | lstrip | rstrip }} + ``` + + - **If a job was already in progress**, the response will be a single error object similar to: + + {% capture code %} + { + "error":{ + "type":"already_running", + "message":"Did not create job for client-id: 116078; connection-id: 233312 because one already exists" + } + } + {% endcapture %} + {% assign description = "Response for POST " | append: example-url | append: " indicating a job is already running" %} + {% include layout/code-snippet.html use-code-block=false language="json" code-description=description code=code %} + + ```json + {{ code | lstrip | rstrip }} + ``` # -------------------------- # # NEXT STEPS # # -------------------------- # next-steps: | - Congratulations on configuring a data source using the Connect API! - - TODO- Add links to other Connect stuff + Congratulations on configuring a data source using the Connect API! Check out the [Tutorials and resources]({{ link.connect.guides.category | prepend: site.baseurl }}) to see what else you can do with Stitch Connect. --- \ No newline at end of file From ac5aff11a3e00ce84b0fc81b9c39cda44bababfb Mon Sep 17 00:00:00 2001 From: Erin Date: Thu, 30 Apr 2020 14:19:50 -0400 Subject: [PATCH 14/17] Add links to create a source endpoint --- .../connect/api/objects/sources/create-a-source.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/_developer-files/connect/api/objects/sources/create-a-source.md b/_developer-files/connect/api/objects/sources/create-a-source.md index 0c2648090..67b2490da 100644 --- a/_developer-files/connect/api/objects/sources/create-a-source.md +++ b/_developer-files/connect/api/objects/sources/create-a-source.md @@ -25,7 +25,10 @@ short: "{{ site.data.connect.core-objects.sources.create.short }}" description: | {{ site.data.connect.core-objects.sources.create.description | flatify }} - Refer to the [Destination and source API availability reference]({{ link.connect.guides.connection-reference | prepend: site.baseurl | append: "#sources-api-availability" }}) for info on the sources that are available in the API. + Refer to these resources for creating and configuring sources: + + - [Destination and source API availability reference]({{ link.connect.guides.connection-reference | prepend: site.baseurl | append: "#sources-api-availability" }}) + - [Create and configure a source using the Connect API]({{ link.connect.guides.create-configure-a-source | prepend: site.baseurl }}) # -------------------------- # From cf76ff712a3f33ff7866ced2bca5cf61016cd5a0 Mon Sep 17 00:00:00 2001 From: Erin Date: Thu, 30 Apr 2020 14:20:04 -0400 Subject: [PATCH 15/17] Updates to data selection guide --- .../guides/api/select-tables-and-fields.md | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/_developer-content/connect/guides/api/select-tables-and-fields.md b/_developer-content/connect/guides/api/select-tables-and-fields.md index de1286af0..3425cc79e 100644 --- a/_developer-content/connect/guides/api/select-tables-and-fields.md +++ b/_developer-content/connect/guides/api/select-tables-and-fields.md @@ -70,7 +70,7 @@ requirements: - item: | **Access to a source with a `field_selection` connection step**. This guide will use a [Shopify SaaS source]({{ site.data.connect.api.section | flatify | prepend: site.baseurl | append: site.data.connect.data-structures.source-form-properties.section | append: "-shopify-object" }}) as an example, but any source type with a `field_selection` step will work. - Refer to the [Source Form Properties documentation]({{ site.data.connect.api.section | flatify | prepend: site.baseurl | append: site.data.connect.data-structures.source-form-properties.section }}) for the sources supported by Connect. To determine if a source has a `field_selection` step, [retrieve its Report Card]({{ link.connect.api | prepend: site.baseurl | append: site.data.connect.core-objects.source-types.get.anchor | flatify }}). + Refer to the [Destination and source API availability reference]({{ link.connect.guides.connection-reference | prepend: site.baseurl | append: "#sources-api-availability" }}) for the sources supported by Connect. To determine if a source has a `field_selection` step, [retrieve its Report Card]({{ link.connect.api | prepend: site.baseurl | append: site.data.connect.core-objects.source-types.get.anchor | flatify }}). # -------------------------- # @@ -87,7 +87,7 @@ steps: {% assign stream-id = "2288758" %} {% assign tap-stream-id = "custom_collections" %} - Create and configure a source. Refer to the [Quick start guide]({{ link.connect.guides.create-configure-a-source | flatify | prepend: site.baseurl }}) for instructions. + Create and configure a source. Refer to steps 1-3 of the [Create and configure a source using the Connect API guide]({{ link.connect.guides.create-configure-a-source | flatify | prepend: site.baseurl }}) for instructions. - title: "Wait for a successful connection check and discovery" anchor: "successful-connection-check-discovery" @@ -288,9 +288,7 @@ steps: {% capture code %}{{ site.data.connect.code-examples.streams.request-bodies.database | rstrip | prepend: quote | append: quote }} {% endcapture %} - {% assign description = "Example request body for a stream with a configurable Replication Method" %} - {% include layout/code-snippet.html code-description=description language="json" code=code %} **Note**: When `replication-method` is set to `INCREMENTAL`, the value of the `replication-key` property must be one of the following: @@ -304,9 +302,7 @@ steps: {% capture code %}{{ site.data.connect.code-examples.streams.saas-stream | rstrip }} {% endcapture %} - {% assign description = "Example metadata for a stream with a forced Replication Method" %} - {% include layout/code-snippet.html code-description=description language="json" code=code %} When the stream's metadata contains the `forced-replication-method` property, its Replication Method cannot be changed. If selected, the stream will use the `forced-replication-method` and the field in `valid-replication-keys` as a Replication Key, if applicable. @@ -315,9 +311,7 @@ steps: {% capture code %}{{ site.data.connect.code-examples.streams.request-bodies.saas | rstrip | prepend: quote | append: quote }} {% endcapture %} - {% assign description = "Example request body for a stream with a forced Replication Method" %} - {% include layout/code-snippet.html code-description=description language="json" code=code %} - title: "Submit the request" @@ -328,9 +322,7 @@ steps: {% assign example-url = site.data.connect.core-objects.streams.update.name %} {% assign request-url = example-url | flatify | replace: "{source_id",source-id | remove: right-bracket | strip_newlines %} - {% assign description = substep.endpoint %} - {% include developers/api-request-examples.html code-description=description header=site.data.connect.request-headers.put.with-body request-url=request-url code=code%} {% capture code %}'{ @@ -367,12 +359,9 @@ steps: {% assign example-url = site.data.connect.core-objects.streams.update.name | strip_newlines %} {% assign request-url = example-url | flatify | replace: "{source_id",source-id | remove: right-bracket | strip_newlines %} - {% assign description = "Selecting a single field via " | append: step.endpoint %} - {% capture code %}{{ site.data.connect.code-examples.field-metadata.request-bodies.saas-field | rstrip | prepend: quote | append: quote }} {% endcapture %} - {% include developers/api-request-examples.html code-description=description header=site.data.connect.request-headers.put.with-body request-url=request-url code=code %} Multiple fields in a stream can be submitted as part of the same request. For each field included in the request body, include a `metadata` object referencing the field. @@ -381,9 +370,7 @@ steps: {% capture code %}{{ site.data.connect.code-examples.field-metadata.request-bodies.saas-fields | rstrip | prepend: quote | append: quote }} {% endcapture %} - {% assign description = "Selecting multiple fields via " | append: step.endpoint %} - {% include developers/api-request-examples.html code-description=description header=site.data.connect.request-headers.put.with-body request-url=request-url code=code %} **Note**: Fields with metadata properties of `inclusion: automatic` or `selected-by-default: true` don't need to be explicitly selected through a request. These fields will be automatically selected for replication regardless of their `selected` value. Refer to the [Field selection and compatibility rules guide]({{ link.connect.guides.field-selection-compatibility-rules | prepend: site.baseurl }}) for more info. From 869e2f3c3ffe08354d30da8901771c690b726389 Mon Sep 17 00:00:00 2001 From: Erin Date: Thu, 30 Apr 2020 14:35:38 -0400 Subject: [PATCH 16/17] Fix section for connection checks --- _data/connect/core-objects/connection-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/connect/core-objects/connection-checks.yml b/_data/connect/core-objects/connection-checks.yml index 178275baa..42c108efb 100644 --- a/_data/connect/core-objects/connection-checks.yml +++ b/_data/connect/core-objects/connection-checks.yml @@ -3,7 +3,7 @@ # ----------------------------- # title: "Connection checks" -section: "#connection-checks" +section: "#connection-checks--section" object: "#connection-check--object" base: "/v4/{connection_type}/{connection_id}/last-connection-check" description: | From 29987a03c9d3d656ab74419f5c737c65e99045ea Mon Sep 17 00:00:00 2001 From: Erin Cochran Date: Thu, 30 Apr 2020 14:37:22 -0400 Subject: [PATCH 17/17] Final fixes --- .../api/create-data-source-connect-api.md | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/_developer-content/connect/guides/api/create-data-source-connect-api.md b/_developer-content/connect/guides/api/create-data-source-connect-api.md index 672ac2c4c..a530b9506 100644 --- a/_developer-content/connect/guides/api/create-data-source-connect-api.md +++ b/_developer-content/connect/guides/api/create-data-source-connect-api.md @@ -94,10 +94,9 @@ steps: Use the [{{ source-types.get.method | upcase }} {{ source-types.get.name | flatify }} endpoint]({{ link.connect.api | append: source-types.get.anchor | prepend: site.baseurl }}) to get the report card for the source. In this example, we're retrieving the report card for a `platform.recurly` source: - {% assign request-url = source-types.get.name %} - - {% assign description = "GET " | append: request-url %} - + {% assign example-url = source-types.get.name %} + {% assign request-url = example-url | flatify | remove: right-bracket | replace:"{source_type","platform.recurly" | strip_newlines %} + {% assign description = "GET " | append: example-url %} {% include developers/api-request-examples.html code-description=description header=site.data.connect.request-headers.get.without-body request-url=request-url %} The response will be a [Source object]({{ link.connect.api | prepend: site.baseurl | append: site.data.connect.core-objects.sources.object }}) with a [Connection step object]({{ link.connect.api | append: site.data.connect.data-structures.connection-steps.section | prepend: site.baseurl }}): @@ -239,16 +238,16 @@ steps: } } {% endcapture %} - - {% assign description = "Response for GET " | append: request-url %} - + {% assign description = "Response for GET " | append: example-url %} {% include layout/code-snippet.html code-description=description code=code %} **Note**: To create the source in your account, the `details.access` property must be `true`. This indicates that the plan your Stitch account is using has access to the source. + {% assign form-property = site.developer-files | where:"key","source-form-properties-recurly-object" | first %} + For Recurly sources, the following steps are required to fully configure the source: - 1. **The `form` step**. Provide values for all required user-provided properties. These properties will have a `is_required: true` value and a `property_type: user_provided` value. Refer to the [{{ form-property.title }} documentation]({{ | append: "#" | append: form-property.key }}) for more info about these properties. + 1. **The `form` step**. Provide values for all required user-provided properties. These properties will have a `is_required: true` value and a `property_type: user_provided` value. Refer to the [{{ form-property.title }} documentation]({{ link.connect.api | prepend: site.baseurl | append: "#" | append: form-property.key }}) for more info about these properties. 2. **The `discover_schema` step.** Stitch runs a [connection check]({{ site.data.connect.core-objects.connection-checks.section | prepend: link.connect.api | prepend: site.baseurl }}) to test the provided `form` properties and detects the streams and fields available in the source. If all `form` properties are valid, including credentials, Stitch will automatically advance to the next step without any action required on your part. @@ -263,14 +262,12 @@ steps: Use the [{{ sources.create.method | upcase }} {{ sources.create.name | flatify }} endpoint]({{ link.connect.api | prepend: site.baseurl | append: sources.create.anchor }}) to create the Recurly source. The request body must include the following properties: - {% assign form-property = site.developer-files | where:"key","source-form-properties-recurly-object" | first %} - {% include developers/api-form-property-fields-logic.html content="source" %} - `type`: The API type of the source. In this example, this value will be `platform.recurly`. - `display_name`: {{ site.data.connect.general.common.attributes.display-name }} - For example: A display name of `Recurly` will create a destination schema named `recurly`. + For example: A display name of `Recurly` will create a destination schema named `recurly`. **Note**: The schema name can't be changed after the source has been created. - `properties`: A [Properties object]({{ site.data.connect.data-structures.properties.section | prepend: link.connect.api | prepend: site.baseurl }}) containing the properties required to configure the source. Refer to the [Source form property documentation]({{ site.data.connect.data-structures.source-form-properties.section | prepend: link.connect.api | prepend: site.baseurl }}) for your source for more info about the required properties. For `platform.recurly`, the properties are: @@ -665,7 +662,7 @@ steps: Now that the source is `fully_configured`, you can start extracting data. - Stitch will automatically schedule a replication job based on the [schedule you set](##create-source-complete-form-step). To see when the next replication job will begin, check the `schedule.next_fire_time` value in the Source object: + Stitch will automatically schedule a replication job based on the [schedule you set](#create-source-complete-form-step). To see when the next replication job will begin, check the `schedule.next_fire_time` value in the Source object: {% capture code %} {