Merge pull request #452 from stitchdata/connect-api-create-source

Create a source using the API

_config.yml

@@ -50,7 +50,7 @@ sass:
   sass_dir: _sass
 
 
-## The syntax highligher used for rendering code blocks
+## The syntax highlighter used for rendering code blocks
 highlighter: rouge
 
 ## Configs for the Markdown processor

_data/connect/common/all-sources.yml

@@ -38,7 +38,7 @@ fields:
 
   - name: *frequency
     type: "string"
-    required: true
+    required: false
     internal: false
     category: "interval-scheduling, anchor-scheduling"
     order: 1

_data/connect/core-objects/connection-checks.yml

@@ -3,7 +3,7 @@
 # ----------------------------- #
 
 title: "Connection checks"
-section: "#connection-checks"
+section: "#connection-checks--section"
 object: "#connection-check--object"
 base: "/v4/{connection_type}/{connection_id}/last-connection-check"
 description: |

_data/connect/core-objects/replication-jobs.yml

@@ -5,19 +5,19 @@
 title: "Replication jobs"
 section: "#replication-jobs"
 object: "#replication-job--object"
-base: "/v4/sources/{source_id}/sync"
+base: &base "/v4/sources/{source_id}/sync"
 description: "An object representing a replication job. A replication job consists of three phases: Extraction, Preparation, and Loading."
 
 post:
-  name: "{{ site.data.connect.api.core-objects.replication-jobs.base }}"
+  name: *base
   title: "Start a replication job"
   method: "post"
   anchor: "#start-a-job"
   description: &post-description "Starts a replication job for a source using the source's unique identifier."
   short: *post-description
 
 delete:
-  name: "{{ site.data.connect.api.core-objects.replication-jobs.base }}"
+  name: *base
   title: "Stop a replication job"
   method: "delete"
   anchor: "#stop-a-job"

_data/taps/versions/recurly.yml

@@ -16,5 +16,5 @@ released-versions:
     status: "deprecated"
     date-released: "September 14, 2016"
     date-last-connection: "August 21, 2019"
-    deprecation-date: ""
+    deprecation-date: "November 21, 2019"
     sunset-date: ""

_data/urls.yaml

@@ -268,7 +268,7 @@ connect:
     category: /developers/stitch-connect/guides
     api-credentials: /developers/stitch-connect/guides/connect-api-credentials
 
-    create-configure-a-source: /developers/stitch-connect/api#quick-start
+    create-configure-a-source: /developers/stitch-connect/guides/create-data-source-with-stitch-connect
     create-import-api-source: /developers/stitch-connect/guides/create-import-api-integration-with-stitch-connect
     manage-import-api-access-tokens: /developers/stitch-connect/guides/manage-import-api-access-tokens
     field-selection-compatibility-rules: /developers/stitch-connect/guides/field-selection-compatibility-rules

_developer-content/connect/guides/api/select-tables-and-fields.md

@@ -25,7 +25,7 @@ layout: tutorial
 ## This is used only on the /stitch-connect/guides page.
 doc-type: "tutorial"
 icon: table-selection
-order: 3
+order: 4
 
 description: "Select streams and fields for replication."
 
@@ -70,7 +70,7 @@ requirements:
   - item: |
       **Access to a source with a `field_selection` connection step**. This guide will use a [Shopify SaaS source]({{ site.data.connect.api.section | flatify | prepend: site.baseurl | append: site.data.connect.data-structures.source-form-properties.section |  append: "-shopify-object" }}) as an example, but any source type with a `field_selection` step will work.
 
-         Refer to the [Source Form Properties documentation]({{ site.data.connect.api.section | flatify | prepend: site.baseurl | append: site.data.connect.data-structures.source-form-properties.section }}) for the sources supported by Connect. To determine if a source has a `field_selection` step, [retrieve its Report Card]({{ link.connect.api | prepend: site.baseurl | append: site.data.connect.core-objects.source-types.get.anchor | flatify }}).
+         Refer to the [Destination and source API availability reference]({{ link.connect.guides.connection-reference | prepend: site.baseurl | append: "#sources-api-availability" }}) for the sources supported by Connect. To determine if a source has a `field_selection` step, [retrieve its Report Card]({{ link.connect.api | prepend: site.baseurl | append: site.data.connect.core-objects.source-types.get.anchor | flatify }}).
 
 
 # -------------------------- #
@@ -87,7 +87,7 @@ steps:
       {% assign stream-id = "2288758" %}
       {% assign tap-stream-id = "custom_collections" %}
 
-      Create and configure a source. Refer to the [Quick start guide]({{ link.connect.guides.create-configure-a-source | flatify | prepend: site.baseurl }}) for instructions.
+      Create and configure a source. Refer to steps 1-3 of the [Create and configure a source using the Connect API guide]({{ link.connect.guides.create-configure-a-source | flatify | prepend: site.baseurl }}) for instructions.
   
   - title: "Wait for a successful connection check and discovery"
     anchor: "successful-connection-check-discovery"
@@ -288,9 +288,7 @@ steps:
 
           {% capture code %}{{ site.data.connect.code-examples.streams.request-bodies.database | rstrip | prepend: quote | append: quote }}
           {% endcapture %}
-
           {% assign description = "Example request body for a stream with a configurable Replication Method" %}
-
           {% include layout/code-snippet.html code-description=description language="json" code=code %}
 
           **Note**: When `replication-method` is set to `INCREMENTAL`, the value of the `replication-key` property must be one of the following:
@@ -304,9 +302,7 @@ steps:
 
           {% capture code %}{{ site.data.connect.code-examples.streams.saas-stream | rstrip }}
           {% endcapture %}
-
           {% assign description = "Example metadata for a stream with a forced Replication Method" %}
-
           {% include layout/code-snippet.html code-description=description language="json" code=code %}
 
           When the stream's metadata contains the `forced-replication-method` property, its Replication Method cannot be changed. If selected, the stream will use the `forced-replication-method` and the field in `valid-replication-keys` as a Replication Key, if applicable.
@@ -315,9 +311,7 @@ steps:
 
           {% capture code %}{{ site.data.connect.code-examples.streams.request-bodies.saas | rstrip | prepend: quote | append: quote }}
           {% endcapture %}
-
           {% assign description = "Example request body for a stream with a forced Replication Method" %}
-
           {% include layout/code-snippet.html code-description=description language="json" code=code %}
 
       - title: "Submit the request"
@@ -328,9 +322,7 @@ steps:
 
           {% assign example-url = site.data.connect.core-objects.streams.update.name %}
           {% assign request-url = example-url | flatify | replace: "{source_id",source-id | remove: right-bracket | strip_newlines %}
-
           {% assign description = substep.endpoint %}
-
           {% include developers/api-request-examples.html code-description=description header=site.data.connect.request-headers.put.with-body request-url=request-url code=code%}
 
           {% capture code %}'{
@@ -367,12 +359,9 @@ steps:
 
       {% assign example-url = site.data.connect.core-objects.streams.update.name | strip_newlines %}
       {% assign request-url = example-url | flatify | replace: "{source_id",source-id | remove: right-bracket | strip_newlines %}
-
       {% assign description = "Selecting a single field via " | append: step.endpoint %}
-
       {% capture code %}{{ site.data.connect.code-examples.field-metadata.request-bodies.saas-field | rstrip | prepend: quote | append: quote }} 
       {% endcapture %}
-
       {% include developers/api-request-examples.html code-description=description header=site.data.connect.request-headers.put.with-body request-url=request-url code=code %}
 
       Multiple fields in a stream can be submitted as part of the same request. For each field included in the request body, include a `metadata` object referencing the field.
@@ -381,9 +370,7 @@ steps:
 
       {% capture code %}{{ site.data.connect.code-examples.field-metadata.request-bodies.saas-fields | rstrip | prepend: quote | append: quote }} 
       {% endcapture %}
-
       {% assign description = "Selecting multiple fields via " | append: step.endpoint %}
-
       {% include developers/api-request-examples.html code-description=description header=site.data.connect.request-headers.put.with-body request-url=request-url code=code %}
 
       **Note**: Fields with metadata properties of `inclusion: automatic` or `selected-by-default: true` don't need to be explicitly selected through a request. These fields will be automatically selected for replication regardless of their `selected` value. Refer to the [Field selection and compatibility rules guide]({{ link.connect.guides.field-selection-compatibility-rules | prepend: site.baseurl }}) for more info.

_developer-files/connect/api/objects/connection-checks/retrieve-sources-last-connection-check-v4.md

@@ -68,5 +68,5 @@ examples:
         code: "{{ site.data.connect.code-examples.connection-checks.successful | lstrip }}"
 
       - title: "A failed connection check"
-        code: "{{ site.data.connect.code-examples.connection-checks.in-progress | lstrip }}"
+        code: "{{ site.data.connect.code-examples.connection-checks.failed | lstrip }}"
 ---

_developer-files/connect/api/objects/connection-steps/connection-step-object.md

@@ -27,10 +27,10 @@ object-attributes:
       The type of step. Possible values are:
 
       - `form` - The first step in every source's creation.
-      - `oauth` - If required, the OAuth step for the source's creation.
+      - `oauth` - If required, the OAuth step for the source's creation. **Note**: OAuth properties may be provided alongside `form` properties in a single `POST` or `PUT` request. A separate request isn't necessary.
       - `profile` - If required, the profile selection step. For example: Selecting a Facebook Ads profile.
-      - `discover_schema` - The step in which Stitch performs a [structure sync]({{ api.terms }}) to detect the tables and attributes available in the source.
-      - `field_selection` - The step in which tables and columns are selected for replication.
+      - `discover_schema` - The step in which Stitch performs a [structure sync]({{ api.terms }}) to detect the streams and fields available in the source.
+      - `field_selection` - The step in which streams and fields are selected for replication.
       - `fully_configured` - Achieved when the source has a successful connection and `field_selection` is complete.
 
   - name: "properties"

_developer-files/connect/api/objects/replication-jobs/start-replication-job.md

@@ -58,7 +58,7 @@ examples:
   - type: "Request"
     request-url: |
       {% assign right-bracket = "}" %}{{ endpoint.short-url | flatify | replace: "{source_id","120643" | remove: right-bracket | strip_newlines }}
-    header: "{{ site.data.connect.request-headers.put.without-body | flatify }}"
+    header: "{{ site.data.connect.request-headers.post.without-body | flatify }}"
     code: ""
 
   - type: "Response"

_developer-files/connect/api/objects/sources/create-a-source.md

@@ -25,7 +25,10 @@ short: "{{ site.data.connect.core-objects.sources.create.short }}"
 description: |
   {{ site.data.connect.core-objects.sources.create.description | flatify }}
 
-  Refer to the [Destination and source API availability reference]({{ link.connect.guides.connection-reference | prepend: site.baseurl | append: "#sources-api-availability" }}) for info on the sources that are available in the API.
+  Refer to these resources for creating and configuring sources:
+
+  - [Destination and source API availability reference]({{ link.connect.guides.connection-reference | prepend: site.baseurl | append: "#sources-api-availability" }})
+  - [Create and configure a source using the Connect API]({{ link.connect.guides.create-configure-a-source | prepend: site.baseurl }})
 
 
 # -------------------------- #

_includes/developers/api-form-property-fields-logic.html

@@ -1,18 +1,20 @@
-<!-- This file contains the logic that's used get a form property's attributes
-        and display them in a table and in the example object section. -->
+{% comment %}
+This file contains the logic that's used get a form property's attributes
+and display them in a table and in the example object section.
 
-<!-- Destination and source form properties are a little different,
-        so we need to run logic specific to each type -->
+Destination and source form properties are a little different,
+so we need to run logic specific to each type
+{% endcomment %}
 
 {% case include.content %}
 
-    <!-- When the form property is a source, run this -->
+<!-- When the form property is a source, run this -->
     {% when 'source' %}
-        <!-- There are two types of form properties: databases and SaaS applications.
+<!-- There are two types of form properties: databases and SaaS applications.
                 Each one is a little different, so we need to run logic specific to
                 each type.  -->
 
-        <!-- This checks if the form property uses start_date and,
+<!-- This checks if the form property uses start_date and,
             if so, creates an array with the property in it. We have to do this
             because not every source uses this. -->
         {% if form-property.uses-start-date == true %}
@@ -21,20 +23,20 @@
             {% assign start-date = null | compact %}
         {% endif %}
 
-        <!-- Get the common fields -->
+<!-- Get the common fields -->
             {% assign all-source-attributes = site.data.connect.common.all-sources.fields | concat: start-date %}
 
         {% case form-property.source-type %}
-            <!-- When the source form property is for a database, run this -->
+<!-- When the source form property is for a database, run this -->
             {% when 'database' %}
                 {% include developers/api-form-property-source-database-fields.html %}
 
-            <!-- When the source form property is for a SaaS source, run this -->
+<!-- When the source form property is for a SaaS source, run this -->
             {% when 'saas' %}
                 {% include developers/api-form-property-source-saas-fields.html %}
         {% endcase %}
 
-    <!-- When the form property is a destination, run this -->
+<!-- When the form property is a destination, run this -->
     {% when 'destination' %}
         {% include developers/api-form-property-destination-fields.html %}
 {% endcase %}

_includes/developers/api-form-property-source-saas-fields.html

@@ -1,8 +1,12 @@
-<!-- This file contains the logic used to pull and display applicable fields
-    for SaaS source form properties. -->
+{% comment %}
+This file contains the logic used to pull and display applicable fields
+    for SaaS source form properties. 
+
+If the form property has its own specific attributes, get them &
+concat them to the common array
+{% endcomment %}
+
 
-<!-- If the form property has its own specific attributes, get them &
-                        concat them to the common array -->
 {% if form-property.object-attributes %}
     {% assign form-object-attributes = form-property.object-attributes %}
 
@@ -16,20 +20,20 @@
 {% assign all-saas-docs = site.saas-integrations | where:"name",form-property.docs-name %}
     {% assign this-saas = all-saas-docs | where:"input",true %}
 
-        {% for saas in this-saas %}
-            {% assign doc-link = saas.url | prepend: site.baseurl %}
-            {% assign replication-frequency-value = saas.frequency | remove: " " | remove: "minutes" | remove: "hours" %}
-
-            <!-- We have to do some math because `frequency_in_minutes` is in minutes, and frequencies in the UI are shown in hours. -->
-                {% if replication-frequency-value == "30" %}
-                    {% assign replication-frequency = replication-frequency-value %}
-                {% else %}
-                    {% assign replication-frequency = replication-frequency-value | times: 60 %}
-                {% endif %}
-
-                {% if saas.cron-scheduling == true %}
-                    {% assign cron-scheduling = true %}
-                {% else %}
-                    {% assign cron-scheduling = false %}
-                {% endif %}
-        {% endfor %}
+{% for saas in this-saas %}
+    {% assign doc-link = saas.url | prepend: site.baseurl %}
+    {% assign replication-frequency-value = saas.frequency | remove: " " | remove: "minutes" | remove: "hours" %}
+
+<!-- We have to do some math because `frequency_in_minutes` is in minutes, and frequencies in the UI are shown in hours. -->
+        {% if replication-frequency-value == "30" %}
+            {% assign replication-frequency = replication-frequency-value %}
+        {% else %}
+            {% assign replication-frequency = replication-frequency-value | times: 60 %}
+        {% endif %}
+
+        {% if saas.cron-scheduling == true %}
+            {% assign cron-scheduling = true %}
+        {% else %}
+            {% assign cron-scheduling = false %}
+        {% endif %}
+{% endfor %}

_saas-integrations/recurly/v1/recurly-v1.md

@@ -13,10 +13,10 @@
 # -------------------------- #
 
 title: Recurly (v1)
-permalink: /integrations/saas/recurly/v1
+permalink: /integrations/saas/recurly
+redirect_from: /integrations/saas/recurly/v1
 keywords: recurly, integration, schema, etl recurly, recurly etl, recurly schema
 layout: singer
-input: false
 
 key: "recurly-setup"
 
@@ -47,6 +47,8 @@ frequency: "1 hour"
 tier: "Standard"
 status-url: "https://status.recurly.com/"
 
+api-type: "platform.recurly"
+
 anchor-scheduling: true
 cron-scheduling: true
 

_saas-integrations/recurly/v14-09-2016/recurly-v14-09-2016.md

@@ -1,13 +1,14 @@
 ---
 title: Recurly
-permalink: /integrations/saas/recurly
+permalink: /integrations/saas/recurly/v14-09-2016
 keywords: recurly, integration, schema, etl recurly, recurly etl, recurly schema
 summary: "Connection instructions and schema details for Stitch's Recurly integration."
 layout: singer
 format: ## controls formatting options in template
   schema-list: true
   table-desc: true
   list: expand
+input: false
 
 key: "recurly-setup"