From 16cbc14f11b0af82c1da470918cc50d3a9822459 Mon Sep 17 00:00:00 2001 From: Aleksa Krolls Date: Thu, 13 Jun 2024 10:39:59 +0200 Subject: [PATCH] Initial commit --- .github/.github/pull_request_template.md | 14 +++ .github/ISSUE_TEMPLATE/bug_report.md | 41 ++++++++ .github/ISSUE_TEMPLATE/feature-request.md | 77 +++++++++++++++ .gitignore | 115 ++++++++++++++++++++++ .prettierrc | 8 ++ README.md | 39 ++++++++ sampleData/payload.json | 3 + workflows/jobs/expression.js | 4 + workflows/lightning-workflow.json | 29 ++++++ 9 files changed, 330 insertions(+) create mode 100644 .github/.github/pull_request_template.md create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature-request.md create mode 100644 .gitignore create mode 100644 .prettierrc create mode 100644 README.md create mode 100644 sampleData/payload.json create mode 100644 workflows/jobs/expression.js create mode 100644 workflows/lightning-workflow.json diff --git a/.github/.github/pull_request_template.md b/.github/.github/pull_request_template.md new file mode 100644 index 0000000..326562a --- /dev/null +++ b/.github/.github/pull_request_template.md @@ -0,0 +1,14 @@ +## Summary + +Add a high-level, single-sentence summary of what this PR changes. + +## Issues + +Reference any related issues here. + +## Review Checklist + +Before merging, the reviewer should check the following items: + +- [ ] Does the PR do what it claims to do (Mention jobs changes)? +- [ ] Does the PR have adaptor version? diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..2e92b54 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,41 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +## Describe the bug and expected behavior +A clear and concise description of what the bug is. Include any error messages from the run logs and the expected behavior. + +## To Reproduce +1. Here is a link to a failed run on OpenFn.org which is indicative of the bug: __________ + +### expression.js +Link to the job itself in Github: _____________ +Adaptor: + + +### state.json + +Either provide state directly, or link to a file. If sensitive information +should be in state, redact it and provide instructions for where it can be +found. + +```json +{ + "configuration": ["SEE LAST PASS: 'client cred'"], + "data": {LINK TO STATE}, + "cursor": "2020-01-19 00:00:00" +} +``` + +## To test/resolve +1. After the desired output is working locally (from the CLI), please [push commits to master || open a pull request]. +2. [Please test the change on OpenFn.org by re-running this run (link) and confirming success.] + +## Toggl +Name of the Toggle project to log work + diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md new file mode 100644 index 0000000..f5f4f16 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -0,0 +1,77 @@ +--- +name: Feature request +about: For new workflows & change requests +title: '' +labels: feature request +assignees: '' + +--- +## Background, context, and business value + +A clear and concise description of what the client wants and WHY. + +For example: [Insert use case here] + +## The specific request, in as few words as possible + +A clear and concise description of what you want to happen. +Things to include as needed: +- The number of workflows needed to be created or updated +- The function of each workflow including specific resources and operations +- Unique identifiers +- Links to mapping specifications, data flow diagrams, sample input/output data, and any API documentation +- Links to the data model of target systems, if available + + +```md +Create a workflow in which OpenFn will: +1. Get new rows from the PostgreSQL database every 1 hour +2. Clean & transform the data according to the specified mapping rules, and then +3. Upsert cases in the Primero case management system via externalId `case_id` +(Note: 1 DB row will = 1 case record.) + +See [links] below for the workflow diagrams, mapping specs, & Primero data model. +``` + +### Data Volumes & Limits +How many records do we think these jobs will need to process in each run? For example: +```md +When you GET data from the DB, this may return up to 1000 records. +There are no known Primero API limits for # of records, but there is API paging to consider. +``` + +## Input + +### Credentials +Which credentials can be used to access the target system(s)? +(Do NOT share credential secrets on this issue -> rather point to where it can be found). + +### Data +Describe how the "input" for this workflow will be generated (e.g., webhook request, timer-based query to be sent). Either provide an example directly, link to a file, or describe how a query can be executed to extract data. + +Be sure to redact any sensitive data and to not paste here. + + +## Workflow Steps + +For each new Workflow, describe the number of steps needed and the high-level function of each step. Also include the trigger on platform and the adaptor needed for each step. +For existing steps, provide a link to the step itself in Github and the high-level changes needed to be made. _Provide the information below for _each_ step that is required._ + + +### 1. Step Name (e.g., `1. Get new DB rows`) + +#### adaptor +E.g., `postgresql` + +#### operation +E.g., Query (`SELECT * from table where created_at >= NOW()`) + +#### output +List of new DB rows + +## How to test +Link to test suite and/or provide examples of input/output scenarios to validate the implementation. + +## Toggl + + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..43d793d --- /dev/null +++ b/.gitignore @@ -0,0 +1,115 @@ +# temp +tmp + +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# TypeScript v1 declaration files +typings/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env +.env.test + +# parcel-bundler cache (https://parceljs.org/) +.cache + +# Next.js build output +.next + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and *not* Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Don't commit state.json +state.json* +.state.json* +._state.json* +**/state.json* +**/.state.json* +**/._state.json* diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..b706630 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,8 @@ +{ + "proseWrap": "always", + "trailingComma": "es5", + "arrowParens": "avoid", + "semi": true, + "singleQuote": true, + "printWidth": 80 +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..506e935 --- /dev/null +++ b/README.md @@ -0,0 +1,39 @@ +# project-name + +Please update this readme when you start a new project. +Summarize the solutions implemented or leverage Github pages to build a documentation site. + +*Note that commits to `main` will be auto-deployed to OpenFn app if the Github Sync is enabled. Always work on a branch!* + +## Getting started with the Repo + +1. create `/tmp` folder locally with a `state.json` file that you don't commit to github +2. start writing and testing jobs locally with the relevant adaptor + +See the [OpenFn CLI docs](https://docs.openfn.org/documentation/cli-walkthrough#7-running-workflows) for more on writing, running, and testing workflows locally. +See the [Platform Docs](https://docs.openfn.org/documentation/build/steps/step-editor) for how to run and test workflows on the app. + +## Project Documentation +The OpenFn team will often leverage [Github Pages](https://pages.github.com/) to configure a documentation site to organize and display project-specific information. +This site can display the content on this README, or be pointed to another markdown `.md` file created on this repo specifically for docs. + +See this example Github pages site that displays this README as a simple site: [https://openfn.github.io/project/](https://openfn.github.io/project/) + +Github Page configuration can be adjusted in the `Settings` of this repository. +Screenshot 2024-05-07 at 2 45 23 PM + + +## Implementation Checklist +[See OpenFn Docs site](https://docs.openfn.org/documentation/get-started/implementation-checklist) or [this workbook](https://docs.google.com/spreadsheets/d/1_XY0nx0OLNUsogrIHnRaSTyZ-KdcSXks-tqwm3ZfMc4/edit#gid=72612093) for a template work plan for the OpenFn implementation, or contact `support@openfn.org` to get the Asana-version of this checklist. + +## Administration +_It's good practice to document on the read me who is responsible for managing & maintaining the OpenFn project._ +### Provisioning, Hosting, & Maintenance +_Document details specific to your OpenFn deployment_ +- Deployment: SaaS or Local? +- Platform Version: OpenFn V1 or V2 +- Maintenance: Specify the team responsible for software maintenance, security updates, etc. - confirm roles & responsibilities across systems + +### Support +_List relevant support point of contacts and system administrators._ +- {Contact1 name, email} diff --git a/sampleData/payload.json b/sampleData/payload.json new file mode 100644 index 0000000..8d6b85c --- /dev/null +++ b/sampleData/payload.json @@ -0,0 +1,3 @@ +{ + "a": 1 +} diff --git a/workflows/jobs/expression.js b/workflows/jobs/expression.js new file mode 100644 index 0000000..4ea4583 --- /dev/null +++ b/workflows/jobs/expression.js @@ -0,0 +1,4 @@ +// language-http + +get('someUrl', { ...params }, callback); +post('someUrl', { body: state => state.data })' diff --git a/workflows/lightning-workflow.json b/workflows/lightning-workflow.json new file mode 100644 index 0000000..ef5ed58 --- /dev/null +++ b/workflows/lightning-workflow.json @@ -0,0 +1,29 @@ +{ + "options": { + "start": "get-data", + "timeout": 1200000 + }, + "workflow": { + "steps": [ + { + "id": "get-data", + "adaptor": "postgresql@3.4.6", + "state": {}, + "configuration": "./tmp/db-credential.json", + "expression": "./jobs/1-getPatientData.js", + "next": { + "post-data": { + "condition": "true" + } + } + }, + { + "id": "post-data", + "adaptor": "http@latest", + "configuration": "./tmp/api-credential.json", + "state": {}, + "expression": "./jobs/2-postData.js" + } + ] + } +}