diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4b28f804657..5e63d85092d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -37,13 +37,11 @@ jobs: env: USE_SSH: true GIT_USER: git - CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }} run: | git config --global user.email "actions@gihub.com" git config --global user.name "gh-actions" yarn install yarn generate-library yarn generate-adaptors - yarn crowdin:sync yarn build npx docusaurus deploy diff --git a/.github/workflows/test-deploy.yml b/.github/workflows/test-deploy.yml index c5752bd3635..9ba58d01666 100644 --- a/.github/workflows/test-deploy.yml +++ b/.github/workflows/test-deploy.yml @@ -21,10 +21,7 @@ jobs: - name: Install dependencies run: yarn install - name: Test build website - env: - CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }} run: | yarn generate-library yarn generate-adaptors - yarn crowdin:sync yarn build diff --git a/crowdin.yml b/crowdin.yml deleted file mode 100644 index 20c6dccadd1..00000000000 --- a/crowdin.yml +++ /dev/null @@ -1,25 +0,0 @@ -project_id: '455970' -api_token_env: 'CROWDIN_PERSONAL_TOKEN' -preserve_hierarchy: true -files: [ - # JSON translation files - { - source: '/i18n/en/**/*', - translation: '/i18n/%two_letters_code%/**/%original_file_name%', - }, - # Docs Markdown files - { - source: '/docs/**/*', - translation: '/i18n/%two_letters_code%/docusaurus-plugin-content-docs/current/**/%original_file_name%', - }, - # Articles Markdown files - # { - # source: '/articles/**/*', - # translation: '/i18n/%two_letters_code%/docusaurus-plugin-content-blog-articles/**/%original_file_name%', - # }, - # Blog Markdown files - # { - # source: '/blog/**/*', - # translation: '/i18n/%two_letters_code%/docusaurus-plugin-content-blog/**/%original_file_name%', - # }, - ] diff --git a/docs/about-lightning.md b/docs/about-lightning.md deleted file mode 100644 index 54b7ab3423f..00000000000 --- a/docs/about-lightning.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: Lightning (Beta) -sidebar_label: Lightning ---- - -## Introducing Lightning - -[OpenFn/Lightning](https://github.com/OpenFn/lightning/) is the v2 of the OpenFn -integration software: a _fully open source_ workflow automation platform -designed for governments and NGOs who need a flexible solution to integrate and -connect _any system_. - -##### Leveraging the tech powering the field-tested enterprise OpenFn platform... - -Lightning brings together the tried and tested technology which we have been -using since 2015 (the OpenFn -[Integration Toolkit](/documentation/getting-started/integration-toolkit)) to -manage the orchestration and execution of integrations in a stable, scalable and -secure way. - -##### ...and providing a fully open source web app with a user-friendly web interface. - -A fully open source web app, it can be deployed anywhere through Docker and -comes with a user-friendly, low-code interface with the full functionality -needed for organizations to build, run and audit their workflows all in one -place. - -![Lightning](/img/lightning_build_run_audit.png) - -### Build - -Empower more users in your organization to have a say in what gets automated and -how. Lightning’s visual interface makes workflows more intelligible to -non-technical users, bridging the gap between the IT specialists that build out -automations and program managers that are the real business/ program experts on -the processes that need automating. - -![Lightning build interface](/img/lightning_build.png) - -### Audit - -Treat every workflow run with the care and attention it deserves. In OpenFn, -each incoming request or transaction that gets processed is more than a piece of -data - it represents a vulnerable child in need of critical support, a farmer -managing their savings to make sure they can afford the next harvest. Lightning -provides users with a dashboard that allows them to monitor the health of their -integrations to make sure no request goes unprocessed. - -![Lightning audit interface](/img/lightning_audit.png) - -## Features - -##### General - -- Deploy Lightning via docker -- Create and delete user accounts -- Create new projects and assign users with different access levels to these - projects (owner/admin/editor/viewer) -- Transfer credential ownership to another user -- View an audit trail of all credential changes (superuser role) -- Set up SSO via an identity provider -- Generate and revoke API tokens -- List projects, jobs and runs via JSON API - -##### Workflow builder - -- Create a new workflow with a webhook or cron trigger -- Create and configure jobs for a workflow with any OpenFn adaptor and operation -- Create credentials through a form -- View all available operations for a given adaptor -- View the metadata from your external system (DHIS2 and Salesforce) -- View the input and output from the last run of each job in a workflow -- Run a job manually - -##### Runs history - -- View all runs grouped by workflow -- Search and filter runs by status, workflow and run logs -- Retry a workflow run from the start (first job) - -##### Project settings - -- Get notified via email on run failure -- Receive a daily, weekly or monthly digest of project activity -- View collaborators for a project -- Update a project name and description - -## Roadmap - -See the [Lightning Roadmap](/documentation/openfn-roadmap) for a detailed list -of features that are in the backlog, planned, and/or in development for the -OpenFn Digital Public Good. - -_You can follow our progress and track delivered features in our -[changelog](https://github.com/OpenFn/Lightning/blob/main/CHANGELOG.md)._ - -## Try it out - -:::danger Please note - -Lightning is still in Beta. - -::: - -You have 3 options for exploring OpenFn/Lightning: - -1. For quick viewing, visit [demo.openfn.org](https://demo.openfn.org/) and log - into our demo account with username: `demo@openfn.org` password: - `welcome123`. (NOTE that any changes made here are lost when the demo resets - every 24 hours. I.e., don't build things you'd like to keep.) -2. To get your own account and start building non-production workflows, register - for an account at [app.openfn.org](https://app.openfn.org/). -3. To install and run Lightning locally follow the instructions in the - [github README](https://github.com/OpenFn/Lightning). - -Go through the -[self-paced user interview](/blog/2023/04/13/lightning-beta#take-15-minutes-to-carry-out-our-user-test) -to learn how OpenFn Lightning works _and_ help us out with feedback in just 15 -minutes. - -## Guiding principles - -Lightning is developed in line with the -[principles for digital development](https://digitalprinciples.org/principles/) -and under the guidance of it's Open Source Steering Committee which you can read -about [here](https://openfn.github.io/governance/OSSC.html). - -On top of this, Lightning follows 4 key principles which determine how it should -be developed: - -### 1. Standards and compliance matter - -Lightning is part of the OpenFn Integration Toolkit which is a certified Digital -Public Good. It is fully open source and even has an Open Source Steering -Committee to make sure our users can influence the roadmap. - -Lightning workflows can be used to automatically enforce and apply data exchange -standards, such as FHIR and ADX. Lightning's design and roadmap are driven by -open standards, and will therefore provide a GovStack- and OpenHIE-compliant -workflow engine. Learn more via the following resources: - -- [Watch this video](https://youtu.be/PTRRZBYtqyc) to learn how OpenFn is an - OpenHIE-compliant workflow engine. -- Check out OpenFn's entry in the - [OpenHIE Reference Technologies page](https://wiki.ohie.org/display/documents/Reference+Technologies). -- Explore the OpenFn-Instant OpenHIE - [reference demo implementation](/documentation/instant-openhie). -- Learn more about the GovStack - [Workflow Building Block](https://govstack.gitbook.io/bb-workflow/2-description) - specification. - -### 2. Interoperability is an ongoing process - -Anyone that has worked on integration projects in the past is well aware that -integrations do break. No matter how well designed they are, the fact is that -**they connect multiple systems that all change over time**: new API versions -get released, data models change, IDs, codes and mappings change, data standards -are updated and the processes themselves evolve. This is why Lightning will -include: - -Enhanced testing and debugging: - -- Save data from workflow runs as test data for robust workflow testing of edge - cases -- View the input and output for each step in a workflow to easily identify where - an error occurred -- Throw custom errors to improve API messages (adaptors) -- Add custom logic to handle a workflow step failure (fail triggers) - -First of class monitoring: - -- Get notified on run failures -- View the status of every run -- Search workflow runs by input/output data and logs -- Filter workflow runs by status, workflow name and date - -Re-processing functionality: - -- Bulk reprocess workflow runs after updating workflow steps to course-correct - if a workflow has been running with flawed logic - -### 3. Collaboration is key - -On one hand, the users that understand what processes need automating are (more -often than not) business analysts, not developers. They’re the experts on what -needs to happen when and where, and they’re very capable of planning out -integrations and putting together mapping specifications and bpmn flows. - -On the other hand, integrations often require custom logic that cannot be -simplified through low-code and therefore must be implemented by software -engineers. - -That’s why **the best integrations are built when non-technical users and -developers collaborate**. Lightning is being developed to bridge the gap between -non-technical and technical users through: - -Intuitive, user-friendly user interface for non-technical users: - -- Understand a workflow in a visual, human-readable format (abstract away from - code to make workflows understandable to non-technical users) -- Build credentials through a form interface (remove the need to read through - confusing API documentation) -- Build API requests through a form interface -- Save mappings used in workflows as constants so they can be easily viewed and - edited without needing to read code -- Clear documentation for users to learn how to plan and build integrations - -Projects-as-code and CLI for a developer interface: - -- Export, import and configure projects as code in the code editor of your - choice -- Run, test and deploy projects through a command line interface -- Review and track changes through version control - -Collaboration functionality: - -- Track changes through version control -- Rollback to a previous version -- Get notified when a workflow is changed -- Share a link to a specific workflow error on the runs history page -- Share a link to a specific workflow step within the builder -- Add collaborators as view-only user or editor to a project -- Audit all changes made to credentials - -### 4. It’s not "just" a request or a piece of data, it’s a person - -OpenFn specializes in integration tooling for the health and humanitarian -sector. This means that behind every piece of data which comes in through a -request lies a person in need of critical services. This is why Lightning -focusses on: - -Accountability: - -- Credential audit trail -- Version control - -Security: - -- Secure credential management (encrypted at REST, credential secrets are - scrubbed from logs, secure credential sharing -- Zero-retention pipelines -- Role-based project access -- Additional authentication rules for webhooks - -## Security - -OpenFn treats security as a top priority, and is trusted to handle information -of the most sensitive nature (for example UNICEF’s child case data). - -To increase transparency and accountability around security, as well to help -other digital public goods think through key aspects of their own organizations’ -security postures, below is a list of the **key aspects of our own security -program**. - -### Organizational security practices - -To ensure a positive security posture at OpenFn, we: - -- Conduct Employee IT security onboarding training & policy -- Run monthly security standups with the whole team -- Conduct an annual security review informed by the OWASP ASVS - -### DevSecOps - -To ensure best practices in our code we: - -- Monitor dependency vulnerabilities via Github’s - [dependabot](https://github.com/features/security) -- Perform static code analysis on each commit with - [Sobelow](https://sobelow.io/) -- Ensure code is clean and standardised through preflight checks -- Monitor code coverage of unit tests and integration tests with Codecov - -### Roles and permissions - -Lightning provides identity and access management for users via various roles -and permissions which determine what level of access they have for resources -across projects and instances (i.e., deployments). - -Lightning has 2 types of access levels: - -1. Instance-wide access levels are managed via an attribute on the `user` - object: - -- **Superusers** are the administrator of the Lightning instance. They can - manage projects and users, configure authentication providers and view the - audit trail. -- **Users** are normal Lightning users. They can manage their own account and - credentials, and have access to projects they are added to. - -2. Project-wide access levels - -- A project **viewer** can view the resources of a project in read-only mode and - configure their own project digest and failure alerts. -- A project **editor** can view, create and edit the jobs and workflows of a - project they have access to, as well as run and rerun jobs. -- A project **admin** has administration access to project members. They can - edit the name and description as well as delete a project. -- A project **owner** can delete a project. - -### Application security - -Lightning is designed to: - -- Scrub credential data from run logs -- Encrypt credentials at REST -- Track credential changes through an audit trail -- Encrypt passwords -- Enforce access controls with deny by default -- Allow users to differentiate between staging and production credentials -- Enable secure credential transfer across users -- Purge credentials and user data on account deletion -- Allow administrators to configure SSO through an identity provider - -### Data residency - -OpenFn Lightning is fully open source and can be deployed in any country. We -offer high-availability managed deployments that are localized to any GCP or AWS -location—guaranteeing that no data ever leaves the selected country. - -### Implementation guidance and recommendations - -To help our users adopt best practices when it comes to the design of their -integrations, we’ve published a -[Security Guidebook for data integration implementations](/documentation/getting-started/security). - -## Get involved - -We are building out in the open, follow our progress on -[Github](https://github.com/OpenFn/lightning) by clicking ‘Watch’ to track -updates and new releases. Ongoing discussions with our Open Source Steering -Committee about Lightning are documented on our -[community forum](https://community.openfn.org/c/ossc/15). Your feedback and -comments are welcome there. If you would like to become a beta user or learn -more about Lightning, book in a call with our product manager here: -https://calendly.com/amber-openfn/short-call. - -![Lightning preview](/img/lightning_preview.png) - -## Lightning FAQ - -#### I can see that Lightning was built recently, is it new? And if so, how can I trust it? - -The Lightning repository may be new, but the technology isn’t. We’ve built out -Lightning by porting the tried and tested code from our proprietary platform. In -other words, Lightning is built with code that has been used in production by -governments and NGOs since 2015 and already handles tens of millions of -transactions a year. Software becomes more robust over time - the more it’s -used, the more edge cases are uncovered and bugs fixed. Over the past 7 years, -every time a bug has come up in our platform, we’ve fixed it and added a test -for it. By bringing over the same tests from platform to Lightning, we’re -essentially guaranteeing the same level of robustness by taking into account -every single edge case or bug that we have ever encountered. - -#### If Lightning was built by open-sourcing code from the OpenFn platform, how is it different? - -Under the hood, Lightning is the same as the OpenFn platform. Integrations are -made up of the same building blocks of triggers, adaptors and job expressions; -requests are executed, retried and reprocessed in exactly the same way. - -What changes in Lightning is how users _build and monitor_ their integrations. - -#### Can I run anything from the OpenFn platform in Lightning? - -Yes, integrations built out on the OpenFn platform are fully compatible with -Lightning. - -#### Who is Lightning for? - -Lightning is for anyone in the government or NGO space that needs to integrate -different systems. - -#### What will I lose by switching from platform to Lightning? - -Right now: version control, authentication rules on webhooks, and the other -features in our roadmap (we’re still in beta). - -Later: nothing - if a feature has proven important to our platform users, it -will be available in Lightning. If there is any feature you require in Lightning -to be able to switch over to it, speak up ! You can reach out to our product -manager Amber via [email](mailto:amber@openfn.org) or even better book some time -with her through her [calendar](https://koalendar.com/e/amber-rignell-openfn). - -#### When will Lightning Beta be ready? - -Lightning is currently in private Beta. You can register for an account on -[app.openfn.org](https://app.openfn.org/). diff --git a/docs/about.md b/docs/about.md deleted file mode 100644 index cff27eec12a..00000000000 --- a/docs/about.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -title: About ---- - -## Open Function Group - -Open Function Group is a team of ICT4D specialists that have been working -exclusively in data integration, automation and interoperability since 2014. - -We maintain OpenFn.org, the sector's leading integration platform as a service, -and a huge number of open-source workflow automation, data integration, and "ETL" tools which you can find on our [Github](https://www.github.com/openfn). - -The platform is trusted by some of the leading development organizations in the -world, including UNICEF, the WHO, the IRC, and Population Council. - -You can learn more about the people at Open Function Group -[here](https://www.openfn.org/leadership). diff --git a/docs/build/inbox.md b/docs/build/inbox.md deleted file mode 100644 index 389ae51b8d8..00000000000 --- a/docs/build/inbox.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -title: The Inbox -sidebar_label: Your Inbox ---- - -## How it works - -On the platform, each project has their own unique inbox URL, something like -`https://www.openfn.org/inbox/54804f1a-4a70-4392-97cb-1f350e98e9c8`. That big -string of numbers and letters is called a `UUID`. It's your address, and the -"place" on the web that you'll send data for processing by OpenFn if you're -doing real-time or "event-based" integration. - -Your project will always be listening, and whenever an HTTP request is received -at that URL, we'll respond with a `202/Accepted` and start processing the data -sent either in the `body` or the `parameters` of that request. - -## `202/Accepted vs 201/Created` - -You've probably heard of `200/OK` or other common "status codes", but the -difference between a `201` and a `202` is very interesting from an integration -perspective. - -The `201/Created` means that we've completed processing whatever data was sent -to us by the requester. Usually, this response is accompanied by a payload with -a new `id` for whatever resource what created. This is _not_ what OpenFn does, -instead we send a `202/Accepted` indicating that your request was acceptable and -we'll get to work. - -:::tip - -OpenFn sends a `202/Accepted` indicating that your request has passed our -initial validation (i.e. the data is valid `JSON` or parseable `XML` and the -inbox URL exists) and that we've enqueued it for processing. - -::: - -Behind the scenes, we've now a system of simple, durable queues that ensure we -don't "drop" this event at any point in time from here on forward. - -1. We'll load it into the database and soon it will appear as a new "message" - record in your "Inbox" page. -2. We'll check the triggers for all the active jobs in your project and if it - matches one of those triggers we'll send it to another queue for job running. -3. We'll make sure your project is configured properly and that you haven't - exceeded your usage limits. -4. We'll start executing a job run, which may itself may hundreds of unique HTTP - requests to other endpoints. -5. And finally we'll report back on the status of that run and soon it will - appear as a new "run" in your "Activity History" page. - -Depending on how many requests your job makes, how much data is being processed, -and the response time of your other systems, all of this could take quite some -time—anywhere from `200ms` to `20 minutes`! - -If the system that sends the data to OpenFn needs to know whether all the -operations in step 4 completed successfully (what do you count as a success with -these various custom actions, by the way?) you should consider implementing a -SAGA pattern, whereby after all this processing is complete you trigger another -request back to the initial system reporting on the downstream tasks. This can -be done in OpenFn with [Flow Triggers](/documentation/jobs/multiple-operations). - -## Synchronous vs. Asynchronous Processing - -On **OpenFn/platform**, processing is asynchronous by default. Multiple complex workflows may be initiated, error handling and notifications all happen downstream. -1. If you send data to OpenFn Inbox, you'll receive a `202` if successful (and `502` if we didn't receive your data/bad request). -2. We'll then load it into the database and soon it will appear as a new "message" - record in your "Inbox" page. -3. We'll check the triggers for all the active jobs in your project and if it - matches one of those triggers we'll send it to another queue for job running. -4. We'll make sure your project is configured properly and that you haven't - exceeded your usage limits. -5. We'll start executing a job run, which may itself may hundreds of unique HTTP - requests to other endpoints. -6. _If you want to then send an update back to the source system... you may configure another job to send requests and updates back to the triggering source system._ - -In **OpenFn/microservice** or using open-source tools, you could create a synchronous system. We've created a way to set up inbox endpoints as -"synchronous", meaning they'll actually hold a connection open _until_ all of -the processing above is completed, and then respond with a `2XX`, `4xx`, or -`5XX`. This is not recommended for high volume systems, but may be a requirement -for some implementations; the sprit of **OpenFn/microservice** is to give as much -control as possible to whoever is deploying it on their servers. diff --git a/docs/build/lightning-quick-start.md b/docs/build/lightning-quick-start.md deleted file mode 100644 index bb378967bd1..00000000000 --- a/docs/build/lightning-quick-start.md +++ /dev/null @@ -1,341 +0,0 @@ ---- -title: Quick-Start ---- - -This tutorial takes ~15 minutes to complete, and teaches you how to build -workfows with OpenFn Lightning. If you get stuck, post a question to our -[community forum](https://community.openfn.org/). - -## 1. Register - -Register for an account on -[app.openfn.org](https://app.openfn.org/users/register) and follow the link sent -to your inbox to confirm your email. - -\*If you already have an account, you can -[login](https://app.openfn.org/users/log_in). - -## 2. Understand the sample workflow - -Click on the 'sample workflow' created for you on registration. - -![lightning-workflows-page](/img/lightning-workflows-page.png) - -:::tip - -A **workflow** is a series of tasks to be carried out _automatically_ (i.e a -process that has been automated). - -::: - -The sample workflow pictured below formats and sends data from a source system -(KoboToolbox, a mobile data-collection app) to a destination system (DHIS2, a -health information management system). It automates patient registration by -taking a patient’s name and age and: - -1. checking if they are over 18 months old; -2. converting it to the same format as DHIS2; -3. uploading it to DHIS2. - -![lightning-sample-workflow](/img/lightning-sample-workflow.png) - -It is made up of 3 _jobs_. - -:::tip - -A **job** is an action to be carried out at a given point in time. It has a -trigger, an adaptor, a credential and a job expression which each define _when, -where, how_ and _what_ to do. - -::: - -Click on Job 3 to view more details about it in the setup and editor tab. - -### [SETUP TAB] - -The SETUP TAB is where you define the when, where and how of your job. - -![lightning_setup](/img/lightning_setup.png) - -**When: trigger** - -The trigger defines when an action should happen. It can be one of the -following: - -- When data is sent to OpenFn Lightning from an external system: **_webhook_** -- At a recurring point in time: **_cron_** -- When the job which comes before it in the workflow succeeds: **_on success_** -- When the job which comes before it in the workflow fails: **_on failure_** - -If you **never** want the job to run, you can disable it by unselecting the -'Enabled' checkbox. - -:::tip - -The trigger for the first job in a workflow will always be either a 'cron' or -'webhook' trigger. All the other jobs will have a trigger of 'on success' or 'on -failure'. - -::: - -**Where: adaptor** - -The adaptor is what helps you communicate with and perform actions in a -particular system. In OpenFn, you can carry out an action in the following -systems: - -- In OpenFn: OpenFn or common adaptors -- In an external system OpenFn has an adaptor for: commcare, DHIS2, google - sheets, kobotoolbox ... -- In any other external system which has an API: http adaptor - -**How: credential** - -Credentials define _how_ a Job is able to perform an action on your behalf, just -as you would need to cover logging in if you were explaining how to carry out an -action manually. - -:::tip - -If you are performing an action in an external system, you'll need to select -_the same_ credential type as your adaptor. - -::: - -### [INPUT TAB] - -The INPUT TAB is where you can see examples of data that has been sent to your -job during previous runs. - -In job 3, we'll be using the data values that are in `names` which are -`"Wycliffe"` and `"Orao"` in this example. Can you see them? - -![lightning_input_data](/img/lightning_input_data.png) - -:::tip - -The _input_ data of a job can be accessed through state. For example, if you -want the `names` values from an input, you can access it at `state.names`. - -::: - -### [EDITOR TAB] - -The EDITOR TAB is where you define _what_ the job should do and which data from -state (which contains your input) to use. - -:::tip - -When you need to use data that comes from your webhook trigger (data sent from -your external system), cron trigger, or a previous job you can find it in -`state`. Learn more [here](https://docs.openfn.org/documentation/jobs/state/). - -::: - -![lightning_editor_1](/img/lightning_editor_1.png) - -In this job, we're using the `names` data from state (which we saw in the Input -tab). - -**What: Job expression** - -The job expression defines what action to carry out and which data values to -use. - -It gets added from the adaptor documentation _below_ the editor as an example -operation, and is then configured to use specific values from the state input -data. (see image below for details) - -![lightning_editor](/img/lightning_editor.png) - -## 3. Run the sample workflow - -:::tip - -A workflow will run when the trigger from the first job (represented as the -first node on the canvas) is called. - -::: - -In the case of the Sample Workflow, this is when data is sent to the webhook -URL. There are three ways of doing this. - -Follow the instructions from _one_ of the options below to run your workflow. - -### Option 1: Manually send data to your first job trigger - -Click on the first job in your workflow, then head to the input tab. Paste the -data below into the `custom input`, then click `run`. - -```json -{ - "data": { - "age_in_months": 19, - "name": "Wycliffe Gigiwe" - } -} -``` - -![lightning_manual_run](/img/lightning_manual_run.png) - -You should now be able to -[see your request on the history page](#4-check-your-request-got-processed-correctly). - -:::tip - -When a job is run, OpenFn adds the input into state (used to get data values in -the job expression), along with the credentials which get added to -configuration. - -::: - -### Option 2: Send data through a curl request - -You can also send data to a webhook URL by making a curl request in your -terminal. - -Copy your webhook URL by clicking on the first node of your workflow, then use -it to replace `YOUR_WEBHOOK_URL` in the command below and run it in your CLI. - -```sh -curl -H 'Content-Type: application/json' \ - -d '{"age_in_months": 19, "name": "Wycliffe Gigiwe"}' \ - -X POST \ - YOUR_WEBHOOK_URL -``` - -You should get a response that looks like this, and be able to -[see your request on the history page](#4-check-your-request-got-processed-correctly). - -```json -{ - "attempt_id": "3602a2e6-cd01-4b48-bfa9-5237e7393c90", - "run_id": "fdebd5a9-3578-4bfd-945e-12e0a24e8c6a", - "work_order_id": "b1899b6f-e420-479f-a6ae-8641189764cd" -} -``` - -### Option 3: Send data from your external system - -:::tip - -You can trigger a workflow from an external system by configuring it's REST -services to send data to your trigger webhook URL. - -::: - -In the case of our Sample Workflow, we're using KoboToolbox as an external -system. - -[Log into](https://kf.kobotoolbox.org/accounts/login/#/) our KoboToolbox demo -account with _username: openfn_demo and password: openfn_demo_. Select the form -you’d like to connect ('Lightning sample workflow') and go to Settings -> REST -services -> Register a new service. - -![kobo](/img/2.3_kobo_rest.png 'Register a REST service with Kobo') - -Set the service name to OpenFn and the URL to the webhook URL (you can copy is -from the first node on your workflow). - -![kobo](/img/2.4_kobo_rest.png 'Set the REST service URL to your OpenFn inbox URL') - -Your form should now be configured to send data to the webhook trigger for your -first job whenever a response is submitted. We can test this out by submitting -some form responses at Form -> Open. - -![kobo form](/img/2.5_open_kobo_form.png 'Open a kobo form') - -Once you've made a form submission, you should be able to -[see your request on the history page](#4-check-your-request-got-processed-correctly). - -## 4. Check your request got processed correctly - -:::tip - -The history page shows you each **work order** or _request for data to be -processed_. - -::: - -Now that you have run your workflow, head to the history page to see the work -order. You'll see it has a status of 'Success' which means it got processed -correctly. - -![lightning_history](/img/lightning_history.png) - -Click on the chevron next to the status to expand it and see each job run. - -![lightning-history_expanded](/img/lightning_history_expanded.png) - -## 5. Make a run that fails, then edit the job and rerun it to make it succeed - -From your workflow page, run the job manually with a patient that is 18 months -old using the data below. - -```json -{ - "data": { - "age_in_months": 18, - "name": "Njoroge Orao" - } -} -``` - -Head to the history page and see that the work order has a status of 'Failure'. -This is because the patient is **not** older than 18 months. - -![lightning_history_failure](/img/lightning_history_failure.png) - -Let's say we made a mistake and _actually_ wanted to register any patient that -is _**both**_ 18 months old _**and**_ above. We want to edit the job logic and -reprocess the request. - -Head to the Editor tab in Job 1 to update the logic by changing the if statement -from `> 18` to `<= 18`. - -Your Job expression should now be the following: - -```js -fn(state => { - if (state.data.age_in_months >= 18) { - console.log('Eligible for program.'); - return state; - } else { - throw 'Error, patient ineligible.'; - } -}); -``` - -Make sure to click save, then head back to your history page and find the work -order you want to reprocess. You can search for "Njoroge Orao" in the search bar -to find it. - -Expand the work order, and click the 'rerun' button next to the first job run. - -![lightning_retry](/img/lightning_retry.png) - -You'll see a new **attempt** created in the same work order, which now succeeds. -The work order status also gets updated to the status of the last attempt to -show 'Success'. - -![lightning_new_attempt](/img/lightning_new_attempt.png) - -Rerun the same work order, this time from 'Job 3 - Upload to DHIS2'. You'll see -the runs for Job 1 and 2 get copied over to the new attempt, so that their -output can be used for the input of Job 3. - -![lightning_rerun_downstream_job](/img/lightning_rerun_downstream_job.png) - -:::tip Note - -When you rerun a workflow from a downstream job, the previous job runs are -copied over to the new attempt, so you can still see where the input from your -downstream job came from. - -::: - -You're all set! If you made it to the end of this tutorial, you should be -familiar with the key concepts you need to start building your own workflow. -Give it a go, and don't forget to post on our -[community forum](https://community.openfn.org/) if you get stuck - or to let us -know what you built. diff --git a/docs/build/triggers-cron.md b/docs/build/triggers-cron.md new file mode 100644 index 00000000000..abec1ba55b1 --- /dev/null +++ b/docs/build/triggers-cron.md @@ -0,0 +1,5 @@ +--- +title: Cron Triggers +--- +# Cron Triggers +Incl. cron expressions diff --git a/docs/build/triggers-webhook.md b/docs/build/triggers-webhook.md new file mode 100644 index 00000000000..3f8e11556ba --- /dev/null +++ b/docs/build/triggers-webhook.md @@ -0,0 +1,5 @@ +--- +title: Webhook Triggers +--- +# Webhook Triggers +Incl. authentication methods diff --git a/docs/build/triggers.md b/docs/build/triggers.md index d6d2db598c6..74844d9a3ac 100644 --- a/docs/build/triggers.md +++ b/docs/build/triggers.md @@ -2,9 +2,8 @@ title: Triggers --- -Triggers are responsible for starting job runs automatically. They come in 4 -types. The most common are "message filter" triggers, but there are also "cron" -triggers, "flow" triggers, and "fail" triggers. +Triggers are responsible for starting workflows automatically. They come in 2 +types: "cron" triggers and "webhook event" triggers. ## Trigger types diff --git a/docs/build/tutorial.md b/docs/build/tutorial.md new file mode 100644 index 00000000000..8564566e0c8 --- /dev/null +++ b/docs/build/tutorial.md @@ -0,0 +1,10 @@ +--- +title: Workflow Tutorial +--- +# Tutorial: Creating your first workflow + +:::warning Under construction + +This docs page is under construction. Check back later for the complete docs, or check out the Docs Version "Platform (v1)". + +::: diff --git a/docs/build/jobs.md b/docs/build/workflows.md similarity index 99% rename from docs/build/jobs.md rename to docs/build/workflows.md index 6ae92379689..659e3443d2c 100644 --- a/docs/build/jobs.md +++ b/docs/build/workflows.md @@ -1,5 +1,5 @@ --- -title: Introduction to Jobs +title: What are "workflows"? --- A job defines the specific series of "operations" (think: tasks or database diff --git a/docs/openfn-roadmap.md b/docs/contributing/openfn-roadmap.md similarity index 99% rename from docs/openfn-roadmap.md rename to docs/contributing/openfn-roadmap.md index c5a53b6f4e1..4c39eaf3c8c 100644 --- a/docs/openfn-roadmap.md +++ b/docs/contributing/openfn-roadmap.md @@ -3,6 +3,12 @@ title: OpenFn Roadmap sidebar_label: OpenFn Roadmap --- +:::warning Under construction + +This docs page is under construction. Check back later for the complete docs, or check out the Docs Version "Platform (v1)". + +::: + ## OpenFn Roadmap This page details the planned roadmaps for the key products in the OpenFn diff --git a/docs/roadmap.md b/docs/contributing/roadmap.md similarity index 100% rename from docs/roadmap.md rename to docs/contributing/roadmap.md diff --git a/docs/style-guide.md b/docs/contributing/style-guide.md similarity index 84% rename from docs/style-guide.md rename to docs/contributing/style-guide.md index cffd4830b3d..b3bde792e12 100644 --- a/docs/style-guide.md +++ b/docs/contributing/style-guide.md @@ -253,45 +253,3 @@ This is a caution This is a warning ::: - -## Tabs - -```mdx-code-block -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -``` - -Note how we import tabs first, then use them as below: - -```jsx -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - - - This is an apple 🍎 - This is an orange 🍊 - This is a banana 🍌 -; -``` - -```mdx-code-block - - This is an apple 🍎 - This is an orange 🍊 - This is a banana 🍌 - -``` diff --git a/docs/writing-code.md b/docs/contributing/writing-code.md similarity index 100% rename from docs/writing-code.md rename to docs/contributing/writing-code.md diff --git a/docs/writing-docs.md b/docs/contributing/writing-docs.md similarity index 100% rename from docs/writing-docs.md rename to docs/contributing/writing-docs.md diff --git a/docs/core.md b/docs/core.md deleted file mode 100644 index be0a0607041..00000000000 --- a/docs/core.md +++ /dev/null @@ -1,58 +0,0 @@ ---- -title: Core ---- - -:::caution Core reaching EOL in 2023. - -OpenFn/core is currently still being used by the v1 platform (www.openfn.org) -but is reaching end-of-life in 2023. - -::: - -## What is core? - -Core is the central job processing program used in the OpenFn platform. It's -what actually executes `jobs` with `state` and `adaptors` to do work for -governments and NGOs all over the world. - -## Where is it used? - -Core is used in OpenFn v1 (the web platform) and by developers who want to test -job execution on their local machines. It's _not_ used in Lightning (OpenFn v2) -which instead makes use of the new runtime. For a local developer experience -using the new runtime, check out [CLI](/documentation/cli). - -## Why might I want to use it now? - -If you've got jobs running on OpenFn v1 and want to test them locally, core will -give you the exact same job running experience as you see on the web. This can -be incredibly helpful for debugging. - -:::tip Using the new CLI. - -If you're a new OpenFn user and want to build or test jobs for Lighting (v2) and beyond in 2023, use the new [CLI](/documentation/cli) instead! - -::: - -## How do I use it? - -Check out the official documentation on -[Github](https://github.com/OpenFn/core). - -The tl;dr: is that you execute jobs from the command line by passing in an -expression, state, and the path to an adaptor. - -```sh -npm install @openfn/core -core execute -l ../language-http.Adaptor -e ./some-exprsesion.js -s ./some-state.json -``` - -The full options are: - -```sh --l, --language resolvable language/adaptor path [required] --e, --expression target expression to execute [required] --s, --state Path to initial state file. [required] --o, --output Path to write result from expression --t, --test Intercepts and logs all HTTP requests to console -``` diff --git a/docs/deploy/options.md b/docs/deploy/options.md index 39841b8417b..be539f9f6a5 100644 --- a/docs/deploy/options.md +++ b/docs/deploy/options.md @@ -11,7 +11,7 @@ outside your country's borders. :::success Portability -Because of OpenFn's [portability specification](/portability.md) and open-source +Because of OpenFn's [portability specification](/documentation/next/deploy/portability) and open-source deployment tools you can transition between these various pathways at any time. We're committed to a **no vendor lock-in** experience. diff --git a/docs/portability-versions.md b/docs/deploy/portability-versions.md similarity index 100% rename from docs/portability-versions.md rename to docs/deploy/portability-versions.md diff --git a/docs/portability.md b/docs/deploy/portability.md similarity index 100% rename from docs/portability.md rename to docs/deploy/portability.md diff --git a/docs/design/api-discovery.md b/docs/design/api-discovery.md new file mode 100644 index 00000000000..23fe8783d8a --- /dev/null +++ b/docs/design/api-discovery.md @@ -0,0 +1,7 @@ +--- +sidebar_label: API Discovery +title: API Discovery for Workflow Design +--- + +# Discovering APIs to inform your workflow automation design +How to analyze API docs and draft technical diagrams (technical WF diagram, update your solution architecture diagram) \ No newline at end of file diff --git a/docs/design/design-workflow.md b/docs/design/design-workflow.md new file mode 100644 index 00000000000..8eddc46cb8d --- /dev/null +++ b/docs/design/design-workflow.md @@ -0,0 +1,7 @@ +--- +sidebar_label: Workflow Design +title: Design your first workflow to automate +--- + +# Designing your first OpenFn workflow +How to diagram; intro to BPMN standard & links to resources \ No newline at end of file diff --git a/docs/design/discovery.md b/docs/design/discovery.md new file mode 100644 index 00000000000..74ce37f13e2 --- /dev/null +++ b/docs/design/discovery.md @@ -0,0 +1,7 @@ +--- +sidebar_label: Discovery & Scoping +title: Discovery & Scoping for OpenFn Projects +--- + +# Discovery & Scoping for OpenFn Projects +Share key scoping questions and introduce solution architecture diagram \ No newline at end of file diff --git a/docs/design/mapping-specs.md b/docs/design/mapping-specs.md new file mode 100644 index 00000000000..3638c8c5ecf --- /dev/null +++ b/docs/design/mapping-specs.md @@ -0,0 +1,7 @@ +--- +sidebar_label: Mapping Specifications +title: Writing Data Element Mapping Specifications +--- + +# Mapping data elements to define data integration & automation rules +Mapping template & process overview \ No newline at end of file diff --git a/docs/design/design-quickstart.md b/docs/design/overview.md similarity index 96% rename from docs/design/design-quickstart.md rename to docs/design/overview.md index 21791e27d2d..ea1396fc902 100644 --- a/docs/design/design-quickstart.md +++ b/docs/design/overview.md @@ -1,8 +1,15 @@ --- -title: Integration Design +sidebar_label: Design Process +title: Design Process for OpenFn Solutions --- +:::warning Under construction -# Getting Started on Integration Design +This docs page is under construction. Check back later for the complete docs, or check out the Docs Version "Platform (v1)". + +::: + +# Getting started with workflow automation design for OpenFn projects +Overview of design process and key outputs/artefacts... **Integration design begins with the functional or business requirements (not the technical bits).** Therefore, you do not need to be an IT consultant or diff --git a/docs/design/when-to-integrate.md b/docs/design/when-to-integrate.md deleted file mode 100644 index 2055f4d6450..00000000000 --- a/docs/design/when-to-integrate.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -Title: When to Integrate ---- - -This article is a stub... it's coming soon. - -In the meantime, check out Aleksa Krolls' -[Three Questions To Ask](articles/2020/06/24/three-questions-to-ask). diff --git a/docs/design/workflow-specs.md b/docs/design/workflow-specs.md new file mode 100644 index 00000000000..70a0cbb67c4 --- /dev/null +++ b/docs/design/workflow-specs.md @@ -0,0 +1,7 @@ +--- +sidebar_label: Workflow Specifications +title: Writing Workflow Automation Specifications +--- + +# Writing specifications for workflow automation solutions +Specifications checklist; hand off to implementers \ No newline at end of file diff --git a/docs/developers/build-with-api.md b/docs/developers/build-with-api.md new file mode 100644 index 00000000000..20743d4224b --- /dev/null +++ b/docs/developers/build-with-api.md @@ -0,0 +1,7 @@ +--- +id: build-with-api +title: Build with the OpenFn API +sidebar_label: OpenFn API +slug: /build-with-api +--- +Build with API (link to Github) / Build an Adaptor \ No newline at end of file diff --git a/docs/cli.md b/docs/developers/cli.md similarity index 99% rename from docs/cli.md rename to docs/developers/cli.md index 270cdfef440..db0a951c8e6 100644 --- a/docs/cli.md +++ b/docs/developers/cli.md @@ -62,7 +62,7 @@ please see the documentation for **[@openfn/core](/documentation/core)** and [kinsta.com/blog/how-to-install-node-js/](https://kinsta.com/blog/how-to-install-node-js/) 3. Have a basic understanding of OpenFn—check out jobs and adaptors, at least, - in the [OpenFn Concepts](getting-started/terminology) of this site. + in the [Intro section](documentation/next) of this site. 4. Install the OpenFn CLI with `npm install -g @openfn/cli` ## Walkthrough & Challenges diff --git a/docs/jobs/each.md b/docs/developers/each.md similarity index 100% rename from docs/jobs/each.md rename to docs/developers/each.md diff --git a/docs/jobs/editing_locally.md b/docs/developers/editing-locally.md similarity index 100% rename from docs/jobs/editing_locally.md rename to docs/developers/editing-locally.md diff --git a/docs/jobs/errors.md b/docs/developers/errors.md similarity index 100% rename from docs/jobs/errors.md rename to docs/developers/errors.md diff --git a/docs/for-devs.md b/docs/developers/for-devs.md similarity index 100% rename from docs/for-devs.md rename to docs/developers/for-devs.md diff --git a/docs/jobs/job-design-intro.md b/docs/developers/job-design-intro.md similarity index 100% rename from docs/jobs/job-design-intro.md rename to docs/developers/job-design-intro.md diff --git a/docs/jobs/job-studio.md b/docs/developers/job-studio.md similarity index 100% rename from docs/jobs/job-studio.md rename to docs/developers/job-studio.md diff --git a/docs/jobs/limits.md b/docs/developers/limits.md similarity index 100% rename from docs/jobs/limits.md rename to docs/developers/limits.md diff --git a/docs/jobs/multiple-operations.md b/docs/developers/multiple-operations.md similarity index 100% rename from docs/jobs/multiple-operations.md rename to docs/developers/multiple-operations.md diff --git a/docs/jobs/operations.md b/docs/developers/operations.md similarity index 100% rename from docs/jobs/operations.md rename to docs/developers/operations.md diff --git a/docs/jobs/state.md b/docs/developers/state.md similarity index 100% rename from docs/jobs/state.md rename to docs/developers/state.md diff --git a/docs/jobs/understanding.md b/docs/developers/understanding.md similarity index 100% rename from docs/jobs/understanding.md rename to docs/developers/understanding.md diff --git a/docs/jobs/working_with_branches.md b/docs/developers/working-with-branches.md similarity index 92% rename from docs/jobs/working_with_branches.md rename to docs/developers/working-with-branches.md index b2e254a2a5c..72343f77eb3 100644 --- a/docs/jobs/working_with_branches.md +++ b/docs/developers/working-with-branches.md @@ -2,9 +2,9 @@ title: Working with branches --- -In the [Editing jobs locally](/documentation/jobs/editing_locally) section we -walked through the process of creating and adding your changes to the `main` -branch of a project. +In the [Editing Workflows Locally](editing-locally) +section we walked through the process of creating and adding your changes to the +`main` branch of a project. However, most code change workflows involve sharing and reviewing changes before deployment. You can do this by creating, testing and sharing your changes on a diff --git a/docs/devtools/home.md b/docs/devtools/home.md deleted file mode 100644 index df6f670975b..00000000000 --- a/docs/devtools/home.md +++ /dev/null @@ -1,361 +0,0 @@ ---- -title: Devtools ---- - -:::caution Devtools is deprecated - -Please note that [OpenFn/devtools](https://github.com/OpenFn/devtools) are being -deprecated and replaced by [OpenFn/cli](/documentation/cli). Learn more about -CLI -[github.com/OpenFn/cli/](https://github.com/OpenFn/kit/tree/main/packages/cli) - -::: - -OpenFn/Devtools is a set of tools for writing & testing job expressions, -managing OpenFn projects, and developing new adaptors. It's how most people work -with OpenFn from their own command lines, outside of OpenFn.org, Microservice, -or Lightning. - -:::info Are you a developer? - -The [Devtools](https://github.com/OpenFn/devtools) repo is a collection of bash -and Node scripts, as well as a _suggested_ (but not necessary) directory -structure for working with OpenFn jobs and adaptors. - -To run OpenFn jobs locally, you only need [Core](/documentation/core) and at -least one adaptor, e.g. [language-http](https://github.com/OpenFn/language-http) -and you may prefer to install core globally via `npm install -g @openfn/core` - -::: - -## Up and running - -1. Install [git](https://git-scm.com/downloads) and - [Node.js](https://nodejs.org/en/download/) (version 14 or greater) - -2. Clone and install devtools to setup core, language-common, and language-http - using either SSH or HTTPS: - -```mdx-code-block -import CodeBlock from '@theme/CodeBlock'; - - - - - git clone git@github.com:OpenFn/devtools.git{'\n'} - cd devtools{'\n'} - ./install.sh ssh - - - - - git clone https://github.com/OpenFn/devtools.git{'\n'} - cd devtools{'\n'} - ./install.sh https - - - -``` - -_Note: If you get a "permission denied" message when running `./install.sh`, try -`run chmod +x ./install.sh ` then retry the install command._ - -## Usage - -Execute takes: - -1. `-l [language-package].Adaptor`: The adaptor being used -2. `-e [expression.js]:` The expression being tested -3. `-s [state.json]`: The message `data: {...}` and credential - `configuration: {...}` -4. `-o [output.json]`: The file to which the output will be written - -### Run a job using bash - -```sh -~/devtools/core/bin/core execute \ - -l ~/devtools/adaptors/language-http \ - -s ./tmp/state.json \ - -o ./tmp/output.json \ - -e ./tmp/expression.js -``` - -### More on Devtools - -```mdx-code-block -import ReactPlayer from 'react-player'; - - -``` - -### Install a specific adaptor version - -To install specific adaptors, run -`./install.sh ${ssh || https} language-${name}` - -When you install a new adaptor, the latest version will be enabled by default. -To switch the adaptor version when running jobs locally, in the root of the -adaptor directory, run: - -`git checkout tags/v2.4.15` (substitute `2.4.15` with the adaptor version you -want) - -### The `--test` option - -```sh -~/devtools/core/bin/core execute \ - -l ~/devtools/adaptors/language-http \ - -s ./tmp/state.json \ - -o ./tmp/output.json \ - -e ./tmp/expression.js \ - --test -``` - -This intercepts all HTTP requests and displays the request information for -debugging. - -#### `.FakeAdaptor` - -Adaptors may provide dummy modules for testing. `language-salesforce` has a -built-in `.FakeAdaptor` which allows a user to test expressions on data without -sending them to a real Salesforce server. - -Instead of using `-l ./language-salesforce.Adaptor`, use -`-l./language-salesforce.FakeAdaptor` to test expressions offline: -`./core/bin/core execute -l ./language-salesforce.FakeAdaptor -s ./tmp/state.json -o ./tmp/output.json -e ./tmp/expression.js` - -#### Offline testing for other adaptors - -For most standard adaptors which make use of HTTP requests, you can add `--test` -to the execute command to intercept all HTTP requests and return a `200`. - -## Hands-on with devtools and the command line - -:::tip - -Check out this example workflow for using devtools in your day-to-day. - -::: - -1. `cd` in the folder containing the repo you're working on. -2. You can keep your job scripts anywhere, but store `state.json` and - `output.json` in a `tmp` folder. In our repos we always add the `tmp` - directory in our `.gitignore` file that tells Github to ignore the specified - paths. Make sure you have your `.gitignore` file and you know what's tracked - by Github and what's not. `state` and `config` may contain sensitive - configuration information and project data so never upload them to Github! -3. The devtools command is a mouthful. You can search your command line history - with `Ctl-r` and typing core to pull it up the devtools command. Notice that - it’s got line breaks and a flag for all the important bits… `-l` for - language-package (adaptor), `-s` for state, `-o` for output, and `-e` for - expression. You can also save your frequently used devtools commands in a - document and just copy-paste. -4. It's quick the change job names or the adaptor in the command. If you put all - your adaptors in the same folder `~/devtools/adaptors/language-_________` you - can quickly swap them in the command, as you can see in the video below. The - Backspace key deletes characters behind your cursor, Delete deletes them in - front. -5. You can use the TAB key to auto-complete the file path as you search for a - job. -6. Once you've changed a couple of characters for the adaptor and expression (in - the video `state` and `output` stayed the same because we're using the `tmp` - convention) press enter and see the results. - -![devtools](/img/devtools.gif) - -## Configure an OpenFn project - -The easiest way to configure a project is via the web interface (you can then -export or `openfn pull` the project as code) but you can also run -`./scripts/generate-project.js` helps you build a project config YAML -interactively, adding your triggers, credentials and jobs to the config. You can -read more about the config file -[here](https://openfn.github.io/microservice/readme.html#sample-configuration) - -If you choose `monolith` mode, all your job code will be included in the YAML. -In `URI` mode, you’ll get a config file with URI-s to your defined jobs. - -![Generate Project](/img/generate-project.gif) - -## Pre-Requisites - -1. [Node](https://nodejs.org/en/download/) is required to run jobs and use many - of the scripts in Devtools (e.g., `npm run build` is required after changes - to adaptors). - -2. A basic working knowledge of NodeJs, promises and asynchronous functions is - essential for writing adaptors. - -## Scripts - -Devtools comes with a collection of scripts to aid in setting up a development -environment for adaptor work, and include commands to quickly clone a large -number of adaptors, create tarballs of adaptors with only production -dependencies included, etc. - -For the kitchen sink, run: - -```sh -./install ssh -./scripts/bootstrap npm-install -``` - -In order to run the scripts, ensure you have cd'd into the project directory and -enter `./scripts/` - -### bootstrap - -Installs all adaptors in `repos` file to the `/adaptors` directory and prepares -the working directory. This needs to be run before running any of the other -scripts. Pass `npm-install` to run npm install for each adaptor also. - -`./scripts/bootstrap npm-install` - to clone, set up hooks and npm install in -each `./scripts/bootstrap`- to clone and set up hooks in each - -### generate-project.js - -`./scripts/generate-project.js` interactively generates a YAML project -configuration file that can be used both on the OpenFn platform and in OpenFn -microservice to define projects. - -### generate-doclets - -Iterates overs all language pack folder names found in the `repos` list and -creates a doclet json file in the `doclets` directory. - -### analyse-doclets - -Iterates overs all doclets found in `doclets` and gives a tree view of the -doclet structure using [jsdoc-query](https://github.com/OpenFn/jsdoc-query). - -## Building adaptors for platform - -All adaptor releases are built inside a `docker container`. The importance of -running the build and release process through a container is to standardize the -build environment across the team. While adaptors can be built and run on lots -of different operating systems and architectures, when we run the platform on -Kubernetes it expects linux boxes running x86... so that's where we build these -official releases. - -Here's how to build and release adaptors: - -1. Reopen your package in **dev-container** by typing `ctrl+shift+p` (or - `cmd+shift+p` on mac) and choosing **Remote-Container: Rebuild and Reopen in - Container**. -2. After the build is finished, open a terminal in vscode and run - `openfn-devtools release .` to build, tag, and push to - [npm](https://www.npmjs.com/). -3. Run `openfn-devtools package-release .` to package everything with production - dependencies and push to [Github](https://github.com/openfn). - -Depending on how you've configured your local environment and your VSCode -installation, you might encounter access issues preventing connections to NPM -and GitHub. - -### Troubleshooting - -There are a number of issues that you may encounter related to sharing settings -that are responsible for passing ssh keys and local configurations from your -host machine into the VSCode container. - -### Git config issues - -An issue can pop up about git config not set, To solve this, you should probably -set your email and name globally using the commands below: - -```sh -git config --global user.email "youremail@something.com" -git config --global user.name "Your Name" -``` - -### SSH key issues - -You may find that you are unable to access your `ssh` keys from inside the -container. - -:::warning Error - -permission denied (publickey) - -::: - -To solve this, first make sure the `ssh agent` is -[up and running](https://code.visualstudio.com/docs/remote/containers#_sharing-git-credentials-with-your-container). -In MacOS, it is running by default. On Linux you can start the agent using the -command - -```sh -eval $(ssh-agent -s) -``` - -Then you can add these line your `~/.bash_profile` or `~/.zprofile` (for Zsh) to -make it run by default. - -```sh -if [ -z "$SSH_AUTH_SOCK" ]; then - RUNNING_AGENT="`ps -ax | grep 'ssh-agent -s' | grep -v grep | wc -l | tr -d '[:space:]'`" - if [ "$RUNNING_AGENT" = "0" ]; then - # Launch a new instance of the agent - ssh-agent -s &> $HOME/.ssh/ssh-agent - fi - eval `cat $HOME/.ssh/ssh-agent` -fi -``` - -Next, run the command below to add your identity to the ssh agent: - -```mdx-code-block -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - - - - - ssh-add - - - - - ssh-add -A - - - -``` - -Finally, configure VSCode to share your local ssh keys with the dev container. -In VSCode, go to `Settings`, and in the search bar, type -`terminal.integrated.inherit`. You should see the option in the image below and -check it if it's unchecked. - -![vscode settings](/img/vscode-settings.png) - -### Github token sharing - -Our release process relies on a `GH_TOKEN` variable. Set up an -[access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) -in Github. - -In your `~/.bash_profile` or `~/.zshrc` file, export the newly created token by -adding this line: - -```sh -export GH_TOKEN= -``` - -## Using a new adaptor in an OpenFn/platform instance - -1. Add your release to the `scripts/install-lp` script. -2. Add the version number to `priv/adaptors.json`. -3. Add the `bodySchema` to `CredentialView.js`. diff --git a/docs/faqs.md b/docs/faqs.md deleted file mode 100644 index 3f5b54dd68b..00000000000 --- a/docs/faqs.md +++ /dev/null @@ -1,262 +0,0 @@ ---- -title: Frequently Asked Questions -sidebar_label: FAQs ---- - -Data integration, interoperability, and workflow automation can be confusing -subjects. Not to mention the fact that there are lots of different terms and -ways of talking about the same concepts. We get it. Here are a few questions -that come up a lot. - -## What is OpenFn? - -OpenFn is an **_integration platform as a service_**. This means our prime -directive is to move data quickly and securely between different software -systems. In most cases: - -1. A source application sends **messages** to your project’s **inbox** when - something happens. - -2. **Jobs** will be triggered, based on your **filters**, and use the data in - those messages to attempt specific actions in destination systems. - -3. The **logs** are recorded so you can see precisely what happened and when and - where it happened to take action in the event of a failed attempt—like - editing the job or even the source message and trying it again. - -## Who uses OpenFn? - -OpenFn is used by organizations big and small, but the individuals interacting -with the platform range from system administrators to Javascript developers. -With a basic understanding of Javascript, the flexibility of the platform is -almost limitless. - -## What is a `job`? - -OpenFn automation centers around jobs, which define the specific series of tasks -or database actions OpenFn should perform. They can be set to be activated -(triggered) at certain time intervals or when data matching specified criteria -is received. You can think of jobs as a set of instructions you might give a -data entry staff member (e.g., create a new Patient record in OpenMRS when a -form containing a newly registered client is received from CommCare, export data -to DHIS2 every week on Friday 11pm, send SMS with payment confirmation number -when payment confirmation message is received etc.). - -:::note - -Jobs are fully configurable and reusable. They can also be chained together to -create [multi-step automation](jobs/multiple-operations) flows, two-way syncs. -and to keep data consistent between multiple applications (using multi-app Saga -patterns). You can read more on two-way synching below. - -::: - -## What is a `run`? - -A run is each individual execution of a job. Imagine that a job is configured to -create a new patient in OpenMRS whenever a case is opened in CommCare. Over the -next week, if 5 cases are opened in CommCare, you’ll see 5 different runs of -this one job in OpenFn. If 4 runs are successful and one has failed, you’ll see -4 new patients in OpenMRS, and your system administrator will have been notified -that one of those patients couldn’t be created (or whatever more robust -error-handling you’ve set up will take place.) - -Note that there’s not always a 1-to-1 mapping between runs and the real-world -things you’re working with. I might define a job that gets all updated event -data from DHIS2 for the last 2 weeks and publishes it to a public map using -CartoDB. This job will be triggered at specified time intervals, every 2 weeks -in this case, and after a month, we’ll only see 2 runs in OpenFn (that’s one run -every two weeks). Each run will have succeeded or failed, and each one might -have processed thousands of events from DHIS2. - -:::note - -For one last example, a single form submission in Open Data Kit might trigger a -job that creates new contacts and attendance records in Salesforce. In this -case, you’ll find a run for each ODK form submission, but each run will create -lots of different records in Salesforce—specifically, at least one contact and N -number of attendance records, corresponding to the number of items in your ODK -form’s “attendance repeat group”. - -::: - -## Is OpenFn open-source? - -OpenFn is a suite of different technologies with different licenses. We have -built and maintain dozens of open-source data transformation and API wrapper -software packages. Those are, for the most part, licensed under the **LGPL** and -can be used freely to extract, transform, and load data from a command line, or -as part of another software application. - -OpenFn also hosts a proprietary web-application that ties these tools together -(www.openfn.org) into an out-of-the-box integration management platform. This -platform is open-core, providing the powerful ETL tools that sit at the heart of -the proprietary OpenFn.org iPaaS as free and open-source software (FOSS). All of -the jobs running on OpenFn.org, as well as all of the underlying adaptors, can -be run offline using our FOSS tools. - -:::note - -OpenFn will also soon offer an enhanced FOSS implementation option called -[OpenFn/microservice](https://openfn.github.io/microservice/). This FOSS -microservice approach is currently in development with funding from the -[DIAL Open Source Center](https://www.osc.dial.community/), -[Digital Square](https://digitalsquare.org/), and the -[FCDO](https://www.gov.uk/government/organisations/foreign-commonwealth-development-office) -(formerly DFID). - -Please note that this pathway does not provide the entire OpenFn platform as -free and open source software (FOSS). In situations where a particular partner -or government is unable to use the proprietary platform (though it can be -deployed on local servers with an unlimited use license), this approach ensures -that all jobs, triggers, and project configuration can be exported from -OpenFn.org and used, in conjunction with OpenFn's FOSS ETL tools to deploy a -microservices-style implementation which incurs zero licence costs and provides -the basic data processing that OpenFn's platform does. While at the outset there -will be no web interface and no ability to reprocess messages, etc., these -features could be built by partners in time to replace the features of the -OpenFn platform. I.E., none of the initial investment in OpenFn will be lost if -the partners choose to build their own, fully open-source integration platform -based on our powerful open-source ETL tools. - -::: - -## How much does OpenFn cost? - -### Design & implementation costs - -OFG offers a range of packages to ensure successful first-time implementations, -which include integration consulting, design, configuration, and -capacity-building. Typical engagements take 1-5 days to complete, and our most -popular package is the Integration QuickStart, in which we spend 1 week to -design and configure ~5 integration flows end-to-end and provide administrative -training to your staff for $5,000. - -### Ongoing costs - -OpenFn.org offers a free plan for users seeking to trial the platform or -implement projects handling low data volumes (up to 100 runs/month). Usage of -OpenFn.org, the proprietary integration-platform-as-a-service (iPaaS), incurs -ongoing costs, which are largely dependent on the expected data volumes to be -processed. OpenFn offers monthly subscriptions, enterprise licenses for annual -and multi-year agreements, as well as unlimited and local deployment options. -Contact enterprise@openfn.org to learn more and for a tailored cost estimate. - -There are also available DIY options, as well as bespoke training services to -develop your capacity to implement and manage OpenFn independently. - -## Can I trial the platform? - -Yes. As a matter of fact, you can use it for free, forever. - -OpenFn.org offers a free plan to all users -([sign up here](https://www.openfn.org/signup)). Try it out using OpenFn Docs, -or contact our team for a free consultation and help getting started. Change -your OpenFn plan at any time (no lock-in!), or contact enterprise@openfn.org to -learn more about annual, enterprise, and unlimited licenses. - -:::tip - -At low volumes, or for prototyping, you can use the hosted platform for free -forever. - -::: - -## How reliable is the hosted service? - -OpenFn has harnessed the extreme stability and scalability of Erlang to -coordinate these actions and provide users with email alerts, project management -tools, and an online job writing IDE. - -We constantly monitor our own status with independent infrastructure at -[status.openfn.org](https://status.openfn.org). You can subscribe to -notifications there or follow [@openfnstatus](https://twitter.com/openfnstatus). - -We've been delivering this service continuously since 2014. - -## Can OpenFn integrate with my custom app? - -Yes, OpenFn can integrate with _any_ application. - -If your technology has a REST endpoint or webhooks service, it will likely work -right out of the box. This covers most web applications (e.g., CommCare, Kobo, -ODK, DHIS2, Salesforce, MS Dynamics, MPesa, etc.). OpenFn can also integrate -with most databases, like Postgres, MySql, and Mongo, custom applications, -legacy government systems, and can even parse CSV files–so long as these can be -accessed from an online location. Read more about -[connecting source applications](source-apps), or check out the Apps page for -applications widely implemented. - -We offer pre-built connectors (called "adaptors") for our users' most popular -apps to make the integration setup quicker and more user-friendly when -connecting with these tools. For example, users can implement language-http to -send basic HTTP requests to any web application, or implement language-dhis2 to -automatically handle DHIS2 authentication and access helper functions like -fetchData()to export DHIS2 datasets. - -## Does OpenFn support two-way syncing? - -Yes, OpenFn can support two-way syncing of applications. Utilizing -[Flow Triggers](build/triggers#flow-triggers), OpenFn jobs can be chained -together to facilitate real-time two-way data sync, -[multi-step automation](jobs/multiple-operations) and data cleaning processes, -and complex branching logic. Users can also implement bi-directional data syncs, -as well as complex Saga Patterns to implement a transaction that spans multiple -applications by configuring webhooks in their endpoint applications and -performing updates in both systems when events take place in either. - -## Do I need to know how to code? - -No, but it helps to have written a formula in MS Excel! Many OpenFn users are -familiar with data, not development, and quickly get comfortable with OpenFn -jobs. - -If your project is leveraging an OpenFn adaptor (e.g., `language-dhis2`), you -have access to pre-built helper functions (e.g., `getPatient`, `update`) so that -you don’t need to write custom code, and rather can use OpenFn documentation or -existing job scripts to write your own job. See OpenFn Github for inspiration -and open-source job code shared by OpenFn users. You’ll notice that these -functions work in the same way that functions do in Excel… `sum(A1, A2, A3)` - -Jobs can be written and extended using raw Javascript for advanced data cleaning -and manipulation. Therefore, you may want to implement Javascript to achieve -specific requirements or to extend existing OpenFn adaptors, which are -open-source! - -## Where is my data stored? - -OpenFn is a middleware provider rather than a data storage system. We move -information from system A to system B, and integrations can be set up to be -compliant with GDPR, HIPAA, and other policies. To make auditing and -reprocessing easy, OpenFn temporarily stores message data and job run history, -but we're not the single source of truth nor the final resting point for these -data. When organizations choose to use our hosted OpenFn platform at OpenFn.org, -no data processed by OpenFn is stored locally and our platform runs on the -Google Cloud Platform (GCP). Read more on our -[Compliance](https://www.openfn.org/compliance) page. - -OpenFn.org currently offers hosting on U.S. and Swiss-based cloud servers. -OpenFn local and in-country cloud deployments are also available upon request. -Contact enterprise@openfn.org to learn more. - -## Is my data secure? - -Yes, OpenFn prioritizes security, stability, and scalability (what we call -[S³](https://www.openfn.org/trust#s3)) above all else, and many of our users -implement OpenFn to comply with GDPR, HIPAA, and other policies. Read more on -our [Trust](https://www.openfn.org/trust), -[Compliance](https://www.openfn.org/compliance), and -[Privacy](https://www.openfn.org/privacy) pages. - -OpenFn.org runs on the Google Cloud Platform, an infrastructure protected by -more than 500 top experts in information, application, and network security. For -organizations with specific compliance and data governance requirements, OpenFn -can also be deployed on designated local- or cloud-infrastructure. - -## What if I have more questions? - -Open Function Group is a team of ICT4D and integration specialists, waiting to -help you get started. Click the chat icon in the bottom right hand corner of -this page to talk now. Or Email our team at admin@openfn.org, chat us on -OpenFn.org, or post a question in our -[Community Forum](https://community.openfn.org). diff --git a/docs/getting-started/commcare-project-walkthrough.md b/docs/getting-started/commcare-project-walkthrough.md deleted file mode 100644 index c8c4e0d76aa..00000000000 --- a/docs/getting-started/commcare-project-walkthrough.md +++ /dev/null @@ -1,306 +0,0 @@ ---- -title: - Walk-through - Syncing your CommCare form submissions to a PostgreSQL database ---- - -**Before starting this tutorial please make sure:** - -- You have signed up for [OpenFn.org](http://openfn.org) (it takes less than a - minute!) -- You have checked out our glossary and have an understanding of basic OpenFn - and API terminology. Check out the pages below to get started - - [OpenFn Concepts](/documentation/getting-started/terminology/) - - [A glossary for data integration](/documentation/getting-started/terminology/) -- You have a CommCare application with at least one form configured. This is - your source system. -- You have a PostgreSQL database configured. This is your destination system. - -**If you don’t have a CommCare application or PostgreSQL database setup, you can -also follow along with the prebuilt solution. Follow along at the links below:** - -1. [Mapping specifications document](https://docs.google.com/spreadsheets/d/1pi_oxImakhtaCCCIENkjTPZeuyWhpFEcNmH7hfvTBgo/edit?usp=sharing) -2. Commcare application to download: - - Username: testuser - - Password: 123 - -![install_cc_app](/img/install_cc_app.png) - -3. [OpenFn project](https://www.openfn.org/projects/commcare-demo/jobs) -4. [Public report that shows records in the PostgreSQL database](https://analytics.openfn.org/public/question/095449a9-5696-463c-a4fb-24614c9f08a5) - -## Getting started - -In this walkthrough, we will be setting up an **automatic data sync between -CommCare and a PostgreSQL database**. We will be syncing submissions coming from -a CommCare `Maternal and Newborn Health` application that has a -`Register a New Patient` form. - -:::tip - -Whenever a CommCare user registers a new patient, the patient details will -automatically be synced to an already configured PostgreSQL database to enable -real-time monitoring and analytics on data collected in the field. For example, -this database can quickly be connected to a dashboard that collects aggregate -data on patients registered! - -::: - -![cc-postgres](/img/cc-postgres.png) - -**This integration can be broken up into two parts:** - -1. Getting data from your source system to your OpenFn inbox so you can inspect - the data structure to inform the job design for part two -2. Transforming and loading this data to your destination system - -… let’s get started! - -## Getting data from CommCare - -**There are two ways to get your CommCare form submissions in your OpenFn inbox -to inspect the data, and to later map it to your destination system.** - -### Option 1: Webhook to forward cases and/or forms in real-time from CommCare to OpenFn using REST service - -CommCareHQ has a native data forwarding feature that provides a webhook/REST -service that can be pointed to the destination of your choice (i.e., your OpenFn -Inbox). When a webhook is configured, any Commcare forms submitted are -**_automatically forwarded_** to the designated endpoint, such as your OpenFn -inbox. After data forwarding is set up, it happens automatically, **_in -real-time for all forms and cases_**. Learn more about configuring a webhook -[here](/adaptors/commcare#webhook-forward-cases-andor-forms-from-commcare-to-openfn-using-rest-service). - -![option1](/img/option1.png) - -### Option 2: Extracting Commcare data via the REST API - -CommCare provides a robust -[REST API](https://confluence.dimagi.com/display/commcarepublic/List+Forms) for -extracting and loading data. This second option involves configuring a job in -OpenFn to fetch CommCare submissions via a `GET` HTTP request with parameters to -filter your data query. Follow along for how to set this job up! - -1. **Create a new project space, or open up an existing one where you have Admin - access.** - -![create_new_project](/img/Create_new_project.gif) - -2. **Create a new “Cron” trigger to schedule this extract job. Consider how - frequently you want this job to run. Daily? Weekly? Every 1 hour?** - -![create_trigger_cc](/img/create_trigger_cc.gif) - -3. **Create a “Raw JSON” credential to input the authentication details for your - CommCare source application.** - -![add_new_cred](/img/add_new_cred.gif) - -In the credential `JSON Configuration`, add your credential as follows: - -```json -{ - "appId": "APPID", - "password": "PASSWORD", - "username": "USERNAME", - "applicationName": "APP NAME", - "hostUrl": "https://www.CommCarehq.org", - "openfnInboxUrl": "INBOXURL" -} -``` - -:::tip - -Check out [this](/documentation/getting-started/terminology/#inbox) docs page on -how to find your OpenFn inbox URL to fill in the configuration above. - -::: - -Now that you've configured the job Trigger and Credential to authenticate… - -4. **Configure a new job. Note that this job will use the HTTP adaptor in order - to connect with the CommCare REST API.** - -![configure_job_cc](/img/configure_job_cc.gif) - -5. **Writing the “FETCH” job expression:** You will want to write a job - expression that sends a `GET` HTTP request to CommCare’s List Forms API. - - `GET /https://www.CommCarecommcarehq.org/a/cc-demo-2/api/v0.5/form` - - We have included the code snippet for replicating this job below. Please - check out the - [CommCare API docs](https://confluence.dimagi.com/display/commcarepublic/List+Forms) - on how to adjust the request query parameters. - -```js -get( - 'https://www.CommCarehq.org/a/cc-demo-2/api/v0.5/form/', - { - query: { - //see API docs to adjust query parameters - limit: 1000, //max limit: 1000 - offset: - state.meta && state.meta.next - ? state.meta.limit + state.meta.offset - : 0, - received_on_start: '2022-02-16', - received_on_end: '2022-02-18', - xmlns: - 'http://openrosa.org/formdesigner/D771417E-354E-4906-A686-DF0BA230F16A', - }, - }, - state => { - //After the CommCare API responds to our GET request, we want to POST the data in the response to our OpenFn Inbox for further inspection - const { meta, objects } = state.data; - const { openfnInboxUrl } = state.configuration; - const forms = objects; - - state.configuration = { baseUrl: 'https://www.openfn.org' }; - console.log('Posting form submissions to OpenFn Inbox...'); - - return each(forms, state => { - return post(`/inbox/${openfnInboxUrl}`, { body: state.data }, state => ({ - ...state, - data: {}, - references: [], - }))(state); - })(state); - } -); -``` - -6. **Once you are finished configuring and writing your job, save and run it!** - -![save_run_job_cc](/img/save_run_job_cc.gif) - -7. **Check out the `Activity History` tab to see if your run succeeded.** If it - succeeded, you should see: - - Successful run log (look for the green!) - - New Messages in your `Inbox` containing data for any forms submitted in the - time frame specified in your query. - -![activity_history_cc](/img/activity_history_cc.png) - -:::info - -**What do do if your run fails:** - -1. Open the run to inspect the error message -2. Adjust the job to issue and re-run the transaction as needed by clicking the - play button in `Activity History` -3. Check out the [PostgreSQL common errors](/adaptors/postgresql/#common-errors) - page for more details! - -::: - -**If you want to replicate this setup and configure your own CommCare -integration, first consider your CommCare extraction options - remember that -there are 2:** - -1. Data forwarding webhook (native CommCare feature) -2. REST API (List Forms API - **_API access requires a paid CommCare plan_**) - - The main advantage of using the webhook is that your data is forwarded to the - destination system in real-time. However, the List Forms API is also - advantageous because it enables users to extract data in bulk on a scheduled - basis, for syncing historical data every month on the 30th, for example. - Deciding on which option to go with depends on your business requirements. - -## Transforming and loading CommCare data to a PostgreSQL database - -1. **You should have a database configured and a username provided for OpenFn to - read and write data in your target DB tables.** For this demo, we have - configured the database - [like this](https://docs.google.com/spreadsheets/d/1pi_oxImakhtaCCCIENkjTPZeuyWhpFEcNmH7hfvTBgo/edit?usp=sharing) - to capture the CommCare form data. Check out the - [design quickstart](/documentation/design/design-quickstart#3-map-data-elements-to-be-exchanged) - for how to create your own `mapping specification document` to map data - elements to be exchanged. - -![db_config](/img/db_config.png) - -2. **Create a new message filter trigger, to run our second job for every new - patient record received in the OpenFn inbox.** Learn more about message - filter triggers - [here](/documentation/build/triggers/#message-filter-triggers). - -![create_new_trgger_db](/img/create_new_trgger_db.gif) - -3. **Create a PostgreSQL credential which will be used by the job to - authenticate with the database.** - -![add_credential_postgres](/img/add_credential_postgres.gif) - -4. **Create a new job with the `postgresql` adaptor for loading the CommCare - data into your destination database.** - -![configure_job_postgres](/img/configure_job_postgres.gif) - -**Writing the job:** For this job we will use the upsert operation to -insert/update records in the destination `patient` table and use `patient_id` as -the primary key. An `upsert` will update an existing row if a specified value -already exists in a table, and insert a new row if the specified value doesn't -already exist. - -```js -upsert('patient', 'ON CONSTRAINT patient_pk', { - patient_id: dataValue('data.patient_name'), - patient_name: dataValue('data.patient_name'), - village_name: dataValue('data.village_name'), - last_menstrual_period: dataValue('data.last_menstrual_period'), - expected_delivery_date: dataValue('data.expected_delivery_date'), - children_alive: dataValue('data.children_alive'), - living_children: dataValue('data.living_children'), - feeling_sick: dataValue('data.feeling_sick'), - total_children: dataValue('data.Total_children'), - risk_level: dataValue('data.Risk_level'), -}); -``` - -Feel free to modify the code above to reflect your CommCare and database -configuration according to your mapping specifications. Check out this -[page](/documentation/jobs/job-studio#job-studio-features) for how to copy the -dataValue for source data fields in the OpenFn job studio. - -:::tip - -Check out the -[design quickstart](/documentation/design/design-quickstart#3-map-data-elements-to-be-exchanged) -for how to create your own `mapping specification document` to map data elements -to be exchanged. - -::: - -6. **Save and turn on the job** - -![save_db_job](/img/save_db_job.gif) - -## Time to test! - -1. Submit a form in CommCare -2. If you have enabled data forwarding, refresh your OpenFn inbox -3. If you have not enabled data forwarding and set up a FETCH job instead, run - the job (ensure the `received_on_start` and `received_on_start` dates in the - FETCH are appropriate). -4. Run the FETCH job–if the fetch job passes, the “Load to DB” job should - automatically run -5. Check out the `Activity History` and ensure that both runs passed (look for - the green checks in the `Status/Action` column). - -![activity_history_final](/img/activity_history_final.png) - -6. **Finally, refresh your database and check out the new submission data!** - -![metabase](/img/metabase.png) - -While this guide is specifically for PostgreSQL databases, you can generally -follow these same steps for other database types (e.g., MS SQL or MySQL)—simply -leverage a different adaptor in your job configuration. - -**Other resources to check out:** - -1. OpenFn Job Library -2. OpenFn Docs ‘App’ pages for CommCare and Postgres - -**Any questions? Comments? New configuration ideas? Please reach out to us with -a post on the [OpenFn Community](https://community.openfn.org/) forum.** diff --git a/docs/getting-started/glossary.md b/docs/getting-started/glossary.md deleted file mode 100644 index d57aa6e204a..00000000000 --- a/docs/getting-started/glossary.md +++ /dev/null @@ -1,141 +0,0 @@ ---- -sidebar_label: Glossary for Integration -title: A glossary for data integration ---- - -Now that we've got a basic understanding of what an integration is, it's -important to establish some of the foundational concepts we need to press -forward. This doesn't mean you can't use OpenFn if you don't know what any of -these words mean prior to reading our documentation, but it does mean that some -of the most important tasks along the OpenFn journey will assume at least a -basic understanding of each of these terms. In -some cases, we also link to further reading if you want a better -understanding of some part of your data integration picture. - -Note: This glossary is meant to be OpenFn-agnostic. The rest of the docs help -you to get a picture of the parts of OpenFn, what we call them, and why, but -this glossary is really meant as a prerequisite to all those other things to aid -users with no experience in this area. - -## API - -API is short for "application programming interface," and it's the part of some -software application that has chosen to make itself visible -(interface) to users outside the application itself. And it's doing that -in a programmatic way, in a way that allows developers of other -applications or data systems to use it the same way each time. - -## API Protocol - -There's no hard and fast rule about how an API gets developed, but over time, -standards have emerged to make it more straightforward for a new user to -interact with Platform X's API, by trying to ensure most applications use one of -a few different formats. That's what an API protocol is. A few of the big names -here are REST, SOAP, JSON, and GraphQL. Rather than reinvent the wheel, -[here's a good primer on how protocols differ, their data formats, and why that all matters.](https://frontend-digest.com/beginners-guide-to-apis-protocols-and-data-formats-f80cf7f30425]) - -## Database - -Any organized collection of data can probably be safely called a database. If -it's got a structure with which to reference all the stuff it's storing, and the -"stuff" is data, then it's a database. - -## Data source - -A data source is an application, database, or table that provides data to some -other platform. Nothing is always a data source. For example, Google -Sheets can be a data source, but it can also pull from data sources (individual -CSV uploads or manual user data entry). We just call it a source when it's doing -the job of sourcing data to some other place. Data sources are the starting -point, temporally, for any integration. - -## Data system - -Sometimes folks get confused about the distinction between a database, a data -source, an application, and a data system. A data system is a more -complex collection of these other things, usually one that allows a user to more -easily interact with all of the data they should have access to. The data system -often serves as an entry point to the myriad databases, applications, tables, -etc. that a user would otherwise have to go 12 different places to find. - -## Encryption - -In this day and age, security is everything. Encryption is the process of taking -something that is readable to anyone and making it only readable to people we -want to read it. OpenFn ensures your data is encrypted every step of the way -while it's in our platform. -[For more on different kinds of encryption, you can look here.](https://ssd.eff.org/en/node/36) - -## File system - -A file system is to files what a data system is to data. It structures your -files in a way that makes it easy for you to retrieve them in a standardized way -(think of your home file system with its file paths on your home computer). File -systems can exist in other contexts too, and sometimes you need to access them -to retrieve a file (a Word doc, CSV, plain text file, etc. might all be relevant -depending on your use case). The only real difference between file systems and -data systems or databases is the kind of information stored, data vs. files. - -## ETL - -ETL stands for extract, transform, and load. These are often thought of as the -three constituent parts of a data integration. First, we extract (push of pull -data from a data source). Then, we transform (make any changes to the data to -make it acceptable to the destination system or application). Then, we load -(send it to the destination). - -## Integration platform - -An integration platform (e.g., OpenFn) is an application (or set of -applications) that help organizations set up, run, and maintain/manage the -integrations between all of their various systems. - -### iPaaS - -You may also see the acronym "iPaaS". This stands for integration platform as a -service and is a type of "software as a service" (or "SaaS"). SaaS is a software -purchasing model in which software is paid for only as it is used (often -month-to-month), rather than purchased up front or given away for free. - -## Metadata - -This is data that tells us about our data. In a table, for example, that's the -name of the columns, the number of rows, etc. Metadata is often brought up in -conversations about privacy—e.g., regulators might want to ensure that _only -metadata_ is moved from Ministry A to Ministry B, as opposed to personally -identifiable information (PII) about individuals themselves. - -## Push, pull, and streaming - -Pushing is when a triggering action in the data source causes it to send -data to the destination. Pulling is the opposite, where the destination -system requests the data from the source based on some triggering action, rather -than waiting for the source to send it on its own. Streaming is a bit -different, and it's when a data source is essentially constantly sending -data to a destination system. - -## Webhook - -A [webhook](/documentation/source-apps#standard-webhook-configuration) (also called a web -callback or HTTP push API — thanks -[SendGrid](https://sendgrid.com/blog/whats-webhook/)!) is a feature of an -application that allows pushing. It's often configured to notify some -external URL when an event occurs. A system administrator might create a -"webhook" which notifies an integration platform whenever some event occurs so -that the iPaaS can start executing some complex workflow. - -## Structured and unstructured data - -Structured data is data that has metadata. Unstructured data has very little -metadata (though probably still has things like time of creation, update, etc.). -Without metadata about the format of the data, unstructured data is more -difficult to interact with programmatically. We need different sorts of rules -when doing ETL on unstructured data to do it well, whereas structured data is an -easier starting point because we know what to expect from a column with a name, -data type, field size, and so on. - -## Writeback - -Refers to a destination system making a change in a data source. When my -destination application receives information from a data source and wants to do -something back to the source in response, that's writeback. diff --git a/docs/getting-started/integrating-using-openfn.md b/docs/getting-started/integrating-using-openfn.md deleted file mode 100644 index 7042578e231..00000000000 --- a/docs/getting-started/integrating-using-openfn.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -title: Integrating using OpenFn - ---- - -In [What's an Integration, Really?], you can read about the how, when, and why of integration design. Now, we take those general concerns and apply them to the OpenFn framework to help you get started with using the platform and knowing the lingo. - - - -When->Triggers -What->Jobs and runs -Why->not our problem/data mapping -How->All of this stuff -How safely->Credentials diff --git a/docs/getting-started/integration-toolkit.md b/docs/getting-started/integration-toolkit.md deleted file mode 100644 index e7828b605ea..00000000000 --- a/docs/getting-started/integration-toolkit.md +++ /dev/null @@ -1,108 +0,0 @@ ---- -title: The Integration Toolkit ---- - -OpenFn's free and open-source Integration Toolkit gives governments and NGOs -around the world more flexibility and freedom to chose how they achieve success -in integration and interoperability projects. The Toolkit is both a recognized -[Digital Public Good](https://digitalpublicgoods.net/) ("DPG") and a -[Digital Square Global Good](https://digitalsquare.org/digital-health-global-goods). - -![DPG](/img/openfn_dpg.png) - -The Toolkit provides a suite of software tools and documentation to help users -design, build, and automate integrations. - -## About the Toolkit - -At the heart of the toolkit is the `project`—a set of jobs, triggers, and -credentials which allow organizations to flexibly define workflows and -integrations across their systems. - -Projects can be ported from the `platform` to `microservice` (the main -deployment pathway for the Integration Toolkit) and back again (see below) but -to really understand the toolkit you've got to first understand Open Function -Group and `platform`, the enterprise iPaaS. - -![Integration Toolkit](/img/integration-toolkit.png) - -Open Function Group has been building free and open source software (FOSS) for -data integration projects in the health, humanitarian, and international -development sectors since in 2014. Their software and services are now in use by -governments, NGOs, and impact-first businesses in over 40 countries. - -OFG's first integration platform was entirely FOSS, but they soon shifted to an -["open-core"](https://en.wikipedia.org/wiki/Open-core_model) (think GitLab) in -order to sustain their impact-focused integration work. Their main hosted -offering, the OpenFn "platform", is _proprietary_ but makes extensive use of the -open-source integration toolkit; in fact, the "platform" may be thought of as an -enterprise/hosted layer running on-top of the basic, open-source building blocks -provided by the Integration Toolkit. - -### Why OFG is driving the development of the Integration Toolkit - -Our mission is to make health & humanitarian interventions more efficient & -effective, and we see investment in the integration toolkit as strategic. - -We'll strive to preserve the integration toolkit as a healthy and bona fide open -source project and sustains its operations through business activities related -to the toolkit and their other proprietary and/or service offerings until it -grows legs of its own and is taken over by the broader community. - -We have designed the tools in the toolkit to be useful as standalone pieces of -software _and_ as modules, used by other applications. Because a substantial -portion of OFG's revenue comes from contracts related to the platform, and -because the platform relies on OpenFn/core, OpenFn/engine, and the OpenFn -adaptors, we hope to ensure that OFG will always be incentivized to continue -their investment in the integration toolkit. - -In other words, we're attempting to ensure that as OFG grows, they will continue -enhancing the open source integration toolkit regardless of whether or not -additional funders and/or stakeholders contribute to the project. - -## What's in the Integration Toolkit - -Separate from "the platform", the integration toolkit is the suite of -applications and modules provided by OFG and the community which enable data -integration, interoperability, and automation solutions via OpenFn-compliant -jobs, triggers, and credentials. The key components of the toolkit are: - -1. OpenFn/docs -2. OpenFn/core -3. OpenFn/engine -4. OpenFn/microservice -5. OpenFn/devtools -6. the OpenFn adaptors -7. _OpenFn/lightning (coming soon...)_ - -:::caution Microservice and devtools are being replaced by Lightning - -Please note that [OpenFn/microservice](https://github.com/OpenFn/microservice) -and [OpenFn/devtools](https://github.com/OpenFn/devtools) are being deprecated -and replaced by [OpenFn/Lightning](https://github.com/OpenFn/lightning), When -lighting is released. - -::: - -### Lightning, coming soon! - -Lightning is an upcoming addition to the Integration Toolkit. It is a _fully -open source_ workflow automation platform designed for governments and NGOs who -need a flexible solution to integrate and connect _any system_. - -You can read all about it [here](/documentation/about-lightning)! - -## Architecture for implementation - -![Lightning architecture](/img/lightning_architecture.png) - -## Open Source Steering Committee (OSSC) - -We've also initiated an Open Source Steering Committee (OSSC) to represent the -OpenFn community of end users and implementers. It reviews and gives feedback on -major roadmap decisions, new designs, specifications, features, and protocol -changes. - -The OSSC's membership and decision making process are defined in the -[OSSC's internal governance policy](https://openfn.github.io/governance/OSSC.html) -if if you're interested in joining, we'd love to hear from you! diff --git a/docs/getting-started/so-you-want-to-integrate.mdx b/docs/getting-started/so-you-want-to-integrate.mdx deleted file mode 100644 index 6bd50e4bbe5..00000000000 --- a/docs/getting-started/so-you-want-to-integrate.mdx +++ /dev/null @@ -1,103 +0,0 @@ ---- -title: So, what is an integration? 🤔 ---- - -import Graph_CommCaretoSF from '/static/js/components/ccsf_graph'; -import Graph_Master_View from '/static/js/components/master_view_graph'; -import Graph_Data_Viz_Flow from '/static/js/components/data_viz_react_flow'; -import ReactFlowProvider from 'react-flow-renderer'; - -OpenFn is an integration platform. And if you found us, you likely came to the -conclusion at some moment prior that you want to integrate technology X with -technology Y (and maybe W and Z while you're at it). But not all of our users -come to OpenFn with a wealth of previous integrations under their belt. So if -this is your first go, this page can help you think through all the different -ways integrations can take shape so that you have a strong understanding of what -it is you really want _before_ you start writing -[(or borrowing)](/adaptors/library) a -[job](/documentation/jobs/job-design-intro/). - -There are plenty of different reasons to integrate your data systems. Maybe you -want one "master" view that you or your clients can trust as a source of truth. - -
- -
- -Maybe you want to automate some data viz that you currently have to do manually. - -
- -
- -Or maybe you just want to expose a small slice of data from one user group to a -different app used exclusively by some other part of your company. - -Regardless of the reason, what every integration boils down to is connecting two -or more disconnected applications. But as you can see, not all integrations look -alike. This basic structure comes in many shapes and sizes. There's plenty of -variety to be found: - -1. Perhaps the most important variation is **why** you move the data. - -This part boils down to end goals. Integration for integration's sake is a waste -of time. What's your reason for wanting X data in Y system? - -It's important to keep that ultimate business requirement in mind when designing -any integration and weigh potential outcomes of design decisions against that -ultimate goal. - -2. **When** you move the data. - -Usually, you can articulate the best case scenario here in plain English pretty -easily. - -> I want Salesforce to \_\_\_ **when** one of our field workers submits a new -> CommCare form. - -
- -
- -or - -> I want Postgres to \_\_\_ **every two weeks.** - -A crucial difference between these two **whens** is that the first turns on an -action, whereas the second is based on a set period of time, regardless of what -happens in that window. - -3. **How** you move the data, namely whether the destination system is pulling - or the source system is pushing (or some other pattern), what format the data - is being transferred in, and what protocol(s) that system is using to do it. - -1 and 2 are really about real world considerations. Sometimes technical -constraints from our source and destination systems can get in the way of our -ideals, but answering the questions in an ideal way doesn't require any serious -thought about the tech behind it. Now that we're at **how**, we have to think -more seriously about the underlying technology. - -There's not space here to explore all the different ways that a platform can -choose to set up to send or receive data, but it's important now to take note -that there are many options, and knowing which ones are available is an -important part of designing a strong integration. For a more in-depth look at -how to answer these questions based on the specifics of your project, check out -\_\_\_. - -4. A final variation to consider here is **how to move the data safely** (sorry - to break the pattern we had going). - -At OpenFn, data security is a first priority. That's also true of many of the -other systems our customers use. What that means is that we often can't just -grab data from one system and put it into another without first assuring each -system that we are someone who's allowed to be there. In general, we talk about -this slice of the world as **authentication**. - -These are all very important questions to consider when designing an -integration. Check out our docs on integration design to learn more about how we -begin to answer these questions and more: - -- **Integration design:** - https://docs.openfn.org/documentation/design/design-quickstart/ -- **Glossary for integration:** - https://docs.openfn.org/documentation/getting-started/glossary/ diff --git a/docs/gsoc.md b/docs/gsoc.md deleted file mode 100644 index f102a018798..00000000000 --- a/docs/gsoc.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -title: Google Summer of Code ---- - -## Overview - -OpenFn provides data integration, automation, and interoperability tools that -are used to scale the world's most promising health and humanitarian -interventions. UNICEF, the World Health Organization, the IRC, and the Wildlife -Conservation Society are just a few of the many organizations that drive -efficiency via OpenFn software. With an open-core model, we've got hosted and -locally-deployed solutions in 40+ countries, and this summer you'll get the -chance to work on leading-edge ETL tools built in Elixir/Erlang, and NodeJs. If -learning about APIs, data transformation, and middleware/automation layers -excites you, OpenFn is the place to be. - -## Mentors - -This summer, you'll get the chance to work with some of the core team at OpenFn, -including [Chaiwa Berian](https://openfn.org/team#chaiwa), -[Mamadou Cissé](https://openfn.org/team#mamadou), -[Stu Corbishley](https://openfn.org/team#stuart), and -[Taylor Downs](https://openfn.org/team#taylor). They're based in Zambia, -Senegal, South Africa, and the United Kingdom, respectively. Between them, -they've got almost 50 years of experience working in software and... a PhD in -Computer Science. (Hey thanks, Mamadou 😉.) - -## Project Ideas - -### OpenFn/microservice Extension - -OpenFn projects (see [`project.yaml`](portability)) can be deployed on the -platform _or_ on microservice, a Phoenix web application. This summer, GSOC -interns will have the opportunity to build out the front-end for this community -supported web app. - -Difficulty level: `medium` - -You'll be working in [`Docker`](https://docs.docker.com/get-started/), -[`Phoenix`](https://www.phoenixframework.org/), -[`Elixir`](https://elixir-lang.org/) and [`Erlang`](https://www.erlang.org/). - -### OpenFn/engine Extension - -Engine is part of the common FOSS toolkit that is used both by `microservice` -and `platform`. It's the software which is responsible for actually executing -calls to `OpenFn/core` and doing neat things like streaming logs back to the -requester. If you're keen on really understanding how Elixir and Erlang work, -getting your hands dirty with OTP apps, engine is where you want to be working -this summer. - -Difficulty level: `high` - -You'll be working in [`Elixir`](https://elixir-lang.org/) and -[`Erlang`](https://www.erlang.org/). - -### OpenFn/core Metrics - -At the bottom of it all, whether we're providing secure patient data transfer -services for ministries of health or making child protection case referrals for -UNICEF, OpenFn relies on spinning up NodeVMs, executing code inside those VMs -safely, and then shutting the down. Welcome to the core. - -This summer you could have the chance to dig into that _sandboxed-VM-in-a-VM_ -magic, learn loads about NodeJs, and provide end-users with better metrics on -exactly what kinds of compute they're using to "get the job done". - -Difficulty level: `medium` - -You'll be working in [`NodeJs`](https://nodejs.dev/learn) and -[`Typescript`](https://www.typescriptlang.org/). - -### Adaptors 2.0 - -Adaptors are the API wrappers that allow OpenFn users to quickly and easily work -with the most common APIs in international development. They provide an -interface for connecting to DHIS2, ODK, CommCare, OpenMRS, etc., etc. - -They're NodeJs modules, but in order to make the adaptor development and _use_ -process better, we want to bring them into the future with Typescript. Adaptors -should tell you how to use them while you use them. - -Check out this thread on -[community.openfn.org](https://community.openfn.org/t/discussion-regarding-adapter-2-0-project) -for more information. - -Difficulty level: `medium` - -You'll be working in [`NodeJs`](https://nodejs.dev/learn) and -[`Typescript`](https://www.typescriptlang.org/). diff --git a/docs/history/activity-history.md b/docs/history/activity-history.md new file mode 100644 index 00000000000..9c9d1031e55 --- /dev/null +++ b/docs/history/activity-history.md @@ -0,0 +1,9 @@ +--- +title: Activity History Overview +sidebar_label: Activity History +--- +:::warning Under construction + +This docs page is under construction. Check back later for the complete docs, or check out the Docs Version "Platform (v1)". + +::: \ No newline at end of file diff --git a/docs/instant-openhie.md b/docs/instant-openhie.md deleted file mode 100644 index 0a36b42ab86..00000000000 --- a/docs/instant-openhie.md +++ /dev/null @@ -1,325 +0,0 @@ ---- -title: Instant OpenHIE ---- - -:::caution Microservice and devtools are being replaced by Lightning - -Please note that OpenFn/microservice and OpenFn/devtools are being deprecated -and replaced by OpenFn/lightning, When Lighting is released, it may be used -within Instant OpenHIE (instead of microservice) as an OpenHIE-compliant workflow engine that can interface with the OpenHIE Interoperability Layer ([learn more](/documentation/about-lightning#standards-and-compliance-matter)). - -::: - -## Overview - -In partnership with [Digital Square][digitalsquare] and _FCDO COVIDaction_, -**OpenFn has been investing in its open source integration toolkit** to provide -robust integration solutions that can connect _any digital health system_ and be -rapidly implemented on any server, in any country, by any organization. - -**[OpenFn/microservice][openfnmicroservice]** is a fully [Instant -OpenHIE][instantopenhie] compliant component which can be used to drive -workflow, achieve compliance with standards, and integrate components of the -[OpenHIE stack][openhiestack]. - -We seek to enhance the value of the [Instant OpenHIE][instantopenhie] project by -developing a package that will include [OpenFn][openfn] as an integration -pathway for connecting with the [OpenHIE architecture][openhiearchitecture]. - -This package aims to enhance the value of [Instant OpenHIE][instantopenhie] by -providing another option for robust information processing, integration, and -business process (workflow) automation. When deploying [Instant -OpenHIE][instantopenhie], implementers now have the option to include -[OpenFn][openfn] as a component. - -[OpenFn][openfn] may also be used as a workflow engine to automate complex -business logic alongside [OpenHIM][openhim] and the [OpenHIE -stack][openhiestack]. Individual [jobs][jobs] in [OpenFn][openfn], sometimes -many in a single microservice deployment, may be used as _“mediators”_ ([see -OpenHIE library of existing mediators][mediators]) to quickly transform and map -data to the [OpenHIE architecture][openhiearchitecture]. - -To demonstrate a real-world use case for how [OpenFn][openfn] might be -implemented in the [OpenHIE architecture][openhiearchitecture], we met with -several community members to identify key use cases for a reference prototype -implementation. - -Visit the [demo repo here][demorepo]. - -## Use Cases for the Prototype Implementation - -We’ve seen that the most common integration use case is that health service -delivery providers, especially large community health worker (“CHW”) -interventions, need to integrate their data and programming into national -eHealth architectures. - -### User stories - -> 1. _As a community health implementer, I want to integrate my CommCare case -> management application used by CHWs with the national patient registry, so -> that I can develop a shared health record and automate reporting -> pipelines._ -> 2. _As a health services provider, I want to integrate my existing application -> with the national HIS, but I want to apply the FHIR standard to my data -> collected before sharing to adhere to compliance and reporting -> requirements._ - -- We therefore decided to build an integration solution that demonstrates how - existing **CHW** applications can be integrated with the national health - infrastructure and leverage a simple job on [OpenFn][openfn] as a - [mediator][mediators] to apply the [FHIR data standard][fhir] and other data - manipulation needed to integrate with [HAPI FHIR][hapifhir]. - -## Implementation Design - -In sum, the prototype sends patient case registration data from mobile data -collection apps ([CommCare][commcare], [KoboToolbox][kobo]) to -[OpenFn/microservice][openfnmicroservice]. [OpenFn][openfn] then transforms the -data and ensures that it adheres to the [FHIR][fhir] [patient][patientspec] and -[encounter][encouterspec] data standards, before sending it onwards to a [FHIR -channel][fhir] in the [OpenHIM][openhim]. [OpenHIM][openhim] is used as a -\_“channel”\_ here for the [OpenHIE architecture][openhiearchitecture] to -validate requests and forward them onto other systems in the **national eHealth -architecture**. In this case, we forward the case data onwards to register the -patients in a [HAPI FHIR][hapifhir] server. - -This implementation design was determined to be the highest value/most in-demand -because it leverages the core functionality of [OpenHIM][openhim] (providing a -reverse proxy and generating an audit trail) without requiring integrators to -build a new [mediator—a][mediators] process that is more complex than -configuring a [job][jobs] within an [OpenFn project][projects]. - -This prototype includes the following components: - -An [Instant OpenHIE][instantopenhie] instance can be spun up which contains -[HAPI FHIR][hapifhir], [OpenHIM][openhim], and a single -[OpenFn/microservice][openfnmicroservice] deployment (a -[project.yaml][projectyaml] file, exported from [OpenFn/platform][openfn]) with -2 different [jobs][jobs]. When data is forwarded to -[OpenFn/microservice][openfnmicroservice] from two distinct form submissions on -[CommCare][commcare] and [Kobo][kobo], it is processed and creates [FHIR][fhir] -patient resources via [OpenHIM][openhim] and [HAPI FHIR][hapifhir]. We’ve opted -for a single [OpenFn/microservice][openfnmicroservice] “project” with two -slightly different [jobs][jobs] and [triggers][triggers] to highlight the -versatility of [OpenFn projects][projects]. - -## Explore the Implementation - -Currently, there are two different ways to explore this demo. The first (the -more traditional _“Instant”_ way) is by **cloning the OpenFn/instant-demo -repo**. Once inside, users type _“yarn setup”_ to get everything up and running. -Running _“yarn test”_ will then demonstrate the -[Kobo][kobo]/[CommCare][commcare] to [OpenFn][openfn] to [OpenHIM][openhim] to -[FHIR][fhir] flows. - -They can explore the various [jobs][jobs], sample payloads, endpoints, and post -data to the various endpoints using either the data forwarding settings in -[CommCare][commcare] and [Kobo][kobo] or via [CURL][curl] (or their HTTP request -agent of choice.) - -Once running, users can see how standard [CommCare][commcare] and [Kobo][kobo] -submissions are transformed by the [OpenFn/microservice][openfnmicroservice] to -adhere to the [FHIR][fhir] specifications for [patients][patientspec] and -[encounters][encouterspec], and then that those subsequent resources are created -on the [HAPI FHIR][hapifhir] server, via a channel on the [OpenHIM][openhim]. - -The second (slightly less conventional way) to explore the [demo][instantdemo], -is via [OpenFn.org][openfn]. Since [OpenFn projects][projects] can be run in -[microservice][openfnmicroservice] or on the [hosted platform][openfn], we’ve -provided a project instance at [OpenFn.org][openfn] that allows users to explore -the configuration required to incorporate [OpenFn][openfn] in an [Instant -OpenHIE][instantopenhie] project. There are three [jobs][examplejobs] which can -be accessed with a **demo user** with _username: demo@openfn.org_ and -_password:guest123_. - -The three jobs will show: - -- How a [CommCare][commcare] submission is transformed and sent to [HAPI - FHIR][hapifhir]; -- How a [Kobo][kobo] submission is transformed and sent to [HAPI - FHIR][hapifhir]; -- And what the final resources that would be sent to [HAPI FHIR][hapifhir] look - like. - -It’s our hope that this will provide a valuable entry-point for [Instant -OpenHIE][instantopenhie] configuration with -[OpenFn/microservices][openfnmicroservice]. - -## About the Implementation Setup - -### Processes - -- We met with [OpenHIE community members][openhiecomm] to understand use cases, - and with [Jembi Health Systems][jembi] to learn about [Instant - OpenHIE][instantopenhie] packages, specifications, and compliance - requirements. - -- Identified sample data sources (real [CommCare][commcare] and [Kobo][kobo] - case registration forms - see here) that we could use to send data to the - **national eHealth architecture**. Here is a [sample submission payload from - CommCare][commcaresample] Here is a [sample submission payload from - Kobo][kobosample] - -- Reviewed [FHIR-HL7][fhir] documentation to determine data standard - requirements for patient data and encounter data. See [FHIR patient - spec][patientspec] and [FHIR encounter spec][encouterspec]. - -- Evaluated [OpenFn][openfn] vs. [OpenHIM][openhim] capabilities to determine - how to use. Determined that using an [OpenHIM channel][openhim] will leverage - the core audit trail functionality from [OpenHIM][openhim], but not require us - to build a [new mediator][mediators]. - -### Project Configuration Steps - -There are two ways to set up a [project.yaml][projectyaml] to run as a -[microservice][openfnmicroservice]. The first is to use the [OpenFn.org -platform][openfn], and the second way is to use [OpenFn/devtools][devtools]. - -These two methods are detailed below: - -1. **Configure a project using the OpenFn.org platform** - - - This option allows organisations to leverage [OpenFn.org][openfn]’s - built-in features for easy [project][projects] setup, [job writing][jobs] - and source code management. - - The [project.yaml][projectyaml] file generated from this project setup will - then be used as the base structure for the [OpenFn - Microservice][openfnmicroservice]. - - The steps to setup the [OpenFn Microservice][openfnmicroservice] project - using the [OpenFn.org platform][openfn] are as below: - - **A. Add [credentials][cred] to the project which will be used to connect - the OpenFn Microservice to OpenHIM.** - - - This is also an opportunity to add [credentials][cred] which [OpenFn - Microservice][openfnmicroservice] may use to connect to source systems - (such as [CommCare][commcare] or [KoBotoolbox][kobo]) . - - **B. Add [triggers][trig] to the project which will be used by the [OpenFn - Microservice][openfnmicroservice] to match payloads from source systems to - [OpenFn Microservice Jobs][jobs].** - - - Note that the [Microservice][openfnmicroservice] is configured to run a - [job][jobs] based on the shape of the incoming payload. - - - For example, a [trigger][trig] may be configured to match payloads, from - [CommCare][commcare], which contain the - `{"@name": "Register New Patient"}` message in their message body. - - - A given [job][jobs] will then match against this message, and will be - invoked by the [OpenFn Microservice][openfnmicroservice] to (a) create a - payload in the [FHIR standard][fhir] containing an [Encounter - Resource][encouterspec] and (b) send the [FHIR Standard][patientspec] - Payload to [OpenHIM][openhim] with instructions to load it to [HAPI - FHIR][hapifhir]. - - **C. Export the [project.yaml][projectyaml] file using the Export Wizard of - the [OpenFn.org][openfn]** - - - The [generated YAML][projectyaml] file will then be used by the [OpenFn - Microservice][openfnmicroservice] to execute the [jobs][jobs] for the - matching payloads. - -2. **Configure a project using the [OpenFn/devtools][devtools]** - - - This option allows organisations to configure the [project][projects] and - host [job expression][jobexpr] source files, for [OpenFn Microservice - projects][projects], without using the [OpenFn platform][openfn]. - - With this option, it is recommended that organisations use source - versioning tools and platforms such as `git` and `github` to manage the - [project][projects] and [job expression][jobexpr] source code/files. - - To configure the [OpenFn Microservice project][openfnmicroservice] using - [OpenFn/devtools][devtools], create a local folder or github repository to - host your project configuration files. Inside this folder, one would then - perform the following actions: - - - Create a credential.json file - - Add credentials as shown in the [sample credential here][samplecred] - - Create the [job expressions][jobexpr]. In this case, one would create the - [CommCare-to-OpenHIM][demoexpr] and [Kobo-to-OpenHIM][demoexpr] - expressions as shown in the demo expressions [here][demoexpr] - - Run the [OpenFn CLI][openfncli] to configure the rest of the project. The - [CLI][openfncli] will assemble the [project.yaml][projectyaml] file from - the different artifacts as provided. See detailed steps in the - documentation site [here][openfncli]. - - - The last step of the [CLI][openfncli] prompts will allow one to export - the [Project YAML file][projectyaml], which will then be used by the - [OpenFn Microservice][openfnmicroservice] to execute the [jobs][jobs] for - matching payloads. - -## Job writing notes - -[OpenFn][openfn] provides two ways of writing jobs: - -- Using the [OpenFn.org’s Job Studio][studio] as detailed in the documentation - site [here][jobs] - - With this option, if editing an existing [Job Expression][jobexpr], one - would be expected to use [OpenFn.org Project Export service][openfn] to - re-generate the [Project YAML][projectyaml] file for the [OpenFn - Microservice][openfnmicroservice]. -- Using [OpenFn/devtools][devtools]. - - This option also allows organisations to write [job expressions][jobexpr] - without using the [OpenFn’s hosted service][openfn]. See detailed - documentation [here][devtools] - - With this option, if editing an existing [Job Expression][jobexpr], one - would be expected to run the [OpenFn CLI][openfncli], to re-generate the - [Project YAML file][projectyaml] for the [OpenFn - Microservice][openfnmicroservice]. - -## System Deployment Steps - -- [OpenFn] provides an automated deployment script that allows system admins to - setup and run the [OpenFn Microservice][openfnmicroservice]. -- For example, to run the [Instant-demo Microservice][instantdemo], the - following steps are recommended: - - Clone the [OpenFn/instant-demo repo][instantdemo] - - Overwrite the [sample “project.yaml”][sampleyaml] file with your newly - generated [project.yaml file][projectyaml], or use the existing [YAML - file][projectyaml] to deploy the demo project. Run the setup command as - described in the documentation [here][instantdemo] - - Verify the system is working by [curling][curl] data (or submitting forms on - [CommCare][commcare]/[Kobo][kobo]) matching their [triggers][triggers] to - the [microservice][openfnmicroservice] endpoint `(localhost:4001/inbox)` and - checking to see that resources are created in [HAPI FHIR][hapifhir]. - - Note how the [test.js file][testfile] handles this verification with the - [sample project.yaml][sampleyaml] - - -[openfn]: https://openfn.org/ -[instantopenhie]: https://wiki.ohie.org/display/resources/Instant+OpenHIE -[openhiestack]: https://openhim.readthedocs.io/en/latest/implementations/openhie.html -[openhiearchitecture]: https://wiki.ohie.org/pages/viewpage.action?pageId=8454157 -[openhim]: http://openhim.org/ -[jobs]: /documentation/build/jobs/ -[mediators]: http://openhim.org/mediator-library/ -[demorepo]: https://github.com/OpenFn/instant-demo -[openfnmicroservice]: /documentation/microservice/home/ -[digitalsquare]: https://digitalsquare.org/ -[fhir]: https://fhir.org/ -[hapifhir]: https://hapifhir.io/ -[commcare]: https://www.commcarehq.org/ -[kobo]: https://www.kobotoolbox.org/ -[projects]: /documentation/build/lightning-quick-start/ -[projectyaml]: https://github.com/OpenFn/microservice/blob/main/project.yaml.example -[triggers]: /documentation/build/triggers/ -[commcaresample]: https://github.com/OpenFn/instant-demo/blob/main/fixtures/commcare_sample.json -[kobosample]: https://github.com/OpenFn/instant-demo/blob/main/fixtures/koboCaseRegistration.json -[patientspec]: https://www.hl7.org/fhir/patient-example.json.html -[encouterspec]: https://www.hl7.org/fhir/encounter-example.json.html -[openhiecomm]: https://ohie.org/tag/community/ -[jembi]: https://www.jembi.org/ -[cred]: /documentation/build/credentials/ -[trig]: /documentation/build/triggers/ -[devtools]: https://github.com/OpenFn/devtools -[testfile]: https://github.com/OpenFn/instant-demo/blob/main/test.js -[instantdemo]: https://github.com/OpenFn/instant-demo -[samplecred]: https://github.com/OpenFn/instant-demo/blob/main/openfn/docker/config/project.yaml#L165-L167 -[openfncli]: /documentation/devtools/home/#configure-an-openfn-project -[demoexpr]: https://github.com/OpenFn/instant-demo/tree/main/expressions -[jobexpr]: /documentation/build/jobs/#a-basic-expression -[sampleyaml]: https://github.com/OpenFn/instant-demo/blob/main/openfn/docker/config/project.yaml -[curl]: https://curl.se/ -[studio]: /documentation/jobs/job-studio/ -[examplejobs]: https://openfn.org/projects/p5pqx3/jobs - diff --git a/versioned_docs/version-legacy/intro.md b/docs/intro/home.md similarity index 96% rename from versioned_docs/version-legacy/intro.md rename to docs/intro/home.md index c6c1f1b2a3f..855e3351bb0 100644 --- a/versioned_docs/version-legacy/intro.md +++ b/docs/intro/home.md @@ -1,9 +1,17 @@ --- title: About +id: home sidebar_label: What is OpenFn? slug: / --- +:::warning Under construction + +This docs page is under construction. Check back later for the complete docs, or +check out the Docs Version "Platform (v1)". + +::: + ## What is OpenFn? :::tip diff --git a/docs/getting-started/implementation-checklist.md b/docs/intro/implementation-checklist.md similarity index 100% rename from docs/getting-started/implementation-checklist.md rename to docs/intro/implementation-checklist.md diff --git a/docs/intro/security-compliance.md b/docs/intro/security-compliance.md new file mode 100644 index 00000000000..44566eace5a --- /dev/null +++ b/docs/intro/security-compliance.md @@ -0,0 +1,7 @@ +--- +sidebar_label: Security +title: Security and compliance in OpenFn projects +--- + +# Security and Compliance in your OpenFn Projects +General OpenFn security language; link to website for SaaS-relevant info? \ No newline at end of file diff --git a/docs/getting-started/security.md b/docs/intro/security.md similarity index 99% rename from docs/getting-started/security.md rename to docs/intro/security.md index f21e28065c3..da95c284b74 100644 --- a/docs/getting-started/security.md +++ b/docs/intro/security.md @@ -1,5 +1,5 @@ --- -sidebar_label: Security +sidebar_label: Security in Implementations title: Security considerations for data integration projects --- diff --git a/docs/standards/openhie.md b/docs/intro/standards.md similarity index 74% rename from docs/standards/openhie.md rename to docs/intro/standards.md index 92e6926f12e..653d17beb52 100644 --- a/docs/standards/openhie.md +++ b/docs/intro/standards.md @@ -1,7 +1,46 @@ --- -title: OpenHIE +sidebar_label: Standards +title: Standards & OpenFn --- +# Digital Public Good +OpenFn is recognised by the +[Ditial Public Goods Alliance](https://digitalpublicgoods.net/) as a Digital +Public Good, or "DPG". + +:::info Digital Public Goods Definition + +Open-source software, open data, open AI models, open standards, and open +content that adhere to privacy and other applicable best practices, do no harm +by design and are of high relevance for attainment of the United Nations 2030 +Sustainable Development Goals (SDGs) + +::: + +You can read more about the DPG standard +[here](https://digitalpublicgoods.net/standard/). + +# Global Good for Health + +OpenFn is one of 36 software applications that have been recognised as a Digital +Square Global Good for Health. + +:::info Global Goods for Health Definition + +A mature digital health software global good is software that is Free and Open +Source Software (FOSS), is supported by a strong community, has a clear +governance structure, is funded by multiple sources, has been deployed at +significant scale, is used across multiple countries, has demonstrated +effectiveness, is designed to be interoperable, and is an emergent standard +application. + +::: + +You can read more about Global Goods for Health +[here](https://digitalsquare.org/digital-health-global-goods). + +# OpenHIE Standard Architecture + _This section assumes you are familiar with the OpenHIE specification–a reference framework that makes sharing health data across information systems possible through a Health Information Exchange (“HIE”). To learn more, check out diff --git a/docs/getting-started/terminology.md b/docs/intro/terminology.md similarity index 100% rename from docs/getting-started/terminology.md rename to docs/intro/terminology.md diff --git a/docs/manage/platform-mgmt.md b/docs/manage/platform-mgmt.md deleted file mode 100644 index 88ed8f59cf3..00000000000 --- a/docs/manage/platform-mgmt.md +++ /dev/null @@ -1,1072 +0,0 @@ ---- -title: Project Management ---- - -:::important - -Currently, this section is specific to **OpenFn/platform**. - -::: - -## Jobs - -This section of the portal allows you to create and manage your jobs. - -### Searching jobs - -For a project with a number of jobs, finding a job can be easily achieved via -the search feature. - -To search for a given job: - -- From the application **menu**, click on **Jobs**. -- Find the **Search jobs** box and type the name of the job in the search box. -- The application will filter and show all jobs matching the portion of text - entered into the search box. - -### Switching on/off a job - -In OpenFn, a job is **off** by default. To **switch on** a given job, follow the -steps below: - -- From the application **menu**, click on **Jobs**. -- Find the job you would like to turn on. -- On the top-right corner of the job card, click on the **switch** button to - turn on/off the job. -- Once switched on, the job's **switch** button will change the color to - **blue**. - -:::info - -Note that once a job is **switched on**, OpenFn will run it automatically, as -[configured](/documentation/build/jobs). If you do not want a job to be run -automatically, by OpenFn, then turn it **off**. - -::: - -### Making a job private - -OpenFn allows you to share jobs to an **open source job library** that other -users can learn from or reuse. All jobs are available for sharing and inherit -project sharing settings, by default. If you do not want a given job to be -available for sharing to [OpenFn's Job Library](/adaptors/library), then you can -mark that job as **private**. To mark a job as private, follow the below steps: - -- From the application **menu**, click on **Jobs**. -- Find the job you would like to mark as **private**. -- On the bottom-left corner of the job card, click on the **View**. -- While on the details page for the selected job, click on the **eye** icon. - -:::info - -Note that once a job is marked as **private**, sharing will be blocked even if -its project is enrolled in the [OpenFn's Job Library](/adaptors/library). You -can toggle this setting back by clicking on the **eye** icon. - -::: - -### Archiving a job - -OpenFn allows you to **archive** a job if it is no longer needed or used. To -archive a job, follow the steps below: - -- From the application **menu**, click on **Jobs**. -- Find the job you would like to **archive**. -- On the bottom-left corner of the job card, click on the **View**. -- While on the details page for the selected job, click on the **archive** icon. -- Confirm archiving in the dialog that pops up after clicking the archive icon. - -:::info - -Note that once **archived**, the job won't appear in your jobs list. Also -messages will not appear to match against it until you restore the job. Also -note that a job **cannot be deleted**, it can only be archived. - -::: - -### Restore archived job - -To restore an archived job, follow the steps below: - -- From the application **menu**, click on **Jobs**. -- While on the jobs list page, click on the **Show archived jobs** button. -- All archived jobs will be shown in the jobs list. -- Find the job you would like to **restore**. -- On the bottom-left corner of the job card, click on the **View**. -- While on the details page for the selected job, click on the **restore** icon. -- The job will now be shown in the list of available jobs. - -### Disabling console logs for a job - -OpenFn allows you to disable `console.log` statements for your job. Disabling -`console.log` ensures that sloppy or malicious code written in the job -expression does not expose sensitive data from the jobs. - -To disable `console.log` for a given job, follow the steps below: - -- From the application **menu**, click on **Jobs**. -- Find the job you would like to **disable console log** for. -- On the bottom-left corner of the job card, click on the **View**. -- While on the details page for the selected job, click on the **lock** icon. - -### Editing a job - -OpenFn allows you to edit or make changes to existing jobs. To edit a given job, -follow the steps below: - -- From the application **menu**, click on **Jobs**. -- Find the job you would like to **edit**. -- On the bottom-left corner of the job card, click on the **View**. -- While on the details page for the selected job, click on the **pencil** icon. -- See details about job editing in [Job Studio here](platform-mgmt#job-studio). - -### Job change history and reverting changes - -If your job is linked to a Github repo, changes made to a job expression can be -reverted to a given git commit. To revert changes made to a job expression, -follow the steps below: - -- From the application **menu**, click on **Jobs**. -- Find the job whose changes you would like to **revert**. -- On the bottom-left corner of the job card, click on the **View**. -- While on the details page for the selected job, scroll down to the bottom of - the job card and click on **View Change History**. -- Select a corresponding change history row. -- Accept the prompts to revert to a previous commit, in the revert dialog. - -:::info - -Note that after the revert dialog confirmation, the job expression will -instantly be reverted to a selected commit. No other jobs will be reverted. To -instantly revert all jobs in for a given project to a previous commit, -[resend the webhook from GitHub](./platform-mgmt#github-version-control). - -::: - -### Creating a new job - -To create a new job, follow the steps below: - -- From the application **menu**, click on **Jobs**. -- Find a **blue** floating button with **+** icon, and click on it. -- Clicking the **+** button will open **Job Studio** for you to enter details - for your new job. -- See details on how to use [Job Studio here](platform-mgmt#job-studio). - -### Job Studio - -**Job Studio** is OpenFn's **Job Editor**. It allows you to create a new job or -edit an exisiting one. It can be accessed by following the steps for -[editing an existing job](platform-mgmt#editing-a-job) or -[creating a new job](platform-mgmt#creating-a-new-job). The instructions below -assume you already know how to open Job Studio by either methods. - -#### Changing Job Studio mode - -Job Studio comes in two editing modes, namely **Wizard mode** and **Fullscreen -mode**. By default, OpenFn JobStudio runs in **wizard mode**. Wizard mode allows -you to configure a job via a step-by-step configuration wizard. On the other -hand, **Fullscreen mode** allows you to quickly configure or edit the job -without the help of the wizard. - -To change from one **Job Studio mode** to another, follow the steps below: - -- While in Job Studio, in the top-right corner, click on the **fullscreen** - icon. -- Depending on the current Job Studio mode, clicking on the **fullscreen** icon - will toggle the editing mode to either **Wizard** or **Fullscreen**. - -:::info - -Note that once you toggle the editing mode, OpenFn updates your user settings -and saves this editing preference as your default Job Studio mode for subsequent -editing sessions. Note, however, that when creating **new jobs**, Job Studio -will always open in **Wizard mode**, regardless of your saved editing mode -preference. - -::: - -#### Configuring a job - -While in Job Studio, if in **Wizard mode**, you will see **four configuration -steps** and an **expression editor**. In **Fullscreen mode**, the **four -configuration steps** appear as regular fields, without a wizard. - -The **four configuration steps** include giving the job a name, defining what -[triggers](/documentation/build/triggers) its execution, selecting an -[adaptor](/adaptors), and providing -[authentication](/documentation/build/credentials) details. - -The **expression editor** is the area where you write your -[job expression](/documentation/build/jobs/#composing-job-expressions). Fill-in -all the details, and click on the **Save** icon in the top-right corner to save -your job's configuration changes. - -#### Inspecting job's initial state - -This feature allows you to view the -[initial state](/documentation/jobs/state/#initial-state) of a selected job. -Note that this feature is currently only available for -[message-triggered jobs](/documentation/build/triggers#message-filter-triggers). - -To view or inspect a job's initial state, click the expression pane splitter and -drag to the right. After dragging, you will see a `json tree` representation of -the matching initial state. To copy a path to a given node in the state, click -on the **_Copy to clipboard_** icon overlaid on the node. The path to that node -will be saved to clipboard, and can then be pasted inside the expression editor -as data path for the job's expression. - -#### Accessing inline adaptor documentation - -For a selected adaptor, OpenFn allows you to view documentation and code -examples for each [adaptor operation](/documentation/jobs/operations). - -To view adaptor documentation, click on the `documentation icon`(first icon) on -the top-right corner of the `Expression Pane`. - -Each adaptor operation has a short description and an example. You can click on -the example expression to copy and use it in your job's expression editor. - -Also note that expression examples or code snippets for adaptor operations can -be auto-generated through the expression editor's autocompletion feature. To -generate a code snippet for a given operation, type the first few letters of the -operation and press the `tab` key. - -#### Changing JobStudio theme - -OpenFn allows you to customize the feel and look of Job Studio. To change Job -Studio's theme from the default one, click the `color palette` icon, and select -a theme of your choice. - -#### Installing a unreleased adaptor version - -In Job Studio, you can install adaptors that are not part of the recommended -adaptors picklist directly from npm. See details -[here](/adaptors#install-on-platform-via-npm) on how to install the unreleased -adaptor version. - -#### Testing a job - -You can test your job without exiting Job Studio, by clicking on the **Save and -Run** button. You can find the **Save and Run** button in the bottom pane of the -Job Studio. - -After clicking on **Save and Run**, the job will be run and its logs will be -streamed to the `Run logs` console. - -:::info - -Note that this feature is currently only available to **message-triggered -jobs**. - -::: - -## Triggers - -This section of the portal allows you to create and manage your Triggers. - -### Searching triggers - -For a project with a number of jobs and a range of trigger criteria, finding a -given trigger can be easily achieved via the search feature. - -Triggers can be filtered/searched by **name** or **criteria**. To search for a -given trigger: - -- From the application **menu**, click on **Triggers**. -- Find the **Search triggers** box and type, in the search box, the **trigger - criteria** for a **message trigger** (e.g, `{"test": "data"}`) or **name** of - the trigger for any other type of trigger. -- The application will filter and show all triggers matching the portion of text - entered into the search box. - -### Editing a trigger - -OpenFn allows you to edit or make changes to existing triggers. To edit a given -trigger, follow the steps below: - -- From the application **menu**, click on **Triggers**. -- Find the trigger you would like to **edit**. -- On the bottom-left corner of the trigger card, click on **Edit**. -- See details about types of triggers and other editing options - [here](/documentation/build/triggers). - -### Deleting a trigger - -OpenFn allows you to delete an existing trigger. To delete a given trigger, -follow the steps below: - -- From the application **menu**, click on **Triggers**. -- Find the trigger you would like to **delete**. -- On the bottom-left corner of the trigger card, click on **Edit**. -- While on the edit page for the selected trigger, click the **trash** icon. -- The application will prompt you to confirm whether you would want to proceed - with deleting the given trigger. - -:::info - -Note that OpenFn will mark this trigger for deletion. You will not be able to -access or edit the trigger once this is done. If there are any jobs linked to -this trigger, they will not run successfully until you assign them new or other -existing triggers. - -::: - -### Creating a trigger - -To create a new trigger, follow the steps below: - -- From the application **menu**, click on **Triggers**. -- Find a **blue** floating button with **+** icon, and click on it. -- Clicking the **+** button will open **New Trigger Form** for you to enter the - details for your new trigger. -- See details about types of triggers and other editing options - [here](/documentation/build/triggers). - -## Credentials - -This section of the portal allows you to create and manage your Credentials. - -### Searching Credentials - -For a project with a number of jobs and a range of credentials, finding a given -credential can be easily achieved via the search feature. - -Credentials can be filtered/searched by **name**. To search for a given -credential: - -- From the application **menu**, click on - [**Credentials** or **My Credentials**](./platform-mgmt#credential-ownership-and-access). -- Find the **Search credentials** box and type, in the search box, **name** of - the credential. -- The application will filter and show all credentials matching the portion of - text entered into the search box. - -:::info - -Note that if you are searching for all credentials you own, then find them via -the **My Credentials** menu item otherwise you can find all credentials assigned -to a given project via the **Credentials** menu item. Also note that not every -credential you own is available to all the projects you are member of. See -details about credential ownership and access -[here](./platform-mgmt#credential-ownership-and-access). - -::: - -### Credential ownership and access - -A credential is owned, by default, by the user who created it. To view all the -credentials you own, follow the steps below: - -- From the application **menu**, click on **My Credentials**. -- A list of all credentials you own will be displayed. - -You can assign a credential to a project, and all users with access to that -project will be able to use it. However, note that a credential can be available -to all users in a given project for use, but only the owner can edit it. - -To view credentials available to a given project, follow the steps below: - -- From the application **menu**, click on **Project Dashboard**. -- Select the **project** for which you would like to see the credentials. -- After the project loads, from the application **menu**, click on - **Credentials**. -- A list of all credentials available to a selected project, will be displayed. - -### Editing a credential - -OpenFn allows you to edit or make changes to existing credentials. To edit a -given credential, follow the steps below: - -- From the application **menu**, click on **Credentials** or **My Credentials**. -- Find the credential you would like to **edit**. -- On the bottom-left corner of the credential card, click on **Edit**. -- See details about types of credentials and other editing options - [here](/documentation/build/credentials). - -### Transferring credential ownership - -In OpenFn, a credential is owned, by default, by the user who created it. -However, OpenFn allows you to change ownership of a credential to another user -of the portal. To transfer credential ownership to another user of the OpenFn -portal, follow the steps below: - -- From the application **menu**, click on **Credentials** or **My Credentials**. -- Find the credential you would like to **transfer ownership**. -- On the bottom-left corner of the credential card, click on **Edit**. -- While on the credential detail page, scroll down to the bottom left corner of - the page and click on **Ownership Transfer**. -- Enter the **email address** and **user number** for the new credential owner. - This information can be found on the recipient's account settings page -- After entering **email address** and **user number**, click on the **Transfer - Ownership** button. -- OpenFn will prompt you to confirm wether to proceed with the transfer or not. - -:::info - -Note that once you proceed with credential ownership transfer, you will lose -access to the credential immediately. The new owner may be able to view or -modify personal or sensitive information stored on this credential.You will not -be able to regain access to this credential without the new owner. - -However, you will still be able to use this credential for jobs in the projects -to which it has been shared until and unless the new owner revokes that -project's access to the credential. - -::: - -### Granting/revoking credential access to a project - -Note that, by default, a credential is available to the project the user had -loaded at the time the user was creating the credential. However, OpenFn allows -you to **grant** or **revoke** access to a credential for one or more projects. -To grant or revoke access to a credential for a project, follow the below steps: - -- From the application **menu**, click on **Credentials** or **My Credentials**. -- Find the credential you would like to **edit project access**. -- On the bottom-left corner of the credential card, click on **Edit**. -- While on the credential detail page, find the **Manage Access** section on the - right side of the page. -- You will see a list of projects that you are a member of. **Mark** the - `checkbox` to **grant** access or **un-mark** the `checkbox` to **revoke** - access for a given project. - -### Deleting a credential - -OpenFn allows you to delete an existing credential if you own it. To delete a -given credential, follow the steps below: - -- From the application **menu**, click on **Credentials**. -- Find the credential you would like to **delete**. -- On the bottom-left corner of the credential card, click on **Edit**. -- While on the edit page for the selected credential, click the **trash** icon - on the top right corner of the page. -- The application will prompt you to confirm whether you would want to proceed - with deleting the given credential. - -:::info - -Note that if you proceed with deleting a given credential, OpenFn will delete -this credential immediately for security reasons. You will not be able to -restore the credential once this is done, but you may create a new credential -with the same login information. If jobs are currently using this credential, -they may not run successfully until you add a new credential and assign it to -those jobs. - -::: - -### Creating a new credential - -To create a new credential, follow the steps below: - -- From the application **menu**, click on **Credentials** or **My Credentials**. -- Find a **blue** floating button with **+** icon, and click on it. -- Clicking the **+** button will prompt you to choose the **type of credential** - you would like to create. -- Note that **credentials** are meant to be used to connect to other systems. So - choose the type of credential that corresponds to the system you will be - integrating with via OpenFn. -- After choosing the type of credential, OpenFn will open the **New Credential - Form** for you to enter the details. -- See details about types of credentials and other editing options - [here](/documentation/build/credentials). - -## Activity - -In this section of the portal, you can view a list of all "runs" - i.e. -individual job runs. This list is essentially a compilation of all jobs, -messages and credentials flowing through your OpenFn account towards your -destination system(s). - -### Runs - -Runs are attempts made on a destination system by running a receipt through a -Job Description. Runs can be viewed and re-processed. Each submission has a -`success`, `started_at`, `finished_at`, `job_description_id`, and `receipt_id` -attribute. `Started_at` and `finished_at` are the timestamps when the submission -began and ended. - -> **Note:** Some runs may take a really long time, particularly if they are -> performing multiple actions in a destination system or if they are fetching -> lots of data from a REST api at the start of a migration. They will appear as -> red if they have failed. In the case of failure, refer to our -> [Troubleshooting](/documentation/manage/troubleshooting-tips-on-platform) -> section below. - -### Filter runs in the Activity view - -You can filter the run logs in the Activity View by: - -- **Text** - Remember to be patient as a full log text search can take time - process. Leverage this feature to search for runs with specific error messages - to support with troubleshooting any failed runs. - -- **Date** - Filter the view to only show runs that failed in the last few - hours/ days/ year – or a custom date range! Note that the default activity - history view shows runs from the last 30 days. - -### Bulk reprocess (retry) runs - -Need to re-process a series of runs? This could be helpful if you had multiple -runs fail due to an error message. - -1. Generate a list of the runs that you want to reprocess by adjusting the - filters—be sure to specify an exact date range, job, status, etc. - -2. Simply click the **Reprocess** button and review the dialog that appears. - This dialog contains important information about the query that will be used - for reprocessing and gives you an approximate number of runs that will be - reprocessed. - -![Retry run button](/img/retrybutton.png) - -3. Click "Reprocess" when you're happy with the query. You'll get feedback on - the number of runs enqueued within seconds, and you should see your project - queue fill up then empty over time as the batch is processed. - -![Retry run button](/img/reprocess-runs.png) - -:::info - -Note that a filtered list of runs will include runs triggered by message -filters, cron, and flow or catch triggers. When you select to reprocess runs -from a filtered list, the runs in that list which can only be triggered by the -successful or failed exit of _another_ run will not be included in the initial -batch. Those jobs will, however, still get run if they are turned on and -successful or failed runs in the batch trigger them. In other words, flow/catch -triggers will behave normally even during a bulk reprocess order. - -::: - -:::note - -Remember that OpenFn plans are run-based, and you can monitor usage in **Project -Settings** to ensure that you don’t hit any run limits when bulk reprocessing! - -::: - -### Export runs to CSV - -You can download and review OpenFn runs data by exporting to a CSV file. - -1. In your activity history view, filter the runs you’d like to export to CSV. - Choose to filter by text, date, job, and status. - -2. Click the **Export as CSV** button to review and confirm the desired export. - -![Export runs button](/img/exportruns.png) - -3. Click the "Export" button to submit the request. A link to download the file - will be sent to your email address shortly. - -![Retry run button](/img/export-runs.png) - -## Inbox - -Your inbox contains the history of all messages that have passed in to your -project, which may or may not have triggered a specific job. Messages are stored -payloads or data that were sent via HTTP post to your inbox. They can be viewed -in formatted JSON, edited, or manually processed (if they did not match a filter -when they were originally delivered.) - -To edit a message, click the "pencil and paper" icon next to that receipt. Be -careful, as no original copy will be persisted. - -### Filter messages in your inbox - -To help you more quickly find relevant messages, you can now filter your inbox -by: - -- **Body Text** - Search your messages for specific text (e.g., find surveys - that contain “India” in the body). As individual projects may have millions of - messages containing tens of thousands of lines of JSON each, we’ve implemented - a “tsvector” search strategy. Please be patient and note that this text-based - search may take a moment to return results.. If you’re curious about how - tsvector works from a technical perspective, check out the - [official documentation](https://www.postgresql.org/docs/10/datatype-textsearch.html#DATATYPE-TSVECTOR). -- **Date** - Choose a relative date range (e.g., “Last 90 Days”) or define a - custom date range yourself. Note that the default inbox view shows “Last 30 - Days”. - -![Image of Inbox Filters](/img/inbox_filter.png) - -### Bulk reprocess messages - -Need to re-run a series of messages? If you had a job fail because of an error -for multiple messages, or need to re-process the data in OpenFn to re-send to a -destination application, then this feature will help you do so more quickly! - -1. Generate a list of the messages that you want to reprocess by adjusting the - filters—be sure to specify an exact date range, the matching trigger, etc. - -2. Simply click the **Reprocess** button and review the dialog that appears. - This dialog contains important information about the query that will be used - for reprocessing and gives you an approximate number of messages that will be - reprocessed. - -![Reprocess button](/img/reprocess_msgs.png) - -3. Click "Reprocess" when you're happy with the query. You'll get feedback on - the number of messages enqueued within seconds, and you should see your - project queue fill up then empty over time as the batch is processed. - -![Retry run button](/img/reprocess-messages.png) - -#### Note when bulk reprocessing messages - -- This simulates the chain of events that starts when messages first arrive in - your inbox. In other words, reprocessed messages will be handled by message - filter triggers for any jobs that have the `autoprocess` setting “on”. If - you've got messages that match certain triggers, but the associated jobs are - switched "off" they will not be run when those messages are reprocessed. - -- Remember that OpenFn plans are run-based, and you can monitor usage in - **Project Settings** to ensure that you don’t hit any run limits when bulk - reprocessing! ![Usage stats chart](/img/usage.png) - -### Export messages to CSV - -You can now download and review OpenFn message data by exporting to a CSV file. - -1. In your inbox, filter the messages you’d like to export to CSV. Choose to - filter by text, date, trigger, and run state. - -2. Click the **Export as CSV** button to review and confirm the desired export. - -![Export CSV button](/img/exportcsv.png) - -3. Click the "Export" button to submit the request. A link to download the file - will be sent to your email address shortly. - -![Retry run button](/img/export-messages.png) - -## Search Console - -The **Search Console** allows users to answer questions such as _Did patient -798123 get successfully referred or Did CommCare submission -123e4567-e89b-12d3-a456-426614174000 get loaded into DHIS2?_ via direct string -search. - -Searches via [Inbox](./platform-mgmt#inbox) and -[Activity History](./platform-mgmt#activity) rely on `JSONB matching` and -`tsvector`, which are more powerful for traversing very large date ranges of -messages or run logs but are less intuitive than string searches. - -The **Search Console** solves this challenge and allows the user to type the -`string` of concern in a **Search Box** and press enter. OpenFn will search in -`message bodies` and `run logs`, by default, and/or in `message headers` if -otherwise specified. - -To use the **Search Console**, follow the below steps: - -1. On the left menu pane, click on **Search Console** link. -2. While on the **Search Console** page, select the **Date Range** and enter the - **text** matching your search in the **Search Box**. -3. Press the **Enter Key** on the _Keyboard_ or click the **Search** button to - search. - -:::note - -OpenFn will limit the results of your search to a maximum of `10` records per -specified search type (i.e. OpenFn will return a maximum of `10` results for -matches found in `bodies`, `logs`, or `headers`). It is therefore recommended to -refine your search to a very _specific string_ and _date range_ for which a -matching result is expected. - -::: - -## Account Management - -### Add a credit card - -OpenFn's hosted iPaaS has a free-forever tier, but if your organization requires -more jobs or runs each month, you can add a credit card and change to a paid -tier. For comprehensive pricing information please visit our -[pricing page](https://openfn.org/pricing). - -To enter your credit card information follow these steps: - -1. Login to your OpenFn account. -2. Click on your profile icon on the top right corner o f the page and select - **Billing**. -3. From the **Billing** page select **Add Card** and enter your credit card - information. - -![Credit Card](/img/add_credit_card.gif) - -### Change plan - -Once your credit card information is entered you can upgrade your plan by -navigating to the Project Settings page and dragging the slider to the right or -left. - -To following these steps: - -1. Login to your OpenFn account. -2. Click on the **Project Settings** link on the left-hand menu of the project - you'd like to modify. (_Or_ click your profile icon on the top right corner - of the page and select **Billing** and select the project that you would like - to upgrade.) -3. This will take you to the **Project Settings** menu. -4. Scroll down on the **Project Settings** page and change plans using the - slider. -5. Once you have selected the desired plan, click **Change to _[plan name]_** - and then confirm the change. - -![Change Plans](/img/change_plan.gif) - -### Lost password - -If at any time you forget the password for your OpenFn account follow these -steps to reset it: - -1. Visit https://openfn.org/login . -2. Enter the email address associated with your account. -3. Click on **Recover Password** (see gif below). This will trigger OpenFn to - send a recovery token to your associated email account. -4. Check your email for the recovery token and make a copy of it. -5. Enter your recovery token and a new password into the OpenFn "Reset Password" - page. - -![Password Reset](/img/recover_password.gif) - -## Project settings - -This section of OpenFn platform allows you to view and update the project -configuration and plan settings. - -### Project Configuration - -To view or update the following project configuration details, follow the steps -below: - -- From the application **menu**, click on **Project Settings** and then find the - **Project Configuration** section. - -#### Changing Project Name - -- Changing your project name will update the URL and, after a 60-day deprecation - period, will break bookmarks or old links to the project. This won't affect - your project's inbox URL, but may impact users with lots of old run or message - links saved offline. - -#### Viewing Project Inbox URL - -- To view the `Inbox URL` for your project, click on the **eye** icon against - the `Unique Inbox URL` label. - -#### Changing Project Description - -- Project description is optional but can be updated under this section. - -#### Changing Concurrency - -- **Concurrency** is the number of jobs that will be run at the same time. - -:::note - -Think of it as the number of workers or employees performing the same task at -your organization. The task may be to convert an inbound patient record to meet -the FHIR standard, then load it into OpenMRS. You could have 10 files waiting to -be processed from separate deliveries. With a concurrency of 10, all ten files -would start to be processed immediately. With a concurrency of 1, they'd be -processed sequentially, the second only being started once your single worker -finished working on the first. - -::: - -- Projects are set to a concurrency of "1" by default. This means that runs will - be processed one-at-a-time and that each subsequent run will be blocked until - the previous run is completed. - -- If your project is subscribed to a paid plan, you have the option of toggling - concurrency from the default "1" all the way up to a concurrency of "10". - -- To change the concurrency level for your project follow these steps: - -1. Login to your OpenFn account. -2. Click on the **Project Settings** link on the left-hand menu of the project - - you'd like to modify. -3. This will take you to the **Project Settings** menu. -4. On the **Project Settings** page change concurrency to the appropriate level - using the slider. -5. Once you have selected the desired concurrency, click **Update Project**. - -![Change Concurrency](/img/change_concurrency.gif) - -#### Changing Notification Threshold - -- By default, OpenFn sends a notification to all project collaborators when - **85%** of the project's allowed runs have been used in a given billing cycle. - You can change this setting by adjusting the **Notification Threshold Slider** - to your desired level. -- Once you have selected the desired notification threshold, click **Update - Project**. - -#### Exporting Project Config - -- OpenFn allows you to run your project as a - [Microservice](/documentation/microservice/home/) . -- There are two options for exporting the project config used in OpenFn - Microservice. Exporting as `project.yaml` will provide you with a `YAML` file - that can be used to run this project with - [OpenFn/engine](/documentation/microservice/home/), - [OpenFn/microservice](/documentation/microservice/home/), or for use in - another OpenFn/platform space. -- Exporting as `microservice.zip` will prepare a `ZIP` file with - `openfn/microservice:latest` (from hub.docker.com) and a your `YAML` file - inside a pre-configured directory structure so that you can run this project - as a microservice via `docker run`. In both cases, your project configuration - will be built asynchronously and you'll receive an email with download link - when it's done. - -### Project Plan - -- In this section, you can view and change your project's pricing plan. - -#### Usage and Subscription - -- This section provides you with a graph that shows your project's current plan - usage limit and current usage pace. -- To view detailed report of the project's usage, find and click on the - **Historical Project Usage** button. - -#### Change Plan - -- By default, OpenFn sets your project's plan to **Free**. -- Before changing your project's subscription plan, you must add a valid credit - card. -- To change the project's plan, find the project plans slider and click on the - plan of your choice. -- After selecting the project plan, click on the **Add Card** button and enter - card details in the form and save. - -### Job Library Sharing - -- The **OpenFn Job Library** is a project supported by the **OpenFn community**. - It's a collection of open-source job code from projects across **ICT4D**. -- You will always able to browse the library so that writing jobs is faster and - easier, but by enabling library contributions for this project, jobs not - marked as "private" will also be published to the library. -- Contribute to the job library, on the top-right corner, find the **library** - icon and click on it. -- Click on the **Yes, contribute to the job library** button of the dialog box - that appears. - -:::note - -Data from **messages** or **runs** are **NEVER** shared. Your job expressions -(which most OpenFn users already keep in public repositories on GitHub) and -other non-sensitive metadata (e.g., adaptor and version, created/updated dates) -will be made searchable to help other organizations and governments write jobs -more quickly and easily if you enable this setting. - -::: - -## User Account Menu - -- You can view and modify your account settings by clicking on the **person** - icon on the top right corner of the **App bar**. - -### Changing User Account Settings - -- To change your user account settings, such as _name_, _IDE Style_, and - _theme_, click on the **Account Settings** of the **User Account Menu**. This - action will take you to the **Account Settings** page. -- While on the **Account Settings** page, make the necessary changes and click - on the **Save** button to save the changes. - -:::note - -- Note also that, while on the **Account Settings** page, you can access - additional features such as _Changing Email_, _Changing Password_, _connecting - and disconnecting to Github_, _Billing Management_, and _Deleting Account_. -- To access these additional features, open the sub-menu by clicking on the - **three dots** on the top-right corner of the **Account Settings** page. - -::: - -### Viewing Billing - -- The **User Account Menu** also allows you to view details of all projects - billed to your account. -- To view a list of all projects billed to your account, click on the - **Billing** menu item. - -### Logging-out - -- You can logout of OpenFn by clicking on the **Logout** menu item of the **User - Account Menu**. - - :::note - - Also note that OpenFn will log you out of your current session after 24 hours - without warning! It also ensures that you are logged-out of all browser tabs, - once your current login session expires. - - ::: - -## Access & Security - -This section covers the **Access & Security** features each OpenFn project has. -To explore these features, on the left hand navigation ribbon click on the -**Access & Security** tab (#1). - -_Please refer to the screenshot below for help navigating the functionality of -this page._ - -![Access&Security Circled](/img/access_security1.png) - -### User Access - -OpenFn provides users with the ability to **add collaborator access**, **revoke -collaborator access**, and, in the event you get stuck and need help from an -implementation specialist, **grant OpenFn support access**. - -#### Add collaborator access - -To add collaborator access to your project from the **Access & Security** page: - -- Enter the e-mail address of your collaborator in the "Add collaborator by -email" field. Note that you will need to select "add as collaborator," or add as -administrator" to add him/her to the project. See the screenshot above for -reference (#2). - - -#### Revoke collaborator access - -To revoke collaborator access to your project from the **Access & Security** -page: - -- Find the collaborator's name in the **User** list and in the **Revoke** column - click on the on the **Revoke** button. See the screenshot above for reference - (#3). - -#### Grant OpenFn support access - -To add OpenFn support team's access to your project from the **Access & -Security** page: - -- Enable the **Grant support access** toggle (#4). - -### Inbox Security - -OpenFn project administrators can choose to configure additional authentication -for any inbound requests made to the project's inbox URL. In the "Access & -Security" page of their OpenFn project, Administrators can choose from API Key -and Basic Auth types, which will prompt administrators to either generate an API -token or to setup a username:password credential. Once this inbox authentication -is configured, any HTTP requests made to the OpenFn Inbox URL must include -either this `x-api-key` token or username:password in the request header. -![inbox security](/img/inbox-security.png) - -#### Rotating auth methods - -Because more than one auth method may be accepted at a given time, some -organizations choose to periodically rotate their auth methods for extra -security and can do so without disrupting live production integrations. To -rotate your inbox auth methods: - -1. Create a _second_ valid auth method with a new token or user:pass - combination. -2. Provide that token to your external systems so that they can start using it - in their webhooks/requests to OpenFn. -3. Once you are certain that all external services are now using the new auth - token, _revoke_ the old auth token. - -You can repeat this process as frequently as is required by your organization's -internal security protocols. - -## GitHub version control - -Managing large numbers of jobs with multiple contributors is complicated. We -developed the GitHub integration so that OpenFn projects can be linked to GitHub -repositories and you can work collaboratively on your jobs, incorporating git -flows for management. - -OK, you're ready to manage your jobs via GitHub, the leading hosted version -control software on the web? Great, this section describes the steps necessary -to get going. - -:::info tl:dr; - -1. If a **commit** is made to a designated branch on GitHub, - - ✅ OpenFn will automatically update the associated job's **expression** to - match the file on GitHub. - -2. If a job's **expression** or **GitHub filepath** is modified on the platform, - - ✅ OpenFn platform will automatically push a **commit** to your Github repo, - updating the linked file to match the expression. - -::: - -Note that if you change a file on GitHub that's _not_ related to any OpenFn -jobs, no update will be made on OpenFn. Likewise, if you edit a job on OpenFn -but _don't_ make any changes to the **expression** or **Github filepath**, no -commit will be made on GitHub. - -:::warning - -As soon as you enter a valid filepath for a job in a project with a connected -Github repo, all modifications made to that job on OpenFn will appear as Github -commits on that branch in that repo. - -Likewise, as soon as you make a commit on Github with a change to a file that is -linked to a job on OpenFn, the contents of that file will overwrite the existing -job on OpenFn. - -⚠️ **PLEASE note** that _before_ you connect Github, there is no version history -for OpenFn jobs on the platform. If you commit something you don't want (like an -empty file) to Github, `autodeploy` is on, and that file is linked to an OpenFn -job, you will **erase your existing job** and you may not be able to retrieve -it. ⚠️ - -For this reason, and because [**OpenFn/cli**](/documentation/cli) provides a -free, open-source, offline testing environment, it's recommended to create your -jobs using a Github repo and test them on your own machine _before_ linking them -to a project on OpenFn. - -::: - -### Setup Steps - -#### Linking your OpenFn account to your Github account - -1. OpenFn: [User Settings](https://www.openfn.org/account): Click the - three-button "action menu" (top right corner of the account card) and select - "Connect to GitHub". -2. GitHub: When prompted by GitHub, grant OpenFn read and write access to - your/your organizations repositories as needed. -3. OpenFn: Once redirected to OpenFn you may be asked to re-authenticate - depending on the domain you originally used to connect to OpenFn. -4. OpenFn: Ensure all changes you've made to your account are saved, and verify - that you see a bright blue check next to "Github OAuth". - -#### Linking projects and jobs to Github repos and files - -1. OpenFn: Project -> Version Control: Specify the repository owner, repository - name and branch for automatic deploys. You can also select to turn on or off - automatic deploys: when _on_ commits to the branch specified will - automatically be written to your jobs on OpenFn. -2. OpenFn: Project -> Jobs -> Job Edit: To link an individual job to a file in a - GitHub repo, edit that job and paste in the path to the job from the root of - your GitHub repo. If your repo looks like this, you'd type `sample_job_1.js` - or `some_folder/some_other_job.js` to link your OpenFn job to the select file - in your repo. - -:::info - -Automated GitHub version control is currently only available for enterprise -users. Contact [enterprise@openfn.org](mailto:enterprise@openfn.org) to build a -custom plan for your needs. - -::: - -### Advanced Version Control - -Using this GitHub integration, you can revert to previous versions of a job by -selecting that version (by its commit date and SHA) on the job view page. A new -commit will be made, updating the job to the state it was in at the time of the -old commit. diff --git a/docs/microservice/home.md b/docs/microservice/home.md deleted file mode 100644 index 304981e2aa9..00000000000 --- a/docs/microservice/home.md +++ /dev/null @@ -1,181 +0,0 @@ ---- -title: Microservice ---- - -:::caution Microservice and devtools are being replaced by Lightning - -Please note that [OpenFn/microservice](https://github.com/OpenFn/microservice) -and [OpenFn/devtools](https://github.com/OpenFn/devtools) are being deprecated -and replaced by [OpenFn/Lightning](https://github.com/OpenFn/lightning), When -lighting is released. - -::: - -## Intent - -OpenFn is used by numerous health and humanitarian organizations around the -world to scale their programs through real-time interoperability, systems -integration, and workflow automation. **OpenFn/microservice** makes use of -OpenFn's open-core technology—namely **OpenFn/core**, **OpenFn/engine**, and the -various OpenFn **adaptors**—to create standalone microservices which can be -deployed on any hardware. - -This microservice approach helps to ensure that governments and NGOs are never -locked-in to OpenFn's SaaS offering, and can port their existing jobs, triggers, -and credentials from [OpenFn.org](https://www.openfn.org) to their own -infrastructure easily. - -## Introduction - -Similar to `platform`, OpenFn/microservice runs on `project.yaml` files. This -means that when organizations or governments have an open-source license -requirement, all their jobs, credentials, and project configurations can be -exported from OpenFn's iPaaS and used to create a microservice deployment. - -While this approach doesn't provide the OpenFn platform front-end with its -various project management and configuration features, it's perfect for groups -with DevOps experience and 100% compatible with the platform. You can even build -and test entire projects on `platform` and then export the `project.yaml` file -to run on your own servers using `microservice`. - -This microservice approach provides flexibility to governments and NGOs, so they -are never locked-in to OpenFn's SaaS platform offering. At any time, an -organization can port their existing jobs, triggers, and credentials from -OpenFn.org to run with our FOSS integration toolkit, using their own -infrastructure. - -## Prerequisites - -Familiarity with other elements of OpenFn's open source integration toolkit is -helpful when considering the microservice approach. - -- [OpenFn/docs](https://docs.openfn.org/) -- [OpenFn/engine](https://github.com/openfn/engine) -- [OpenFn/core](https://github.com/openFn/core) -- [OpenFn/devtools](https://openfn.github.io/devtools/) - -## Docker up and running - -Assuming you've got an `.env` and a sample project at `./sample-project` -directory with a `project.yaml` spec: - -```sh -docker-compose up -``` - -You can configure either the compose file or the .env, or run the container -using `docker run`: - -```sh -docker run -v :/home/microservice/ \ - --env-file \ - --network host \ - openfn/microservice:v0.3.2 -``` - -## Development up and running guide - -- Clone this repo with `git clone git@github.com:OpenFn/microservice.git` -- Enter the directory with `cd microservice` -- Install dependencies with `mix setup` -- Run the tests with `mix test` -- Make a project directory to hold your project artifacts with - `mkdir sample-project` -- Create a new project specification with - `cp project.yaml.example ./sample-project/project.yaml` -- Create a `.env` file with `cp .env.example .env` -- Install necessary adaptors via - `npm install @openfn/language-http --prefix priv/openfn/runtime/node_modules --no-save --no-package-lock --global-style` -- Start your microservice server with - `env $(cat .env | grep -v "#" | xargs ) iex -S mix phx.server` - -### Up and running inside Docker - -- Build a docker image with `docker build -t openfn/microservice:v0.3.0 .` -- Run with the [docker run command](#Docker-run) - -## Project configuration - -You can configure the jobs, triggers, credentials and language packs used in -your microservice in the `project.yaml` config file. - -### First setup using the sample config - -The -[sample project configuration file](https://github.com/OpenFn/microservice/blob/main/project.yaml.example) -describes an example project setup to help you get acquainted with this -structure. - -By default microservice is configured with 4 sample jobs: - -1. `job-1` is triggered when a matching message arrives to the inbox (see - `trigger-1`). -2. `recurring-job` is a timed job scheduled to run every minute and is linked to - the `every-minute` cron trigger. -3. `flow-job` and `catch-job` run after the `success` and `failure` of job-1, - respectively. - -All of the jobs are configured with the language pack `openfn/language-common`. - -In the default sample configuration a new message posted to -`localhost:4000/inbox` that matches `trigger-1` (i.e. the message contains -`"number":2`) is greeted with an asynchronous acknowledgement receipt -(`HTTP 202` `Data accepted and processing has begun`) and will trigger `job-1` -to run. - -You can try this out with the following snippet: - -```sh -curl -X POST -H "Content-Type: application/json" \ - -d '{ - "number":2, - "surveyId": 37479 -}' \ - http://localhost:4000/inbox -``` - -Posting a message not matching any of the triggers (e.g. `“number”:3`) equally -prompts an acknowledgement but doesn’t trigger any jobs. - -Example message post for this non-match scenario: - -```sh -curl -X POST -H "Content-Type: application/json" \ - -d '{ - "number":3, - "surveyId": 37479 -}' \ - http://localhost:4000/inbox -``` - -HTTP `post` requests made to -[`localhost:4000/inbox`](http://localhost:4000/inbox) will be processed by the -`Receiver`, according to the `credential`, `expression`, and `adaptor` defined -in the project configuration `YAML` file. - -Time-based jobs will be run by `Engine` according to the `credential`, -`expression`, and `adaptor` defined in your `project.yaml` file. - -### Setup from your existing OpenFn platform project - -If you have a project configured on OpenFn, you have two ways for exporting your -config on the Project Settings page and running your project in microservice. - -1. If you export as `project.yaml`, you can download your settings in `yaml` - format from your platform project Download page or from a link in the - auto-generated email sent to your address. You can plug this file into your - environment as set up using the - [Development Up and Running Guide](#Development-up-and-running-guide). - -2. If you export as `microservice.zip`, you'll get your microservice folder - ready to run with `docker`, containing - -- a `docker-compose.yaml` config file -- a project folder containing `project.yaml` -- `.env` file with the default environment variables for docker -- a `Readme` file - -`cd` into the folder and run the project with `docker-compose up`. If you don't -have the docker image, it will be auto-pulled from `hub.docker.com`. - -![Export Microservice Zip](/img/microservice_zip_export.gif) diff --git a/docs/projects/platform-mgmt.md b/docs/projects/platform-mgmt.md new file mode 100644 index 00000000000..e1b3bb42a21 --- /dev/null +++ b/docs/projects/platform-mgmt.md @@ -0,0 +1,9 @@ +--- +title: Project Management +--- + +:::warning Under construction + +This docs page is under construction. Check back later for the complete docs, or check out the Docs Version "Platform (v1)". + +::: \ No newline at end of file diff --git a/docs/manage/troubleshooting-tips-on-platform.md b/docs/projects/troubleshooting-tips-on-platform.md similarity index 100% rename from docs/manage/troubleshooting-tips-on-platform.md rename to docs/projects/troubleshooting-tips-on-platform.md diff --git a/docs/source-apps.md b/docs/source-apps.md deleted file mode 100644 index 5895f4afd0a..00000000000 --- a/docs/source-apps.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -title: Generic Data Sources -sidebar_label: Generic Sources ---- - -## Standard webhook configuration - -This section describes how to enable push notifications from selected source -applications or how to configure pull jobs to fetch data from those sources. If -you don't see yours in the alphabetical list below feel free to add it with a -pull request. - -Every OpenFn project has a unique **Inbox URL** address that can be used as an -endpoint for any JSON webhook. To set up a data source, configure that source to -make a POST over HTTPS to your Inbox URL. See [Your Inbox](/build/inbox.md). - -To connect an application with standard JSON webhooks, copy your inbox URL from -the "Inbox" page or your "Project Settings" screen and use it as the destination -URL on your source application. Unless you have specifically configured it on -the "Access & Security" page, no authentication is required. - -**_N.B.: This is by no means an exhaustive list._** It is merely a list of -common sources that external contributors have added. Remember that anything -with a REST api or a JSON-based notification service can be used with OpenFn. - -## How webhooks enable real time integration - -Webhooks services (sometimes called “REST Services”) are services that your -users can configure on your application which make posts to other REST -endpoints. The most common example we’ll come across is a form, submission, or -case forwarding service that will send a copy of a submission to an external -API. - -## Providing a UI for your webhook? - -This is likely the most end-user interactive part of your API, and you’ll -probably want a feature in your user-interface that allows them to turn on and -off these various services. See the below example from Kobo Toolbox (left) and -CommCare (right). - -![kobo_to_commcare](/img/webhooks1.png) - -## When to send? - -Consider whether to set up watches or triggers at the DB level (this seems like -overkill but is provided by some databases relatively inexpensively) or at -several key interfaces in your application. What types of -updates/submissions/changes in your application might other applications need to -be notified of in real time? A new submission is the most common, but updates to -a “case”, changes to UAM, or any other events could be valuable. - -## What to send? - -The whole resource, please. This anticipates our thoughts on sector-wide data -standards slightly, but (within reason) it makes sense to expose everything your -end-user will need to run the next step in their logic. Some interfaces allow -the user to control which fields (and even which related resources) are sent in -a given payload, but often the default is to send everything and let them pick -and choose what they want to use. diff --git a/docs/standards/digital-public-goods.md b/docs/standards/digital-public-goods.md deleted file mode 100644 index 558cfbe2994..00000000000 --- a/docs/standards/digital-public-goods.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -title: Digital Public Goods ---- - -OpenFn is recognised by the -[Ditial Public Goods Alliance](https://digitalpublicgoods.net/) as a Digital -Public Good, or "DPG". - -:::info Digital Public Goods Definition - -Open-source software, open data, open AI models, open standards, and open -content that adhere to privacy and other applicable best practices, do no harm -by design and are of high relevance for attainment of the United Nations 2030 -Sustainable Development Goals (SDGs) - -::: - -You can read more about the DPG standard -[here](https://digitalpublicgoods.net/standard/). diff --git a/docs/standards/global-goods.md b/docs/standards/global-goods.md deleted file mode 100644 index 00fa3bd279c..00000000000 --- a/docs/standards/global-goods.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -title: Global Goods ---- - -OpenFn is one of 36 software applications that have been recognised as a Digital -Square Global Good for Health. - -:::info Global Goods for Health Definition - -A mature digital health software global good is software that is Free and Open -Source Software (FOSS), is supported by a strong community, has a clear -governance structure, is funded by multiple sources, has been deployed at -significant scale, is used across multiple countries, has demonstrated -effectiveness, is designed to be interoperable, and is an emergent standard -application. - -::: - -You can read more about Global Goods for Health -[here](https://digitalsquare.org/digital-health-global-goods). diff --git a/docs/support/support.md b/docs/support/support.md new file mode 100644 index 00000000000..3877fe864d2 --- /dev/null +++ b/docs/support/support.md @@ -0,0 +1,11 @@ +--- +title: Support for OpenFn Implementations +sidebar_label: Support +--- +:::warning Under construction + +This docs page is under construction. Check back later for the complete docs, or check out the Docs Version "Platform (v1)". + +::: + + \ No newline at end of file diff --git a/docs/tutorials/kobo-to-dhis2.md b/docs/tutorials/kobo-to-dhis2.md new file mode 100644 index 00000000000..695f2b29942 --- /dev/null +++ b/docs/tutorials/kobo-to-dhis2.md @@ -0,0 +1,12 @@ +--- +sidebar_label: Kobo to DHIS2 +title: Kobo to DHIS2 Reporting Workflow +--- +:::warning Under construction + +This docs page is under construction. Check back later for the complete docs, or check out the Docs Version "Platform (v1)". + +::: + +# Tutorial: Kobo to DHIS2 Reporting Workflow + diff --git a/docs/users/user-profile.md b/docs/users/user-profile.md new file mode 100644 index 00000000000..a404372ef7c --- /dev/null +++ b/docs/users/user-profile.md @@ -0,0 +1,11 @@ +--- +title: Update User Profile +sidebar_label: User Profile +--- +:::warning Under construction + +This docs page is under construction. Check back later for the complete docs, or check out the Docs Version "Platform (v1)". + +::: + + \ No newline at end of file diff --git a/docusaurus.config.js b/docusaurus.config.js index d4077025896..e42bcf43ece 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -13,10 +13,6 @@ module.exports = { favicon: 'img/favicon.ico', organizationName: 'openfn', projectName: 'docs', - i18n: { - defaultLocale: 'en', - locales: ['en', 'fr'], - }, markdown: { mermaid: true, }, @@ -36,10 +32,10 @@ module.exports = { }, items: [ { - to: 'documentation/', - activeBasePath: 'documentation', - label: 'Docs', + type: 'doc', position: 'left', + docId: 'intro/home', + label: 'Docs', }, { to: 'adaptors', @@ -56,10 +52,6 @@ module.exports = { label: 'Blog', position: 'left', }, - { - type: 'localeDropdown', - position: 'right', - }, { type: 'docsVersionDropdown', position: 'right', @@ -83,21 +75,22 @@ module.exports = { { title: 'This Site', items: [ + // { + // type: 'doc', + // docId: 'intro', + // label: 'Docs', + // }, { - label: 'Documentation', - to: 'documentation', - }, - { - label: 'Adaptors', to: 'adaptors', + label: 'Adaptors', }, { - label: 'Articles', to: 'articles', + label: 'Articles', }, { - label: 'Blog', to: 'blog', + label: 'Blog', }, ], }, @@ -147,11 +140,11 @@ module.exports = { versions: { current: { banner: 'unreleased', - label: "Lightning 🚧" + label: 'Lightning 🚧', }, - 'legacy': { + legacy: { banner: 'none', - label: "Platform (v1)" + label: 'Platform (v1)', }, }, }, diff --git a/package.json b/package.json index 5fead54fdca..ff5c39f9060 100644 --- a/package.json +++ b/package.json @@ -12,15 +12,10 @@ "swizzle": "docusaurus swizzle", "deploy": "docusaurus deploy", "serve": "docusaurus serve", - "clear": "docusaurus clear", - "crowdin": "crowdin", - "write-translations": "docusaurus write-translations", - "write-heading-ids": "docusaurus write-heading-ids", - "crowdin:sync": "docusaurus write-translations && crowdin upload && crowdin download" + "clear": "docusaurus clear" }, "dependencies": { "@babel/helper-module-transforms": "^7.17.7", - "@crowdin/cli": "3.7.8", "@docusaurus/core": "2.4.3", "@docusaurus/plugin-google-gtag": "2.4.3", "@docusaurus/preset-classic": "2.4.3", diff --git a/sidebars-main.js b/sidebars-main.js index 33c50641302..93196bba68a 100644 --- a/sidebars-main.js +++ b/sidebars-main.js @@ -2,114 +2,81 @@ module.exports = { docs: [ { type: 'category', - label: 'Introduction', + label: 'Intro', items: [ - 'intro', - 'getting-started/integration-toolkit', - 'about-lightning', - 'getting-started/so-you-want-to-integrate', + 'intro/home', + // 'intro/terminology', + // 'intro/standards', + // 'intro/security-compliance', + // 'intro/security', + // 'intro/implementation-checklist', ], }, { type: 'category', - label: 'Getting Started', + label: 'Designing Workflows', items: [ - 'build/lightning-quick-start', - 'getting-started/terminology', - 'getting-started/implementation-checklist', - 'getting-started/security', + 'design/overview', + //'design/discovery' ], }, { type: 'category', - label: 'Design', - items: ['design/design-quickstart', 'getting-started/glossary'], + label: 'Tutorials', + items: ['tutorials/kobo-to-dhis2'], }, { type: 'category', - label: 'Build', + label: 'Build Workflows', items: [ - { - type: 'category', - label: 'Jobs', - items: [ - 'build/jobs', - 'jobs/job-design-intro', - 'jobs/understanding', - 'jobs/operations', - 'jobs/multiple-operations', - 'jobs/state', - 'jobs/each', - 'jobs/job-studio', - 'jobs/editing_locally', - 'jobs/working_with_branches', - ], - }, - 'build/triggers', - 'build/credentials', - { type: 'link', label: 'Adaptors', href: '/adaptors' }, - { - type: 'category', - label: 'Live Data', - items: ['build/inbox', 'source-apps'], - }, - 'cli', - 'build/troubleshooting', - { - type: 'category', - label: 'Old Versions', - items: ['core', 'devtools/home', 'microservice/home'], - }, + 'build/tutorial', + //'build/triggers' ], }, { type: 'category', - label: 'Deploy', + label: 'Developers', items: [ - 'deploy/options', - 'deploy/requirements', - 'portability', - 'instant-openhie', + 'developers/cli', + //'developers/for-devs' ], }, { type: 'category', - label: 'Manage', - items: [ - 'manage/platform-mgmt', - 'manage/troubleshooting-tips-on-platform', - 'jobs/errors', - 'jobs/limits', - 'release-notes', - ], + label: 'Local Deployment', + items: ['deploy/options', 'deploy/requirements', 'deploy/portability'], }, + { type: 'category', - label: 'Standards', - items: [ - 'standards/digital-public-goods', - 'standards/global-goods', - 'standards/openhie', - ], + label: 'Monitor Activity History', + items: ['history/activity-history'], }, { - type: 'doc', - id: 'faqs', + type: 'category', + label: 'Manage Projects', + items: ['projects/platform-mgmt'], }, { type: 'category', - label: 'Contributing', - items: [ - 'openfn-roadmap', - 'for-devs', - 'gsoc', - 'writing-docs', - 'style-guide', - ], + label: 'User Management', + items: ['users/user-profile'], }, { - type: 'doc', - id: 'about', + type: 'category', + label: 'Support', + items: ['support/support'], + }, + { + type: 'category', + label: 'Docs & Roadmap', + items: [ + 'contributing/openfn-roadmap', + // 'contributing/roadmap', + // 'contributing/writing-code', + // 'contributing/writing-docs', + // 'contributing/style-guide', + ], }, { type: 'link', diff --git a/src/pages/index.js b/src/pages/index.js index 1b9ee66275d..a4338f3c503 100644 --- a/src/pages/index.js +++ b/src/pages/index.js @@ -220,7 +220,6 @@ function Home() {
Get Started diff --git a/docs/intro.md b/versioned_docs/version-legacy/intro/home.md similarity index 99% rename from docs/intro.md rename to versioned_docs/version-legacy/intro/home.md index c6c1f1b2a3f..e0b21b8da27 100644 --- a/docs/intro.md +++ b/versioned_docs/version-legacy/intro/home.md @@ -1,5 +1,6 @@ --- title: About +id: home sidebar_label: What is OpenFn? slug: / --- diff --git a/versioned_sidebars/version-legacy-sidebars.json b/versioned_sidebars/version-legacy-sidebars.json index 5758a32f663..5f483529b6d 100644 --- a/versioned_sidebars/version-legacy-sidebars.json +++ b/versioned_sidebars/version-legacy-sidebars.json @@ -4,7 +4,7 @@ "type": "category", "label": "Introduction", "items": [ - "intro", + "intro/home", "getting-started/integration-toolkit", "about-lightning", "getting-started/so-you-want-to-integrate" diff --git a/yarn.lock b/yarn.lock index 35ae1006b82..3791842b459 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1765,18 +1765,6 @@ __metadata: languageName: node linkType: hard -"@crowdin/cli@npm:3.7.8": - version: 3.7.8 - resolution: "@crowdin/cli@npm:3.7.8" - dependencies: - njre: ^0.2.0 - shelljs: ^0.8.4 - bin: - crowdin: jdeploy-bundle/jdeploy.js - checksum: 4898403bc034d5058197f0e975a924aaa9c871ca77012add7a2819c597a81a0f9b89d365bab01b651bf978e775809b91eac5d161edc01e77fc08c8bd6efd89d2 - languageName: node - linkType: hard - "@discoveryjs/json-ext@npm:0.5.7": version: 0.5.7 resolution: "@discoveryjs/json-ext@npm:0.5.7" @@ -2562,7 +2550,6 @@ __metadata: resolution: "@openfn/docs@workspace:." dependencies: "@babel/helper-module-transforms": ^7.17.7 - "@crowdin/cli": 3.7.8 "@docusaurus/core": 2.4.3 "@docusaurus/module-type-aliases": 2.4.3 "@docusaurus/plugin-google-gtag": 2.4.3 @@ -4005,13 +3992,6 @@ __metadata: languageName: node linkType: hard -"buffer-crc32@npm:~0.2.3": - version: 0.2.13 - resolution: "buffer-crc32@npm:0.2.13" - checksum: 06252347ae6daca3453b94e4b2f1d3754a3b146a111d81c68924c22d91889a40623264e95e67955b1cb4a68cbedf317abeabb5140a9766ed248973096db5ce1c - languageName: node - linkType: hard - "buffer-from@npm:^1.0.0": version: 1.1.2 resolution: "buffer-from@npm:1.1.2" @@ -4238,13 +4218,6 @@ __metadata: languageName: node linkType: hard -"chownr@npm:^1.1.4": - version: 1.1.4 - resolution: "chownr@npm:1.1.4" - checksum: 115648f8eb38bac5e41c3857f3e663f9c39ed6480d1349977c4d96c95a47266fcacc5a5aabf3cb6c481e22d72f41992827db47301851766c4fd77ac21a4f081d - languageName: node - linkType: hard - "chownr@npm:^2.0.0": version: 2.0.0 resolution: "chownr@npm:2.0.0" @@ -4426,13 +4399,6 @@ __metadata: languageName: node linkType: hard -"command-exists-promise@npm:^2.0.2": - version: 2.0.2 - resolution: "command-exists-promise@npm:2.0.2" - checksum: f6674abbc7d9cc704255999adb42e8b8fe165d941de207d1df8177e1ccea2afb9ff57aad7a70f9235056dd317e45b31f9726ecb2c39826f9a1f98a50f4de1b31 - languageName: node - linkType: hard - "commander@npm:7, commander@npm:^7.2.0": version: 7.2.0 resolution: "commander@npm:7.2.0" @@ -6031,15 +5997,6 @@ __metadata: languageName: node linkType: hard -"fd-slicer@npm:~1.1.0": - version: 1.1.0 - resolution: "fd-slicer@npm:1.1.0" - dependencies: - pend: ~1.2.0 - checksum: c8585fd5713f4476eb8261150900d2cb7f6ff2d87f8feb306ccc8a1122efd152f1783bdb2b8dc891395744583436bfd8081d8e63ece0ec8687eeefea394d4ff2 - languageName: node - linkType: hard - "feed@npm:^4.2.2": version: 4.2.2 resolution: "feed@npm:4.2.2" @@ -6229,15 +6186,6 @@ __metadata: languageName: node linkType: hard -"fs-minipass@npm:^1.2.7": - version: 1.2.7 - resolution: "fs-minipass@npm:1.2.7" - dependencies: - minipass: ^2.6.0 - checksum: 40fd46a2b5dcb74b3a580269f9a0c36f9098c2ebd22cef2e1a004f375b7b665c11f1507ec3f66ee6efab5664109f72d0a74ea19c3370842214c3da5168d6fdd7 - languageName: node - linkType: hard - "fs-minipass@npm:^2.0.0, fs-minipass@npm:^2.1.0": version: 2.1.0 resolution: "fs-minipass@npm:2.1.0" @@ -6273,7 +6221,7 @@ __metadata: "fsevents@patch:fsevents@~2.3.2#~builtin": version: 2.3.2 - resolution: "fsevents@patch:fsevents@npm%3A2.3.2#~builtin::version=2.3.2&hash=df0bf1" + resolution: "fsevents@patch:fsevents@npm%3A2.3.2#~builtin::version=2.3.2&hash=18f3a7" dependencies: node-gyp: latest conditions: os=darwin @@ -8048,7 +7996,7 @@ __metadata: languageName: node linkType: hard -"minimist@npm:^1.2.0, minimist@npm:^1.2.5, minimist@npm:^1.2.6": +"minimist@npm:^1.2.0, minimist@npm:^1.2.5": version: 1.2.8 resolution: "minimist@npm:1.2.8" checksum: 75a6d645fb122dad29c06a7597bddea977258957ed88d7a6df59b5cd3fe4a527e253e9bbf2e783e4b73657f9098b96a5fe96ab8a113655d4109108577ecf85b0 @@ -8106,16 +8054,6 @@ __metadata: languageName: node linkType: hard -"minipass@npm:^2.6.0, minipass@npm:^2.9.0": - version: 2.9.0 - resolution: "minipass@npm:2.9.0" - dependencies: - safe-buffer: ^5.1.2 - yallist: ^3.0.0 - checksum: 077b66f31ba44fd5a0d27d12a9e6a86bff8f97a4978dedb0373167156b5599fadb6920fdde0d9f803374164d810e05e8462ce28e86abbf7f0bea293a93711fc6 - languageName: node - linkType: hard - "minipass@npm:^3.0.0, minipass@npm:^3.1.1, minipass@npm:^3.1.6": version: 3.3.6 resolution: "minipass@npm:3.3.6" @@ -8132,15 +8070,6 @@ __metadata: languageName: node linkType: hard -"minizlib@npm:^1.3.3": - version: 1.3.3 - resolution: "minizlib@npm:1.3.3" - dependencies: - minipass: ^2.9.0 - checksum: b0425c04d2ae6aad5027462665f07cc0d52075f7fa16e942b4611115f9b31f02924073b7221be6f75929d3c47ab93750c63f6dc2bbe8619ceacb3de1f77732c0 - languageName: node - linkType: hard - "minizlib@npm:^2.1.1, minizlib@npm:^2.1.2": version: 2.1.2 resolution: "minizlib@npm:2.1.2" @@ -8151,17 +8080,6 @@ __metadata: languageName: node linkType: hard -"mkdirp@npm:^0.5.5": - version: 0.5.6 - resolution: "mkdirp@npm:0.5.6" - dependencies: - minimist: ^1.2.6 - bin: - mkdirp: bin/cmd.js - checksum: 0c91b721bb12c3f9af4b77ebf73604baf350e64d80df91754dc509491ae93bf238581e59c7188360cec7cb62fc4100959245a42cfe01834efedc5e9d068376c2 - languageName: node - linkType: hard - "mkdirp@npm:^1.0.3, mkdirp@npm:^1.0.4": version: 1.0.4 resolution: "mkdirp@npm:1.0.4" @@ -8234,18 +8152,6 @@ __metadata: languageName: node linkType: hard -"njre@npm:^0.2.0": - version: 0.2.0 - resolution: "njre@npm:0.2.0" - dependencies: - command-exists-promise: ^2.0.2 - node-fetch: ^2.5.0 - tar: ^4.4.8 - yauzl: ^2.10.0 - checksum: b5ec0e31237ff437956a4390c3bdf63ec3f911e83f6fc671d12448ae053a30546c2e82000d815636b118ff20b71395f97f45cc7c331ba15972b6d45ea678da1d - languageName: node - linkType: hard - "no-case@npm:^3.0.4": version: 3.0.4 resolution: "no-case@npm:3.0.4" @@ -8279,20 +8185,6 @@ __metadata: languageName: node linkType: hard -"node-fetch@npm:^2.5.0": - version: 2.6.9 - resolution: "node-fetch@npm:2.6.9" - dependencies: - whatwg-url: ^5.0.0 - peerDependencies: - encoding: ^0.1.0 - peerDependenciesMeta: - encoding: - optional: true - checksum: acb04f9ce7224965b2b59e71b33c639794d8991efd73855b0b250921382b38331ffc9d61bce502571f6cc6e11a8905ca9b1b6d4aeb586ab093e2756a1fd190d0 - languageName: node - linkType: hard - "node-forge@npm:^1": version: 1.3.1 resolution: "node-forge@npm:1.3.1" @@ -8754,13 +8646,6 @@ __metadata: languageName: node linkType: hard -"pend@npm:~1.2.0": - version: 1.2.0 - resolution: "pend@npm:1.2.0" - checksum: 6c72f5243303d9c60bd98e6446ba7d30ae29e3d56fdb6fae8767e8ba6386f33ee284c97efe3230a0d0217e2b1723b8ab490b1bbf34fcbb2180dbc8a9de47850d - languageName: node - linkType: hard - "picocolors@npm:^1.0.0": version: 1.0.0 resolution: "picocolors@npm:1.0.0" @@ -10068,7 +9953,7 @@ __metadata: "resolve@patch:resolve@^1.1.6#~builtin, resolve@patch:resolve@^1.14.2#~builtin, resolve@patch:resolve@^1.3.2#~builtin": version: 1.22.1 - resolution: "resolve@patch:resolve@npm%3A1.22.1#~builtin::version=1.22.1&hash=c3c19d" + resolution: "resolve@patch:resolve@npm%3A1.22.1#~builtin::version=1.22.1&hash=07638b" dependencies: is-core-module: ^2.9.0 path-parse: ^1.0.7 @@ -10180,7 +10065,7 @@ __metadata: languageName: node linkType: hard -"safe-buffer@npm:5.2.1, safe-buffer@npm:>=5.1.0, safe-buffer@npm:^5.1.0, safe-buffer@npm:^5.1.2, safe-buffer@npm:^5.2.1, safe-buffer@npm:~5.2.0": +"safe-buffer@npm:5.2.1, safe-buffer@npm:>=5.1.0, safe-buffer@npm:^5.1.0, safe-buffer@npm:~5.2.0": version: 5.2.1 resolution: "safe-buffer@npm:5.2.1" checksum: b99c4b41fdd67a6aaf280fcd05e9ffb0813654894223afb78a31f14a19ad220bba8aba1cb14eddce1fcfb037155fe6de4e861784eb434f7d11ed58d1e70dd491 @@ -10460,7 +10345,7 @@ __metadata: languageName: node linkType: hard -"shelljs@npm:^0.8.4, shelljs@npm:^0.8.5": +"shelljs@npm:^0.8.5": version: 0.8.5 resolution: "shelljs@npm:0.8.5" dependencies: @@ -10896,21 +10781,6 @@ __metadata: languageName: node linkType: hard -"tar@npm:^4.4.8": - version: 4.4.19 - resolution: "tar@npm:4.4.19" - dependencies: - chownr: ^1.1.4 - fs-minipass: ^1.2.7 - minipass: ^2.9.0 - minizlib: ^1.3.3 - mkdirp: ^0.5.5 - safe-buffer: ^5.2.1 - yallist: ^3.1.1 - checksum: 423c8259b17f8f612cef9c96805d65f90ba9a28e19be582cd9d0fcb217038219f29b7547198e8fd617da5f436376d6a74b99827acd1238d2f49cf62330f9664e - languageName: node - linkType: hard - "tar@npm:^6.1.11, tar@npm:^6.1.2": version: 6.1.13 resolution: "tar@npm:6.1.13" @@ -11522,11 +11392,11 @@ __metadata: "typescript@patch:typescript@^4.1.3#~builtin": version: 4.9.5 - resolution: "typescript@patch:typescript@npm%3A4.9.5#~builtin::version=4.9.5&hash=289587" + resolution: "typescript@patch:typescript@npm%3A4.9.5#~builtin::version=4.9.5&hash=a1c5e5" bin: tsc: bin/tsc tsserver: bin/tsserver - checksum: 1f8f3b6aaea19f0f67cba79057674ba580438a7db55057eb89cc06950483c5d632115c14077f6663ea76fd09fce3c190e6414bb98582ec80aa5a4eaf345d5b68 + checksum: 2eee5c37cad4390385db5db5a8e81470e42e8f1401b0358d7390095d6f681b410f2c4a0c496c6ff9ebd775423c7785cdace7bcdad76c7bee283df3d9718c0f20 languageName: node linkType: hard @@ -12303,7 +12173,7 @@ __metadata: languageName: node linkType: hard -"yallist@npm:^3.0.0, yallist@npm:^3.0.2, yallist@npm:^3.1.1": +"yallist@npm:^3.0.2": version: 3.1.1 resolution: "yallist@npm:3.1.1" checksum: 48f7bb00dc19fc635a13a39fe547f527b10c9290e7b3e836b9a8f1ca04d4d342e85714416b3c2ab74949c9c66f9cebb0473e6bc353b79035356103b47641285d @@ -12324,16 +12194,6 @@ __metadata: languageName: node linkType: hard -"yauzl@npm:^2.10.0": - version: 2.10.0 - resolution: "yauzl@npm:2.10.0" - dependencies: - buffer-crc32: ~0.2.3 - fd-slicer: ~1.1.0 - checksum: 7f21fe0bbad6e2cb130044a5d1d0d5a0e5bf3d8d4f8c4e6ee12163ce798fee3de7388d22a7a0907f563ac5f9d40f8699a223d3d5c1718da90b0156da6904022b - languageName: node - linkType: hard - "yocto-queue@npm:^0.1.0": version: 0.1.0 resolution: "yocto-queue@npm:0.1.0"