Add toUTF8 function in salesforce (#441)

* add toUTF8 function

* add changeset

* add unit test

* fix typo

* update ast

* salesforce: fix any-ascii import

* salesforce: fix any-ascii import

* salesforce: add initialiser to not blow up simple-ast

* bump version and update changelog

---------

Co-authored-by: Joe Clark <[email protected]>

packages/salesforce/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @openfn/language-salesforce
 
+## 4.3.0
+
+### Minor Changes
+
+- 1d5b62f: Add `toUTF8` function
+
 ## 4.2.2
 
 ### Patch Changes

packages/salesforce/ast.json

@@ -831,6 +831,44 @@
         ]
       },
       "valid": true
+    },
+    {
+      "name": "toUTF8",
+      "params": [
+        "input"
+      ],
+      "docs": {
+        "description": "Transliterates unicode characters to their best ASCII representation",
+        "tags": [
+          {
+            "title": "public",
+            "description": null,
+            "type": null
+          },
+          {
+            "title": "example",
+            "description": "fn((state) => {\n  const s = toUTF8(\"άνθρωποι\");\n  console.log(s); // anthropoi\n  return state;\n});"
+          },
+          {
+            "title": "param",
+            "description": "A string with unicode characters",
+            "type": {
+              "type": "NameExpression",
+              "name": "string"
+            },
+            "name": "input"
+          },
+          {
+            "title": "returns",
+            "description": "ASCII representation of input string",
+            "type": {
+              "type": "NameExpression",
+              "name": "String"
+            }
+          }
+        ]
+      },
+      "valid": true
     }
   ],
   "exports": [],

packages/salesforce/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-salesforce",
-  "version": "4.2.2",
+  "version": "4.3.0",
   "description": "Salesforce Language Pack for OpenFn",
   "homepage": "https://docs.openfn.org",
   "exports": {
@@ -32,6 +32,7 @@
   ],
   "dependencies": {
     "@openfn/language-common": "workspace:*",
+    "any-ascii": "^0.3.2",
     "axios": "^0.21.4",
     "jsforce": "^1.11.1",
     "lodash": "^4.17.21"

packages/salesforce/src/Adaptor.js

@@ -24,6 +24,14 @@ import { expandReferences as newExpandReferences } from '@openfn/language-common
 import jsforce from 'jsforce';
 import flatten from 'lodash/flatten';
 
+// use a dynamic import because any-ascii is pure ESM and doesn't play well with CJS
+// Note that technically we should await this, but in practice the module will be loaded
+// before execute is called
+let anyAscii = undefined;
+import('any-ascii').then(m => {
+  anyAscii = m.default;
+});
+
 /**
  * Adds a lookup relation or 'dome insert' to a record.
  * @public
@@ -759,6 +767,22 @@ export function steps(...operations) {
   return flatten(operations);
 }
 
+/**
+ * Transliterates unicode characters to their best ASCII representation
+ * @public
+ * @example
+ * fn((state) => {
+ *   const s = toUTF8("άνθρωποι");
+ *   console.log(s); // anthropoi
+ *   return state;
+ * });
+ * @param {string} input - A string with unicode characters
+ * @returns {String} - ASCII representation of input string
+ */
+export function toUTF8(input) {
+  return anyAscii(input);
+}
+
 // Note that we expose the entire axios package to the user here.
 import axios from 'axios';
 

packages/salesforce/test/Adaptor.test.js

@@ -6,6 +6,7 @@ import {
   createIf,
   upsert,
   upsertIf,
+  toUTF8,
   steps,
   each,
   field,
@@ -170,4 +171,20 @@ describe('Adaptor', () => {
         .catch(done);
     });
   });
+
+  describe('toUTF8', () => {
+    it('Transliterate unicode to ASCII representation', () => {
+      expect(toUTF8('άνθρωποι')).to.eql('anthropoi');
+      // Misc
+      expect(toUTF8('☆ ♯ ♰ ⚄ ⛌')).to.equal('* # + 5 X');
+      // Emojis
+      expect(toUTF8('👑 🌴')).to.eql(':crown: :palm_tree:');
+      // Letterlike
+      expect(toUTF8('№ ℳ ⅋ ⅍')).to.eql('No M & A/S');
+      // Ordinal coordinator
+      expect(toUTF8('Nhamaonha 6ª Classe 2023-10-09')).to.eql(
+        'Nhamaonha 6a Classe 2023-10-09'
+      );
+    });
+  });
 });