update to user process script

nunocoracao · Nov 29, 2024 · 30062aa · 30062aa
1 parent 82f5a07
commit 30062aa
Show file tree

Hide file tree

Showing 54 changed files with 159 additions and 109 deletions.
diff --git a/exampleSite/content/users/0327ac1993f378aa2fc5bbb8a840c9cb/feature.jpg b/exampleSite/content/users/0327ac1993f378aa2fc5bbb8a840c9cb/feature.jpg
diff --git a/exampleSite/content/users/033c87b7e950b82ea500c552c9523367/feature.jpg b/exampleSite/content/users/033c87b7e950b82ea500c552c9523367/feature.jpg
diff --git a/exampleSite/content/users/039ca94b0aeeb9ab4c95226615bfb840/feature.jpg b/exampleSite/content/users/039ca94b0aeeb9ab4c95226615bfb840/feature.jpg
diff --git a/exampleSite/content/users/069e8f01e7ef48830ebea5239e2189b4/feature.jpg b/exampleSite/content/users/069e8f01e7ef48830ebea5239e2189b4/feature.jpg
diff --git a/exampleSite/content/users/080a851b486ff5be4201f6fbe1f9616d/feature.jpg b/exampleSite/content/users/080a851b486ff5be4201f6fbe1f9616d/feature.jpg
diff --git a/exampleSite/content/users/1314bf3d7ce70ad90f1fc7666032e147/feature.jpg b/exampleSite/content/users/1314bf3d7ce70ad90f1fc7666032e147/feature.jpg
diff --git a/exampleSite/content/users/15cf92fba36f0319bd4b176da554fdb9/feature.jpg b/exampleSite/content/users/15cf92fba36f0319bd4b176da554fdb9/feature.jpg
diff --git a/exampleSite/content/users/1a882ded73613252bb07c2c13f40b9e3/feature.jpg b/exampleSite/content/users/1a882ded73613252bb07c2c13f40b9e3/feature.jpg
diff --git a/exampleSite/content/users/1f94add7b0ba58983ff507632b822401/feature.jpg b/exampleSite/content/users/1f94add7b0ba58983ff507632b822401/feature.jpg
diff --git a/exampleSite/content/users/24577d47b9474b6bc72eba9a6009dba4/feature.jpg b/exampleSite/content/users/24577d47b9474b6bc72eba9a6009dba4/feature.jpg
diff --git a/exampleSite/content/users/48ff442c7de299911eab10b72b02cc84/feature.jpg b/exampleSite/content/users/48ff442c7de299911eab10b72b02cc84/feature.jpg
diff --git a/exampleSite/content/users/4b8c5f7ee72baf8411dae867f9206b63/feature.jpg b/exampleSite/content/users/4b8c5f7ee72baf8411dae867f9206b63/feature.jpg
diff --git a/exampleSite/content/users/4de384e83ae8af3834cad28f6dfc6966/feature.jpg b/exampleSite/content/users/4de384e83ae8af3834cad28f6dfc6966/feature.jpg
diff --git a/exampleSite/content/users/5440c85ea880d93157088032515962d3/feature.jpg b/exampleSite/content/users/5440c85ea880d93157088032515962d3/feature.jpg
diff --git a/exampleSite/content/users/5801e67070cd59e592c82693a1bda773/feature.jpg b/exampleSite/content/users/5801e67070cd59e592c82693a1bda773/feature.jpg
diff --git a/exampleSite/content/users/5901a99bd64ec242dcb34b4de5282827/feature.jpg b/exampleSite/content/users/5901a99bd64ec242dcb34b4de5282827/feature.jpg
diff --git a/exampleSite/content/users/64faf41f22317a170125dd7b2cd642e2/feature.jpg b/exampleSite/content/users/64faf41f22317a170125dd7b2cd642e2/feature.jpg
diff --git a/exampleSite/content/users/66a61706d45b77b559869cd1247fe8fc/feature.jpg b/exampleSite/content/users/66a61706d45b77b559869cd1247fe8fc/feature.jpg
diff --git a/exampleSite/content/users/681eec9c776b8417d9ed157ba32d1cca/feature.jpg b/exampleSite/content/users/681eec9c776b8417d9ed157ba32d1cca/feature.jpg
diff --git a/exampleSite/content/users/6a603c5d465ecb2422ed7a96fd55c99c/feature.jpg b/exampleSite/content/users/6a603c5d465ecb2422ed7a96fd55c99c/feature.jpg
diff --git a/exampleSite/content/users/6e0c28f8b70015dd35f332d927960d06/feature.jpg b/exampleSite/content/users/6e0c28f8b70015dd35f332d927960d06/feature.jpg
diff --git a/exampleSite/content/users/6e57b74d392f64f3698aee7046cc2606/feature.jpg b/exampleSite/content/users/6e57b74d392f64f3698aee7046cc2606/feature.jpg
diff --git a/exampleSite/content/users/6ed0dcb17b6e24c166f7da0bdfb0c1f5/feature.jpg b/exampleSite/content/users/6ed0dcb17b6e24c166f7da0bdfb0c1f5/feature.jpg
diff --git a/exampleSite/content/users/72d71e304faaa40b3d547d5357edc450/feature.jpg b/exampleSite/content/users/72d71e304faaa40b3d547d5357edc450/feature.jpg
diff --git a/exampleSite/content/users/7596a225b244e0859da338621e680481/feature.jpg b/exampleSite/content/users/7596a225b244e0859da338621e680481/feature.jpg
diff --git a/exampleSite/content/users/7d028665dc1cf54485e6e82a7083ce69/feature.jpg b/exampleSite/content/users/7d028665dc1cf54485e6e82a7083ce69/feature.jpg
diff --git a/exampleSite/content/users/8032d46bf9c96ae4b59c419032c7ad89/feature.jpg b/exampleSite/content/users/8032d46bf9c96ae4b59c419032c7ad89/feature.jpg
diff --git a/exampleSite/content/users/807a03786a63504a144bb3b947a2969c/feature.jpg b/exampleSite/content/users/807a03786a63504a144bb3b947a2969c/feature.jpg
diff --git a/exampleSite/content/users/82b432ad2bfc873ec725001f0b3ac495/feature.jpg b/exampleSite/content/users/82b432ad2bfc873ec725001f0b3ac495/feature.jpg
diff --git a/exampleSite/content/users/8fc3ba43309eb4210c5da59847dab5ac/feature.jpg b/exampleSite/content/users/8fc3ba43309eb4210c5da59847dab5ac/feature.jpg
diff --git a/exampleSite/content/users/9327a7a2a271822354f23963b0557368/feature.jpg b/exampleSite/content/users/9327a7a2a271822354f23963b0557368/feature.jpg
diff --git a/exampleSite/content/users/954584ca9b811d77ac600cf7c426be45/feature.jpg b/exampleSite/content/users/954584ca9b811d77ac600cf7c426be45/feature.jpg
diff --git a/exampleSite/content/users/98f15180691d43fbfcbaf685e1b9ab06/feature.jpg b/exampleSite/content/users/98f15180691d43fbfcbaf685e1b9ab06/feature.jpg
diff --git a/exampleSite/content/users/9d30279ebc54617897215f9164a2eb27/feature.jpg b/exampleSite/content/users/9d30279ebc54617897215f9164a2eb27/feature.jpg
diff --git a/exampleSite/content/users/9eed597b03512bc7b7b4b3cf81814a37/feature.jpg b/exampleSite/content/users/9eed597b03512bc7b7b4b3cf81814a37/feature.jpg
diff --git a/exampleSite/content/users/c0a065e9e761d745b1ff12b90254cee0/feature.jpg b/exampleSite/content/users/c0a065e9e761d745b1ff12b90254cee0/feature.jpg
diff --git a/...1a99bd64ec242dcb34b4de5282827/index.it.md → ...065e9e761d745b1ff12b90254cee0/index.it.md b/...1a99bd64ec242dcb34b4de5282827/index.it.md → ...065e9e761d745b1ff12b90254cee0/index.it.md
@@ -1,8 +1,8 @@
 ---
-                title: "albertolvera.com"
-                tags: [Sito personale]
-                externalUrl: "https://albertolvera.com"
-                weight: 81
+                title: "さばっこ"
+                tags: [Sito personale, Blog]
+                externalUrl: "https://sabac.co"
+                weight: 961
                 showDate: false
                 showAuthor: false
                 showReadingTime: false

diff --git a/...1a99bd64ec242dcb34b4de5282827/index.ja.md → ...065e9e761d745b1ff12b90254cee0/index.ja.md b/...1a99bd64ec242dcb34b4de5282827/index.ja.md → ...065e9e761d745b1ff12b90254cee0/index.ja.md
@@ -1,8 +1,8 @@
 ---
-                title: "albertolvera.com"
-                tags: [個人サイト]
-                externalUrl: "https://albertolvera.com"
-                weight: 81
+                title: "さばっこ"
+                tags: [個人サイト, ブログ]
+                externalUrl: "https://sabac.co"
+                weight: 961
                 showDate: false
                 showAuthor: false
                 showReadingTime: false

diff --git a/...5901a99bd64ec242dcb34b4de5282827/index.md → ...c0a065e9e761d745b1ff12b90254cee0/index.md b/...5901a99bd64ec242dcb34b4de5282827/index.md → ...c0a065e9e761d745b1ff12b90254cee0/index.md
@@ -1,8 +1,8 @@
 ---
-                title: "albertolvera.com"
-                tags: [Personal site]
-                externalUrl: "https://albertolvera.com"
-                weight: 81
+                title: "さばっこ"
+                tags: [Personal site,Blog]
+                externalUrl: "https://sabac.co"
+                weight: 961
                 showDate: false
                 showAuthor: false
                 showReadingTime: false

diff --git a/...9bd64ec242dcb34b4de5282827/index.zh-cn.md → ...e9e761d745b1ff12b90254cee0/index.zh-cn.md b/...9bd64ec242dcb34b4de5282827/index.zh-cn.md → ...e9e761d745b1ff12b90254cee0/index.zh-cn.md
@@ -1,8 +1,8 @@
 ---
-                title: "albertolvera.com"
-                tags: [个人网站]
-                externalUrl: "https://albertolvera.com"
-                weight: 81
+                title: "さばっこ"
+                tags: [个人网站, 博客]
+                externalUrl: "https://sabac.co"
+                weight: 961
                 showDate: false
                 showAuthor: false
                 showReadingTime: false

diff --git a/exampleSite/content/users/c506ded8078df151fc8ce943e7ac1c8a/feature.jpg b/exampleSite/content/users/c506ded8078df151fc8ce943e7ac1c8a/feature.jpg
diff --git a/exampleSite/content/users/d52c3523ad809eb1d9ddbcbe32e92661/feature.jpg b/exampleSite/content/users/d52c3523ad809eb1d9ddbcbe32e92661/feature.jpg
diff --git a/exampleSite/content/users/d6d2aa4cd9e351e749136df67a27c136/feature.jpg b/exampleSite/content/users/d6d2aa4cd9e351e749136df67a27c136/feature.jpg
diff --git a/exampleSite/content/users/d8ebb1396d8c7d2233b33a6b3ca0adc4/feature.jpg b/exampleSite/content/users/d8ebb1396d8c7d2233b33a6b3ca0adc4/feature.jpg
diff --git a/exampleSite/content/users/db20512c1ee8daab423ea18268cb5910/feature.jpg b/exampleSite/content/users/db20512c1ee8daab423ea18268cb5910/feature.jpg
diff --git a/exampleSite/content/users/e07582d573aa187c3c15bb042872ca4a/feature.jpg b/exampleSite/content/users/e07582d573aa187c3c15bb042872ca4a/feature.jpg
diff --git a/exampleSite/content/users/e263f9bb6dcec27003a8f5f947ba5abb/feature.jpg b/exampleSite/content/users/e263f9bb6dcec27003a8f5f947ba5abb/feature.jpg
diff --git a/exampleSite/content/users/e263f9bb6dcec27003a8f5f947ba5abb/index.it.md b/exampleSite/content/users/e263f9bb6dcec27003a8f5f947ba5abb/index.it.md
@@ -0,0 +1,14 @@
+---
+                title: "Academy Customize"
+                tags: [Sito aziendale]
+                externalUrl: "https://acamize.com/"
+                weight: 951
+                showDate: false
+                showAuthor: false
+                showReadingTime: false
+                showEdit: false
+                showLikes: false
+                showViews: false
+                layoutBackgroundHeaderSpace: false
+                ---
+

diff --git a/exampleSite/content/users/e263f9bb6dcec27003a8f5f947ba5abb/index.ja.md b/exampleSite/content/users/e263f9bb6dcec27003a8f5f947ba5abb/index.ja.md
@@ -0,0 +1,14 @@
+---
+                title: "Academy Customize"
+                tags: [事業所]
+                externalUrl: "https://acamize.com/"
+                weight: 951
+                showDate: false
+                showAuthor: false
+                showReadingTime: false
+                showEdit: false
+                showLikes: false
+                showViews: false
+                layoutBackgroundHeaderSpace: false
+                ---
+

diff --git a/exampleSite/content/users/e263f9bb6dcec27003a8f5f947ba5abb/index.md b/exampleSite/content/users/e263f9bb6dcec27003a8f5f947ba5abb/index.md
@@ -0,0 +1,13 @@
+---
+                title: "Academy Customize"
+                tags: [Business site]
+                externalUrl: "https://acamize.com/"
+                weight: 951
+                showDate: false
+                showAuthor: false
+                showReadingTime: false
+                showEdit: false
+                showLikes: false
+                showViews: false
+                layoutBackgroundHeaderSpace: false
+                ---

diff --git a/exampleSite/content/users/e263f9bb6dcec27003a8f5f947ba5abb/index.zh-cn.md b/exampleSite/content/users/e263f9bb6dcec27003a8f5f947ba5abb/index.zh-cn.md
@@ -0,0 +1,14 @@
+---
+                title: "Academy Customize"
+                tags: [企业网站]
+                externalUrl: "https://acamize.com/"
+                weight: 951
+                showDate: false
+                showAuthor: false
+                showReadingTime: false
+                showEdit: false
+                showLikes: false
+                showViews: false
+                layoutBackgroundHeaderSpace: false
+                ---
+

diff --git a/exampleSite/content/users/fec44f5dd2be29e37c305696d018f99f/feature.jpg b/exampleSite/content/users/fec44f5dd2be29e37c305696d018f99f/feature.jpg
diff --git a/exampleSite/content/users/users.json b/exampleSite/content/users/users.json
@@ -64,14 +64,6 @@
       "College club site"
     ]
   },
-  {
-    "title": "albertolvera.com",
-    "url": "https://albertolvera.com",
-    "source": "n/a",
-    "tags": [
-      "Personal site"
-    ]
-  },
   {
     "title": "fahru.my.id",
     "url": "https://www.fahru.my.id",

diff --git a/processUsers.js b/processUsers.js
@@ -1,5 +1,5 @@
 const fs = require('fs');
-const crypto = require("crypto") 
+const crypto = require("crypto")
 const puppeteer = require("puppeteer");
 
 const configDir = "./exampleSite/config/_default";
@@ -12,107 +12,114 @@ var targetLangs = []
 
 const configFiles = fs.readdirSync(configDir);
 configFiles.forEach(file => {
-    if (file.indexOf("languages.") > -1) {
-        var lang = file.split(".")[1];
-        if (lang != defaultLang) {
-            targetLangs.push(lang);
-        }
+  if (file.indexOf("languages.") > -1) {
+    var lang = file.split(".")[1];
+    if (lang != defaultLang) {
+      targetLangs.push(lang);
     }
+  }
 });
 
 const indexFiles = fs.readdirSync(usersFolderPath);
 for (var i in targetLangs) {
-    var targetFile = '_index.' + targetLangs[i] + '.md';
-    if (indexFiles.indexOf(targetFile) == -1) {
-        fs.copyFileSync(usersFolderPath + '_index.md', usersFolderPath + targetFile);
-    }
+  var targetFile = '_index.' + targetLangs[i] + '.md';
+  if (indexFiles.indexOf(targetFile) == -1) {
+    fs.copyFileSync(usersFolderPath + '_index.md', usersFolderPath + targetFile);
+  }
 }
 
 let rawdata = fs.readFileSync(usersFolderPath + 'users.json');
 let users = JSON.parse(rawdata);
 let userDict = {}
 for (var i in users) {
-    userDict[users[i].title.replaceAll("/", "-")] = true;
+  userDict[generateDirName(users[i].url)] = true;
 }
 
 const files = fs.readdirSync(usersFolderPath);
 
 
 for (file in files) {
-    let stats = fs.statSync(usersFolderPath + files[file]);
-    if (files[file] != 'users.json' && files[file].indexOf("_index.") == -1) {
-        if (stats.isDirectory()) {
-            if (!userDict[files[file].replaceAll("/", "-")]) {
-                console.log('deleting: ', files[file]);
-                fs.rmdirSync(usersFolderPath + files[file], { recursive: true, force: true });
-            }
-        } else {
-            console.log('deleting: ', files[file]);
-            fs.unlinkSync(usersFolderPath + files[file]);
-        }
+  let stats = fs.statSync(usersFolderPath + files[file]);
+  if (files[file] != 'users.json' && files[file].indexOf("_index.") == -1) {
+    if (stats.isDirectory()) {
+      if (!userDict[files[file]]) {
+        console.log('deleting: ', files[file]);
+        fs.rmdirSync(usersFolderPath + files[file], { recursive: true, force: true });
+      }
+    } else {
+      console.log('deleting: ', files[file]);
+      fs.unlinkSync(usersFolderPath + files[file]);
     }
+  }
 }
 
 var cache = {};
 
+function generateDirName(seed) {
+  var hash = crypto.createHash('md5');
+  hash.update(seed);
+  var dir = hash.digest('hex');
+  return dir;
+}
+
 async function convert(text, from, to) {
-    var options = {
-        from: from,
-        to: to
-    };
-    if (!cache[to])
-        cache[to] = {};
-    if (cache[to][text])
-        return cache[to][text];
-    var translated_text = await translate(text, options);
-    cache[to][text] = translated_text.text;
-    return translated_text.text;
+  var options = {
+    from: from,
+    to: to
+  };
+  if (!cache[to])
+    cache[to] = {};
+  if (cache[to][text])
+    return cache[to][text];
+  var translated_text = await translate(text, options);
+  cache[to][text] = translated_text.text;
+  return translated_text.text;
 }
 
 async function translateFrontMatterTags(block, targetLang, tags) {
-    var array = block.split("\n");
-    var translatedBlock = "";
-    for (var i = 0; i < array.length; i++) {
-        if (array[i].indexOf(":") > -1) {
-            var elements = array[i].split(":");
-            var newElement = "";
-            if (elements[0].indexOf("tags") != -1) {
-                translatedTags = []
-                for (var j in tags) {
-                    var tempTag = await convert(tags[j], defaultLang, targetLang);
-                    translatedTags.push(tempTag);
-                }
-                var trasnlatedTagsString = translatedTags.join(", ");
-                newElement = elements[0] + ": [" + trasnlatedTagsString + "]";
-            } else {
-                newElement = array[i];
-            }
-        } else {
-            newElement = array[i];
+  var array = block.split("\n");
+  var translatedBlock = "";
+  for (var i = 0; i < array.length; i++) {
+    if (array[i].indexOf(":") > -1) {
+      var elements = array[i].split(":");
+      var newElement = "";
+      if (elements[0].indexOf("tags") != -1) {
+        translatedTags = []
+        for (var j in tags) {
+          var tempTag = await convert(tags[j], defaultLang, targetLang);
+          translatedTags.push(tempTag);
         }
-        translatedBlock += newElement + "\n";
+        var trasnlatedTagsString = translatedTags.join(", ");
+        newElement = elements[0] + ": [" + trasnlatedTagsString + "]";
+      } else {
+        newElement = array[i];
+      }
+    } else {
+      newElement = array[i];
     }
-    return translatedBlock;
+    translatedBlock += newElement + "\n";
+  }
+  return translatedBlock;
 }
 
 puppeteer
-    .launch({
-        defaultViewport: {
-            width: 1280,
-            height: 800,
-        },
-    })
-    .then(async (browser) => {
+  .launch({
+    defaultViewport: {
+      width: 1280,
+      height: 800,
+    },
+  })
+  .then(async (browser) => {
 
-        const page = await browser.newPage();
+    const page = await browser.newPage();
 
-        for (var i in users) {
+    for (var i in users) {
 
-            var userMDFile = "---\n\
+      var userMDFile = "---\n\
                 title: \""+ users[i].title + "\"\n\
                 tags: ["+ users[i].tags + "]\n\
                 externalUrl: \""+ users[i].url + "\"\n\
-                weight: "+ (i+1) +"\n\
+                weight: "+ (i + 1) + "\n\
                 showDate: false\n\
                 showAuthor: false\n\
                 showReadingTime: false\n\
@@ -121,26 +128,22 @@ puppeteer
                 showViews: false\n\
                 layoutBackgroundHeaderSpace: false\n\
                 \r---\n";
-
-            //digest the title to create a folder
-            var hash = crypto.createHash('md5');
-            hash.update(users[i].url);
-            var dir = usersFolderPath + hash.digest('hex');
-
-
-            if (!fs.existsSync(dir)) {
-                fs.mkdirSync(dir);
-                console.log(i, users[i].title);
-                fs.writeFileSync(dir + '/index.md', userMDFile);
-                for (var j in targetLangs) {
-                    var content = await translateFrontMatterTags(userMDFile, targetLangs[j], users[i].tags);
-                    fs.writeFileSync(dir + '/index.' + targetLangs[j] + '.md', content);
-                }
-                await page.goto(users[i].url);
-                await page.screenshot({ path: dir + "/feature.jpg" });
-            }
+
+      var dir = usersFolderPath + generateDirName(users[i].url);
+
+      if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir);
+        console.log(i, users[i].title);
+        fs.writeFileSync(dir + '/index.md', userMDFile);
+        for (var j in targetLangs) {
+          var content = await translateFrontMatterTags(userMDFile, targetLangs[j], users[i].tags);
+          fs.writeFileSync(dir + '/index.' + targetLangs[j] + '.md', content);
         }
+        await page.goto(users[i].url);
+        await page.screenshot({ path: dir + "/feature.jpg" });
+      }
+    }
 
-        await browser.close();
+    await browser.close();
 
-    });
+  });