Add DEBUG env to print memory info, and improve file download avoid…

… crash when file download file or 404.

feishu-pages/package.json

@@ -1,6 +1,6 @@
 {
   "name": "feishu-pages",
-  "version": "0.5.12",
+  "version": "0.5.13",
   "description": "Generate Feishu Wiki into a Markdown for work with Static Page Generators.",
   "repository": {
     "url": "https://github.com/longbridgeapp/feishu-pages"
@@ -24,7 +24,7 @@
     "@types/node": "^20.5.7",
     "axios": "^1.5.0",
     "dotenv": "^16.3.1",
-    "feishu-docx": "0.5.6",
+    "feishu-docx": "0.5.7",
     "mime-types": "^2.1.35",
     "typescript": "^5.2.2"
   },

feishu-pages/src/doc.ts

@@ -3,7 +3,7 @@ import fs from 'fs';
 import yaml from 'js-yaml';
 import path from 'path';
 import { CACHE_DIR, Doc, feishuFetchWithIterator } from './feishu';
-import { writeTemplfile } from './utils';
+import { printMemoryUsage, writeTemplfile } from './utils';
 
 /**
  * Fetch doc content
@@ -53,9 +53,11 @@ export const fetchDocBody = async (fileDoc: Doc) => {
   };
 
   doc.blocks = await fetchDocBlocks(document_id);
+  printMemoryUsage('loaded doc blocks');
 
   const render = new MarkdownRenderer(doc as any);
   const content = render.parse();
+  printMemoryUsage('MarkdownRenderer parsed');
   const fileTokens = render.fileTokens;
   const meta = render.meta;
 

feishu-pages/src/feishu.ts

@@ -242,6 +242,11 @@ export const feishuDownload = async (fileToken: string, localPath: string) => {
         }
       )
       .then((res: AxiosResponse) => {
+        if (res.status !== 200) {
+          console.error(' -> Failed to download image:', fileToken, res.status);
+          return null;
+        }
+
         // Write cache info
         fs.writeFileSync(cacheFileMetaPath, JSON.stringify(res.headers));
         res.data.pipe(fs.createWriteStream(cacheFilePath));
@@ -291,8 +296,12 @@ export const feishuDownload = async (fileToken: string, localPath: string) => {
   if (!hasCache) {
     console.info(' -> Writing file:', localPath);
   }
-  fs.copyFileSync(cacheFilePath, localPath);
+  if (!fs.existsSync(cacheFilePath)) {
+    console.warn('file not found,', cacheFilePath, 'may be is download 404');
+    return null;
+  }
 
+  fs.copyFileSync(cacheFilePath, localPath);
   return localPath;
 };
 

feishu-pages/src/utils.ts

@@ -137,3 +137,22 @@ export function cleanupTmpFiles() {
     fs.rmSync(tmpDir, { recursive: true });
   }
 }
+
+export function printMemoryUsage(prefix?: string) {
+  if (process.env.DEBUG !== '1' && process.env.DEBUG !== 'true') {
+    return;
+  }
+
+  const used = process.memoryUsage();
+  if (prefix) {
+    prefix = prefix + ' ';
+  }
+
+  console.log(
+    `${prefix}${humanizeFileSize(used.rss)} RSS, ${humanizeFileSize(
+      used.heapTotal
+    )} heapTotal, ${humanizeFileSize(
+      used.heapUsed
+    )} heapUsed, ${humanizeFileSize(used.external)} external`
+  );
+}

yarn.lock

@@ -931,6 +931,14 @@
     "@types/tough-cookie" "*"
     parse5 "^7.0.0"
 
+"@types/node-fetch@^2.6.9":
+  version "2.6.9"
+  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.9.tgz#15f529d247f1ede1824f7e7acdaa192d5f28071e"
+  integrity sha512-bQVlnMLFJ2d35DkPNjEPmd9ueO/rh5EiaZt2bhqiSarPjZIuIV6bPQVqcrEyvNo+AfTrRGVazle1tl597w3gfA==
+  dependencies:
+    "@types/node" "*"
+    form-data "^4.0.0"
+
 "@types/node@*", "@types/node@^20.5.7":
   version "20.5.7"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-20.5.7.tgz#4b8ecac87fbefbc92f431d09c30e176fc0a7c377"
@@ -2626,6 +2634,11 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/mark.js/-/mark.js-8.11.1.tgz#180f1f9ebef8b0e638e4166ad52db879beb2ffc5"
   integrity sha512-1I+1qpDt4idfgLQG+BNWmrqku+7/2bi5nLf4YwF8y8zXvmfiTBY3PV3ZibfrjBueCByROpuBjLLFCajqkgYoLQ==
 
+marked-xhtml@^1.0.7:
+  version "1.0.7"
+  resolved "https://registry.yarnpkg.com/marked-xhtml/-/marked-xhtml-1.0.7.tgz#a16d5ee6fe176783ff3c0da07db18aeec068765e"
+  integrity sha512-UAdMhRgS5K+OVqCI9BDRp42ioMs8TqdNdoSMCVf+7kyyRCvYySMec2mRPud1IrMrdQ1Vs8cCTk32xQ9LB0utFA==
+
 marked@^9.1.2:
   version "9.1.2"
   resolved "https://registry.yarnpkg.com/marked/-/marked-9.1.2.tgz#a54ca772d2b5a43de7d8ed40111354b4b7985527"
@@ -2702,6 +2715,13 @@ natural-compare@^1.4.0:
   resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
   integrity sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==
 
+node-fetch@^2:
+  version "2.7.0"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d"
+  integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==
+  dependencies:
+    whatwg-url "^5.0.0"
+
 node-int64@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b"
@@ -3323,6 +3343,11 @@ tr46@^4.1.1:
   dependencies:
     punycode "^2.3.0"
 
+tr46@~0.0.3:
+  version "0.0.3"
+  resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
+  integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==
+
 ts-interface-checker@^0.1.9:
   version "0.1.13"
   resolved "https://registry.yarnpkg.com/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz#784fd3d679722bc103b1b4b8030bcddb5db2a699"
@@ -3460,6 +3485,11 @@ walker@^1.0.8:
   dependencies:
     makeerror "1.0.12"
 
+webidl-conversions@^3.0.0:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"
+  integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==
+
 webidl-conversions@^7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-7.0.0.tgz#256b4e1882be7debbf01d05f0aa2039778ea080a"
@@ -3485,6 +3515,14 @@ whatwg-url@^12.0.0, whatwg-url@^12.0.1:
     tr46 "^4.1.1"
     webidl-conversions "^7.0.0"
 
+whatwg-url@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d"
+  integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==
+  dependencies:
+    tr46 "~0.0.3"
+    webidl-conversions "^3.0.0"
+
 which@^2.0.1:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1"