improve decomposition of input in la sutysisku

lagleki · Jul 23, 2021 · 5d7b401 · 5d7b401
1 parent c5e2dfd
commit 5d7b401
Show file tree

Hide file tree

Showing 20 changed files with 10,798 additions and 633 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -7,7 +7,12 @@ RUN apt-get update
 
 RUN apt-get install -y build-essential software-properties-common curl
 
-RUN apt-get install -y python3 python3-dev python3-pip nodejs npm
+RUN apt-get install -y python3 python3-dev
+
+# get install script and pass it to execute: 
+RUN curl -sL https://deb.nodesource.com/setup_16.x | bash
+# and install node 
+RUN apt-get install nodejs
 
 RUN apt-get install -y rsync
 

diff --git a/dev.sh b/dev.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 docker kill livla ; docker rm livla
-docker build -f ./dev/Dockerfile-dev -t livla .
+docker build --no-cache -f ./dev/Dockerfile-dev -t livla .
 
 docker run \
   -d -it \

diff --git a/dev/Dockerfile-dev b/dev/Dockerfile-dev
@@ -7,7 +7,12 @@ RUN apt-get update
 
 RUN apt-get install -y build-essential software-properties-common curl
 
-RUN apt-get install -y python3 python3-dev python3-pip nodejs npm
+RUN apt-get install -y python3 python3-dev
+
+# get install script and pass it to execute: 
+RUN curl -sL https://deb.nodesource.com/setup_16.x | bash
+# and install node 
+RUN apt-get install nodejs
 
 RUN apt-get install -y rsync
 

diff --git a/docker_start.sh b/docker_start.sh
@@ -5,7 +5,7 @@ docker kill livla ; docker rm livla
 docker run \
   -it \
   --name livla \
-  --memory 2g \
+  --memory 3g \
   -v $(pwd)/src:/livla/src/:Z \
   -v $(pwd)/config:/livla/config/:Z \
   -v $(pwd)/node_modules:/livla/node_modules/:Z \

diff --git a/src/livla/index.js b/src/livla/index.js
@@ -762,9 +762,48 @@ function splitToChunks(array, parts) {
   return result;
 }
 
-const ningau_paladeksi_sutysisku = async ({ json, tegerna }) => {
-  const { compress } = require('compress-json')
+function workerGenerateChunk({ tegerna, chunk, index }) {
+  const fs = require('fs-extra')
+  const path = require('path-extra')
+  const brotli = require('brotli-wasm');
+  let result = []
+  const outp = {
+    formatName: 'dexie',
+    formatVersion: 1,
+    data: {
+      databaseName: 'sorcu1',
+      databaseVersion: 1,
+      tables: [
+        {
+          name: 'valsi',
+          schema: '++id, bangu, w, y, d, n, t, *s, g, *r, *cache',
+          rowCount: chunk.length,
+        },
+      ],
+      data: [
+        {
+          tableName: 'valsi',
+          inbound: true,
+          rows: chunk,
+        },
+      ],
+    },
+  }
+
+  let t = path.join(
+    '/livla/build/sutysisku/data',
+    `parsed-${tegerna}-${index}.bin`
+  )
+  try {
+    fs.writeFileSync(t, brotli.compress(Buffer.from(JSON.stringify(outp))))
+    result.push(`saving ${tegerna}-${index}`);
+  } catch (error) {
+    result.push(`coudlnt save ${tegerna}-${index}`);
+  }
+  return result
+}
 
+const ningau_paladeksi_sutysisku = async ({ json, tegerna }) => {
   const keys = Object.keys(json)
 
   let arr = []
@@ -788,36 +827,14 @@ const ningau_paladeksi_sutysisku = async ({ json, tegerna }) => {
   arr.sort((x, y) => (order.indexOf(x.t) === -1) ? 1 : order.indexOf(x.t) - order.indexOf(y.t));
   const hash = require('object-hash')(arr)
 
-  splitToChunks(arr, 5).forEach((chunk, index) => {
-    const outp = {
-      formatName: 'dexie',
-      formatVersion: 1,
-      data: {
-        databaseName: 'sorcu1',
-        databaseVersion: 1,
-        tables: [
-          {
-            name: 'valsi',
-            schema: '++id, bangu, w, y, d, n, t, *s, g, *r, *cache',
-            rowCount: chunk.length,
-          },
-        ],
-        data: [
-          {
-            tableName: 'valsi',
-            inbound: true,
-            rows: chunk,
-          },
-        ],
-      },
-    }
-
-    let t = path.join(
-      '/livla/build/sutysisku/data',
-      `parsed-${tegerna}-${index}.blobz.json`
-    )
-    fs.writeFileSync(t, JSON.stringify(compress(outp)))
-  })
+  let index = 0
+  for (const chunk of splitToChunks(arr, 5)) {
+    const { makeSimpleWorker } = require("inline-webworker-functional/node")
+    const generateChunk = makeSimpleWorker(workerGenerateChunk)
+    const result = await generateChunk({ tegerna, chunk, index })
+    for (const el of result) console.log(el);
+    index++
+  }
   const versio = '/livla/build/sutysisku/data/versio.json'
   let jsonTimes = {}
   try {
@@ -826,7 +843,6 @@ const ningau_paladeksi_sutysisku = async ({ json, tegerna }) => {
   jsonTimes[tegerna] = hash
   fs.writeFileSync(versio, JSON.stringify(jsonTimes))
   console.log(`updated ${tegerna} dexie.js sutysisku`);
-
 }
 
 const ningau_palasutysisku = async (language, lojbo) => {

diff --git a/src/mahantufa/cmaxes.html b/src/mahantufa/cmaxes.html
@@ -5,7 +5,7 @@
   <script type="text/javascript" src="https://code.jquery.com/jquery-3.5.1.min.js"></script>
   <script src='https://cdnjs.cloudflare.com/ajax/libs/socket.io/2.0.4/socket.io.js'></script>
   <script>
-    var socket = io.connect('http://vrici.lojban.org:3020');
+    var socket = io.connect('http://stodi.lojban.org:3020');
 
     function updateJs(a) {
       var s = document.createElement("script");