Add g-zip size

Author: sonic16x

package-lock.json

@@ -84,6 +84,7 @@
     "eslint-plugin-storybook": "^0.11.1",
     "file-loader": "^6.2.0",
     "globals": "^15.12.0",
+    "gzip-size": "^7.0.0",
     "html-webpack-plugin": "^5.6.3",
     "jest": "^29.7.0",
     "jsdom": "^25.0.1",

package.json

@@ -4,6 +4,7 @@ import { IAlphabets } from 'reducers'
 
 import {
     convertCases,
+    deduplicate,
     filterLatin,
     filterNiqqud,
     getCaseTips,
@@ -235,20 +236,22 @@ class DictionaryClass {
 
                     switch (lang) {
                         case 'isv':
-                            splittedField = this
-                                .splittedMap['isv-src']
-                                .get(id)
-                                .map((word) => this.searchPrepare(lang, word))
-                                .filter((item, i, ar) => ar.indexOf(item) === i)
-                            
+                            splittedField = deduplicate(
+                                this
+                                    .splittedMap['isv-src']
+                                    .get(id)
+                                    .map((word) => this.searchPrepare(lang, word))
+                            )
+
                             break
                         case 'isv-src':
-                            splittedField = this
-                                .splitWords(fromField)
-                                .concat(getWordForms(item))
-                                .map((word) => this.searchPrepare('isv-src', getLatin(word, '2')))
-                                .filter((item, i, ar) => ar.indexOf(item) === i)
-                            
+                            splittedField = deduplicate(
+                                this
+                                    .splitWords(fromField)
+                                    .concat(getWordForms(item))
+                                    .map((word) => this.searchPrepare('isv-src', getLatin(word, '2')))
+                            )
+
                             break
                         default:
                             splittedField = this.splitWords(fromField).map((word) => this.searchPrepare(lang, word))

src/services/dictionary.ts

@@ -6,6 +6,8 @@ import { Dictionary, loadTablesData } from 'services'
 
 import { getColumnName, transposeMatrix } from 'utils'
 
+import { gzipSizeSync } from 'gzip-size'
+
 
 loadTablesData.then(({ data, columns }) => {
     Dictionary.init(data)
@@ -24,7 +26,7 @@ loadTablesData.then(({ data, columns }) => {
         }
     })
 
-    const basicData = transposeMatrix(basicDataTransposed)
+    const basicData = transposeMatrix<string>(basicDataTransposed)//.sort((a, b) => a[1].localeCompare(b[1]))
 
     const searchIndexBasic = [
         'isv-src',
@@ -49,6 +51,10 @@ loadTablesData.then(({ data, columns }) => {
     fs.writeFileSync('./static/data/basic.json', jsonDataStr)
     fs.writeFileSync('./static/data/translateStatistic.json', translateStatisticStr)
 
+    const sizeMB = gzipSizeSync(jsonDataStr) / 1000000
+    // eslint-disable-next-line no-console
+    console.info(`basic.json g-zip size: ${sizeMB.toFixed(2)} MB`)
+
     addLangs.forEach((lang) => {
         const langDataTransposed = [
             columns.find(([fieldName]) => fieldName === lang),

src/services/generateDictionary.ts

@@ -0,0 +1 @@
+export const deduplicate = (arr) => Array.from(new Set(arr))

src/utils/deduplicate/deduplicate.ts

@@ -0,0 +1 @@
+export * from './deduplicate'

src/utils/deduplicate/index.ts

@@ -1,6 +1,7 @@
 export * from './bcp47'
 export * from './convertCases'
 export * from './createTaggedTemplate'
+export * from './deduplicate'
 export * from './filterLatin'
 export * from './filterNiqqud'
 export * from './getAllDataFromResults'