Feat(extract): unused keys

package.json

@@ -17,7 +17,7 @@
     "lint": "eslint \"src/**/*.ts*\"",
     "preview": "qwik build preview && vite preview --open",
     "qwik-speak-extract": "node ./packages/qwik-speak/extract/cli.js --supportedLangs=en-US,it-IT,de-DE --assetsPath=i18n",
-    "qwik-speak-extract-with-autokeys": "node ./packages/qwik-speak/extract/cli.js --supportedLangs=en-US,it-IT,de-DE --assetsPath=i18n --autoKeys=true",
+    "qwik-speak-extract-with-autokeys": "node ./packages/qwik-speak/extract/cli.js --supportedLangs=en-US,it-IT,de-DE --assetsPath=i18n --autoKeys=true --unusedKeys=true --runtimeAssets=runtime",
     "qwik-speak-extract-fallback": "node fallback.js",
     "start": "vite --open --mode ssr",
     "test": "vitest test --run",

packages/qwik-speak/tools/core/merge.ts

@@ -57,3 +57,26 @@ export function merge(target: Translation, source: Translation) {
   target = { ...target, ...source };
   return target;
 }
+
+export function deleteExtraProperties(
+  source: Translation,
+  target: Translation,
+  keySeparator: string = '.',
+  parentPath = ''
+): string[] {
+  const deletedPaths: string[] = [];
+
+  for (const key in source) {
+    const currentPath = parentPath ? `${parentPath}${keySeparator}${key}` : key;
+
+    if (typeof source[key] === 'object' && typeof target[key] === 'object') {
+      deletedPaths.push(...deleteExtraProperties(source[key], target[key], keySeparator, currentPath));
+      if (Object.keys(source[key]).length === 0) delete source[key];
+    } else if (!(key in target)) {
+      delete source[key];
+      deletedPaths.push(currentPath);
+    }
+  }
+
+  return deletedPaths;
+}

packages/qwik-speak/tools/core/types.ts

@@ -43,10 +43,19 @@ export interface QwikSpeakExtractOptions {
    */
   keyValueSeparator?: string;
   /**
-   * Automatically handle keys for each string. Default is false
+   * Automatically handle keys for each string. Default is false.
    * Make sure to set autoKeys: true in the vite plugin options for qwik inline
    */
   autoKeys?: boolean;
+  /**
+   * Automatically remove unused keys from assets,
+   * except in runtime assets
+   */
+  unusedKeys?: boolean;
+  /**
+   * Comma-separated list of runtime assets to preserve
+   */
+  runtimeAssets?: string[];
 }
 
 /**
@@ -86,7 +95,7 @@ export interface QwikSpeakInlineOptions {
    */
   keyValueSeparator?: string;
   /**
-   * Automatically handle keys for each string. Default is false
+   * Automatically handle keys for each string. Default is false.
    * Make sure to enable --autoKeys=true when running the extractor
    */
   autoKeys?: boolean;

packages/qwik-speak/tools/extract/cli.ts

@@ -5,7 +5,8 @@ import { qwikSpeakExtract } from './index';
 const assertType = (value: any, type: string): boolean => {
   if (type === value) return true;
   if (type === 'array' && Array.isArray(value)) return true;
-  if (type === 'string' && typeof (value) === 'string') return true;
+  if (type === 'string' && typeof value === 'string') return true;
+  if (type === 'boolean' && typeof value === 'boolean') return true;
   return false;
 };
 
@@ -62,7 +63,18 @@ for (const arg of args) {
       else errors.push(wrongOption(key, value));
       break;
     case 'autoKeys':
-      if (assertType(value, 'string') && (value === 'true' || value === 'false')) options.autoKeys = value === 'true'
+      if (assertType(value, 'boolean')) options.autoKeys = value;
+      else if (assertType(value, 'string') && (value === 'true' || value === 'false')) options.autoKeys = value === 'true';
+      else errors.push(wrongOption(key, value));
+      break;
+    case 'unusedKeys':
+      if (assertType(value, 'boolean')) options.unusedKeys = value;
+      else if (assertType(value, 'string') && (value === 'true' || value === 'false')) options.unusedKeys = value === 'true';
+      else errors.push(wrongOption(key, value));
+      break;
+    case 'runtimeAssets':
+      if (assertType(value, 'array')) options.runtimeAssets = value;
+      else if (assertType(value, 'string')) options.runtimeAssets = [value];
       else errors.push(wrongOption(key, value));
       break;
     case 'error':

packages/qwik-speak/tools/extract/index.ts

@@ -12,7 +12,7 @@ import {
   parseJson,
   parseSequenceExpressions
 } from '../core/parser';
-import { deepClone, deepMerge, deepMergeMissing, deepSet, merge } from '../core/merge';
+import { deepClone, deepMerge, deepMergeMissing, deepSet, deleteExtraProperties, merge } from '../core/merge';
 import { sortTarget, toJsonString } from '../core/format';
 import { getOptions, getRules } from '../core/intl-parser';
 import { generateAutoKey, isExistingKey, isObjectPath } from '../core/autokeys';
@@ -34,6 +34,8 @@ export async function qwikSpeakExtract(options: QwikSpeakExtractOptions) {
     keySeparator: options.keySeparator ?? '.',
     keyValueSeparator: options.keyValueSeparator ?? '@@',
     autoKeys: options.autoKeys ?? false,
+    unusedKeys: options.unusedKeys ?? false,
+    runtimeAssets: options.runtimeAssets ?? []
   }
 
   // Logs
@@ -232,9 +234,15 @@ export async function qwikSpeakExtract(options: QwikSpeakExtractOptions) {
 
       if (existsSync(baseAssets)) {
 
-        const files = await readdir(baseAssets);
+        let files = await readdir(baseAssets);
 
         if (files.length > 0) {
+          // Do not include runtime assets
+          if (resolvedOptions.runtimeAssets.length > 0) {
+            files = files.filter(filename => !resolvedOptions.runtimeAssets.includes(parse(filename).name));
+          }
+          if (files.length === 0) return [assetsData, assetsFilenames];
+
           const ext = extname(files[0]);
 
           let data: Translation = {};
@@ -383,6 +391,21 @@ export async function qwikSpeakExtract(options: QwikSpeakExtractOptions) {
     }
   }
 
+  /* Drop unused keys */
+  if (resolvedOptions.unusedKeys) {
+    const deletedPaths = new Set<string>();
+    for (const lang of resolvedOptions.supportedLangs) {
+      const asset = assetsData.get(lang);
+      if (asset) {
+        const paths = deleteExtraProperties(asset, translation[lang], resolvedOptions.keySeparator);
+        for (const path of paths) {
+          deletedPaths.add(path);
+        }
+      }
+    }
+    stats.set('unused keys', (stats.get('unused keys') ?? 0) + deletedPaths.size);
+  }
+
   /* Deep merge translation data */
   if (assetsData.size > 0) {
     for (const [lang, data] of assetsData) {
@@ -413,6 +436,9 @@ export async function qwikSpeakExtract(options: QwikSpeakExtractOptions) {
       case 'dynamic plural':
         console.log('\x1b[32m%s\x1b[0m', `plurals skipped due to dynamic keys/options: ${value}`);
         break;
+      case 'unused keys':
+        console.log('\x1b[32m%s\x1b[0m', `unused keys removed: ${value}`);
+        break;
     }
   }
 }

packages/qwik-speak/tools/tests/merge.test.ts

@@ -1,6 +1,6 @@
 import { test, describe, expect } from 'vitest';
 
-import { deepMerge, deepMergeMissing, deepSet } from '../core/merge';
+import { deepMerge, deepMergeMissing, deepSet, deleteExtraProperties } from '../core/merge';
 
 describe('merge', () => {
   test('deepSet', () => {
@@ -22,7 +22,7 @@ describe('merge', () => {
     const target2 = { key1: { key2: '' } };
     const source2 = { key1: { key2: [{ subkey1: 'Subkey1', subkey2: 'Subkey2' }] } };
     deepMerge(target2, source2);
-    expect(target2).toEqual({ key1: { key2: [{ subkey1: 'Subkey1', subkey2: 'Subkey2' }] } } );
+    expect(target2).toEqual({ key1: { key2: [{ subkey1: 'Subkey1', subkey2: 'Subkey2' }] } });
     const target3 = { key1: '' };
     const source3 = { key1: { subkey1: 'Subkey1', subkey2: 'Subkey2' } };
     deepMerge(target3, source3);
@@ -45,4 +45,45 @@ describe('merge', () => {
     deepMergeMissing(target2, source2);
     expect(target2).toEqual({ key1: { subkey1: 'Subkey1', subkey2: 'Subkey2' } });
   });
+  test('deleteExtraProperties', () => {
+    // Expect add value
+    const target = {
+      a: {
+        b: {
+          c: {
+            d: 'Test d'
+          }
+        },
+        f: 'Test f'
+      },
+      a1: 'Test a1',
+      b2: 'Test b2'
+    };
+    const source = {
+      a: {
+        b: {
+          c: {
+            d: 'Test d',
+            e: 'Test e'
+          }
+        },
+        f: 'Test f'
+      },
+      a1: 'Test a1',
+      a2: 'Test a2'
+    };
+    const deletedPaths = deleteExtraProperties(source, target);
+    expect(source).toEqual({
+      a: {
+        b: {
+          c: {
+            d: 'Test d'
+          }
+        },
+        f: 'Test f'
+      },
+      a1: 'Test a1'
+    });
+    expect(deletedPaths).toEqual(['a.b.c.e', 'a2']);
+  });
 });