From b2d9703f6800270e08f34a9b50c31398c22f945e Mon Sep 17 00:00:00 2001 From: Roberto Simonetti Date: Fri, 2 Feb 2024 22:50:13 +0100 Subject: [PATCH] Feat(extract): unused keys --- package.json | 2 +- packages/qwik-speak/tools/core/merge.ts | 23 ++++++++++ packages/qwik-speak/tools/core/types.ts | 13 +++++- packages/qwik-speak/tools/extract/cli.ts | 16 ++++++- packages/qwik-speak/tools/extract/index.ts | 30 ++++++++++++- packages/qwik-speak/tools/tests/merge.test.ts | 45 ++++++++++++++++++- 6 files changed, 120 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index 019f127..fbc71f6 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "lint": "eslint \"src/**/*.ts*\"", "preview": "qwik build preview && vite preview --open", "qwik-speak-extract": "node ./packages/qwik-speak/extract/cli.js --supportedLangs=en-US,it-IT,de-DE --assetsPath=i18n", - "qwik-speak-extract-with-autokeys": "node ./packages/qwik-speak/extract/cli.js --supportedLangs=en-US,it-IT,de-DE --assetsPath=i18n --autoKeys=true", + "qwik-speak-extract-with-autokeys": "node ./packages/qwik-speak/extract/cli.js --supportedLangs=en-US,it-IT,de-DE --assetsPath=i18n --autoKeys=true --unusedKeys=true --runtimeAssets=runtime", "qwik-speak-extract-fallback": "node fallback.js", "start": "vite --open --mode ssr", "test": "vitest test --run", diff --git a/packages/qwik-speak/tools/core/merge.ts b/packages/qwik-speak/tools/core/merge.ts index d9436be..e040b18 100644 --- a/packages/qwik-speak/tools/core/merge.ts +++ b/packages/qwik-speak/tools/core/merge.ts @@ -57,3 +57,26 @@ export function merge(target: Translation, source: Translation) { target = { ...target, ...source }; return target; } + +export function deleteExtraProperties( + source: Translation, + target: Translation, + keySeparator: string = '.', + parentPath = '' +): string[] { + const deletedPaths: string[] = []; + + for (const key in source) { + const currentPath = parentPath ? `${parentPath}${keySeparator}${key}` : key; + + if (typeof source[key] === 'object' && typeof target[key] === 'object') { + deletedPaths.push(...deleteExtraProperties(source[key], target[key], keySeparator, currentPath)); + if (Object.keys(source[key]).length === 0) delete source[key]; + } else if (!(key in target)) { + delete source[key]; + deletedPaths.push(currentPath); + } + } + + return deletedPaths; +} diff --git a/packages/qwik-speak/tools/core/types.ts b/packages/qwik-speak/tools/core/types.ts index 82164a6..9e945f7 100644 --- a/packages/qwik-speak/tools/core/types.ts +++ b/packages/qwik-speak/tools/core/types.ts @@ -43,10 +43,19 @@ export interface QwikSpeakExtractOptions { */ keyValueSeparator?: string; /** - * Automatically handle keys for each string. Default is false + * Automatically handle keys for each string. Default is false. * Make sure to set autoKeys: true in the vite plugin options for qwik inline */ autoKeys?: boolean; + /** + * Automatically remove unused keys from assets, + * except in runtime assets + */ + unusedKeys?: boolean; + /** + * Comma-separated list of runtime assets to preserve + */ + runtimeAssets?: string[]; } /** @@ -86,7 +95,7 @@ export interface QwikSpeakInlineOptions { */ keyValueSeparator?: string; /** - * Automatically handle keys for each string. Default is false + * Automatically handle keys for each string. Default is false. * Make sure to enable --autoKeys=true when running the extractor */ autoKeys?: boolean; diff --git a/packages/qwik-speak/tools/extract/cli.ts b/packages/qwik-speak/tools/extract/cli.ts index b2df774..07a3013 100644 --- a/packages/qwik-speak/tools/extract/cli.ts +++ b/packages/qwik-speak/tools/extract/cli.ts @@ -5,7 +5,8 @@ import { qwikSpeakExtract } from './index'; const assertType = (value: any, type: string): boolean => { if (type === value) return true; if (type === 'array' && Array.isArray(value)) return true; - if (type === 'string' && typeof (value) === 'string') return true; + if (type === 'string' && typeof value === 'string') return true; + if (type === 'boolean' && typeof value === 'boolean') return true; return false; }; @@ -62,7 +63,18 @@ for (const arg of args) { else errors.push(wrongOption(key, value)); break; case 'autoKeys': - if (assertType(value, 'string') && (value === 'true' || value === 'false')) options.autoKeys = value === 'true' + if (assertType(value, 'boolean')) options.autoKeys = value; + else if (assertType(value, 'string') && (value === 'true' || value === 'false')) options.autoKeys = value === 'true'; + else errors.push(wrongOption(key, value)); + break; + case 'unusedKeys': + if (assertType(value, 'boolean')) options.unusedKeys = value; + else if (assertType(value, 'string') && (value === 'true' || value === 'false')) options.unusedKeys = value === 'true'; + else errors.push(wrongOption(key, value)); + break; + case 'runtimeAssets': + if (assertType(value, 'array')) options.runtimeAssets = value; + else if (assertType(value, 'string')) options.runtimeAssets = [value]; else errors.push(wrongOption(key, value)); break; case 'error': diff --git a/packages/qwik-speak/tools/extract/index.ts b/packages/qwik-speak/tools/extract/index.ts index 800405b..b746ef7 100644 --- a/packages/qwik-speak/tools/extract/index.ts +++ b/packages/qwik-speak/tools/extract/index.ts @@ -12,7 +12,7 @@ import { parseJson, parseSequenceExpressions } from '../core/parser'; -import { deepClone, deepMerge, deepMergeMissing, deepSet, merge } from '../core/merge'; +import { deepClone, deepMerge, deepMergeMissing, deepSet, deleteExtraProperties, merge } from '../core/merge'; import { sortTarget, toJsonString } from '../core/format'; import { getOptions, getRules } from '../core/intl-parser'; import { generateAutoKey, isExistingKey, isObjectPath } from '../core/autokeys'; @@ -34,6 +34,8 @@ export async function qwikSpeakExtract(options: QwikSpeakExtractOptions) { keySeparator: options.keySeparator ?? '.', keyValueSeparator: options.keyValueSeparator ?? '@@', autoKeys: options.autoKeys ?? false, + unusedKeys: options.unusedKeys ?? false, + runtimeAssets: options.runtimeAssets ?? [] } // Logs @@ -232,9 +234,15 @@ export async function qwikSpeakExtract(options: QwikSpeakExtractOptions) { if (existsSync(baseAssets)) { - const files = await readdir(baseAssets); + let files = await readdir(baseAssets); if (files.length > 0) { + // Do not include runtime assets + if (resolvedOptions.runtimeAssets.length > 0) { + files = files.filter(filename => !resolvedOptions.runtimeAssets.includes(parse(filename).name)); + } + if (files.length === 0) return [assetsData, assetsFilenames]; + const ext = extname(files[0]); let data: Translation = {}; @@ -383,6 +391,21 @@ export async function qwikSpeakExtract(options: QwikSpeakExtractOptions) { } } + /* Drop unused keys */ + if (resolvedOptions.unusedKeys) { + const deletedPaths = new Set(); + for (const lang of resolvedOptions.supportedLangs) { + const asset = assetsData.get(lang); + if (asset) { + const paths = deleteExtraProperties(asset, translation[lang], resolvedOptions.keySeparator); + for (const path of paths) { + deletedPaths.add(path); + } + } + } + stats.set('unused keys', (stats.get('unused keys') ?? 0) + deletedPaths.size); + } + /* Deep merge translation data */ if (assetsData.size > 0) { for (const [lang, data] of assetsData) { @@ -413,6 +436,9 @@ export async function qwikSpeakExtract(options: QwikSpeakExtractOptions) { case 'dynamic plural': console.log('\x1b[32m%s\x1b[0m', `plurals skipped due to dynamic keys/options: ${value}`); break; + case 'unused keys': + console.log('\x1b[32m%s\x1b[0m', `unused keys removed: ${value}`); + break; } } } diff --git a/packages/qwik-speak/tools/tests/merge.test.ts b/packages/qwik-speak/tools/tests/merge.test.ts index 8f61d9a..445cc6e 100644 --- a/packages/qwik-speak/tools/tests/merge.test.ts +++ b/packages/qwik-speak/tools/tests/merge.test.ts @@ -1,6 +1,6 @@ import { test, describe, expect } from 'vitest'; -import { deepMerge, deepMergeMissing, deepSet } from '../core/merge'; +import { deepMerge, deepMergeMissing, deepSet, deleteExtraProperties } from '../core/merge'; describe('merge', () => { test('deepSet', () => { @@ -22,7 +22,7 @@ describe('merge', () => { const target2 = { key1: { key2: '' } }; const source2 = { key1: { key2: [{ subkey1: 'Subkey1', subkey2: 'Subkey2' }] } }; deepMerge(target2, source2); - expect(target2).toEqual({ key1: { key2: [{ subkey1: 'Subkey1', subkey2: 'Subkey2' }] } } ); + expect(target2).toEqual({ key1: { key2: [{ subkey1: 'Subkey1', subkey2: 'Subkey2' }] } }); const target3 = { key1: '' }; const source3 = { key1: { subkey1: 'Subkey1', subkey2: 'Subkey2' } }; deepMerge(target3, source3); @@ -45,4 +45,45 @@ describe('merge', () => { deepMergeMissing(target2, source2); expect(target2).toEqual({ key1: { subkey1: 'Subkey1', subkey2: 'Subkey2' } }); }); + test('deleteExtraProperties', () => { + // Expect add value + const target = { + a: { + b: { + c: { + d: 'Test d' + } + }, + f: 'Test f' + }, + a1: 'Test a1', + b2: 'Test b2' + }; + const source = { + a: { + b: { + c: { + d: 'Test d', + e: 'Test e' + } + }, + f: 'Test f' + }, + a1: 'Test a1', + a2: 'Test a2' + }; + const deletedPaths = deleteExtraProperties(source, target); + expect(source).toEqual({ + a: { + b: { + c: { + d: 'Test d' + } + }, + f: 'Test f' + }, + a1: 'Test a1' + }); + expect(deletedPaths).toEqual(['a.b.c.e', 'a2']); + }); });