Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 | 1x 1x 4x 4x 11x 11x 1x 1x 1x 1x 1x 1x 1x 1x 8x 8x 18x 18x 8x 10x 8x 1x 1x 1x 13x 11x 7x 1x 8x 10x 8x 8x 8x 17x 5x 12x 3x 9x 8x 1x 1x 1x 1x 1x 1x 4x 4x 2x 4x | /**
* Renders the CSV string tables as TypeScript source, and finds keys in them that no source file
* references.
*
* This runs at build time, not at runtime. Nothing in the component runtime imports it, so its Node
* and CSV dependencies never reach a browser bundle. It's published rather than kept in scripts/ so
* that applications sharing these string tables can drive the same conversion and checks.
*/
import { parse } from 'csv-parse/sync';
import { readFile, readdir, writeFile } from 'node:fs/promises';
import path from 'node:path';
import { format, resolveConfig } from 'prettier';
export type StringsMap = Record<string, string>;
const GENERATED_HEADER = '/* Generated from the CSV files in csv/ by `yarn generate-strings`. Do not edit. */';
/**
* Loads a CSV strings file into an object, taking keys and values from the first two columns.
*/
export const csvToStrings = (csvData: string): StringsMap => {
const rows: string[][] = parse(csvData, {
// Skip header row
from: 2,
// Only rows for strings that have comments have 3 fields.
relax_column_count_less: true,
// Skip empty "row" that is really just the terminating linefeed
skip_empty_lines: true,
});
return rows.reduce<StringsMap>((result, row) => {
result[row[0]] = row[1];
return result;
}, {});
};
/**
* Renders a strings object as a TypeScript module that exports a constant called "strings",
* formatted the way the repository's Prettier configuration wants it so that generating a table and
* formatting the tree don't fight over the result.
*/
export const stringsToTypeScript = async (stringsMap: StringsMap, targetPath: string): Promise<string> => {
const source = `${GENERATED_HEADER}\nexport const strings = ${JSON.stringify(stringsMap, null, 2)};\n`;
const prettierConfig = await resolveConfig(targetPath);
return format(source, { ...prettierConfig, filepath: targetPath });
};
/**
* Transforms the English strings table into gibberish, which makes untranslated text obvious
* without waiting on a translation.
*
* 1. Split the English string into whitespace-delimited words.
* 2. Reverse the order of the words.
* 3. Render each word as a base64 encoding of its UTF-8 representation, except for words that look
* like format string placeholders.
*/
export const generateGibberish = (english: StringsMap): StringsMap =>
Object.fromEntries(
Object.entries(english).map(([key, value]) => [
key,
value
.split(' ')
.reverse()
.map((word) => (word.startsWith('{') ? word : Buffer.from(word, 'utf-8').toString('base64').replace(/=/g, '')))
.join(' '),
])
);
const exportStrings = async (
englishStrings: StringsMap,
localizedStrings: StringsMap,
locale: string,
targetDir: string,
defaultToEnglish: boolean
): Promise<void> => {
const stringsMap: StringsMap = {};
for (const key of Object.keys(englishStrings)) {
if (key in localizedStrings) {
stringsMap[key] = localizedStrings[key];
} else {
console.warn(`Locale ${locale} has no translation for ${key}`);
if (defaultToEnglish) {
stringsMap[key] = englishStrings[key];
}
}
}
const targetPath = path.resolve(targetDir, `strings-${locale}.ts`);
await writeFile(targetPath, await stringsToTypeScript(stringsMap, targetPath), { encoding: 'utf-8' });
};
/**
* Converts a CSV strings file to a TypeScript source file that exports a constant called "strings".
* This will be an object that has the same keys as the English strings file; the English strings
* will be used for any keys that aren't translated yet.
*
* The filename is assumed to be the locale code with a ".csv" suffix. Converting the English file
* also writes the gibberish table, which is derived from English rather than translated.
*/
export const convertCsvFile = async (csvPath: string, targetDir: string, defaultToEnglish = true): Promise<void> => {
if (!csvPath.endsWith('.csv')) {
throw new Error('Cannot convert a non-CSV file');
}
const locale = path.basename(csvPath, '.csv');
const stringsMap = csvToStrings(await readFile(csvPath, { encoding: 'utf-8' }));
let englishStringsMap: StringsMap;
if (locale === 'en') {
englishStringsMap = stringsMap;
} else {
const englishPath = path.resolve(path.dirname(csvPath), 'en.csv');
englishStringsMap = csvToStrings(await readFile(englishPath, { encoding: 'utf-8' }));
}
await exportStrings(englishStringsMap, stringsMap, locale, targetDir, defaultToEnglish);
if (locale === 'en') {
await exportStrings(englishStringsMap, generateGibberish(englishStringsMap), 'gx', targetDir, defaultToEnglish);
}
};
/**
* Converts the CSV files for all locales to TypeScript source files. The list of locales is
* determined by the presence of CSV files.
*/
export const convertAllLocales = async (
csvDir: string,
stringsDir: string,
defaultToEnglish = true
): Promise<string[]> => {
const csvFiles = (await readdir(csvDir)).filter((filename) => filename.endsWith('.csv'));
await Promise.all(
csvFiles.map((filename) => convertCsvFile(path.join(csvDir, filename), stringsDir, defaultToEnglish))
);
return csvFiles;
};
export type UnusedStrings = {
/** Keys that are never read from the strings table and don't appear anywhere else either. */
unused: string[];
/**
* Keys that are never read from the strings table but whose name appears elsewhere in the source,
* e.g. as an unrelated enum value, or as a key held in a variable and looked up with
* strings[variable]. They need a manual check.
*/
possiblyUnused: string[];
};
export type FindUnusedStringsOptions = {
csvPath: string;
sourceDir: string;
extensions?: string[];
/** Directory names to skip at any depth. Defaults to "strings", where the generated tables live. */
excludedDirectories?: string[];
};
const DEFAULT_EXTENSIONS = ['.ts', '.js', '.tsx', '.jsx'];
const DEFAULT_EXCLUDED_DIRECTORIES = ['strings'];
export const findSourceFiles = async (
directory: string,
extensions: string[] = DEFAULT_EXTENSIONS,
excludedDirectories: string[] = DEFAULT_EXCLUDED_DIRECTORIES
): Promise<string[]> => {
const entries = await readdir(directory, { withFileTypes: true });
const nested = await Promise.all(
entries.map(async (entry) => {
const entryPath = path.join(directory, entry.name);
if (entry.isDirectory()) {
return excludedDirectories.includes(entry.name)
? []
: findSourceFiles(entryPath, extensions, excludedDirectories);
}
return extensions.includes(path.extname(entry.name)) ? [entryPath] : [];
})
);
return nested.flat().sort();
};
const STRINGS_PROPERTY_ACCESS = /\bstrings\s*\??\.\s*(\w+)/g;
const STRINGS_INDEX_ACCESS = /\bstrings\s*(?:\?\.)?\s*\[([^\]]*)\]/g;
const QUOTED_KEY = /(['"`])(\w+)\1/g;
/**
* Returns the keys a source file reads from the strings table, either as properties
* (strings.SAVE) or as quoted keys inside an index expression (strings[isRead ? 'READ' : 'UNREAD']).
*/
export const findAccessedKeys = (source: string): string[] => [
...Array.from(source.matchAll(STRINGS_PROPERTY_ACCESS), (match) => match[1]),
...Array.from(source.matchAll(STRINGS_INDEX_ACCESS)).flatMap((match) =>
Array.from(match[1].matchAll(QUOTED_KEY), (literal) => literal[2])
),
];
export const classifyKeys = (keys: string[], sources: string[]): UnusedStrings => {
const accessedKeys = new Set(sources.flatMap(findAccessedKeys));
const identifiers = new Set(sources.flatMap((source) => source.match(/\w+/g) ?? []));
const unused: string[] = [];
const possiblyUnused: string[] = [];
keys.forEach((key) => {
if (accessedKeys.has(key)) {
return;
}
if (identifiers.has(key)) {
possiblyUnused.push(key);
} else {
unused.push(key);
}
});
return { unused, possiblyUnused };
};
export const findUnusedStrings = async ({
csvPath,
sourceDir,
extensions,
excludedDirectories,
}: FindUnusedStringsOptions): Promise<UnusedStrings> => {
const keys = Object.keys(csvToStrings(await readFile(csvPath, { encoding: 'utf-8' })));
const files = await findSourceFiles(sourceDir, extensions, excludedDirectories);
const sources = await Promise.all(files.map((file) => readFile(file, { encoding: 'utf-8' })));
return classifyKeys(keys, sources);
};
export const formatUnusedStrings = ({ unused, possiblyUnused }: UnusedStrings): string => {
const lines = unused.length === 0 ? ['No unused entries found.'] : ['Unused entries found:', ...unused];
if (possiblyUnused.length > 0) {
lines.push(
'',
'Possibly unused entries (never read from strings, but the name appears elsewhere; check manually):',
...possiblyUnused
);
}
return `${lines.join('\n')}\n`;
};
|