Sync missing translation keys in app/locales/*.json files to a Google Sheet

.gitignore

@@ -6,6 +6,8 @@ yarn-debug.log*
 yarn-error.log*
 lerna-debug.log*
 
+*-credentials.json
+
 # Diagnostic reports (https://nodejs.org/api/report.html)
 report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
 

ops/lost-in-translation.js

@@ -0,0 +1,193 @@
+/* eslint-disable no-await-in-loop, global-require */
+const { parse } = require('path');
+const { readdirSync } = require('fs');
+const { spawnSync } = require('child_process');
+const { GoogleSpreadsheet } = require('google-spreadsheet');
+
+/**
+ * This script finds all the english translation keys across
+ * all commits (including branches and pull requests) for each
+ * locale.json in app/locales. Afterwards it looks at each locale
+ * separately to see if it's missing any one the english keys.
+ *
+ * Important: You need to run it from the root directory.
+ *
+ * Usage:
+ * node ops/lost-in-translation.js
+ */
+
+/**
+ * Calls git cli with arguments and returns stdout.
+ */
+function git(...args) {
+ const { stdout } = spawnSync('git', args, { encoding: 'utf-8' });
+ return stdout;
+}
+
+/**
+ * Returns the commit hashes for a file path across all local branches.
+ */
+function findCommitHashesForFile(filePath) {
+ const output = git('log', '--pretty=format:"%h"', '--all', '--', filePath);
+ const hashes = output.split('\n').map(line => line.replace(/"/g, ''));
+ return hashes;
+}
+
+/**
+ * Returns the JSON representation of the contents of a file at certain commit hash.
+ */
+function retrieveJSONForFileAtCommitHash(filePath, commitHash) {
+ const contentsAtCommitHash = git('show', `${commitHash}:${filePath}`);
+ try {
+ return JSON.parse(contentsAtCommitHash);
+ } catch (error) {
+ // Sometimes the file is not in a proper JSON format. Simply return nothing in that case.
+ return {};
+ }
+}
+
+/**
+ * Normalizes translation key like we do in app/server.ts.
+ */
+function normalizeTranslationKey(translationKey) {
+ return translationKey.replace(/[\s\n\t]+/g, ' ').trim();
+}
+
+/**
+ * Find all the locales (ie. en-IN, no, se) in the provided directory.
+ */
+function retrieveAllLocales(directoryPath) {
+ const filenames = readdirSync(directoryPath);
+ const locales = filenames.map(filename => parse(filename).name);
+ return locales;
+}
+
+/**
+ * Add rows to a Google Spreadsheet that are not already added.
+ */
+async function addUniqueRowsToGoogleSheet(sheet, rowsToAdd) {
+ const alreadyAddedRows = await sheet.getRows({ limit: 1000 });
+ const uniqueRowsToAdd = rowsToAdd.filter(
+ rowToAdd =>
+ !alreadyAddedRows.find(
+ alreadyAddedRow => alreadyAddedRow.key === rowToAdd.key
+ )
+ );
+ await sheet.addRows(uniqueRowsToAdd);
+ return uniqueRowsToAdd;
+}
+
+/**
+ * Retrieve all the sheets in a Google Spreadsheet document.
+ */
+async function retrieveSheetsInDocument(doc) {
+ const sheets = [];
+ for (let sheetIndex = 0; sheetIndex < doc.sheetCount; sheetIndex += 1) {
+ const sheet = await doc.sheetsByIndex[sheetIndex];
+ sheets.push(sheet);
+ }
+ return sheets;
+}
+
+/**
+ * Step 1: Find all the (english) translation keys across all branches and PRs.
+ *
+ * If a Dutch developer has made a feature in a branch, we expect that him/her added a key
+ * to one or many locale files (usually the english locale (en.json) or to their own locale
+ * file (nl.json), or both).
+ *
+ * But they probably haven't added translations to all the other locales, because they don't
+ * speak the other languages. And this is the problem, because now we need find people to help
+ * translate the added keys to all the other locales.
+ *
+ * So what we do here is simply to find *all* the english translation keys across the entire
+ * project. That means looking at all the english translations keys in all the locale files
+ * since the start of the project across all branches and PRs. We throw them into a set that
+ * we use in the next step.
+ */
+const allLocales = retrieveAllLocales('app/locales/');
+const allEnglishTranslationKeys = new Set([]);
+for (const locale of allLocales) {
+ const filePath = `app/locales/${locale}.json`;
+ for (const commitHash of findCommitHashesForFile(filePath)) {
+ const translation = retrieveJSONForFileAtCommitHash(filePath, commitHash);
+ for (const translationKey of Object.keys(translation)) {
+ allEnglishTranslationKeys.add(normalizeTranslationKey(translationKey));
+ }
+ }
+}
+
+/**
+ * Step 2: For each locale file check if there are any missing english translation keys
+ * across all branches and PRs and all commits/changes.
+ *
+ * If there's a missing english translation key, we know that we're most likely missing
+ * a translation for that locale. So what we need to do is to translate it, or get help
+ * to translate it.
+ *
+ * We throw it into a dictionary where the key is the locale and the value is a set of
+ * missing translations from english to that locale.
+ */
+const translationKeysByLocale = {};
+for (const locale of allLocales) {
+ const filePath = `app/locales/${locale}.json`;
+ for (const commitHash of findCommitHashesForFile(filePath)) {
+ const translation = retrieveJSONForFileAtCommitHash(filePath, commitHash);
+ const translationKeys = Object.keys(translation).map(key =>
+ normalizeTranslationKey(key)
+ );
+ if (translationKeysByLocale[locale] === undefined) {
+ translationKeysByLocale[locale] = new Set([]);
+ }
+ translationKeysByLocale[locale] = new Set([
+ ...translationKeysByLocale[locale],
+ ...translationKeys
+ ]);
+ }
+}
+
+/**
+ * Step 3: Add rows of missing translations to Google Spreadsheet.
+ *
+ * https://docs.google.com/spreadsheets/d/1ILFfc1DX4ujMnLnf9UqhwQGM9Ke3s1cAWciy8VqMHZw
+ */
+(async () => {
+ const doc = new GoogleSpreadsheet(
+ '1ILFfc1DX4ujMnLnf9UqhwQGM9Ke3s1cAWciy8VqMHZw'
+ );
+ await doc.useServiceAccountAuth(
+ require('./coronastatus-translation-486cef09736e-credentials.json')
+ );
+ await doc.loadInfo();
+ const sheets = await retrieveSheetsInDocument(doc);
+
+ for (const locale of allLocales) {
+ // Create sheet for this locale if it doesn't already exist.
+ let matchingSheet = sheets.find(sheet => sheet.title === locale);
+ if (!matchingSheet) {
+ matchingSheet = await doc.addSheet({
+ title: locale,
+ headerValues: ['key', 'translation']
+ });
+ }
+
+ // Find rows that don't already exist by looking at the key column.
+ const rows = [];
+ for (const englishTranslationKey of allEnglishTranslationKeys) {
+ if (!translationKeysByLocale[locale].has(englishTranslationKey)) {
+ const row = { key: englishTranslationKey, translation: '' };
+ rows.push(row);
+ }
+ }
+
+ // Add and print out how many rows we added.
+ const addedRows = await addUniqueRowsToGoogleSheet(matchingSheet, rows);
+ console.log(
+ `Added ${addedRows.length} of ${rows.length} missing translations to the ${locale} sheet.`
+ );
+
+ // Avoid getting rate limited by Google's API (max writes per 100 seconds).
+ console.log('Waiting before processing the next sheet.');
+ await new Promise(resolve => setTimeout(resolve, 20 * 1000));
+ }
+})();

package.json

@@ -24,6 +24,7 @@
  "ejs": "^3.0.1",
  "express": "^4.17.1",
  "express-rate-limit": "^5.1.1",
+ "google-spreadsheet": "^3.0.10",
  "i18n": "^0.8.6",
  "node-fetch": "^2.6.0",
  "pg": "^7.18.2",