Merge branch 'master' into support-ts-config-file

README.md

@@ -127,8 +127,8 @@ export default {
   // Default namespace used in your i18next config
 
   defaultValue: '',
-  // Default value to give to empty keys
-  // You may also specify a function accepting the locale, namespace, and key as arguments
+  // Default value to give to keys with no value
+  // You may also specify a function accepting the locale, namespace, key, and value as arguments
 
   indentation: 2,
   // Indentation of the catalog files
@@ -183,15 +183,6 @@ export default {
   sort: false,
   // Whether or not to sort the catalog. Can also be a [compareFunction](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/sort#parameters)
 
-  skipDefaultValues: false,
-  // Whether to ignore default values
-  // You may also specify a function accepting the locale and namespace as arguments
-
-  useKeysAsDefaultValue: false,
-  // Whether to use the keys as the default value; ex. "Hello": "Hello", "World": "World"
-  // This option takes precedence over the `defaultValue` and `skipDefaultValues` options
-  // You may also specify a function accepting the locale and namespace as arguments
-
   verbose: false,
   // Display info about the parsing including some stats
 

index.d.ts

@@ -0,0 +1,116 @@
+import EventEmitter from "events";
+
+export type SupportedLexer = "HandlebarsLexer" | "HTMLLexer" | "JavascriptLexer" | "JsxLexer" | "VueLexer";
+
+// BaseLexer is not importable therefore this is the best if done simple
+export class CustomLexerClass extends EventEmitter {}
+export type CustomLexer = typeof CustomLexerClass;
+
+export interface CustomLexerConfig extends Record<string, unknown> {
+  lexer: CustomLexer;
+}
+
+export interface HandlebarsLexerConfig {
+  lexer: "HandlebarsLexer";
+  functions?: string[];
+}
+
+export interface HTMLLexerConfig {
+  lexer: "HTMLLexer";
+  functions?: string[];
+  attr?: string;
+  optionAttr?: string;
+}
+
+export interface JavascriptLexerConfig {
+  lexer: "JavascriptLexer";
+  functions?: string[];
+  namespaceFunctions?: string[];
+  attr?: string;
+  parseGenerics?: false;
+  typeMap?: Record<string, unknown>;
+}
+
+export interface JavascriptWithTypesLexerConfig {
+  lexer: "JavascriptLexer";
+  functions?: string[];
+  namespaceFunctions?: string[];
+  attr?: string;
+  parseGenerics: true;
+  typeMap: Record<string, unknown>;
+}
+
+export interface JsxLexerConfig {
+  lexer: "JsxLexer";
+  functions?: string[];
+  namespaceFunctions?: string[];
+  attr?: string;
+  transSupportBasicHtmlNodes?: boolean;
+  transKeepBasicHtmlNodesFor?: string[];
+  parseGenerics?: false;
+  typeMap?: Record<string, unknown>;
+}
+
+export interface JsxWithTypesLexerConfig {
+  lexer: "JsxLexer";
+  functions?: string[];
+  namespaceFunctions?: string[];
+  attr?: string;
+  transSupportBasicHtmlNodes?: boolean;
+  transKeepBasicHtmlNodesFor?: string[];
+  parseGenerics: true;
+  typeMap: Record<string, unknown>;
+}
+
+export interface VueLexerConfig {
+  lexer: "VueLexer";
+  functions?: string[];
+}
+
+export type LexerConfig =
+  | HandlebarsLexerConfig
+  | HTMLLexerConfig
+  | JavascriptLexerConfig
+  | JavascriptWithTypesLexerConfig
+  | JsxLexerConfig
+  | JsxWithTypesLexerConfig
+  | VueLexerConfig
+  | CustomLexerConfig;
+
+export interface UserConfig {
+  contextSeparator?: string;
+  createOldCatalogs?: boolean;
+  defaultNamespace?: string;
+  defaultValue?: string | ((locale?: string, namespace?: string, key?: string) => string);
+  indentation?: number;
+  keepRemoved?: boolean;
+  keySeparator?: string | false;
+  lexers?: {
+    hbs?: (SupportedLexer | CustomLexer | LexerConfig)[];
+    handlebars?: (SupportedLexer | CustomLexer | LexerConfig)[];
+    htm?: (SupportedLexer | CustomLexer | LexerConfig)[];
+    html?: (SupportedLexer | CustomLexer | LexerConfig)[];
+    mjs?: (SupportedLexer | CustomLexer | LexerConfig)[];
+    js?: (SupportedLexer | CustomLexer | LexerConfig)[];
+    ts?: (SupportedLexer | CustomLexer | LexerConfig)[];
+    jsx?: (SupportedLexer | CustomLexer | LexerConfig)[];
+    tsx?: (SupportedLexer | CustomLexer | LexerConfig)[];
+    default?: (SupportedLexer | CustomLexer | LexerConfig)[];
+  };
+  lineEnding?: "auto" | "crlf" | "\r\n" | "cr" | "\r" | "lf" | "\n";
+  locales?: string[];
+  namespaceSeparator?: string | false;
+  output?: string;
+  pluralSeparator?: string;
+  input?: string | string[];
+  sort?: boolean | ((a: string, b: string) => -1 | 0 | 1);
+  skipDefaultValues?: boolean | ((locale?: string, namespace?: string) => boolean);
+  useKeysAsDefaultValue?: boolean | ((locale?: string, namespace?: string) => boolean);
+  verbose?: boolean;
+  failOnWarnings?: boolean;
+  failOnUpdate?: boolean;
+  customValueTemplate?: Record<string, string> | null;
+  resetDefaultValueLocale?: string | null;
+  i18nextOptions?: Record<string, unknown> | null;
+  yamlOptions?: Record<string, unknown> | null;
+}

package.json

@@ -35,7 +35,7 @@
     "esbuild": "^0.15.13",
     "fs-extra": "^10.0.0",
     "gulp-sort": "^2.0.0",
-    "i18next": "^21.2.0",
+    "i18next": "^22.0.4",
     "js-yaml": "4.1.0",
     "lilconfig": "^2.0.6",
     "rsvp": "^4.8.2",

src/helpers.js

@@ -40,33 +40,12 @@ function dotPathToHash(entry, target = {}, options = {}) {
   }
 
   const defaultValue =
-    typeof options.value === 'function'
-      ? options.value(options.locale, entry.namespace, key)
-      : options.value
-
-  const skipDefaultValues =
-    typeof options.skipDefaultValues === 'function'
-      ? options.skipDefaultValues(options.locale, entry.namespace)
-      : options.skipDefaultValues
-
-  const useKeysAsDefaultValue =
-    typeof options.useKeysAsDefaultValue === 'function'
-      ? options.useKeysAsDefaultValue(options.locale, entry.namespace)
-      : options.useKeysAsDefaultValue
+    entry[`defaultValue${options.suffix}`] || entry.defaultValue || ''
 
   let newValue =
-    entry[`defaultValue${options.suffix}`] ||
-    entry.defaultValue ||
-    defaultValue ||
-    ''
-
-  if (skipDefaultValues) {
-    newValue = ''
-  }
-
-  if (useKeysAsDefaultValue) {
-    newValue = key
-  }
+    typeof options.value === 'function'
+      ? options.value(options.locale, entry.namespace, key, defaultValue)
+      : options.value || defaultValue
 
   if (path.endsWith(separator)) {
     path = path.slice(0, -separator.length)
@@ -316,6 +295,40 @@ function yamlConfigLoader(filepath, content) {
   return yaml.load(content)
 }
 
+// unescape common html entities
+// code from react-18next taken from
+// https://github.com/i18next/react-i18next/blob/d3247b5c232f5d8c1a154fe5dd0090ca88c82dcf/src/unescape.js
+function unescape(text) {
+  const matchHtmlEntity =
+    /&(?:amp|#38|lt|#60|gt|#62|apos|#39|quot|#34|nbsp|#160|copy|#169|reg|#174|hellip|#8230|#x2F|#47);/g
+  const htmlEntities = {
+    '&': '&',
+    '&': '&',
+    '&lt;': '<',
+    '&#60;': '<',
+    '&gt;': '>',
+    '&#62;': '>',
+    '&apos;': "'",
+    '&#39;': "'",
+    '&quot;': '"',
+    '&#34;': '"',
+    '&nbsp;': ' ',
+    '&#160;': ' ',
+    '&copy;': '©',
+    '&#169;': '©',
+    '&reg;': '®',
+    '&#174;': '®',
+    '&hellip;': '…',
+    '&#8230;': '…',
+    '&#x2F;': '/',
+    '&#47;': '/',
+  }
+
+  const unescapeHtmlEntity = (m) => htmlEntities[m]
+
+  return text.replace(matchHtmlEntity, unescapeHtmlEntity)
+}
+
 export {
   dotPathToHash,
   mergeHashes,
@@ -327,4 +340,5 @@ export {
   esConfigLoader,
   tsConfigLoader,
   yamlConfigLoader,
+  unescape,
 }

src/lexers/jsx-lexer.js

@@ -1,5 +1,6 @@
 import JavascriptLexer from './javascript-lexer.js'
 import ts from 'typescript'
+import { unescape } from '../helpers.js'
 
 export default class JsxLexer extends JavascriptLexer {
   constructor(options = {}) {
@@ -91,18 +92,6 @@ export default class JsxLexer extends JavascriptLexer {
       const entry = {}
       entry.key = getKey(tagNode)
 
-      const defaultsProp = getPropValue(tagNode, 'defaults')
-      const defaultValue =
-        defaultsProp || this.nodeToString.call(this, node, sourceText)
-
-      if (defaultValue !== '') {
-        entry.defaultValue = defaultValue
-
-        if (!entry.key) {
-          entry.key = entry.defaultValue
-        }
-      }
-
       const namespace = getPropValue(tagNode, 'ns')
       if (namespace) {
         entry.namespace = namespace
@@ -143,6 +132,22 @@ export default class JsxLexer extends JavascriptLexer {
         } else entry[property.name.text] = true
       })
 
+      const defaultsProp = getPropValue(tagNode, 'defaults')
+      let defaultValue =
+        defaultsProp || this.nodeToString.call(this, node, sourceText)
+
+      if (entry.shouldUnescape === true) {
+        defaultValue = unescape(defaultValue)
+      }
+
+      if (defaultValue !== '') {
+        entry.defaultValue = defaultValue
+
+        if (!entry.key) {
+          entry.key = entry.defaultValue
+        }
+      }
+
       return entry.key ? entry : null
     } else if (tagNode.tagName.text === 'Interpolate') {
       const entry = {}

src/transform.js

@@ -36,9 +36,7 @@ export default class i18nTransform extends Transform {
       output: 'locales/$LOCALE/$NAMESPACE.json',
       resetDefaultValueLocale: null,
       sort: false,
-      useKeysAsDefaultValue: false,
       verbose: false,
-      skipDefaultValues: false,
       customValueTemplate: null,
       failOnWarnings: false,
       yamlOptions: null,
@@ -171,8 +169,6 @@ export default class i18nTransform extends Transform {
           separator: this.options.keySeparator,
           pluralSeparator: this.options.pluralSeparator,
           value: this.options.defaultValue,
-          useKeysAsDefaultValue: this.options.useKeysAsDefaultValue,
-          skipDefaultValues: this.options.skipDefaultValues,
           customValueTemplate: this.options.customValueTemplate,
         })
 

test/helpers/dotPathToHash.test.js

@@ -35,16 +35,6 @@ describe('dotPathToHash helper function', () => {
     done()
   })
 
-  it('supports custom separator when `useKeysAsDefaultValue` is true', (done) => {
-    const { target } = dotPathToHash(
-      { keyWithNamespace: 'namespace-two-three' },
-      {},
-      { separator: '-', useKeysAsDefaultValue: true }
-    )
-    assert.deepEqual(target, { namespace: { two: { three: 'two-three' } } })
-    done()
-  })
-
   it('handles an empty namespace', (done) => {
     const { target, duplicate } = dotPathToHash({
       keyWithNamespace: 'ns.',

test/lexers/jsx-lexer.test.js

@@ -314,7 +314,7 @@ describe('JsxLexer', () => {
         done()
       })
 
-      it('invalid interpolation gets stripped', (done) => {
+      it('strips invalid interpolation', (done) => {
         const Lexer = new JsxLexer()
         const content =
           '<Trans count={count}>before{{ key1, key2 }}after</Trans>'
@@ -379,6 +379,16 @@ describe('JsxLexer', () => {
         )
         done()
       })
+
+      it('supports the shouldUnescape options', (done) => {
+        const Lexer = new JsxLexer()
+        const content = '<Trans shouldUnescape>I&apros;m Cielquan</Trans>'
+        assert.equal(
+          Lexer.extract(content)[0].defaultValue,
+          'I&apros;m Cielquan'
+        )
+        done()
+      })
     })
   })
 })