feat: filesTemplate option to simplify the push command

eslint.config.js

@@ -12,7 +12,12 @@ export default tseslint.config({
     'scripts/**/*.{js,ts,mjs,cjs,tsx,jsx}',
     '*.config.{js,ts}',
   ],
-  ignores: ['**/*.generated.ts', 'dist/**/*', 'dist-types/**/*'],
+  ignores: [
+    '**/*.generated.ts',
+    'dist/**/*',
+    'dist-types/**/*',
+    'src/schema.d.ts',
+  ],
   extends: [
     eslint.configs.recommended,
     ...tseslint.configs.recommended,

package.json

@@ -37,7 +37,7 @@
     "json5": "^2.2.3",
     "jsonschema": "^1.4.1",
     "openapi-fetch": "0.13.1",
-    "tinyglobby": "^0.2.10",
+    "tinyglobby": "^0.2.12",
     "unescape-js": "^1.1.4",
     "vscode-oniguruma": "^2.0.1",
     "vscode-textmate": "^9.1.0",

schema.json

@@ -42,8 +42,20 @@
     "push": {
       "type": "object",
       "properties": {
+        "filesTemplate": {
+          "description": "A template that describes the structure of the local files and their location with file [structure template format](https://docs.tolgee.io/tolgee-cli/push-pull-strings#file-structure-template-format).\n\nExample: `./public/{namespace}/{languageTag}.json`",
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "array",
+              "items": { "type": "string" }
+            }
+          ]
+        },
         "files": {
-          "description": "Define, which files should be pushed and attach language/namespace to them. By default Tolgee pushes all files specified here, you can filter them by languages and namespaces properties.",
+          "description": "More explicit alternative to `filesTemplate`. Define, which files should be pushed and attach language/namespace to them. By default Tolgee pushes all files specified here, you can filter them by languages and namespaces properties.",
           "type": "array",
           "items": { "$ref": "#/$defs/fileMatch" }
         },
@@ -128,7 +140,7 @@
           "type": "boolean"
         },
         "fileStructureTemplate": {
-          "description": "Defines exported file structure: https://tolgee.io/tolgee-cli/push-pull-strings#file-structure-template-format",
+          "description": "Defines exported file structure: https://docs.tolgee.io/tolgee-cli/push-pull-strings#file-structure-template-format",
           "type": "string"
         },
         "emptyDir": {

src/commands/push.ts

@@ -19,6 +19,8 @@ import { mapImportFormat } from '../utils/mapImportFormat.js';
 import { TolgeeClient, handleLoadableError } from '../client/TolgeeClient.js';
 import { BodyOf } from '../client/internal/schema.utils.js';
 import { components } from '../client/internal/schema.generated.js';
+import { findFilesByTemplate } from '../utils/filesTemplate.js';
+import { valueToArray } from '../utils/valueToArray.js';
 
 type ImportRequest = BodyOf<
   '/v2/projects/{projectId}/single-step-import',
@@ -46,6 +48,7 @@ type PushOptions = BaseOptions & {
   namespaces?: string[];
   tagNewKeys?: string[];
   removeOtherKeys?: boolean;
+  filesTemplate?: string[];
 };
 
 async function allInPattern(pattern: string) {
@@ -147,11 +150,25 @@ const pushHandler = (config: Schema) =>
   async function (this: Command) {
     const opts: PushOptions = this.optsWithGlobals();
 
-    if (!config.push?.files) {
-      exitWithError('Missing option `push.files` in configuration file.');
+    let allMatchers: FileMatch[] = [];
+
+    const filesTemplate = opts.filesTemplate;
+
+    if (!filesTemplate && !config.push?.files?.length) {
+      exitWithError('Missing option `push.filesTemplate` or `push.files`.');
+    }
+
+    if (filesTemplate) {
+      for (const template of filesTemplate) {
+        allMatchers = allMatchers.concat(
+          ...(await findFilesByTemplate(template))
+        );
+      }
     }
 
-    const filteredMatchers = config.push.files.filter((r) => {
+    allMatchers = allMatchers.concat(...(config.push?.files || []));
+
+    const filteredMatchers = allMatchers.filter((r) => {
       if (
         r.language &&
         opts.languages &&
@@ -226,6 +243,12 @@ export default (config: Schema) =>
   new Command()
     .name('push')
     .description('Pushes translations to Tolgee')
+    .addOption(
+      new Option(
+        '-ft, --files-template <templates...>',
+        'A template that describes the structure of the local files and their location with file structure template format (more at: https://docs.tolgee.io/tolgee-cli/push-pull-strings#file-structure-template-format).\n\nExample: `./public/{namespace}/{languageTag}.json`\n\n'
+      ).default(valueToArray(config.push?.filesTemplate))
+    )
     .addOption(
       new Option(
         '-f, --force-mode <mode>',

src/config/tolgeerc.ts

@@ -8,14 +8,15 @@ import { CosmiconfigResult } from 'cosmiconfig/dist/types.js';
 import { error, exitWithError } from '../utils/logger.js';
 import { existsSync } from 'fs';
 import { Schema } from '../schema.js';
+import { valueToArray } from '../utils/valueToArray.js';
 
 const explorer = cosmiconfig('tolgee', {
   loaders: {
     noExt: defaultLoaders['.json'],
   },
 });
 
-function parseConfig(input: Schema, configDir: string): Schema {
+function parseConfig(input: Schema, configDir: string) {
   const rc = { ...input };
 
   if (rc.apiUrl !== undefined) {
@@ -60,6 +61,13 @@ function parseConfig(input: Schema, configDir: string): Schema {
     }));
   }
 
+  // convert relative paths in config to absolute
+  if (rc.push?.filesTemplate) {
+    rc.push.filesTemplate = valueToArray(rc.push.filesTemplate)?.map(
+      (template) => resolve(configDir, template).replace(/\\/g, '/')
+    );
+  }
+
   // convert relative paths in config to absolute
   if (rc.pull?.path !== undefined) {
     rc.pull.path = resolve(configDir, rc.pull.path).replace(/\\/g, '/');

src/schema.d.ts

@@ -103,7 +103,15 @@ export interface Schema {
   parser?: "react" | "vue" | "svelte" | "ngx";
   push?: {
     /**
-     * Define, which files should be pushed and attach language/namespace to them. By default Tolgee pushes all files specified here, you can filter them by languages and namespaces properties.
+     * A template that describes the structure of the local files and their location with file [structure template format](https://docs.tolgee.io/tolgee-cli/push-pull-strings#file-structure-template-format).
+     *
+     * Example: `./public/{namespace}/{languageTag}.json`
+     */
+    filesTemplate?: string | string[];
+    /**
+     * A template that describes the structure of the local files and their location with [file structure template format](http://localhost:3001/tolgee-cli/push-pull-strings#file-structure-template-format).
+     *
+     * Example: `./public/{namespace}/{languageTag}.json`
      */
     files?: FileMatch[];
     /**
@@ -162,7 +170,7 @@ export interface Schema {
      */
     supportArrays?: boolean;
     /**
-     * Defines exported file structure: https://tolgee.io/tolgee-cli/push-pull-strings#file-structure-template-format
+     * Defines exported file structure: https://docs.tolgee.io/tolgee-cli/push-pull-strings#file-structure-template-format
      */
     fileStructureTemplate?: string;
     /**

src/utils/filesTemplate.ts

@@ -0,0 +1,153 @@
+import { glob } from 'tinyglobby';
+import { exitWithError } from './logger.js';
+import { FileMatch } from '../schema.js';
+
+const GLOB_EXISTING_DOUBLE_STAR = /(\*\*)/g;
+const GLOB_EXISTING_STAR = /(\*)/g;
+const GLOB_EXISTING_ENUM = /\{([^}]*?,[^}]*?)\}/g;
+
+const PLACEHOLDER_DOUBLE_ASTERISK = '__double_asterisk';
+const PLACEHOLDER_ASTERISK = '__asterisk';
+const PLACEHOLDER_ENUM_PREFIX = '__enum:';
+
+export class FileMatcherException extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = this.constructor.name;
+  }
+}
+
+function splitToParts(template: string) {
+  return template.split(/(\{.*?\})/g).filter(Boolean);
+}
+
+function getVariableName(part: string) {
+  if (part.startsWith('{') && part.endsWith('}')) {
+    return part.substring(1, part.length - 1).trim();
+  }
+  return false;
+}
+
+export function sanitizeTemplate(template: string) {
+  let value = template;
+  const matchedEnums = [...(value.match(GLOB_EXISTING_ENUM)?.values() || [])];
+  matchedEnums.forEach((val) => {
+    value = value.replace(
+      val,
+      `{${PLACEHOLDER_ENUM_PREFIX}${getVariableName(val)}}`
+    );
+  });
+  value = value.replaceAll(
+    GLOB_EXISTING_DOUBLE_STAR,
+    '{' + PLACEHOLDER_DOUBLE_ASTERISK + '}'
+  );
+  value = value.replaceAll(
+    GLOB_EXISTING_STAR,
+    '{' + PLACEHOLDER_ASTERISK + '}'
+  );
+  return value;
+}
+
+export function getFileMatcher(file: string, template: string) {
+  let fileName = file;
+  const allVariables: Record<string, string> = {};
+  const templateParts = splitToParts(template);
+  for (const [i, part] of templateParts.entries()) {
+    const variable = getVariableName(part);
+    if (!variable) {
+      if (fileName.startsWith(part)) {
+        fileName = fileName.substring(part.length);
+      } else {
+        throw new FileMatcherException(`Unexpected part "${part}"`);
+      }
+    } else {
+      const next = templateParts[i + 1];
+      if (next) {
+        const variableEnd = fileName.indexOf(next);
+        if (getVariableName(next) || variableEnd === -1) {
+          throw new FileMatcherException(
+            `Can't have two variables without separator (${part} + ${next})`
+          );
+        } else {
+          allVariables[variable] = fileName.substring(0, variableEnd);
+          fileName = fileName.substring(variableEnd);
+        }
+      } else {
+        allVariables[variable] = fileName;
+      }
+    }
+  }
+
+  const result: FileMatch = { path: file };
+  for (const [variable, value] of Object.entries(allVariables)) {
+    if (variable === 'languageTag') {
+      result.language = value;
+    } else if (variable === 'snakeLanguageTag') {
+      result.language = value.replaceAll('_', '-');
+    } else if (variable === 'androidLanguageTag') {
+      if (value[3] === 'r') {
+        result.language =
+          value.substring(0, 3) + value.substring(4, value.length);
+      } else {
+        result.language = value;
+      }
+    } else if (variable === 'namespace') {
+      result.namespace = value;
+    } else if (
+      variable !== 'extension' &&
+      ![PLACEHOLDER_ASTERISK, PLACEHOLDER_DOUBLE_ASTERISK].includes(variable) &&
+      !variable.startsWith(PLACEHOLDER_ENUM_PREFIX)
+    ) {
+      throw new FileMatcherException(`Unknown variable "${variable}"`);
+    }
+  }
+  return result;
+}
+
+export function getGlobPattern(template: string) {
+  let value = template.replaceAll(
+    GLOB_EXISTING_DOUBLE_STAR,
+    '{__double_asterisk}'
+  );
+  value = value.replaceAll(GLOB_EXISTING_STAR, '{__asterisk}');
+  const parts = splitToParts(value);
+  const globPattern = parts
+    .map((part) => {
+      const variableName = getVariableName(part);
+      if (variableName) {
+        if (variableName === PLACEHOLDER_DOUBLE_ASTERISK) {
+          return '**';
+        } else if (variableName.startsWith(PLACEHOLDER_ENUM_PREFIX)) {
+          return (
+            '{' + variableName.substring(PLACEHOLDER_ENUM_PREFIX.length) + '}'
+          );
+        } else {
+          return '*';
+        }
+      } else {
+        return part;
+      }
+    })
+    .join('');
+
+  return globPattern;
+}
+
+export async function findFilesByTemplate(
+  template: string
+): Promise<FileMatch[]> {
+  try {
+    const sanitized = sanitizeTemplate(template);
+    const globPattern = getGlobPattern(sanitized);
+    const files = await glob(globPattern, { onlyFiles: true, absolute: true });
+    return files.map((file) => {
+      return getFileMatcher(file, sanitized);
+    });
+  } catch (e) {
+    if (e instanceof FileMatcherException) {
+      exitWithError(e.message + ` in template ${template}`);
+    } else {
+      throw e;
+    }
+  }
+}

src/utils/valueToArray.ts

@@ -0,0 +1,9 @@
+export function valueToArray<T>(
+  value: T[] | T | undefined | null
+): T[] | undefined | null {
+  if (Array.isArray(value) || value === undefined || value == null) {
+    return value as any;
+  } else {
+    return [value];
+  }
+}

test/__fixtures__/differentFormatsProject/android-xml.json

@@ -2,11 +2,6 @@
   "$schema": "../../../schema.json",
   "format": "ANDROID_XML",
   "push": {
-    "files": [
-      {
-        "path": "./android-xml/values-en/**",
-        "language": "en"
-      }
-    ]
+    "filesTemplate": "./android-xml/values-{languageTag}/**"
   }
 }