index.ts

import AdminForth, { Filters, suggestIfTypo, AdminForthDataTypes, RAMLock, filtersTools, AdminForthFilterOperators } from "adminforth";
import type { IAdminForth, IHttpServer, AdminForthComponentDeclaration, AdminForthResourceColumn, AdminForthResource, BeforeLoginConfirmationFunction, AdminForthConfigMenuItem, AdminUser } from "adminforth";
import type { PluginOptions, SupportedLanguage } from './types.js';
import { I18nBase, getCountryCodeFromLangCode, getPrimaryLanguageCode, requiresSlavicPlurals, SLAVIC_PLURAL_EXAMPLES, SLAVIC_PLURAL_FORMS, SLAVIC_PLURAL_FORMS_HINT, SOURCE_LANGUAGE } from './base.js';
import { z } from "zod";
import iso6391 from 'iso-639-1';

const updateFieldBodySchema = z.object({
  resourceId: z.string(),
  recordId: z.union([z.string(), z.number()]),
  field: z.string(),
  value: z.unknown(),
  reviewed: z.boolean().nullish(),
}).strict();

const translateSelectedBodySchema = z.object({
  selectedLanguages: z.array(z.string()).optional(),
  selectedIds: z.array(z.union([z.string(), z.number()])),
}).strict();

const getFilteredIdsBodySchema = z.object({
  filters: z.any(),
}).passthrough();
import { iso31661Alpha2ToAlpha3 } from 'iso-3166';
import path from 'path';
import fs from 'fs-extra';
import chokidar from 'chokidar';
import  { AsyncQueue } from '@sapphire/async-queue';
import { parse } from 'bcp-47';
import pLimit from 'p-limit';
import { randomUUID } from 'crypto';
import { afLogger } from "adminforth";

const processFrontendMessagesQueue = new AsyncQueue();

interface IFailedTranslation {
  lang: SupportedLanguage;
  en_string: string;
  failedReason: string;
}


interface ICachingAdapter {
  get(key: string): Promise<any>;
  set(key: string, value: any): Promise<void>;
  clear(key: string): Promise<void>;
}

class CachingAdapterMemory implements ICachingAdapter {
  cache: Record<string, any> = {};
  async get(key: string) {
    return this.cache[key];
  }
  async set(key: string, value: any) {
    this.cache[key] = value;
  }
  async clear(key: string) {
    if (this.cache[key]) {
      delete this.cache[key];
    }
  }
}

function ensureTemplateHasAllParams(template, newTemplate) {
  // string ensureTemplateHasAllParams("a {b} c {d}", "я {b} і {d} в") // true
  // string ensureTemplateHasAllParams("a {b} c {d}", "я і {d} в") // false
  // string ensureTemplateHasAllParams("a {b} c {d}", "я {bb} і {d} в") // false
  const existingParams = template.match(/{[^}]+}/g);
  const newParams = newTemplate.match(/{[^}]+}/g);
  const existingParamsSet = new Set(existingParams);
  const newParamsSet = new Set(newParams);
  return existingParamsSet.size === newParamsSet.size && [...existingParamsSet].every(p => newParamsSet.has(p));
}

class AiTranslateError extends Error {
  constructor(message: string) {
    super(message);
    this.name = 'AiTranslateError';
  }
}
export default class I18nPlugin extends I18nBase<PluginOptions> {
  emailField: AdminForthResourceColumn;
  passwordField: AdminForthResourceColumn;
  authResource: AdminForthResource;
  emailConfirmedField?: AdminForthResourceColumn;
  trFieldNames: Partial<Record<SupportedLanguage, string>>;
  enFieldName: string;
  cache: ICachingAdapter;
  primaryKeyFieldName: string;
  menuItemWithBadgeId: string;

  externalAppOnly: boolean;

  // sorted by name list of all supported languages, without en e.g. 'al|ro|uk'
  fullCompleatedFieldValue: string;

  constructor(options: PluginOptions) {
    super(options);
    this.cache = new CachingAdapterMemory();
    this.trFieldNames = {};
  }

  // translated languages joined by '|', e.g. 'uk|ja', stored in completedFieldName to filter not fully translated strings
  completedLangsValue(isTranslated: (lang: SupportedLanguage) => boolean): string {
    return this.targetLanguages().filter(isTranslated).join('|');
  }

  computeCompletedFieldValue(record: any): string {
    return this.completedLangsValue((lang) => Boolean(record[this.trFieldNames[lang]]));
  }

  // all translations of category for frontend_messages
  categoryCacheKey(category: string, lang: string): string {
    return `${this.resourceConfig.resourceId}:${category}:${lang}`;
  }

  // one string translated by backend tr
  messageCacheKey(category: string, lang: string, enString: string): string {
    return `${this.categoryCacheKey(category, lang)}:${enString}`;
  }

  clearTranslationCache(category: string, lang: string, enString: string) {
    this.cache.clear(this.categoryCacheKey(category, lang));
    this.cache.clear(this.messageCacheKey(category, lang, enString));
  }

  async updateUntranslatedMenuBadge() {
    if (this.menuItemWithBadgeId) {
      const resource = this.adminforth.resource(this.resourceConfig.resourceId);
      const count = await resource.count([Filters.NEQ(this.options.completedFieldName, this.fullCompleatedFieldValue)]);

      this.adminforth.websocket.publish(`/opentopic/update-menu-badge/${this.menuItemWithBadgeId}`, {
        badge: count || null
      });
    }
  }


  async modifyResourceConfig(adminforth: IAdminForth, resourceConfig: AdminForthResource) {
    super.modifyResourceConfig(adminforth, resourceConfig);

    if (!this.adminforth.config.componentsToExplicitRegister) {
      this.adminforth.config.componentsToExplicitRegister = [];
    }
    this.adminforth.config.componentsToExplicitRegister.push(
      {
        file: this.componentPath('TranslationJobViewComponent.vue')
      }
    );

    if (this.options.translateLangAsBCP47Code) {
      for (const [lang, bcp47] of Object.entries(this.options.translateLangAsBCP47Code)) {
        if (!this.options.supportedLanguages.includes(lang as SupportedLanguage)) {
          throw new Error(`Invalid language code ${lang} in translateLangAsBCP47Code. It must be one of the supportedLanguages.`);
        }
      }
    }

    this.externalAppOnly = this.options.externalAppOnly === true;

    // find primary key field
    this.primaryKeyFieldName = resourceConfig.columns.find(c => c.primaryKey)?.name;

    if (!this.primaryKeyFieldName) {
      throw new Error(`Primary key field not found in resource ${resourceConfig.resourceId}`);
    }

    // parse trFieldNames
    for (const lang of this.targetLanguages()) {
      if (this.options.translationFieldNames?.[lang]) {
        this.trFieldNames[lang] = this.options.translationFieldNames[lang];
      } else {
        this.trFieldNames[lang] = lang + '_string';
      }
      // find column by name
      const column = resourceConfig.columns.find(c => c.name === this.trFieldNames[lang]);
      if (!column) {
        throw new Error(`Field ${this.trFieldNames[lang]} not found for storing translation for language ${lang}
           in resource ${resourceConfig.resourceId}, consider adding it to columns or change trFieldNames option to remap it to existing column`);
      }

      column.components = column.components || {};
      
      // set edit and create custom component - SingleMultiInput.vue
      column.components.edit = {
        file: this.componentPath('SingleMultiInput.vue'),
      };
      column.components.create = {
        file: this.componentPath('SingleMultiInput.vue'),
      };

      // set ListCell for list
      column.components.list = {
        file: this.componentPath('ListCell.vue'),
        meta: {
          pluginInstanceId: this.pluginInstanceId,
          lang,
          reviewedCheckboxesFieldName: this.options.reviewedCheckboxesFieldName,
          enFieldName: this.enFieldName,
        },
      };
    }

    this.enFieldName = this.trFieldNames[SOURCE_LANGUAGE] || `${SOURCE_LANGUAGE}_string`;

    this.fullCompleatedFieldValue = this.completedLangsValue(() => true);

    // if not enFieldName column is not found, throw error
    const enColumn = resourceConfig.columns.find(c => c.name === this.enFieldName);
    if (!enColumn) {
      throw new Error(`Field ${this.enFieldName} not found column to store english original string in resource ${resourceConfig.resourceId}`);
    }

    enColumn.components = enColumn.components || {};
    enColumn.components.edit = {
      file: this.componentPath('SingleMultiInput.vue'),
    };
    enColumn.components.create = {
      file: this.componentPath('SingleMultiInput.vue'),
    };
    enColumn.components.list = {
      file: this.componentPath('ListCell.vue'),
      meta: {
        enFieldName: this.enFieldName,
      }
    };
    enColumn.editReadonly = true;

    // if sourceFieldName defined, check it exists
    if (this.options.sourceFieldName) {
      if (!resourceConfig.columns.find(c => c.name === this.options.sourceFieldName)) {
        throw new Error(`Field ${this.options.sourceFieldName} not found in resource ${resourceConfig.resourceId}`);
      }
    }

    // if completedFieldName defined, check it exists and should be string
    if (this.options.completedFieldName) {
      const column = resourceConfig.columns.find(c => c.name === this.options.completedFieldName);
      if (!column) {
        const similar = suggestIfTypo(resourceConfig.columns.map((col) => col.name), this.options.completedFieldName);
        throw new Error(`Field ${this.options.completedFieldName} not found in resource ${resourceConfig.resourceId}${similar ? `Did you mean '${similar}'?` : ''}`);
      }

      // if showIn is not defined, add it as empty
      column.showIn = {
        show: false,
        list: false,
        edit: false,
        create: false,
        filter: false,
      };

      // add virtual field for incomplete
      resourceConfig.columns.unshift({
        name: 'fully_translated',
        label: 'Fully translated',
        virtual: true,
        filterOptions: {
          multiselect: false,
        },
        showIn: {
          show: true,
          list: true,
          edit: false,
          create: false,
          filter: true,
        },
        type: AdminForthDataTypes.BOOLEAN,
      });
    }

    // add underLogin component
    if (!this.externalAppOnly) {
      this.injectLanguageSelectors(adminforth);
    }

    // disable create allowedActions for translations
    resourceConfig.options.allowedActions.create = false;

    // add hook to validate user did not screw up with template params
    resourceConfig.hooks.edit.beforeSave.push(async ({ updates, oldRecord }: { updates: any, oldRecord?: any }): Promise<{ ok: boolean, error?: string }> => {
      for (const lang of this.targetLanguages()) {
        if (updates[this.trFieldNames[lang]]) {  // if user set '', it will have '' in updates, then it is fine, we shoudl nto check it
          if (!ensureTemplateHasAllParams(oldRecord[this.enFieldName], updates[this.trFieldNames[lang]])) {
            return { ok: false, error: `Template params mismatch for ${updates[this.enFieldName]}. Template param names should be the same as in original string. E. g. 'Hello {name}', should be 'Hola {name}' and not 'Hola {nombre}'!` };
          }
        }
      }
      return { ok: true };
    });

    // add hook on edit of any translation
    resourceConfig.hooks.edit.afterSave.push(async ({ updates, oldRecord }: { updates: any, oldRecord?: any }): Promise<{ ok: boolean, error?: string }> => {
      if (oldRecord) {
        // find lang which changed
        const langsChanged = this.targetLanguages().filter((lang) => updates[this.trFieldNames[lang]] !== undefined);

        // clear frontend cache for all langsChanged
        for (const lang of langsChanged) {
          this.clearTranslationCache(oldRecord[this.options.categoryFieldName], lang, oldRecord[this.enFieldName]);
        }
        this.updateUntranslatedMenuBadge();
      }
      // clear frontend cache for all lan
      return { ok: true };
    });

    // add hook on delete of any translation to reset cache
    resourceConfig.hooks.delete.afterSave.push(async ({ record }: { record: any }): Promise<{ ok: boolean, error?: string }> => {
      for (const lang of this.options.supportedLanguages) {
        this.clearTranslationCache(record[this.options.categoryFieldName], lang, record[this.enFieldName]);
      }
      this.updateUntranslatedMenuBadge();
      return { ok: true };
    });

    if (this.options.completedFieldName) {
      // on show and list add a list hook which will add incomplete field to record if translation is missing for at least one language
      const addIncompleteField = (record: any) => {
        // form list of all langs, sorted by alphabet, without en, to get 'al|ro|uk'
        
      
        record.fully_translated = this.fullCompleatedFieldValue === record[this.options.completedFieldName];
      }
      resourceConfig.hooks.list.afterDatasourceResponse.push(async ({ response }: { response: any[] }): Promise<{  ok: boolean, error?: string }> => {
        response.forEach(addIncompleteField);
        return { ok: true }
      });
      resourceConfig.hooks.show.afterDatasourceResponse.push(async ({ response }: { response: any }): Promise<{  ok: boolean, error?: string }> => {
        addIncompleteField(response.length && response[0]);
        return { ok: true }
      });

      // also add edit hook beforeSave to update completedFieldName
      resourceConfig.hooks.edit.beforeSave.push(async ({ record, oldRecord }: { record: any, oldRecord: any }): Promise<{ ok: boolean, error?: string }> => {
        const futureRecord = { ...oldRecord, ...record };
        
        const futureCompletedFieldValue = await this.computeCompletedFieldValue(futureRecord);
    
        record[this.options.completedFieldName] = futureCompletedFieldValue;
        return { ok: true };
      });

      // add list hook to support filtering by fully_translated virtual field
      resourceConfig.hooks.list.beforeDatasourceRequest.push(async ({ query }: { query: any }): Promise<{ ok: boolean, error?: string }> => {
        if (!query.filters || query.filters.length === 0) {
          query.filters = [];
        }

        // get fully_translated field from filter if it is there
        const fullyTranslatedFilter = query.filters.find((f: any) => f.field === 'fully_translated');
        if (fullyTranslatedFilter) {
          // remove it from filters because it is virtual field
          query.filters = query.filters.filter((f: any) => f.field !== 'fully_translated');
          if (fullyTranslatedFilter.value) {
            query.filters.push({
              field: this.options.completedFieldName,
              value: this.fullCompleatedFieldValue,
              operator: 'eq',
            });
          } else {
            query.filters.push({
              field: this.options.completedFieldName,
              value: this.fullCompleatedFieldValue,
              operator: 'ne',
            });
          }
        }

        return { ok: true };
      });
    }

    // add bulk action

    const pageInjection = {
      file: this.componentPath('BulkActionButton.vue'),
      meta: {
        supportedLanguages: this.options.supportedLanguages,
        countryCodes: Object.fromEntries(this.options.supportedLanguages.map((lang) => [lang, getCountryCodeFromLangCode(lang)])),
        pluginInstanceId: this.pluginInstanceId,
      }
    }

    if (!resourceConfig.options.pageInjections) {
      resourceConfig.options.pageInjections = {};
    }
    if (!resourceConfig.options.pageInjections.list) {
      resourceConfig.options.pageInjections.list = {};
    }
    if (!resourceConfig.options.pageInjections.list.threeDotsDropdownItems) {
      resourceConfig.options.pageInjections.list.threeDotsDropdownItems = [];
    }
    if (this.options.completeAdapter) {
      (resourceConfig.options.pageInjections.list.threeDotsDropdownItems as AdminForthComponentDeclaration[]).push(pageInjection);
    }

    // if there is menu item with resourceId, add .badge function showing number of untranslated strings
    const addBadgeCountToMenuItem = (menuItem: AdminForthConfigMenuItem) => {
      this.menuItemWithBadgeId = menuItem.itemId;
      menuItem.badge = async () => {
        const resource = adminforth.resource(menuItem.resourceId);
        const count = await resource.count([Filters.NEQ(this.options.completedFieldName, this.fullCompleatedFieldValue)]);
        return count ? `${count}` : null;
      };
      menuItem.badgeTooltip = 'Untranslated count';
    }
    adminforth.config.menu.forEach((menuItem) => {
      if (menuItem.resourceId === resourceConfig.resourceId && !menuItem.badge) {
        addBadgeCountToMenuItem(menuItem);
      }
      if (menuItem.children) {
        menuItem.children.forEach((child) => {
          if (child.resourceId === resourceConfig.resourceId && !child.badge) {
            addBadgeCountToMenuItem(child);
          }
        });
      }
    });
  }

  async generateAndSaveBunch (
    prompt: string,
    strings: { en_string: string, category: string, schemaKey?: string }[], 
    translations: any[],
    updateStrings: Record<string, { updates: any, category: string, strId: string, enStr: string, translatedStr: string }> = {},
    lang: string,
    failedToTranslate: IFailedTranslation[],
    needToTranslateByLang: Record<string, any> = {},
    jobId: string,
    promptCost: number,
  ): Promise<{ ok: boolean, error?: string }>{

    // return [];
    const jsonSchemaProperties = {};
    const schemaKeyToEnString = new Map<string, string>();
    const enStringToSchemaKey = new Map<string, string>();
    strings.forEach(s => {
      if (!enStringToSchemaKey.has(s.en_string)) {
        enStringToSchemaKey.set(s.en_string, s.schemaKey || `text_${enStringToSchemaKey.size}`);
      }
    });
    enStringToSchemaKey.forEach((schemaKey, enString) => {
      schemaKeyToEnString.set(schemaKey, enString);
      jsonSchemaProperties[schemaKey] = { 
        type: 'string',
        minLength: 1,
      };
    });

    const dedupRequired = Array.from(schemaKeyToEnString.keys());
    // call OpenAI
    const resp = await this.options.completeAdapter.complete({
      content: prompt,
      maxTokens: prompt.length * 2,
      outputSchema: {
        name: "translation_response",
        schema: {
          type: "object",
          properties: jsonSchemaProperties,
          required: dedupRequired,
          additionalProperties: false,
        },
      },
    });
    
    process.env.HEAVY_DEBUG && console.log(`🪲🔪LLM resp >> ${prompt.length}, <<${resp.content.length} :\n\n`, JSON.stringify(resp));
    
    if (resp.error) {
      return { ok: false, error: resp.error };
    }

    let res;
    try {
      res = resp.content;
    } catch (e) {
      console.error(`Error in parsing LLM resp: ${resp}\n Prompt was: ${prompt}\n Resp was: ${JSON.stringify(resp)}`, );
      strings.forEach(s => {
        failedToTranslate.push({
          lang: lang as SupportedLanguage,
          en_string: s.en_string,
          failedReason: "Error in parsing LLM response"
        });
      });
      return { ok: false, error: 'Error in parsing LLM response' };
    }

    const backgroundJobsPlugin = this.adminforth.getPluginByClassName<any>('BackgroundJobsPlugin');

    backgroundJobsPlugin.updateJobFieldsAtomically(jobId, async () => {
      // do all set / get fields in this function to make state update atomic and there is no conflicts when 2 tasks in parallel do get before set.
      // don't do long awaits in this callback, since it has exclusive lock.
      let totalUsedTokens = await backgroundJobsPlugin.getJobField(jobId, 'totalUsedTokens');
      totalUsedTokens += promptCost;
      await backgroundJobsPlugin.setJobField(jobId, 'totalUsedTokens', totalUsedTokens);
    })


    try {
      res = JSON.parse(res);
    } catch (e) {
      console.error(`Error in parsing LLM resp json: ${resp}\n Prompt was: ${prompt}\n Resp was: ${JSON.stringify(resp)}`, );
      strings.forEach(s => {
        failedToTranslate.push({
          lang: lang as SupportedLanguage,
          en_string: s.en_string,
          failedReason: "Error in parsing LLM response JSON"
        });
      });
      return { ok: false, error: 'Error in parsing LLM response JSON' };
    }


    for (const [schemaKey, translatedStr] of Object.entries(res) as [string, string][]) {
      const enStr = schemaKeyToEnString.get(schemaKey) || schemaKey;
      const translationsTargeted = translations.filter(t => t[this.enFieldName] === enStr);
      // might be several with same en_string
      for (const translation of translationsTargeted) {
        //translation[this.trFieldNames[lang]] = translatedStr;
        if (!updateStrings[translation[this.primaryKeyFieldName]]) {

          updateStrings[translation[this.primaryKeyFieldName]] = {
            updates: {},
            translatedStr,
            enStr,
            category: translation[this.options.categoryFieldName],
            strId: translation[this.primaryKeyFieldName],
          };
        }
        // make sure LLM did not screw up with template params
        if (translation[this.enFieldName].includes('{') && !ensureTemplateHasAllParams(translation[this.enFieldName], translatedStr)) {
          console.warn(`LLM Screwed up with template params mismatch for "${translation[this.enFieldName]}"on language ${lang}, it returned "${translatedStr}"`);
          continue;
        }
        updateStrings[
          translation[this.primaryKeyFieldName]
        ].updates[this.trFieldNames[lang]] = translatedStr;
      }
    }

    const langsInvolved = new Set(Object.keys(needToTranslateByLang));

    // here we need to save updateStrings
    await Promise.all(
      Object.entries(updateStrings).map(
        async ([_, { updates, strId }]: [string, { updates: any, category: string, strId: string }]) => {
          // get old full record
          const oldRecord = await this.adminforth.resource(this.resourceConfig.resourceId).get([Filters.EQ(this.primaryKeyFieldName, strId)]);

          // because this will translate all languages, we can set completedLangs to all languages
          const futureCompletedFieldValue = await this.computeCompletedFieldValue({ ...oldRecord, ...updates });

          await this.adminforth.resource(this.resourceConfig.resourceId).update(strId, {
            ...updates,
            [this.options.completedFieldName]: futureCompletedFieldValue,
          });
        }
      )
    );
    

    for (const lang of langsInvolved) {
      for (const { enStr, category } of Object.values(updateStrings)) {
        this.clearTranslationCache(category, lang, enStr);
      }
    }

    return { ok: true };
  }

  async getTranslateToLangTasks (
    langIsoCode: SupportedLanguage, 
    strings: { id: string, en_string: string, category: string }[], 
    plurals=false,
    translations: any[],
    updateStrings: Record<string, { updates: any, category: string, strId: string, enStr: string, translatedStr: string }> = {},
    needToTranslateByLang: Record<string, any> = {},
    adminUser: AdminUser,
  ): Promise<any> {

    const maxInputTokens = this.options.inputTokensPerBatch ?? 30000;

    const limit = pLimit(30); 
    const enStringsTokenLengthCache: Record<string, any> = {};
    let tokenCountingFailed = false;

    // Counting tokens is only used to split requests into batches. Some completion
    // adapters cannot provide a token count, so use a conservative estimate instead
    // of preventing the translation job from being created.
    const getTokenLength = async (value: string): Promise<number> => {
      try {
        const tokenLength = await this.options.completeAdapter.measureTokensCount(value);
        if (!Number.isFinite(tokenLength) || tokenLength < 0) {
          throw new Error(`Invalid token count: ${tokenLength}`);
        }
        return tokenLength;
      } catch (error) {
        if (!tokenCountingFailed) {
          tokenCountingFailed = true;
          afLogger.warn(`Could not count tokens for translation batches in plugin ${this.constructor.name}; using a character-based estimate instead: ${String(error)}`);
        }
        // English text normally uses about 4 characters per token. Three gives
        // batches some extra headroom for JSON formatting and unusual characters.
        return Math.max(1, Math.ceil(value.length / 3));
      }
    };
    

    const tokenLengthPerString = async ({ str, id }: { str: string; id: string }): Promise<void> => {
      const objectToPush = {
        en_string: str,
        numOfTokens: await getTokenLength(`"${str}":"", \n`)
      };
      enStringsTokenLengthCache[id] = objectToPush;
    }
    const promises = strings.map(s => limit(() => tokenLengthPerString({ str: s.en_string, id: s.id })));

    await Promise.all(promises);
    if (strings.length === 0) {
      return;
    }

    const replacedLanguageCodeForTranslations = this.options.translateLangAsBCP47Code && langIsoCode.length === 2 ? this.options.translateLangAsBCP47Code[langIsoCode as any] : null;
    const langSchema = parse(String(langIsoCode), 
      { 
        normalize: true, 
      }
    );
    const langCode = replacedLanguageCodeForTranslations ? replacedLanguageCodeForTranslations : langIsoCode;
    const lang = langIsoCode;
    const primaryLang = getPrimaryLanguageCode(lang);
    const langName = iso6391.getName(primaryLang);
    const requestSlavicPlurals = requiresSlavicPlurals(lang) && plurals;
    const region = langSchema.region;
    const basePrompt = `
      I need to translate strings in JSON to ${langName} language ${replacedLanguageCodeForTranslations || lang.length > 2 ? `BCP-47 code ${langCode}` : `ISO 639-1 code ${langIsoCode}`} from English for my web app.

      
      ${region ? `Use the regional conventions for ${langCode} (region ${region}), including spelling, punctuation, and formatting.` : ''}
      ${requestSlavicPlurals ? `You should provide ${SLAVIC_PLURAL_FORMS} slavic forms (in format "${SLAVIC_PLURAL_FORMS_HINT}") e.g. "apple | apples" should become "${SLAVIC_PLURAL_EXAMPLES[primaryLang]}"` : ''}
      JSON keys are internal identifiers. Keep keys as is and write translations into values. If values have variables (in curly brackets), then translated strings should have them as well (variables itself should not be translated). Here are the strings:
      \`\`\`json
      {

      }
      \`\`\`
    `;

    const failedToTranslate: IFailedTranslation[] = [];
    const basePromptTokenLength = await getTokenLength(basePrompt);
    const allowedTokensAmountForFields = maxInputTokens - basePromptTokenLength;
    const stringsToTranslate: Record<string, { id: string; en_string: string; category: string }> = Object.fromEntries(strings.map(s => [s.id, s]));
    const generationTasksInitialData = []
    while (Object.keys(stringsToTranslate).length !== 0) {
      const stringBanch = [];
      const stringIdsInBatch = [];
      let banchTokens = 0;
      for (const id of Object.keys(stringsToTranslate)) {
        const { en_string } = stringsToTranslate[id];
        const numberOfTokensForString = enStringsTokenLengthCache[id]?.numOfTokens || 0;
        if( banchTokens + numberOfTokensForString <= allowedTokensAmountForFields ) {
          stringBanch.push( en_string );
          stringIdsInBatch.push(id);
          banchTokens += numberOfTokensForString;
        } else {
          continue;
        }
      }
      if ( stringBanch.length === 0 ) {
        Object.values(stringsToTranslate).forEach(s => {
          failedToTranslate.push({
            lang,
            en_string: s.en_string,
            failedReason: "Not enough input generation tokens"
          });
          generationTasksInitialData.push(
            { 
              state: {
                failedToTranslate,
              }
            }
          )
        });
        break;
      }

      for ( const id of stringIdsInBatch) {
        delete stringsToTranslate[id];
      }

      const promptToGenerate = basePrompt.split(`\`\`\`json`)[0] + 
        `\`\`\`json
        {
          ${
            Array.from(new Set(stringBanch))
              .map((s, index) => `"text_${index}": ${JSON.stringify(s)}`)
              .join(",\n")
          }
        }
        \`\`\``;
      const stringBanchCopy = [...stringBanch];
      const schemaKeyByEnString = Object.fromEntries(
        Array.from(new Set(stringBanch)).map((s, index) => [s, `text_${index}`])
      );
      generationTasksInitialData.push(
        {
          state: {
            taskName: `Translate ${strings.length} strings`,
            prompt: promptToGenerate,
            strings: strings
              .filter(s => stringBanchCopy.includes(s.en_string))
              .map(s => ({
                ...s,
                schemaKey: schemaKeyByEnString[s.en_string],
              })),
            translations: translations.filter(t => stringBanchCopy.includes(t.en_string)),
            updateStrings,
            lang,
            countryCode: getCountryCodeFromLangCode(lang as SupportedLanguage),
            failedToTranslate,
            needToTranslateByLang,
            promptCost: basePromptTokenLength + banchTokens,
          }
        }
      ) 
    }

    return generationTasksInitialData;
  }


  // returns translated count
  async bulkTranslate({ selectedIds, selectedLanguages, adminUser }: 
    { 
      selectedIds: string[], 
      selectedLanguages?: SupportedLanguage[],
      adminUser: AdminUser,
    }): Promise<string>
    {
    const needToTranslateByLang : Partial<
      Record<
        SupportedLanguage,
        {
          id: string;
          en_string: string;
          category: string;
        }[]
      >
    > = {};

    const translations = await this.adminforth.resource(this.resourceConfig.resourceId).list(Filters.IN(this.primaryKeyFieldName, selectedIds));
    const languagesToProcess = selectedLanguages || this.options.supportedLanguages;
    for (const lang of languagesToProcess) {
      if (lang === SOURCE_LANGUAGE) {
        // all strings are in source language, no need to translate
        continue;
      }
      for (const translation of translations) {
        if (!translation[this.trFieldNames[lang]]) {
          if (!needToTranslateByLang[lang]) {
            needToTranslateByLang[lang] = [];
          }
          needToTranslateByLang[lang].push({
            id: translation[this.primaryKeyFieldName],
            'en_string': translation[this.enFieldName],
            category: translation[this.options.categoryFieldName],
          })
        }
      }
    }

    const updateStrings: Record<string, { 
      updates: any, 
      category: string,
      strId: string,
      enStr: string,
      translatedStr: string
    }> = {};


    let generationTasksInitialData = [];
    await Promise.all(
      Object.entries(needToTranslateByLang).map(
        async ([lang, strings]: [SupportedLanguage, { id: string, en_string: string, category: string }[]]) => {
          // first translate without plurals
          const stringsWithoutPlurals = strings.filter(s => !s.en_string.includes('|'));
          const noPluralTranslationsTasks = await this.getTranslateToLangTasks(lang, stringsWithoutPlurals, false, translations, updateStrings, needToTranslateByLang, adminUser);

          const stringsWithPlurals = strings.filter(s => s.en_string.includes('|'));
          const pluralTranslationsTasks = await this.getTranslateToLangTasks(lang, stringsWithPlurals, true, translations, updateStrings, needToTranslateByLang, adminUser);
          generationTasksInitialData = generationTasksInitialData.concat(noPluralTranslationsTasks || []).concat(pluralTranslationsTasks || []);
        }
      )
    );


    const totalTranslationTokenCost = generationTasksInitialData.reduce((a, b) => a + (b.state?.promptCost || 0), 0);
    const backgroundJobsPlugin = this.adminforth.getPluginByClassName<any>('BackgroundJobsPlugin');
    const jobId = await backgroundJobsPlugin.startNewJob(
      `Translate ${selectedIds.length} items`, //job name
      adminUser, // adminuser
      generationTasksInitialData, //initial tasks
      'translation_job_handler', //job handler name
    );

    afLogger.info(`Started background job with ID ${jobId} `);
    await backgroundJobsPlugin.setJobField(jobId, 'totalTranslationTokenCost', totalTranslationTokenCost);
    await backgroundJobsPlugin.setJobField(jobId, 'totalUsedTokens', 0);

    return jobId
  }

  
  async processExtractedMessages(adminforth: IAdminForth, filePath: string) {
    await processFrontendMessagesQueue.wait();
    // messages file is in i18n-messages.json
    let messages;
    try {
      messages = await fs.readJson(filePath);
      process.env.HEAVY_DEBUG && console.info('🐛 Messages file found');

    } catch (e) {
      process.env.HEAVY_DEBUG && console.error('🐛 Messages file not yet exists, probably npm run i18n:extract not finished/started yet, might be ok');
      return;
    }
    // loop over missingKeys[i].path and add them to database if not exists
    const messagesForFeed = messages.missingKeys.map((mk) => {
      return {
        en_string: mk.path,
        source: mk.file,
      };
    });

    await this.feedCategoryTranslations(messagesForFeed, 'frontend')
    
    
  }


  async tryProcessAndWatch(adminforth: IAdminForth) {
    const serveDir = adminforth.codeInjector.getServeDir();
    // messages file is in i18n-messages.json
    const messagesFile = path.join(serveDir, 'i18n-messages.json');
    process.env.HEAVY_DEBUG && console.log('🪲🔔messagesFile read started', messagesFile);
    this.processExtractedMessages(adminforth, messagesFile);
    // we use watcher because file can't be yet created when we start - bundleNow can be done in build time or can be done now
    // that is why we make attempt to process it now and then watch for changes
    const w = chokidar.watch(messagesFile, { 
      persistent: true,
      ignoreInitial: true, // don't trigger 'add' event for existing file on start
    });
    w.on('change', () => {
      process.env.HEAVY_DEBUG && console.log('🪲🔔messagesFile change', messagesFile);
      this.processExtractedMessages(adminforth, messagesFile);
    });
    w.on('add', () => {
      process.env.HEAVY_DEBUG && console.log('🪲🔔messagesFile add', messagesFile);
      this.processExtractedMessages(adminforth, messagesFile);
    });
  }
  
  validateConfigAfterDiscover(adminforth: IAdminForth, resourceConfig: AdminForthResource) {
    // optional method where you can safely check field types after database discovery was performed
    // ensure each trFieldName (apart from enFieldName) is nullable column of type string
    const backgroundJobsPlugin = adminforth.getPluginByClassName<any>('BackgroundJobsPlugin');

    if (!backgroundJobsPlugin) {
      throw new Error(`BackgroundJobsPlugin is required for ${this.constructor.name} to work, please add it to your plugins`);
    }

    backgroundJobsPlugin.registerTaskHandler({
      // job handler name
      jobHandlerName: 'translation_job_handler',
      //handler function
      handler: async ({ jobId, setTaskStateField, getTaskStateField }) => {
        const initialState: {    
          taskName: string,
          prompt?: string,
          strings?: { en_string: string, category: string }[], 
          translations?: any[],
          updateStrings?: Record<string, { updates: any, category: string, strId: string, enStr: string, translatedStr: string }>,
          lang?: string,
          failedToTranslate?: IFailedTranslation[],
          needToTranslateByLang?: Record<string, any>,
          promptCost?: number,
        } = await getTaskStateField();
        let bunchResult: { ok: boolean, error?: string } | undefined;

        if ( initialState.prompt && initialState.strings && initialState.translations && initialState.updateStrings && initialState.lang && initialState.failedToTranslate && initialState.needToTranslateByLang) {
          bunchResult = await this.generateAndSaveBunch(
            initialState.prompt,
            initialState.strings,
            initialState.translations,
            initialState.updateStrings,
            initialState.lang,
            initialState.failedToTranslate,
            initialState.needToTranslateByLang,
            jobId,
            initialState.promptCost
          );
        }
        afLogger.debug(`Translation task for language ${initialState.lang} completed.`);

        const stateToSave = {
          taskName: initialState.taskName,
          lang: initialState.lang,
          failedToTranslate: initialState.failedToTranslate,
        }
        await setTaskStateField(stateToSave);

        this.adminforth.websocket.publish('/translation_progress', {});
        if (bunchResult && !bunchResult.ok) {
          afLogger.error(`Failed to translate bunch for language ${initialState.lang} in plugin ${this.constructor.name}: ${bunchResult.error}`);
          throw new AiTranslateError(bunchResult.error);
        }
        if (initialState.failedToTranslate.length > 0) {
          afLogger.error(`Failed to translate some strings for language ${initialState.lang} in plugin ${this.constructor.name}:, ${initialState.failedToTranslate}`);
          throw new Error(`Failed to translate some strings for language ${initialState.lang}, check job details for more info`);
        }
      },
      //limit of tasks, that are running in parallel
      parallelLimit: this.options.parallelTranslationLimit || 20,
    })

    backgroundJobsPlugin.registerTaskDetailsComponent({
      jobHandlerName: 'translation_job_handler', // Handler name
      component: { 
        file: this.componentPath('TranslationJobViewComponent.vue')  //custom component for the job details
      },
    })


    if (this.options.completeAdapter) {
      this.options.completeAdapter.validate();
    }

    for (const lang of this.targetLanguages()) {
      const column = resourceConfig.columns.find(c => c.name === this.trFieldNames[lang]);
      if (!column) {
        throw new Error(`Field ${this.trFieldNames[lang]} not found for storing translation for language ${lang}
           in resource ${resourceConfig.resourceId}, consider adding it to columns or change trFieldNames option to remap it to existing column`);
      }
      if (column.required.create || column.required.edit) {
        throw new Error(`Field ${this.trFieldNames[lang]} should be not required in resource ${resourceConfig.resourceId}`);
      }
    }

    if (this.options.reviewedCheckboxesFieldName) {
      // ensure type is JSON
      const column = resourceConfig.columns.find(c => c.name === this.options.reviewedCheckboxesFieldName);
      if (!column) {
        const similar = suggestIfTypo(resourceConfig.columns.map((col) => col.name), this.options.reviewedCheckboxesFieldName);
        throw new Error(`Field ${this.options.reviewedCheckboxesFieldName} not found in resource ${resourceConfig.resourceId}${similar ? `Did you mean '${similar}'?` : ''}`);
      }
      if (column.type !== AdminForthDataTypes.JSON) {
        throw new Error(`Field ${this.options.reviewedCheckboxesFieldName} should be of type JSON in resource ${resourceConfig.resourceId}, but it is ${column.type}`);
      }
    }


    // ensure categoryFieldName defined and is string
    if (!this.options.categoryFieldName) {
      throw new Error(`categoryFieldName option is not defined. It is used to categorize translations and return only specific category e.g. to frontend`);
    }
    const categoryColumn = resourceConfig.columns.find(c => c.name === this.options.categoryFieldName);
    if (!categoryColumn) {
      throw new Error(`Field ${this.options.categoryFieldName} not found in resource ${resourceConfig.resourceId}`);
    }
    if (categoryColumn.type !== AdminForthDataTypes.STRING && categoryColumn.type !== AdminForthDataTypes.TEXT) {
      throw new Error(`Field ${this.options.categoryFieldName} should be of type string in resource ${resourceConfig.resourceId}, but it is ${categoryColumn.type}`);
    }

    // in this plugin we will use plugin to fill the database with missing language messages
    if (!this.externalAppOnly) {
      this.tryProcessAndWatch(adminforth);

      this.installBackendTranslator(adminforth, async (msg, category, lang) => {
        if (category === 'frontend') {
          throw new Error(`Category 'frontend' is reserved for frontend messages, use any other category for backend messages`);
        }

        // try to get translation from cache
        const cacheKey = this.messageCacheKey(category, lang, msg);
        const cached = await this.cache.get(cacheKey);
        if (cached) {
          return cached;
        }
        const resource = adminforth.resource(resourceConfig.resourceId);
        const translation = await resource.get([Filters.EQ(this.enFieldName, msg), Filters.EQ(this.options.categoryFieldName, category)]);
        if (!translation) {
          await resource.create({
            [this.enFieldName]: msg,
            [this.options.categoryFieldName]: category,
          });
          this.updateUntranslatedMenuBadge();
        }

        // do this check here, to faster register missing translations
        const result = lang === SOURCE_LANGUAGE ? msg : translation?.[this.trFieldNames[lang]] || msg;
        // cache so even if key does not exist, we will not hit database
        await this.cache.set(cacheKey, result);
        return result;
      });
    }
  }

  async getCategoryTranslations(category: string, lang: string): Promise<Record<string, string>> {
    const resource = this.adminforth.resource(this.resourceConfig.resourceId);
    const cacheKey = this.categoryCacheKey(category, lang);
    const cached = await this.cache.get(cacheKey);
    if (cached) {
      return cached;
    }
    const allTranslations = await resource.list([Filters.EQ(this.options.categoryFieldName, category)]);
    const translations = this.fallbackToOriginal(Object.fromEntries(
      allTranslations.map((tr) => [tr[this.enFieldName], tr[this.trFieldNames[lang]]]),
    ));
    await this.cache.set(cacheKey, translations);
    return translations;
  }

  async feedCategoryTranslations(messages: {
    en_string: string;
    source: string;
  }[], category: string): Promise<void> {
    const adminforth = this.adminforth;
    const missingKeysDeduplicated = messages.reduce((acc: any[], missingKey: any) => {
      if (!acc.find((a) => a.en_string === missingKey.en_string)) {
        acc.push(missingKey);
      }
      return acc;
    }, []);

    await Promise.all(missingKeysDeduplicated.map(async (missingKey: any) => {
      const key = missingKey.en_string;
      const source = missingKey.source;
      const exists = await adminforth.resource(this.resourceConfig.resourceId).count([
        Filters.EQ(this.enFieldName, key), Filters.EQ(this.options.categoryFieldName, category)
      ]);
      if (exists) {
        return;
      }
      if (!key) {
        console.error(`Faced an empty key in feeding ${category} messages, source ${source}`);
      }
      const record = {
        [this.enFieldName]: key,
        [this.options.categoryFieldName]: category,
        ...(this.options.sourceFieldName ? { [this.options.sourceFieldName]: source } : {}),
      };
      try {
        await adminforth.resource(this.resourceConfig.resourceId).create(record);
      } catch (e) {
        console.error('🐛 Error creating record', e);
      }
    }));

    // updateBadge
    this.updateUntranslatedMenuBadge();
  }


  setupEndpoints(server: IHttpServer) {
    super.setupEndpoints(server);

    const lock = new RAMLock();

    server.endpoint({
      method: 'POST',
      path: `/plugin/${this.pluginInstanceId}/update-field`,
      request_schema: updateFieldBodySchema,
      handler: async ({ body, adminUser, headers, response }) => {
        const data = body as z.infer<typeof updateFieldBodySchema>;
        const { resourceId, recordId, field, value, reviewed } = data;
        if (resourceId !== this.resourceConfig.resourceId) {
          return { error: 'Invalid resourceId' };
        }
        if (!field) {
          return { error: 'No field provided' };
        }
        if (recordId === undefined || recordId === null) {
          return { error: 'No recordId provided' };
        }
        const resource = this.adminforth.config.resources.find(r => r.resourceId === resourceId);
        // Create update object with just the single field
        const updateRecord = { [field]: value };
        
        // Use AdminForth's built-in update method
        const connector = this.adminforth.connectors[resource.dataSource];

        let oldRecord;
        let result;
        await lock.run(`edit-trans-${recordId}`,  async () => {
          // put into lock so 2 editors will not update the same record at the same time
          oldRecord = await connector.getRecordByPrimaryKey(resource, recordId as string)

          if (!oldRecord) {
            result = { error: 'Record not found' };
            return;
          }

          if (this.options.reviewedCheckboxesFieldName) {
            let oldValue;
            if (!oldRecord[this.options.reviewedCheckboxesFieldName]) {
              oldValue = {}
            } else {
              oldValue = {...oldRecord[this.options.reviewedCheckboxesFieldName]};
            }
            oldValue[field] = reviewed;
            updateRecord[this.options.reviewedCheckboxesFieldName] = { ...oldValue };
          }

          result = await this.adminforth.updateResourceRecord({
            resource,
            recordId,
            record: updateRecord,
            oldRecord,
            adminUser
          });
        });

        if (result.error) {
          return { error: result.error };
        }

        const updatedRecord = await connector.getRecordByPrimaryKey(resource, recordId as string);

        return { record: updatedRecord };
      }
    });

    server.endpoint({
      method: 'POST',
      path: `/plugin/${this.pluginInstanceId}/translate-selected-to-languages`,
      noAuth: false,
      request_schema: translateSelectedBodySchema,
      handler: async ({ body, tr, adminUser, response }) => {
        const data = body as z.infer<typeof translateSelectedBodySchema>;
        const selectedLanguages = data.selectedLanguages;
        const selectedIds = data.selectedIds;

        if (!Array.isArray(selectedIds) || selectedIds.length === 0) {
          return { ok: false, error: 'No records selected' };
        }

        const jobId = await this.bulkTranslate({
          selectedIds: selectedIds as string[],
          selectedLanguages: selectedLanguages as SupportedLanguage[] | undefined,
          adminUser,
        });

        return { 
          ok: true, 
          jobId: jobId,
        };
      }
    });

    server.endpoint({
      method: 'POST',
      path: `/plugin/${this.pluginInstanceId}/get_filtered_ids`,
      request_schema: getFilteredIdsBodySchema,
      handler: async ({ body, adminUser, headers, query, cookies, requestUrl, response }) => {
        const resource = this.resourceConfig;

        for (const hook of resource.hooks?.list?.beforeDatasourceRequest || []) {
          const filterTools = filtersTools.get(body);
          body.filtersTools = filterTools;
          const resp = await hook({
            resource,
            query: body,
            adminUser,
            //@ts-ignore
            filtersTools: filterTools, 
            extra: {
              body, query, headers, cookies, requestUrl
            },
            adminforth: this.adminforth,
          });
          if (!resp || (!resp.ok && !resp.error)) {
            throw new Error(`Hook must return object with {ok: true} or { error: 'Error' } `);
          }
          if (resp.error) {
            return { error: resp.error };
          }
        }
        const filters = body.filters;

        const normalizedFilters = { operator: AdminForthFilterOperators.AND, subFilters: [] };
        if (filters) {
          if (typeof filters !== 'object') {
            throw new Error(`Filter should be an array or an object`);
          }
          if (Array.isArray(filters)) {
            // if filters are an array, they will be connected with "AND" operator by default
            normalizedFilters.subFilters = filters;
          } else if (filters.field) {
            // assume filter is a SingleFilter
            normalizedFilters.subFilters = [filters];
          } else if (filters.subFilters) {
            // assume filter is a AndOr filter
            normalizedFilters.operator = filters.operator;
            normalizedFilters.subFilters = filters.subFilters;
          } else {
            // wrong filter
            throw new Error(`Wrong filter object value: ${JSON.stringify(filters)}`);
          }
        }

        const records = await this.adminforth.resource(this.resourceConfig.resourceId).list(normalizedFilters);
        if (!records) {
          return { ok: true, recordIds: [] };
        }
        const primaryKeyColumn = this.resourceConfig.columns.find((col) => col.primaryKey);

        const recordIds = records.map(record => record[primaryKeyColumn.name]);

        return { ok: true, recordIds }
      }
    });

  }

}

export { I18nBase, getCountryCodeFromLangCode, getPrimaryLanguageCode, isValidSupportedLanguageTag, requiresSlavicPlurals, SOURCE_LANGUAGE, PLURAL_SEPARATOR, SLAVIC_PLURAL_FORMS, SLAVIC_PLURAL_FORMS_HINT } from './base.js';
export type { I18nBaseOptions, SupportedLanguage } from './types.js';