diff --git a/README.md b/README.md index 08c4e41..6173d5c 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ volumes: (re)Build and run your container. Once it is running, run the following command (from host) to install the extension's columns in the database and add the config folder. - + Replace the `my-directus` with the name of your service running directus if it is different ```bash @@ -125,6 +125,7 @@ onImport: async (item, itemsSrv) => { | Variable | Description | Default | | -------- | ----------- | ------- | | `SCHEMA_SYNC` | Set to automatically do **IMPORT**, **EXPORT** or **BOTH** | `null` | +| `SCHEMA_SYNC_PATH` | Path to the `schema-sync` folder | (Directus root)/schema-sync | | `SCHEMA_SYNC_CONFIG` | (optional) An additional config file to use in addition, eg. `test_config.js` | `null` | | `SCHEMA_SYNC_SPLIT` | (optional) Splits the schema file into multiple files once per collection | `true` | | `SCHEMA_SYNC_MERGE` | (optional) Only insert and update items found in the import set (including duplicates). Does not remove items in the DB that are not in the import set. | `false` | @@ -151,7 +152,7 @@ Besides auto importing and exporting, you can also run the commands manually. Update the `schema-sync/directus_config.js` file with the following: Replace `directus_roles` -Add `directus_policies` +Add `directus_policies` Replace `directus_permissions` Add `directus_access` diff --git a/src/collectionExporter.ts b/src/collectionExporter.ts index 699fd90..558d0dc 100644 --- a/src/collectionExporter.ts +++ b/src/collectionExporter.ts @@ -3,7 +3,8 @@ import type { ApiExtensionContext } from '@directus/extensions'; import { mkdir, readFile, rm, writeFile } from 'fs/promises'; import { condenseAction } from './condenseAction.js'; import type { CollectionExporterOptions, IExporter, IGetItemsService, ItemsService, ToUpdateItemDiff } from './types'; -import { ExportHelper, getDiff, sortObject } from './utils.js'; +import { ExportMeta } from './exportMeta.js'; +import { getDiff, sortObject, fileExists } from './utils.js'; import { glob } from 'glob'; type PARTIAL_CONFIG = { count: number; groupedBy: string[]; partial: true }; @@ -18,6 +19,7 @@ const DEFAULT_COLLECTION_EXPORTER_OPTIONS: CollectionExporterOptions = { class CollectionExporter implements IExporter { protected _getService: () => Promise; + protected _exportMeta: ExportMeta; protected collection: string; protected options: CollectionExporterOptions; @@ -44,13 +46,14 @@ class CollectionExporter implements IExporter { this.collection = collectionName; + this._exportMeta = new ExportMeta(options.path); const fileName = this.options.prefix ? `${this.options.prefix}_${collectionName}` : collectionName; - this.filePath = `${ExportHelper.dataDir}/${fileName}.json`; + this.filePath = `${this._exportMeta.dataDir}/${fileName}.json`; } protected ensureCollectionGroupDir = async () => { - if (!(await ExportHelper.fileExists(`${ExportHelper.dataDir}/${this.collection}`))) { - await mkdir(`${ExportHelper.dataDir}/${this.collection}`, { recursive: true }); + if (!(await fileExists(`${this._exportMeta.dataDir}/${this.collection}`))) { + await mkdir(`${this._exportMeta.dataDir}/${this.collection}`, { recursive: true }); } else { // Clean up old files const files = await glob(this.groupedFilesPath('*')); @@ -71,7 +74,7 @@ class CollectionExporter implements IExporter { protected groupedFilesPath(fileName: string) { fileName = `${this.options.prefix || '_'}_${fileName}`; - return `${ExportHelper.dataDir}/${this.collection}/${fileName}.json`; + return `${this._exportMeta.dataDir}/${this.collection}/${fileName}.json`; } get name() { @@ -100,7 +103,7 @@ class CollectionExporter implements IExporter { throw new Error(`Collection ${this.name} has invalid JSON: ${json}`); } - return await this.loadGroupedItems(parsedJSON, merge); + return await this.loadGroupedItems(parsedJSON, merge); } protected exportCollectionToFile = async () => { diff --git a/src/exportManager.ts b/src/exportManager.ts index f63f09e..f69d475 100644 --- a/src/exportManager.ts +++ b/src/exportManager.ts @@ -6,7 +6,10 @@ import { ExportCollectionConfig, IExporterConfig, IGetItemsService } from './typ export class ExportManager { protected exporters: IExporterConfig[] = []; - constructor(protected logger: ApiExtensionContext['logger']) {} + constructor( + protected path: string | undefined, + protected logger: ApiExtensionContext['logger'] + ) {} // FIRST: Add exporters public addExporter(exporterConfig: IExporterConfig) { @@ -15,7 +18,10 @@ export class ExportManager { public addCollectionExporter(config: ExportCollectionConfig, getItemsService: IGetItemsService) { for (let collectionName in config) { - const opts = config[collectionName]!; + const opts = { + ...config[collectionName]!, + path: this.path + }; this.exporters.push({ watch: opts.watch, exporter: new CollectionExporter(collectionName, getItemsService, opts, this.logger), diff --git a/src/exportMeta.ts b/src/exportMeta.ts new file mode 100644 index 0000000..a6f1915 --- /dev/null +++ b/src/exportMeta.ts @@ -0,0 +1,67 @@ +import { createHash } from 'crypto'; +import { readFile, readdir, writeFile } from 'fs/promises'; +import { resolve } from 'path'; + +export class ExportMeta { + public schemaDir: string; + + constructor(schemaDir?: string) { + this.schemaDir = resolve(process.cwd(), schemaDir ?? 'schema-sync') + } + + get dataDir() { + return resolve(this.schemaDir, 'data'); + } + + get hashFile() { + return resolve(this.schemaDir, 'hash.txt'); + } + + async updateExportMeta() { + const hasher = createHash('sha256'); + const files = await readdir(this.dataDir); + for (const file of files) { + if (file.endsWith('.json')) { + const json = await readFile(`${this.dataDir}/${file}`, { encoding: 'utf8' }); + hasher.update(json); + } + } + const hash = hasher.digest('hex'); + + const { hash: previousHash } = await this.getExportMeta() || {}; + + // Only update hash file if it has changed + if (hash === previousHash) return false; + + const ts = utcTS(); + const txt = hash + '@' + ts; + + await writeFile(this.hashFile, txt); + return { + hash, + ts, + }; + } + + async getExportMeta() { + try { + const content = await readFile(this.hashFile, { encoding: 'utf8' }); + const [hash, ts] = content.split('@'); + + if (hash && ts && new Date(ts).toString() !== 'Invalid Date') { + return { + hash, + ts, + }; + } + } catch { + // ignore + } + return null; + } +} + + +export function utcTS(isoTimestamp: string = new Date().toISOString()) { + return isoTimestamp.replace('T', ' ').replace(/\.\d*Z/, ''); +} diff --git a/src/index.ts b/src/index.ts index 3c356da..b2b2716 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,15 +4,17 @@ import type { SchemaOverview } from '@directus/types'; import { condenseAction } from './condenseAction'; import { copyConfig } from './copyConfig'; import { ExportManager } from './exportManager'; +import { ExportMeta } from './exportMeta.js'; import { SchemaExporter } from './schemaExporter'; import type { ExportCollectionConfig, IGetItemsService, ItemsService } from './types'; import { UpdateManager } from './updateManager'; -import { ADMIN_ACCOUNTABILITY, ExportHelper, nodeImport } from './utils'; +import { ADMIN_ACCOUNTABILITY, nodeImport } from './utils'; const registerHook: HookConfig = async ({ action, init }, { env, services, database, getSchema, logger }) => { const { SchemaService, ItemsService } = services; - const schemaOptions = { + const schemaExportOptions = { + path: env.SCHEMA_SYNC_PATH, split: typeof env.SCHEMA_SYNC_SPLIT === 'boolean' ? env.SCHEMA_SYNC_SPLIT : true, }; @@ -37,21 +39,22 @@ const registerHook: HookConfig = async ({ action, init }, { env, services, datab // We need to do this in async in order to load the config files let _exportManager: ExportManager; + const exportMeta = new ExportMeta(env.SCHEMA_SYNC_PATH); const createExportManager = async (dataOnly = false) => { - const exportMng = new ExportManager(logger); + const exportMng = new ExportManager(env.SCHEMA_SYNC_PATH, logger); if (!dataOnly) { exportMng.addExporter({ watch: ['collections', 'fields', 'relations'], - exporter: new SchemaExporter(getSchemaService, logger, schemaOptions), + exporter: new SchemaExporter(getSchemaService, logger, schemaExportOptions), }); } - const { syncDirectusCollections } = (await nodeImport(ExportHelper.schemaDir, 'directus_config.js')) as { + const { syncDirectusCollections } = (await nodeImport(exportMeta.schemaDir, 'directus_config.js')) as { syncDirectusCollections: ExportCollectionConfig; }; - const { syncCustomCollections } = (await nodeImport(ExportHelper.schemaDir, 'config.js')) as { + const { syncCustomCollections } = (await nodeImport(exportMeta.schemaDir, 'config.js')) as { syncCustomCollections: ExportCollectionConfig; }; exportMng.addCollectionExporter(syncDirectusCollections, getItemsService); @@ -59,7 +62,7 @@ const registerHook: HookConfig = async ({ action, init }, { env, services, datab // Additional config if (env.SCHEMA_SYNC_CONFIG) { - const { syncCustomCollections } = (await nodeImport(ExportHelper.schemaDir, env.SCHEMA_SYNC_CONFIG)) as { + const { syncCustomCollections } = (await nodeImport(exportMeta.schemaDir, env.SCHEMA_SYNC_CONFIG)) as { syncCustomCollections: ExportCollectionConfig; }; if (syncCustomCollections) { @@ -71,7 +74,7 @@ const registerHook: HookConfig = async ({ action, init }, { env, services, datab return exportMng; } - + const exportManager = async (dataOnly = false) => { if (dataOnly && env.SCHEMA_SYNC_DATA_ONLY !== true) { return await createExportManager(true); @@ -85,7 +88,7 @@ const registerHook: HookConfig = async ({ action, init }, { env, services, datab }; const updateMeta = condenseAction(async (saveToDb = true) => { - const meta = await ExportHelper.updateExportMeta(); + const meta = await exportMeta.updateExportMeta(); if (saveToDb && meta && (await updateManager.lockForUpdates(meta.hash, meta.ts))) { await updateManager.commitUpdates(); } @@ -106,7 +109,7 @@ const registerHook: HookConfig = async ({ action, init }, { env, services, datab if (env.SCHEMA_SYNC === 'BOTH' || env.SCHEMA_SYNC === 'IMPORT') { init('app.before', async () => { try { - const meta = await ExportHelper.getExportMeta(); + const meta = await exportMeta.getExportMeta(); if (!meta) return logger.info('Nothing exported yet it seems'); if (!(await updateManager.lockForUpdates(meta.hash, meta.ts))) return; // Schema is locked / no change, nothing to do @@ -141,7 +144,7 @@ const registerHook: HookConfig = async ({ action, init }, { env, services, datab const exportSchema = new SchemaExporter( getSchemaService, logger, - args && 'split' in args ? args : schemaOptions + args && 'split' in args ? args : schemaExportOptions ); await exportSchema.export(); @@ -156,10 +159,10 @@ const registerHook: HookConfig = async ({ action, init }, { env, services, datab .description('Import only the schema file') .action(async () => { logger.info('Importing schema...'); - const meta = await ExportHelper.getExportMeta(); + const meta = await exportMeta.getExportMeta(); if (!meta) return logger.info('Nothing exported yet it seems'); - const exportSchema = new SchemaExporter(getSchemaService, logger, schemaOptions); + const exportSchema = new SchemaExporter(getSchemaService, logger, schemaExportOptions); await exportSchema.load(); await updateManager.forceCommitUpdates(meta.hash, meta.ts); @@ -198,7 +201,7 @@ const registerHook: HookConfig = async ({ action, init }, { env, services, datab .option('--data', 'Only import data and not schema') .action(async ({ merge, data }: { merge: boolean; data: boolean }) => { try { - logger.info(`Importing everything from: ${ExportHelper.dataDir}`); + logger.info(`Importing everything from: ${exportMeta.dataDir}`); const expMng = await exportManager(data); await expMng.loadAll(merge); @@ -215,7 +218,7 @@ const registerHook: HookConfig = async ({ action, init }, { env, services, datab .description('Export the schema and all data as configured from DB to file') .action(async () => { try { - logger.info(`Exporting everything to: ${ExportHelper.dataDir}`); + logger.info(`Exporting everything to: ${exportMeta.dataDir}`); const expMng = await exportManager(); await expMng.exportAll(); diff --git a/src/schemaExporter.ts b/src/schemaExporter.ts index 78285de..e4411fe 100644 --- a/src/schemaExporter.ts +++ b/src/schemaExporter.ts @@ -5,10 +5,12 @@ import { readFile, writeFile, mkdir, rm } from 'fs/promises'; import { glob } from 'glob'; import { condenseAction } from './condenseAction.js'; import { exportHook } from './schemaExporterHooks.js'; +import { ExportMeta } from './exportMeta.js'; import type { IExporter } from './types'; -import { ExportHelper } from './utils.js'; +import { fileExists } from './utils.js'; export class SchemaExporter implements IExporter { + protected _exportMeta: ExportMeta; protected _filePath: string; protected _getSchemaService: () => any; protected _exportHandler = condenseAction(() => this.createAndSaveSnapshot()); @@ -17,15 +19,22 @@ export class SchemaExporter implements IExporter { constructor( getSchemaService: () => any, protected logger: ApiExtensionContext['logger'], - protected options = { split: true } + protected options = { + path: undefined, + split: true + } as { + path?: string, + split?: boolean + } ) { this._getSchemaService = () => getSchemaService(); - this._filePath = `${ExportHelper.dataDir}/schema.json`; + this._exportMeta = new ExportMeta(options.path); + this._filePath = `${this._exportMeta.dataDir}/schema.json`; } protected ensureSchemaFilesDir = async () => { - if (!(await ExportHelper.fileExists(`${ExportHelper.dataDir}/schema`))) { - await mkdir(`${ExportHelper.dataDir}/schema`, { recursive: true }); + if (!(await fileExists(`${this._exportMeta.dataDir}/schema`))) { + await mkdir(`${this._exportMeta.dataDir}/schema`, { recursive: true }); } else { // Clean up old schema files const files = await glob(this.schemaFilesPath('*')); @@ -34,7 +43,7 @@ export class SchemaExporter implements IExporter { }; protected schemaFilesPath(collection: string) { - return `${ExportHelper.dataDir}/schema/${collection}.json`; + return `${this._exportMeta.dataDir}/schema/${collection}.json`; } get name() { diff --git a/src/types.ts b/src/types.ts index 9a614c3..7e076c4 100644 --- a/src/types.ts +++ b/src/types.ts @@ -41,6 +41,9 @@ export type CollectionExporterOptions = { // Specify additional query options to filter, sort and limit the exported items query?: Pick; + // Path to the export folder + path?: string; + // Prefix to add to the exported file name prefix?: string; onExport?: (item: Item, srv: ItemsService) => Promise; diff --git a/src/updateManager.ts b/src/updateManager.ts index 7ebe431..38dd17e 100644 --- a/src/updateManager.ts +++ b/src/updateManager.ts @@ -1,5 +1,4 @@ import type { Knex } from 'knex'; -import { ExportHelper } from './utils'; export class UpdateManager { protected db: Knex; diff --git a/src/utils.ts b/src/utils.ts index e1bb736..0a56824 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,5 +1,4 @@ -import { createHash } from 'crypto'; -import { access, readFile, readdir, writeFile } from 'fs/promises'; +import { access } from 'fs/promises'; import { resolve } from 'path'; import { pathToFileURL } from 'url'; @@ -9,73 +8,12 @@ export function nodeImport(dir: string, file: string) { return import(pathToFileURL(resolve(dir, file)).href); } -export class ExportHelper { - static get schemaDir() { - return resolve(process.cwd(), 'schema-sync'); - } - - static get dataDir() { - return resolve(ExportHelper.schemaDir, 'data'); - } - - static get hashFile() { - return resolve(ExportHelper.schemaDir, 'hash.txt'); - } - - static utcTS(isoTimestamp: string = new Date().toISOString()) { - return isoTimestamp.replace('T', ' ').replace(/\.\d*Z/, ''); - } - - static async updateExportMeta() { - const hasher = createHash('sha256'); - const files = await readdir(ExportHelper.dataDir); - for (const file of files) { - if (file.endsWith('.json')) { - const json = await readFile(`${ExportHelper.dataDir}/${file}`, { encoding: 'utf8' }); - hasher.update(json); - } - } - const hash = hasher.digest('hex'); - - const { hash: previousHash } = await ExportHelper.getExportMeta() || {}; - - // Only update hash file if it has changed - if (hash === previousHash) return false; - - const ts = ExportHelper.utcTS(); - const txt = hash + '@' + ts; - - await writeFile(this.hashFile, txt); - return { - hash, - ts, - }; - } - - static async fileExists(path: string) { - try { - await access(path); - return true; - } catch { - return false; - } - } - - static async getExportMeta() { - try { - const content = await readFile(this.hashFile, { encoding: 'utf8' }); - const [hash, ts] = content.split('@'); - - if (hash && ts && new Date(ts).toString() !== 'Invalid Date') { - return { - hash, - ts, - }; - } - } catch { - // ignore - } - return null; +export async function fileExists(path: string) { + try { + await access(path); + return true; + } catch { + return false; } } @@ -131,4 +69,4 @@ export function sortObject | T[]>(obj: T): T { sortedObj[key] = sortObject((obj as Record)[key]); }); return sortedObj as T; -} +} \ No newline at end of file