Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .talismanrc
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
fileignoreconfig:
- filename: package-lock.json
checksum: d3b93fad9630655f037e36b78fea3354f1a038988562254afdad0f6e54ece12d
checksum: d3a79961a2cb2744e3a1e6ec33a195a26c0edc8a242c539358de0cb0608bbf20
- filename: pnpm-lock.yaml
checksum: aa6177859aaa87caf2892e8034657fd485c3abe7c13a833fd28449a1d33fa950
- filename: packages/contentstack-import-setup/test/unit/backup-handler.test.ts
Expand Down Expand Up @@ -211,4 +211,6 @@ fileignoreconfig:
checksum: 49dd8e754a0d3635585a74e943ab097593f061089a7cddc22683ec6caddbb3c5
- filename: packages/contentstack-export/test/unit/export/modules/personalize.test.ts
checksum: 83cf034fabee00b42b4243a8c0b8ba280ab7c1e68ffd741c49c31aaee8ca0315
- filename: packages/contentstack-export/src/commands/cm/stacks/export.ts
checksum: 00037f0c47b1c183175a5e37db786788155a146e72c7e2817775067ecd8c7bb3
version: "1.0"
2,075 changes: 1,036 additions & 1,039 deletions package-lock.json

Large diffs are not rendered by default.

17 changes: 17 additions & 0 deletions packages/contentstack-export/src/commands/cm/stacks/export.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,19 +119,36 @@ export default class ExportCommand extends Command {
let exportDir: string = pathValidator('logs');
try {
const { flags } = await this.parse(ExportCommand);
log.info('Starting export command execution', { stackApiKey: flags['stack-api-key'] || flags.alias });
log.debug('Parsed command flags', { flags: JSON.stringify(flags) });

const exportConfig = await setupExportConfig(flags);
log.debug('Export configuration setup completed', {
apiKey: exportConfig.apiKey,
module: exportConfig.moduleName,
hasQuery: !!exportConfig.query,
});

// Prepare the context object
const context = this.createExportContext(exportConfig.apiKey, exportConfig.authenticationMethod);
exportConfig.context = { ...context };
//log.info(`Using Cli Version: ${this.context?.cliVersion}`, exportConfig.context);
log.debug('Export context created', exportConfig.context);

// Assign exportConfig variables
this.assignExportConfig(exportConfig);

exportDir = sanitizePath(exportConfig.cliLogsPath || exportConfig.data || exportConfig.exportDir);
log.info(`Export directory set to: ${exportDir}`, exportConfig.context);
log.debug('Initializing management API client', exportConfig.context);

const managementAPIClient: ContentstackClient = await managementSDKClient(exportConfig);
log.debug('Management API client initialized successfully', exportConfig.context);

log.info('Starting module exporter', exportConfig.context);
const moduleExporter = new ModuleExporter(managementAPIClient, exportConfig);
await moduleExporter.start();
log.debug('Module exporter completed successfully', exportConfig.context);
if (!exportConfig.branches?.length) {
writeExportMetaFile(exportConfig);
}
Expand Down
13 changes: 10 additions & 3 deletions packages/contentstack-export/src/export/modules/assets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,9 @@ export default class ExportAssets extends BaseClass {
...this.commonQueryParam,
query: { is_dir: true },
};


log.debug('Applying query filters for asset folders export', this.exportConfig.context);
this.applyQueryFilters(queryParam, 'assets');
log.debug(`Fetching asset folders with query: ${JSON.stringify(queryParam)}`, this.exportConfig.context);

const onSuccess = ({ response: { items } }: any) => {
Expand Down Expand Up @@ -137,7 +139,9 @@ export default class ExportAssets extends BaseClass {
include_publish_details: true,
except: { BASE: this.assetConfig.invalidKeys },
};
log.debug('Applying query filters for assets export', this.exportConfig.context);
this.applyQueryFilters(queryParam, 'assets');
log.debug(`Assets query parameters after applying filters: ${JSON.stringify(queryParam)}`, this.exportConfig.context);

if (this.assetConfig.includeVersionedAssets) {
const customHandler = (array: Array<any>) => {
Expand Down Expand Up @@ -364,8 +368,11 @@ export default class ExportAssets extends BaseClass {
time: 5000,
length: data.headers['content-length'],
});
str.on('progress', function (progressData) {
console.log(`${asset.filename}: ${Math.round(progressData.percentage)}%`);
str.on('progress', (progressData) => {
log.debug(
`${asset.filename}: ${Math.round(progressData.percentage)}%`,
{ ...this.exportConfig.context, uid: asset.uid, filename: asset.filename },
);
});
data.pipe(str).pipe(assetWriterStream);
} else {
Expand Down
31 changes: 29 additions & 2 deletions packages/contentstack-export/src/export/modules/base-class.ts
Original file line number Diff line number Diff line change
Expand Up @@ -140,10 +140,11 @@ export default abstract class BaseClass {
);

if (this.exportConfig.modules.assets.displayExecutionTime) {
console.log(
log.debug(
`Time taken to execute: ${exeTime} milliseconds; wait time: ${
exeTime < 1000 ? 1000 - exeTime : 0
} milliseconds`,
this.exportConfig.context,
);
}

Expand Down Expand Up @@ -194,13 +195,39 @@ export default abstract class BaseClass {
protected applyQueryFilters(requestObject: any, moduleName: string): any {
if (this.exportConfig.query?.modules?.[moduleName]) {
const moduleQuery = this.exportConfig.query.modules[moduleName];
log.debug(
`Applying query filters for module: ${moduleName}`,
this.exportConfig.context,
);
log.debug(
`Query filters to apply: ${JSON.stringify(moduleQuery)}`,
this.exportConfig.context,
);
// Merge the query parameters with existing requestObject
if (moduleQuery) {
if (!requestObject.query) {
requestObject.query = moduleQuery;
log.debug(
`Initialized query object with filters for module: ${moduleName}`,
this.exportConfig.context,
);
} else {
log.debug(
`Merging query filters with existing query for module: ${moduleName}`,
this.exportConfig.context,
);
Object.assign(requestObject.query, moduleQuery);
}
Object.assign(requestObject.query, moduleQuery);
log.debug(
`Query filters applied successfully for module: ${moduleName}. Final query: ${JSON.stringify(requestObject.query)}`,
this.exportConfig.context,
);
}
} else {
log.debug(
`No query filters found for module: ${moduleName}`,
this.exportConfig.context,
);
}
return requestObject;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,9 @@ export default class ContentTypesExport extends BaseClass {
}

// Add after existing qs setup and before contentTypesDirPath
log.debug('Applying query filters for content types export', this.exportConfig.context);
this.applyQueryFilters(this.qs, 'content-types');
log.debug(`Content types query parameters after applying filters: ${JSON.stringify(this.qs)}`, this.exportConfig.context);

this.contentTypesDirPath = path.resolve(
sanitizePath(exportConfig.data),
Expand Down
4 changes: 4 additions & 0 deletions packages/contentstack-export/src/export/modules/entries.ts
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,10 @@ export default class EntriesExport extends BaseClass {
locale: options.locale,
},
};
log.debug(
`Applying query filters for entries export (content type: ${options.contentType}, locale: ${options.locale})`,
this.exportConfig.context,
);
this.applyQueryFilters(requestObject, 'entries');

log.debug(`Fetching entries with request: ${JSON.stringify(requestObject)}`, this.exportConfig.context);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@ export default class ExportExtensions extends BaseClass {
this.extensions = {};
this.extensionConfig = exportConfig.modules.extensions;
this.qs = { include_count: true };
log.debug('Applying query filters for extensions export', this.exportConfig.context);
this.applyQueryFilters(this.qs, 'extensions');
log.debug(`Extensions query parameters after applying filters: ${JSON.stringify(this.qs)}`, this.exportConfig.context);
this.exportConfig.context.module = 'extensions';
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,9 @@ export default class GlobalFieldsExport extends BaseClass {
sanitizePath(this.globalFieldsConfig.dirName),
);
this.globalFields = [];
log.debug('Applying query filters for global fields export', this.exportConfig.context);
this.applyQueryFilters(this.qs, 'global-fields');
log.debug(`Global fields query parameters after applying filters: ${JSON.stringify(this.qs)}`, this.exportConfig.context);
this.exportConfig.context.module = 'global-fields';
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,9 @@ export default class ExportTaxonomies extends BaseClass {
this.taxonomiesConfig = exportConfig.modules.taxonomies;
this.qs = { include_count: true, limit: this.taxonomiesConfig.limit || 100, skip: 0 };

log.debug('Applying query filters for taxonomies export', this.exportConfig.context);
this.applyQueryFilters(this.qs, 'taxonomies');
log.debug(`Taxonomies query parameters after applying filters: ${JSON.stringify(this.qs)}`, this.exportConfig.context);
this.exportConfig.context.module = 'taxonomies';
this.localesFilePath = pResolve(
sanitizePath(exportConfig.data),
Expand Down
25 changes: 25 additions & 0 deletions packages/contentstack-export/src/utils/export-config-handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -118,19 +118,44 @@ const setupConfig = async (exportCmdFlags: any): Promise<ExportConfig> => {

// Handle query flag - can be inline JSON or file path
if (exportCmdFlags['query']) {
log.info('Query parameter detected, parsing query configuration...', { queryInput: exportCmdFlags['query'] });
try {
const queryInput = exportCmdFlags['query'];

// Check if it's a file path (contains .json extension or path separators)
if (queryInput.includes('.json') || queryInput.includes('/') || queryInput.includes('\\')) {
log.debug(`Reading query from file: ${queryInput}`);
// Try to read as file path
config.query = await readFile(queryInput);
log.info(`Successfully loaded query configuration from file: ${queryInput}`);
log.debug(`Query configuration loaded: ${JSON.stringify(config.query)}`);
} else {
log.debug('Parsing query as inline JSON string');
config.query = JSON.parse(queryInput);
log.info('Successfully parsed inline query configuration');
log.debug(`Query configuration parsed: ${JSON.stringify(config.query)}`);
}

// Log which modules have query filters
if (config.query?.modules) {
const modulesWithFilters = Object.keys(config.query.modules);
log.info(
`Query filters configured for ${modulesWithFilters.length} module(s): ${modulesWithFilters.join(', ')}`,
);
modulesWithFilters.forEach((moduleName) => {
log.debug(
`Query filters for module '${moduleName}': ${JSON.stringify(config.query.modules[moduleName])}`,
);
});
} else {
log.debug('No module-specific query filters found in query configuration');
}
} catch (error) {
log.error('Failed to parse query configuration', { error: error.message, queryInput: exportCmdFlags['query'] });
throw new Error(`Invalid query format: ${error.message}`);
}
} else {
log.debug('No query parameter provided, proceeding without query filters');
}

// Add authentication details to config for context tracking
Expand Down
4 changes: 2 additions & 2 deletions packages/contentstack-export/src/utils/file-helper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import * as fs from 'fs';
import * as path from 'path';
import mkdirp from 'mkdirp';
import bigJSON from 'big-json';
import { FsUtility, sanitizePath } from '@contentstack/cli-utilities';
import { FsUtility, sanitizePath, log } from '@contentstack/cli-utilities';

export const readFileSync = function (filePath: string, parse: boolean): unknown {
let data;
Expand Down Expand Up @@ -47,7 +47,7 @@ export const readLargeFile = function (filePath: string, options: { type?: strin
resolve(data);
});
parseStream.on('error', (error: Error) => {
console.log('error', error);
log.error('Error parsing large file', { filePath, error: error.message });
reject(error);
});
readStream.pipe(parseStream);
Expand Down
Loading