From 66a9bf9e23b1b5543c383c9d7bd4cdac1c42e5c9 Mon Sep 17 00:00:00 2001 From: naman-contentstack Date: Fri, 3 Jul 2026 18:11:16 +0530 Subject: [PATCH] feat: enhance AM with asset publishing --- .talismanrc | 26 +-- .../src/utils/cs-assets-api-adapter.ts | 12 +- .../src/import/modules/assets.ts | 183 ++++++++++++++++++ .../src/import/modules/base-class.ts | 19 +- 4 files changed, 211 insertions(+), 29 deletions(-) diff --git a/.talismanrc b/.talismanrc index e302b6ed9..0ff9ec6fd 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,28 +1,6 @@ fileignoreconfig: - filename: pnpm-lock.yaml checksum: 49856e7bac5d502aea70519fb31c86e6a0deb6afdfdd911162dfd43188fe3938 -- filename: packages/contentstack-apps-cli/eslint.config.js - checksum: 22e7f364a33612f0100f7b5f6e2f1d28491298cbaca1debf806cd0a988c8bb2c -- filename: packages/contentstack-asset-management/eslint.config.js - checksum: b951a153138f42ee34ab7c0e17827637e25e2d7cb89e150ba378ba533f6d23a7 -- filename: packages/contentstack-import-setup/eslint.config.js - checksum: d487ec978f0dc2471c68c618bf77f9cc7feb1d745d4bb1d84c28f218f132a2b3 -- filename: packages/contentstack-import/eslint.config.js - checksum: 2aeebb2c8d4836490b8aacdda15a9951df41a405c72d5758ef656fe8a31314cc -- filename: packages/contentstack-export/eslint.config.js - checksum: bb451c301e84929aca8c284cb74636de5fd851b18654e07d0eae5e88b0a729ac -- filename: packages/contentstack-clone/eslint.config.js - checksum: 148993140399d12ae033602585f84c06c6a04b0a96b8d7811303211543962ba7 -- filename: packages/contentstack-query-export/eslint.config.js - checksum: b87cecdd9c351066fbda88fc5984a48cade05b227a5ce5fbb84aed36805bf9ed -- filename: packages/contentstack-branches/eslint.config.js - checksum: 7b043a59fc9c523d5f772c1b81d6d4b6c65fb7f8edb8df73e48ba821e7298f0b -- filename: packages/contentstack-content-type/eslint.config.js - checksum: 26da78717a38d8e7464a069626213dd3010efa6e50f91efbc996f26b18346948 -- filename: packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts - checksum: 22708ea1e27a48a5741426a8e17e5d8b243864d877066861bc275d82393002eb -- filename: packages/contentstack-asset-management/src/export/assets.ts - checksum: b169481a31393a9036fbe4d41429bfee3d0f321629f01a72089469ddf5e8826d -- filename: packages/contentstack-asset-management/src/export/assets.ts - checksum: 0a4e04bc91f65cb695a4ca0415dc042b64e6563f0b4a3b718cd9a0ac0d1d7fab +- filename: packages/contentstack-import/src/import/modules/assets.ts + checksum: d2d3cb113b88cf5c9bfc93bd1ff1061b12978abfd240ced161b26a02ce4455b4 version: '1.0' diff --git a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts index 28b0e7a11..fdc0b8a31 100644 --- a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts +++ b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts @@ -419,7 +419,12 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { pageSize = FALLBACK_AM_API_PAGE_SIZE, fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY, ): Promise { - const baseParams: Record = workspaceUid ? { workspace: workspaceUid } : {}; + // include_publish_details=true so each asset carries its `publish_details` array (env/api_key/ + // locale) — persisted in the chunk files and consumed by the import publish step. + const baseParams: Record = { + include_publish_details: 'true', + ...(workspaceUid ? { workspace: workspaceUid } : {}), + }; return this.paginate( spaceUid, `/api/spaces/${encodeURIComponent(spaceUid)}/assets`, @@ -488,7 +493,10 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { } async getWorkspaceAssets(spaceUid: string, workspaceUid?: string, pageSize = FALLBACK_AM_API_PAGE_SIZE, fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY): Promise { - const baseParams: Record = workspaceUid ? { workspace: workspaceUid } : {}; + const baseParams: Record = { + include_publish_details: 'true', + ...(workspaceUid ? { workspace: workspaceUid } : {}), + }; const items = await this.fetchAllPages( spaceUid, `/api/spaces/${encodeURIComponent(spaceUid)}/assets`, diff --git a/packages/contentstack-import/src/import/modules/assets.ts b/packages/contentstack-import/src/import/modules/assets.ts index c65345d91..81588e993 100644 --- a/packages/contentstack-import/src/import/modules/assets.ts +++ b/packages/contentstack-import/src/import/modules/assets.ts @@ -138,6 +138,10 @@ export default class ImportAssets extends BaseClass { await this.linkImportedAmSpacesToBranch(spaceMappings); + if (!this.importConfig.skipAssetsPublish) { + await this.publishAmSpaces(spaceMappings); + } + this.completeProgressWithMessage(); return; } @@ -248,6 +252,185 @@ export default class ImportAssets extends BaseClass { } } + /** + * Returns true when an AM asset will actually be published, so counting and enqueuing stay in sync + * with the ticks emitted per publish attempt. Requires all three: + * - a UID mapping (old AM UID → new AM UID); without it the asset was never uploaded and cannot be + * published — counting it would leave the progress bar short a tick, + * - at least one publish_details entry for the source stack (matching api_key) — an AM asset is + * shared and may be published into multiple stacks; only this export's stack is replayed, + * - that entry targets an environment present in the source environments map (so we can map it). + */ + private isAmAssetPublishable(asset: Record, sourceStack: string): boolean { + if (!asset?.uid || !this.assetsUidMap?.[asset.uid]) { + return false; + } + return ( + filter( + asset?.publish_details, + (pd: any) => pd?.api_key === sourceStack && this.environments?.hasOwnProperty(pd?.environment), + ).length > 0 + ); + } + + /** + * Publishes imported AM (Contentstack Assets / spaces) assets, mirroring the legacy `publish()` + * but re-pointed at each space's chunk store under `spaces/{oldSpaceUid}/assets`. + * + * Environments and asset UIDs are resolved from the same maps the legacy path uses: + * - `this.environments` (source env UID → { name }) loaded in the constructor, + * - `this.assetsUidMap` (old AM UID → new AM UID) from `mapper/assets/uid-mapping.json`, which the + * AM import already wrote. + * Only publish_details for the source stack (`config.source_stack`) are honored — see + * {@link isAmAssetPublishable}. + * + * @param {SpaceMapping[]} spaceMappings mappings produced by the AM import + */ + private async publishAmSpaces(spaceMappings: SpaceMapping[]): Promise { + const sourceStack = this.importConfig.source_stack; + if (!sourceStack) { + log.warn( + 'Skipping CS Assets publish: source stack API key (stack/stack.json) not found, so publish_details cannot be scoped to this stack.', + this.importConfig.context, + ); + return; + } + + if (isEmpty(this.assetsUidMap)) { + log.debug('Loading asset UID mappings from file for CS Assets publish', this.importConfig.context); + this.assetsUidMap = (this.fs.readFile(this.assetUidMapperPath, true) as Record) || {}; + } + + const assetsFileName = this.assetConfig.fileName; + + // Resolve each space's on-disk assets dir (spaces/{oldSpaceUid}/assets), matching where the AM + // import read from. Skip spaces without an assets index (empty/reused). + const spaceAssetDirs = spaceMappings + .map(({ oldSpaceUid }) => join(this.importConfig.contentDir, 'spaces', oldSpaceUid, 'assets')) + .filter((dir) => existsSync(join(dir, assetsFileName))); + + if (spaceAssetDirs.length === 0) { + // Imported spaces exist but none expose an assets index at the expected on-disk path. This is + // usually a layout change in the AM export (a silently-skipped publish would look like success), + // so surface it loudly rather than at debug. + if (spaceMappings.length > 0) { + log.warn( + `CS Assets publish skipped: no assets index found under spaces/{spaceUid}/${assetsFileName} for ${spaceMappings.length} imported space(s). Assets were imported but not published.`, + this.importConfig.context, + ); + } else { + log.debug('No CS Assets spaces to publish', this.importConfig.context); + } + return; + } + + // Pass 1: count publishable assets (source-stack scoped) for the progress row total. + let publishableCount = 0; + for (const assetsDir of spaceAssetDirs) { + const fsUtil = new FsUtility({ basePath: assetsDir, indexFileName: assetsFileName }); + for (const _ of values(fsUtil.indexFileContent)) { + const chunkData = await fsUtil.readChunkFiles.next().catch(() => ({})); + publishableCount += filter(values(chunkData as Record[]), (asset) => + this.isAmAssetPublishable(asset, sourceStack), + ).length; + } + } + + if (publishableCount === 0) { + log.info('No CS Assets to publish for the source stack', this.importConfig.context); + return; + } + + this.progressManager?.addProcess(PROCESS_NAMES.ASSET_PUBLISH, publishableCount); + this.progressManager + ?.startProcess(PROCESS_NAMES.ASSET_PUBLISH) + .updateStatus(PROCESS_STATUS[PROCESS_NAMES.ASSET_PUBLISH].PUBLISHING, PROCESS_NAMES.ASSET_PUBLISH); + + const onSuccess = ({ apiData: { uid, title } = undefined }: any) => { + this.progressManager?.tick(true, `published: ${title || uid}`, null, PROCESS_NAMES.ASSET_PUBLISH); + log.success(`Asset '${uid}: ${title}' published successfully`, this.importConfig.context); + }; + + const onReject = ({ error, apiData: { uid, title } = undefined }: any) => { + this.progressManager?.tick( + false, + `publish failed: ${title || uid}`, + error?.message || PROCESS_STATUS[PROCESS_NAMES.ASSET_PUBLISH].FAILED, + PROCESS_NAMES.ASSET_PUBLISH, + ); + log.error(`Asset '${uid}: ${title}' not published`, this.importConfig.context); + handleAndLogError(error, { ...this.importConfig.context, uid, title }); + }; + + const serializeData = (apiOptions: ApiOptions) => { + const { apiData: asset } = apiOptions; + const publishDetails = filter( + asset.publish_details, + (pd: any) => pd?.api_key === sourceStack && this.environments?.hasOwnProperty(pd?.environment), + ); + + if (!publishDetails.length) { + apiOptions.entity = undefined; + return apiOptions; + } + + const environments = uniq(map(publishDetails, ({ environment }) => this.environments[environment].name)); + const locales = uniq(map(publishDetails, 'locale')); + + if (environments.length === 0 || locales.length === 0) { + log.debug(`Skipping publish for asset ${asset.uid} - no valid environments/locales`, this.importConfig.context); + apiOptions.entity = undefined; + return apiOptions; + } + + asset.locales = locales; + asset.environments = environments; + apiOptions.apiData.publishDetails = { locales, environments }; + + apiOptions.uid = this.assetsUidMap[asset.uid] as string; + if (!apiOptions.uid) { + log.debug(`Skipping publish for asset ${asset.uid} - no UID mapping found`, this.importConfig.context); + apiOptions.entity = undefined; + } + + return apiOptions; + }; + + // Pass 2: publish, one space's chunks at a time. Only source-stack-scoped assets are enqueued so + // every ticked item is a real publish attempt. + for (const assetsDir of spaceAssetDirs) { + const fsUtil = new FsUtility({ basePath: assetsDir, indexFileName: assetsFileName }); + const indexer = fsUtil.indexFileContent; + const indexerCount = values(indexer).length; + + for (const index in indexer) { + const apiContent = filter(values(await fsUtil.readChunkFiles.next()), (asset) => + this.isAmAssetPublishable(asset, sourceStack), + ); + log.debug(`Found ${apiContent.length} publishable CS Assets in chunk ${index}`, this.importConfig.context); + + await this.makeConcurrentCall({ + apiContent, + indexerCount, + currentIndexer: +index, + processName: 'cs-assets publish', + apiParams: { + serializeData, + reject: onReject, + resolve: onSuccess, + entity: 'publish-assets', + includeParamOnCompletion: true, + // CS Assets publish requires api_version 3.2 (see base-class 'publish-assets'). + additionalInfo: { api_version: '3.2' }, + }, + concurrencyLimit: this.assetConfig.uploadAssetsConcurrency, + }); + } + } + + this.progressManager?.completeProcess(PROCESS_NAMES.ASSET_PUBLISH, true); + } + /** * @method importFolders * @returns {Promise} Promise diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index 2ed881a97..fe6710bf6 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -371,12 +371,25 @@ export default abstract class BaseClass { .replace(pick(apiData, [...this.modulesConfig.assets.validKeys, 'upload']) as AssetData) .then(onSuccess) .catch(onReject); - case 'publish-assets': - return this.stack - .asset(uid) + case 'publish-assets': { + const assetClient = this.stack.asset(uid); + // CS Assets (spaces) publish must go to api_version 3.2. The SDK's asset `publish()` takes no + // api_version arg and only forwards `stackHeaders` as request headers, so inject it into this + // per-call instance's headers (a fresh object — the shared stack headers are untouched). + // `additionalInfo.api_version` is set only by the AM publish path; legacy asset publish omits + // it and is unaffected. + const publishApiVersion = additionalInfo?.api_version; + if (publishApiVersion) { + (assetClient as any).stackHeaders = { + ...((assetClient as any).stackHeaders ?? {}), + api_version: publishApiVersion, + }; + } + return assetClient .publish(pick(apiData, ['publishDetails']) as PublishConfig) .then(onSuccess) .catch(onReject); + } case 'create-extensions': return this.stack .extension()