diff --git a/api/src/constants/index.ts b/api/src/constants/index.ts index 785fc58db..29b917e31 100644 --- a/api/src/constants/index.ts +++ b/api/src/constants/index.ts @@ -347,6 +347,7 @@ export const DATABASE_FILES = { CONTENT_TYPES_MAPPER: 'contentTypesMapper.json', FIELD_MAPPER: 'field-mapper.json', ENTRY_MAPPER: 'entry-mapper.json', + ASSET_MAPPER: 'asset-mapper.json', UID_MAPPER: 'uid-mapper.json', UPDATED_ENTRIES: 'updated-entries.json', ASSET_METADATA: 'asset-metadata.json', diff --git a/api/src/controllers/projects.contentMapper.controller.ts b/api/src/controllers/projects.contentMapper.controller.ts index de654c834..1481a87ad 100644 --- a/api/src/controllers/projects.contentMapper.controller.ts +++ b/api/src/controllers/projects.contentMapper.controller.ts @@ -188,6 +188,30 @@ const updateEntryStatus = async (req: Request, res: Response): Promise => res.status(resp?.status).json(resp); }; +/** + * Retrieves the asset mapping for a project. + * + * @param req - The request object. + * @param res - The response object. + * @returns A Promise that resolves to void. + */ +const getAssetMapping = async (req: Request, res: Response): Promise => { + const resp = await contentMapperService.getAssetMapping(req); + res.status(resp?.status).json(resp); +}; + +/** + * Toggles the reuse/re-import decision for the given asset mapper rows. + * + * @param req - The request object. + * @param res - The response object. + * @returns A Promise that resolves to void. + */ +const updateAssetStatus = async (req: Request, res: Response): Promise => { + const resp = await contentMapperService.updateAssetStatus(req); + res.status(resp?.status).json(resp); +}; + export const contentMapperController = { getContentTypes, getFieldMapping, @@ -203,5 +227,7 @@ export const contentMapperController = { getExistingGlobalFields, getSingleGlobalField, getEntryMapping, - updateEntryStatus + updateEntryStatus, + getAssetMapping, + updateAssetStatus }; diff --git a/api/src/models/assetMapper.ts b/api/src/models/assetMapper.ts new file mode 100644 index 000000000..cae1c5a1e --- /dev/null +++ b/api/src/models/assetMapper.ts @@ -0,0 +1,60 @@ +import { JSONFile } from "lowdb/node"; +import LowWithLodash from "../utils/lowdb-lodash.utils.js"; +import path from "path"; +import fs from 'node:fs'; +import { DATABASE_FILES } from "../constants/index.js"; +import { sanitizeProjectId } from "../utils/sanitize-path.utils.js"; + +/** + * Represents an asset mapper object. Rows exist only for assets whose source + * uid is stable across export runs; isUpdate=true means "update the existing + * Contentstack asset in place (same UID, new file)" on a delta iteration, + * isUpdate=false means "keep/reuse the existing asset as-is". + */ +export interface AssetMapper { + asset_mapper: { + id: string; + projectId: string; + otherCmsAssetUid: string; + filename: string; + title: string; + file_size: number | string; + assetPath: string; + isUpdate: boolean; + contentstackAssetUid: string; + isChanged: boolean; + }[]; +} + +const defaultData: AssetMapper = { asset_mapper: [] }; + +/** + * Creates and returns a database instance for the asset mapper for a specific + * project and iteration. + * @param projectId - The unique identifier of the project + * @param iteration - The migration iteration the mapping belongs to + * @returns The database instance for the asset mapper + */ +const getAssetMapperDb = (projectId: string, iteration: number) => { + // projectId is HTTP-derived in several routes; validate it via an allowlist + // before using it as a path segment to prevent path traversal (CWE-23). + // sanitizeProjectId returns null for unsafe input (e.g. containing "..", "/"). + const safeProjectId = sanitizeProjectId(projectId); + if (safeProjectId === null) { + throw new Error("Invalid projectId"); + } + const dir = path.join( + process.cwd(), + DATABASE_FILES.DIRECTORY, + safeProjectId, + iteration.toString() + ); + fs.mkdirSync(dir, { recursive: true }); + const db = new LowWithLodash( + new JSONFile(path.join(dir, DATABASE_FILES.ASSET_MAPPER)), + defaultData + ); + return db; +}; + +export default getAssetMapperDb; diff --git a/api/src/routes/contentMapper.routes.ts b/api/src/routes/contentMapper.routes.ts index d1dcbfb4f..d0391fdf7 100644 --- a/api/src/routes/contentMapper.routes.ts +++ b/api/src/routes/contentMapper.routes.ts @@ -118,6 +118,24 @@ router.put( asyncRouter(contentMapperController.updateEntryStatus) ); +/** + * Get Asset Mapping List + * @route GET /assetMapping/:projectId/:skip/:limit/:searchText? + */ +router.get( + "/assetMapping/:projectId/:skip/:limit/:searchText?", + asyncRouter(contentMapperController.getAssetMapping) +); + +/** + * Update Asset Status (toggle reuse/re-import per asset) + * @route PUT /updateAssetStatus/:projectId + */ +router.put( + "/updateAssetStatus/:projectId", + asyncRouter(contentMapperController.updateAssetStatus) +); + /** * Get Single Global Field data * @route GET /:projectId/:globalFieldUid diff --git a/api/src/services/aem.service.ts b/api/src/services/aem.service.ts index 0c11df8dd..740cae9b0 100644 --- a/api/src/services/aem.service.ts +++ b/api/src/services/aem.service.ts @@ -3,6 +3,7 @@ /* eslint-disable no-unsafe-optional-chaining */ import fs from 'fs'; import path from 'path'; +import { createHash } from 'node:crypto'; import read from 'fs-readdir-recursive'; import _ from "lodash"; import { v4 as uuidv4 } from 'uuid'; @@ -265,7 +266,15 @@ function getCurrentLocale(parseData: any): string | undefined { } function getLocaleFromMapper(mapper: Record, locale: string): string | undefined { - return Object.keys(mapper).find(key => mapper[key] === locale); + const target = locale?.toLowerCase?.(); + const exact = Object.keys(mapper).find(key => mapper[key]?.toLowerCase?.() === target); + if (exact) return exact; + // Fall back to base-language match: source "en-US" should match mapper value "en" + const base = target?.split?.('-')?.[0]; + return Object.keys(mapper).find(key => { + const v = mapper[key]?.toLowerCase?.(); + return v === base || v?.split?.('-')?.[0] === base; + }); } const deepFlattenObject = (obj: any, prefix = '', res: any = {}) => { @@ -400,6 +409,7 @@ const createAssets = async ({ const pathToUidMap: Record = {}; // Path to UID mapping const seenFilenames = new Map(); const pathToFilenameMap = new Map(); + const usedAssetUids = new Set(); // Discover assets and deduplicate by filename for await (const fileName of read(assetsDir)) { @@ -429,7 +439,25 @@ const createAssets = async ({ pathToFilenameMap.set(value, filename); // Only create asset ONCE per unique filename if (!seenFilenames?.has(filename)) { - const uid = uuidv4?.()?.replace?.(/-/g, ''); + // Derive a uid that is stable across export runs so delta + // dedupe (asset-metadata/uid-mapper lookups) can match + // prior iterations: jcr:uuid → hashed DAM path → random. + const jcrUuid = parseData?._raw?.assetNode?.['jcr:uuid']; + let uid = + typeof jcrUuid === 'string' && jcrUuid.trim() !== '' + ? jcrUuid.replace(/-/g, '').toLowerCase() + : ''; + if (!uid || usedAssetUids.has(uid)) { + const assetPath = parseData?.asset?.path; + uid = + typeof assetPath === 'string' && assetPath.trim() !== '' + ? createHash('sha256').update(assetPath).digest('hex').slice(0, 32) + : ''; + } + if (!uid || usedAssetUids.has(uid)) { + uid = uuidv4?.()?.replace?.(/-/g, ''); + } + usedAssetUids.add(uid); const blobPath = firstJson?.replace?.('.metadata.json', ''); seenFilenames?.set(filename, { @@ -1063,6 +1091,38 @@ function processFieldsRecursive( const uid = getLastKey(field?.contentstackFieldUid); const actualUid = getActualFieldUid(uid, field?.uid); + const isAemComponentFallback = + typeof field?.otherCmsType === 'string' && field.otherCmsType.includes('/components/'); + const valueLooksLikeAemComponent = + value && typeof value === 'object' && !Array.isArray(value) && ':type' in (value as any); + + // An unconfigured AEM folder node (nt:folder) is an empty placeholder, not + // real content. Leaking it as a raw {":type":"nt:folder"} object corrupts a + // field that is also mapped as a reference under the same uid, which then + // crashes the import audit-fix ("entry.map is not a function"). Emit null. + if (value && typeof value === 'object' && (value as any)?.[':type'] === 'nt:folder') { + obj[actualUid] = null; + break; + } + + // A 'json' field is always created as a JSON-RTE in the content-type schema + // (allow_json_rte + rich_text_type: "advanced"), so the importer walks this + // value as an RTE document (gatherJsonRteAssetIds -> value.children.forEach). + // A raw AEM component object (e.g. {":type": "..."}) or an array has no + // `children`, so leaking it here crashes the import with + // "Cannot read properties of undefined (reading 'forEach')". Keep the value + // only if it is already a valid JSON-RTE doc; otherwise emit null (the + // importer safely skips null/falsy RTE values). + if (isAemComponentFallback || valueLooksLikeAemComponent) { + const isValidJsonRte = + !!value && + typeof value === 'object' && + !Array.isArray(value) && + Array.isArray((value as any).children); + obj[actualUid] = isValidJsonRte ? value : null; + break; + } + let htmlContent = ''; if (typeof value === 'string') { @@ -1301,6 +1361,7 @@ const createEntry = async ({ const entriesData: Record> = {}; const allLocales: object = { ...project?.master_locale, ...project?.locales }; const entryMapping: Record = {}; + const usedEntryUids = new Set(); // Process each entry file for await (const fileName of read(entriesDir)) { @@ -1310,12 +1371,33 @@ const createEntry = async ({ } const content: unknown = await fs.promises.readFile(filePath, 'utf-8'); if (typeof content === 'string') { - const uid = uuidv4?.()?.replace?.(/-/g, ''); const parseData = JSON.parse(content); + // Use the page model's stable "id" as the entry uid so uid-mapper keys + // stay consistent across delta iterations; random uuid only as fallback. + let modelId = typeof parseData?.id === 'string' && parseData.id.trim() !== '' + ? uidCorrector(parseData.id) + : ''; + // Template-based entries (experience fragments like xf-web-variation, and + // pages like content-page) carry no stable page "id"; derive a stable uid + // from title + templateType (or just templateType when there's no title) + // so they track across iterations (must match extractEntries in + // upload-api's migration-aem). + if (!modelId && parseData?.templateType) { + modelId = parseData?.title + ? uidCorrector(`${parseData.title}_${parseData.templateType}`) + : uidCorrector(parseData.templateType); + } + const uid = modelId && !usedEntryUids.has(modelId) + ? modelId + : uuidv4?.()?.replace?.(/-/g, ''); + usedEntryUids.add(uid); const title = getTitle(parseData); const isEFragment = isExperienceFragment(parseData); const templateUid = isEFragment?.isXF ? parseData?.title : parseData?.templateName ?? parseData?.templateType; - const contentType = (contentTypes as ContentType[] | undefined)?.find?.((element) => element?.otherCmsUid === templateUid); + let contentType = (contentTypes as ContentType[] | undefined)?.find?.((element) => element?.otherCmsUid === templateUid); + if (!contentType && parseData?.title) { + contentType = (contentTypes as ContentType[] | undefined)?.find?.((element) => element?.otherCmsUid === parseData?.title); + } const locale = getCurrentLocale(parseData); const mappedLocale = locale ? getLocaleFromMapper(allLocales as Record, locale) : Object?.keys?.(project?.master_locale ?? {})?.[0]; const items = parseData?.[':items']?.root?.[':items']; @@ -1343,6 +1425,18 @@ const createEntry = async ({ ); addEntryToEntriesData(entriesData, resolvedCtUid, data, mappedLocale); addUidToEntryMapping(entryMapping, resolvedCtUid, uid); + } else { + const reason = !contentType?.contentstackUid + ? `no content type matched (templateUid="${templateUid}", title="${parseData?.title}")` + : !mappedLocale + ? `no mapped locale for "${locale}" (available: ${Object.values(allLocales as Record).join(', ') || 'none'})` + : 'no entry data produced'; + await customLogger( + projectId, + destinationStackId, + 'warn', + getLogMessage(srcFunc, `Skipped entry from "${fileName}": ${reason}.`, {}) + ); } } } diff --git a/api/src/services/contentMapper.service.ts b/api/src/services/contentMapper.service.ts index 3ecdf71d0..b6cd69f1d 100644 --- a/api/src/services/contentMapper.service.ts +++ b/api/src/services/contentMapper.service.ts @@ -28,9 +28,11 @@ import FieldMapperModel from '../models/FieldMapper.js'; import { v4 as uuidv4 } from 'uuid'; import getFieldMapperDb from "../models/FieldMapper.js"; import getEntryMapperDb, { EntryMapper } from "../models/EntryMapper.js"; +import getAssetMapperDb, { AssetMapper } from "../models/assetMapper.js"; import getContentTypesMapperDb, { ContentTypesMapper } from "../models/contentTypesMapper-lowdb.js"; import getUidMapperDb from "../models/uidMapper.js"; import { isDuplicateEntry } from '../utils/entry-duplicate.utils.js'; +import { loadPreviousAssetMetadata } from '../utils/asset-update.utils.js'; const idCorrector = ({ id }: { id: string }) => { @@ -258,6 +260,48 @@ const putTestData = async (req: Request) => { data.entry_mapper = allEntries; }); + // Store asset mapping rows when the connector provides them (connectors + // that can derive stable asset uids at upload time, e.g. AEM). A matched, + // changed asset defaults to isUpdate=true ("update the existing asset in + // place"); unchanged matched assets default to reuse; brand-new assets are + // imported normally. + if (Array.isArray(req?.body?.assetMapping)) { + const AssetMapperModel = getAssetMapperDb(projectId, iteration); + await AssetMapperModel.read(); + const prevAssetMetadata: Record = + iteration > 1 ? loadPreviousAssetMetadata(projectId, iteration - 1) : {}; + + const assetRows = req.body.assetMapping + .filter(Boolean) + .map((asset: any) => { + const sourceUid = (asset?.otherCmsAssetUid ?? asset?.id ?? '') as string; + const contentstackAssetUid = + (uidMapperCurrent?.data as any)?.assets?.[sourceUid] ?? + (uidMapperPrev?.data as any)?.assets?.[sourceUid] ?? + ''; + const prev = prevAssetMetadata?.[sourceUid]; + const isChanged = prev + ? String(prev?.filename ?? '') !== String(asset?.filename ?? '') || + String(prev?.file_size ?? '') !== String(asset?.file_size ?? '') + : false; + + return { + ...asset, + id: String(asset?.id ?? sourceUid ?? uuidv4()).replace(/[{}]/g, '').toLowerCase(), + projectId, + otherCmsAssetUid: sourceUid, + contentstackAssetUid, + isChanged, + isUpdate: Boolean(contentstackAssetUid) && isChanged, + }; + }) + .filter((row: any) => row?.otherCmsAssetUid); + + await AssetMapperModel.update((data: any) => { + data.asset_mapper = assetRows; + }); + } + await ContentTypesMapperModelLowdb.update((data: any) => { // Simple approach: just replace with new content types data.ContentTypesMappers = contentType; @@ -2208,6 +2252,166 @@ const enrichEntriesWithUidMapper = async ( +const updateAssetStatus = async (req: Request) => { + const { projectId } = req?.params; + const { ids } = req?.body; + const validatedUids: string[] = Array.isArray(ids) ? ids : []; + const srcFunc = "updateAssetStatus"; + if (isEmpty(validatedUids)) { + logger.error( + getLogMessage( + srcFunc, + "Invalid ids" + ) + ); + return { + status: HTTP_CODES?.BAD_REQUEST, + data: { + message: "Invalid ids", + }, + }; + } + try { + await ProjectModelLowdb.read(); + const projectData = ProjectModelLowdb.chain + .get("projects") + .find({ id: projectId }) + .value(); + const iteration = projectData?.iteration || 1; + const AssetMapperModel = getAssetMapperDb(projectId, iteration); + await AssetMapperModel.read(); + const foundAssets: AssetMapper["asset_mapper"] = []; + await AssetMapperModel.update((data: any) => { + data?.asset_mapper?.forEach((asset: any) => { + if (validatedUids.includes(asset?.id)) { + asset.isUpdate = !asset.isUpdate; + foundAssets.push(asset); + } + }); + }); + + if (foundAssets.length > 0) { + return { + status: HTTP_CODES?.OK, + data: foundAssets + }; + } + + return { + status: HTTP_CODES?.NOT_FOUND, + data: { + message: "Asset not found", + }, + }; + + } catch (error: any) { + logger.error( + getLogMessage( + srcFunc, + "Error occurred while updating asset mapping", + error + ) + ); + throw new ExceptionFunction( + error?.message || HTTP_TEXTS.INTERNAL_ERROR, + error?.statusCode || error?.status || HTTP_CODES.SERVER_ERROR, + ); + } +}; + +const getAssetMapping = async (req: Request) => { + const srcFunc = "getAssetMapping"; + const projectId = req?.params?.projectId; + const skip: any = req?.params?.skip; + const limit: any = req?.params?.limit; + const search: string = req?.params?.searchText?.toLowerCase(); + + let result: any[] = []; + let filteredResult = []; + let totalCount = 0; + + try { + await ProjectModelLowdb.read(); + const projectData = ProjectModelLowdb.chain + .get("projects") + .find({ id: projectId }) + .value(); + const iteration = projectData?.iteration || 1; + + const AssetMapperModel = getAssetMapperDb(projectId, iteration); + await AssetMapperModel.read(); + let assetMapping = AssetMapperModel.chain + .get("asset_mapper") + .filter({ projectId }) + .value(); + + // Fallback: right after a restart and before a re-upload the current + // iteration has no rows yet — show the previous iteration's mapping. + if ((!assetMapping || assetMapping?.length === 0) && iteration > 1) { + const PrevAssetMapperModel = getAssetMapperDb(projectId, iteration - 1); + await PrevAssetMapperModel.read(); + assetMapping = PrevAssetMapperModel.chain + .get("asset_mapper") + .filter({ projectId }) + .value(); + } + + // Fill missing contentstackAssetUid from uid-mapper (current first, then + // the previous iteration) so rows saved before the import resolve later. + const uidMapperCurrent = getUidMapperDb(projectId, iteration); + await uidMapperCurrent.read(); + let uidMapperPrev: any = null; + if (iteration > 1) { + uidMapperPrev = getUidMapperDb(projectId, iteration - 1); + await uidMapperPrev.read(); + } + const enrichedMapping = (assetMapping ?? []).map((item: any) => { + if (!item) return item; + const existing = item?.contentstackAssetUid; + if (existing != null && String(existing).trim() !== '') { + return item; + } + const resolved = + (uidMapperCurrent?.data as any)?.assets?.[item?.otherCmsAssetUid] ?? + (uidMapperPrev?.data as any)?.assets?.[item?.otherCmsAssetUid]; + return resolved ? { ...item, contentstackAssetUid: resolved } : item; + }); + + if (!isEmpty(enrichedMapping)) { + if (search) { + filteredResult = enrichedMapping?.filter?.((item: any) => + item?.filename?.toLowerCase().includes(search) || + item?.title?.toLowerCase().includes(search) + ); + totalCount = filteredResult?.length; + result = filteredResult?.slice(skip, Number(skip) + Number(limit)); + } else { + totalCount = enrichedMapping?.length; + result = enrichedMapping?.slice(skip, Number(skip) + Number(limit)); + } + } + return { + status: HTTP_CODES?.OK, + count: totalCount, + assetMapping: result + }; + + } catch (error: any) { + logger.error( + getLogMessage( + srcFunc, + "Error occurred while getting asset mapping of projects", + error + ) + ); + + throw new ExceptionFunction( + error?.message || HTTP_TEXTS.INTERNAL_ERROR, + error?.statusCode || error?.status || HTTP_CODES.SERVER_ERROR + ); + } +}; + export const contentMapperService = { putTestData, getContentTypes, @@ -2226,4 +2430,6 @@ export const contentMapperService = { getExistingExtensions, getEntryMapping, updateEntryStatus, + getAssetMapping, + updateAssetStatus, }; diff --git a/api/src/services/migration.service.ts b/api/src/services/migration.service.ts index f90205c06..7bdbc533a 100644 --- a/api/src/services/migration.service.ts +++ b/api/src/services/migration.service.ts @@ -50,8 +50,8 @@ import { import { aemService } from './aem.service.js'; import { requestWithSsoTokenRefresh } from '../utils/sso-request.utils.js'; import { utilsUpdateCli } from './updateEntryCli.service.js'; -import { clearStaleEntries, enrichConfigWithAssetMapping, removeEntriesFromDatabase } from '../utils/entry-update.utils.js'; -import { removeExistingAssets, saveAssetMetadata } from '../utils/asset-update.utils.js'; +import { clearStaleEntries, enrichConfigWithAssetMapping, enrichConfigWithAssetUpdates, ensureUpdateConfigFile, removeEntriesFromDatabase } from '../utils/entry-update.utils.js'; +import { removeExistingAssets, saveAssetMetadata, AssetUpdate } from '../utils/asset-update.utils.js'; /** * Creates a test stack. @@ -1135,6 +1135,7 @@ const startMigration = async (req: Request): Promise => { .value(); const iteration = projectData?.iteration || 1; let configFilePath: string | null = null; + let assetUpdates: AssetUpdate[] = []; let safeDeltaMigrationLogPath: string | undefined; const destinationStackId = project?.destination_stack_id; @@ -1216,12 +1217,22 @@ const startMigration = async (req: Request): Promise => { } } - saveAssetMetadata(indexData, projectId, iteration, safeDeltaMigrationLogPath); + // projectId is HTTP-derived and gets interpolated into database//... + // paths below; the resulting config file is later read with fs.readFileSync. + // Sanitize once and bail on invalid input so a traversal value (e.g. "../../etc") + // can never reach the filesystem. sanitizeProjectId rebuilds the value + // char-by-char from an allowlist, which breaks the taint chain. + if (!safePid) { + await customLogger(projectId, destinationStackId, 'error', 'Invalid project id; skipping delta asset/entry processing.'); + return; + } + + saveAssetMetadata(indexData, safePid, iteration, safeDeltaMigrationLogPath); if (iteration > 1) { - await removeExistingAssets(projectId, safeDeltaMigrationLogPath); + assetUpdates = await removeExistingAssets(safePid, safeDeltaMigrationLogPath); configFilePath = await removeEntriesFromDatabase( - projectId, + safePid, safeDeltaMigrationLogPath ); await customLogger(projectId, destinationStackId, 'info', `Config file generated at ${configFilePath}`); @@ -1237,13 +1248,24 @@ const startMigration = async (req: Request): Promise => { loggerPath ); + // Make sure an update config exists when there are asset updates but no + // entry updates, so the asset-replace step still runs. + if (!configFilePath && assetUpdates.length) { + configFilePath = ensureUpdateConfigFile(safePid, iteration); + } + if (configFilePath) { enrichConfigWithAssetMapping( configFilePath, - projectId, + safePid, iteration, safeDeltaMigrationLogPath ); + enrichConfigWithAssetUpdates( + configFilePath, + assetUpdates, + safeDeltaMigrationLogPath + ); await utilsUpdateCli?.updateEntryCli( region, user_id, diff --git a/api/src/utils/asset-update.utils.ts b/api/src/utils/asset-update.utils.ts index b87a1f31e..f2b9cdbb5 100644 --- a/api/src/utils/asset-update.utils.ts +++ b/api/src/utils/asset-update.utils.ts @@ -1,4 +1,5 @@ import ProjectModelLowdb from "../models/project-lowdb.js"; +import getAssetMapperDb from "../models/assetMapper.js"; import path from "path"; import fs from "node:fs"; import { MIGRATION_DATA_CONFIG, DATABASE_FILES } from "../constants/index.js"; @@ -28,10 +29,27 @@ interface AssetMetadata { url: string; } +/** + * A matched asset the user chose to update in place: the existing Contentstack + * asset (kept at the same `uid`) whose binary should be replaced with the file + * at `filePath` after the import completes. + */ +export interface AssetUpdate { + uid: string; + filePath: string; + filename: string; + title: string; +} + /** * Traverses an object and replaces any asset reference whose uid matches * a source asset ID with the corresponding Contentstack asset UID. - * Asset references are objects with a "uid" property matching a known source asset ID. + * Two reference shapes are handled: + * 1. Object refs — `{ uid: "", ... }` (mapped "file" fields). + * 2. Bare string refs — `{ src: "" }` etc. AEM stores many asset + * references (e.g. carousel `image[].src`) as the plain uid string, not an + * object; without this branch those refs keep the source uid and the import + * rejects them with "is not a valid upload." */ const replaceAssetRefsInObject = ( obj: any, @@ -43,6 +61,14 @@ const replaceAssetRefsInObject = ( for (const key of Object.keys(obj)) { const value = obj[key]; + + // Bare string asset uid reference, e.g. { src: "" }. + if (typeof value === "string" && assetUidMap?.has(value)) { + obj[key] = assetUidMap?.get(value); + modified = true; + continue; + } + if (!value || typeof value !== "object") continue; if (value?.uid && assetUidMap?.has(value?.uid)) { @@ -98,7 +124,7 @@ export const saveAssetMetadata = ( /** * Loads asset metadata from a previous iteration. */ -const loadPreviousAssetMetadata = ( +export const loadPreviousAssetMetadata = ( projectId: string, prevIteration: number, ): Record => { @@ -179,24 +205,30 @@ const hasAssetChanged = ( }; /** - * Removes existing (already-migrated) assets from cmsMigrationData to prevent duplicates. + * Reconciles already-migrated assets in cmsMigrationData against the user's + * Asset Mapper decisions so matched assets are never re-imported as duplicates. * * For iteration 1: only saves asset metadata for future comparisons. * For iteration 2+: * 1. Reads uid-mapper.assets from previous iteration * 2. Reads asset-metadata.json from previous iteration * 3. For each asset in current index.json: - * - If NOT in uid-mapper → new asset, keep it - * - If in uid-mapper AND metadata matches → unchanged, replace refs with CS UID, remove from import - * - If in uid-mapper BUT metadata differs → updated asset, keep it for re-import - * 4. Replaces asset references in entry JSON files with Contentstack UIDs - * 5. Removes deduplicated asset entries from index.json and their file folders + * - If NOT in uid-mapper → new asset, keep it for import + * - If in uid-mapper and the user chose "update" (isUpdate=true; default + * when the file changed) → keep its existing CS UID, drop it from the + * import and collect it for an in-place binary replace after import + * - If in uid-mapper and the user chose "reuse" (isUpdate=false) → keep + * its existing CS UID, drop it from the import, leave the asset untouched + * 4. Repoints asset references in entry JSON files to the existing CS UID + * 5. Removes matched assets from index.json (and reused assets' file folders) * 6. Saves current asset metadata for the next iteration + * + * @returns the assets to replace in place (empty when there are none). */ export const removeExistingAssets = async ( projectId: string, loggerPath?: string, -): Promise => { +): Promise => { await ProjectModelLowdb.read(); const projectData = ProjectModelLowdb.chain .get("projects") @@ -212,7 +244,7 @@ export const removeExistingAssets = async ( "removeExistingAssets", loggerPath, ); - return; + return []; } const assetsDir = path.join( @@ -231,7 +263,7 @@ export const removeExistingAssets = async ( "removeExistingAssets", loggerPath, ); - return; + return []; } writeLogEntry( `Assets index.json found at ${indexPath}`, @@ -244,7 +276,7 @@ export const removeExistingAssets = async ( const raw = fs.readFileSync(indexPath, "utf-8"); if (!raw.trim()) { console.error(`Assets index.json is empty at ${indexPath}`); - return; + return []; } indexData = JSON.parse(raw); } catch (error) { @@ -252,7 +284,7 @@ export const removeExistingAssets = async ( `Failed to parse assets index.json at ${indexPath}:`, error instanceof Error ? error.message : String(error), ); - return; + return []; } saveAssetMetadata(indexData, projectId, iteration, loggerPath); @@ -263,7 +295,7 @@ export const removeExistingAssets = async ( "removeExistingAssets", loggerPath, ); - return; + return []; } writeLogEntry( `Iteration ${iteration} found, loading previous asset uid map and metadata.`, @@ -286,49 +318,115 @@ export const removeExistingAssets = async ( "removeExistingAssets", loggerPath, ); - return; + return []; } writeLogEntry( `Previous asset metadata loaded from ${prevIteration} iteration.`, "removeExistingAssets", loggerPath, ); - const assetsToReuse = new Map(); + // User decisions from the Asset Mapper screen (present when the connector + // provides upload-time asset rows, e.g. AEM). isUpdate=true means "update the + // existing Contentstack asset in place (same UID, new file)"; isUpdate=false + // means "keep/reuse the existing asset as-is". Assets without a row fall back + // to automatic filename+size change detection (changed → update). + const AssetMapperModel = getAssetMapperDb(projectId, iteration); + await AssetMapperModel.read(); + const decisionByUid = new Map(); + for (const row of (AssetMapperModel.data as any)?.asset_mapper ?? []) { + if (row?.otherCmsAssetUid) { + decisionByUid.set(row.otherCmsAssetUid, Boolean(row.isUpdate)); + } + } + + // Both reused and updated assets keep their existing Contentstack UID, so + // their entry references are repointed to it and they are dropped from the + // fresh import (no duplicate). Reused assets need nothing more; updated assets + // additionally have their binary replaced in place after the import runs. + const assetUidReplacements = new Map(); const assetsToRemoveFromIndex: string[] = []; + const assetsToDeleteFiles: string[] = []; + const assetUpdates: AssetUpdate[] = []; + + const filesDir = path.join(assetsDir, "files"); for (const [assetId, assetData] of Object.entries(indexData)) { const contentstackUid = prevAssetUidMap[assetId]; if (!contentstackUid) continue; - if (!hasAssetChanged(assetId, assetData, prevMetadata)) { - assetsToReuse.set(assetId, contentstackUid); + const decision = decisionByUid.get(assetId); + + // No explicit Asset Mapper decision (e.g. connectors that don't populate + // the asset mapper): keep the legacy automatic behavior — unchanged assets + // are reused, changed assets are re-imported as new. + if (decision === undefined) { + if (hasAssetChanged(assetId, assetData, prevMetadata)) { + writeLogEntry( + `Asset "${assetId}" changed (no mapper decision) → re-import`, + "removeExistingAssets", + loggerPath, + ); + continue; // leave it in index.json for a normal import + } + assetUidReplacements.set(assetId, contentstackUid); assetsToRemoveFromIndex.push(assetId); + assetsToDeleteFiles.push(assetId); writeLogEntry( - `Asset "${assetId}" unchanged → reuse CS UID "${contentstackUid}"`, + `Asset "${assetId}" unchanged (no mapper decision) → reuse existing CS UID "${contentstackUid}"`, "removeExistingAssets", loggerPath, ); + continue; + } + + // Explicit user decision. Either way the asset keeps its existing CS UID and + // is dropped from the fresh import; its references are repointed to that UID. + assetUidReplacements.set(assetId, contentstackUid); + assetsToRemoveFromIndex.push(assetId); + + const filename = assetData?.filename ?? ""; + const filePath = path.join(filesDir, assetId, filename); + + if (decision && filename && fs.existsSync(filePath)) { + assetUpdates.push({ + uid: contentstackUid, + filePath, + filename, + title: assetData?.title ?? filename, + }); writeLogEntry( - `Asset "${assetId}" has been reused from previous migration`, + `Asset "${assetId}" → update existing CS UID "${contentstackUid}" in place`, "removeExistingAssets", loggerPath, ); } else { - writeLogEntry( - `Asset "${assetId}" changed → will re-import`, - "removeExistingAssets", - loggerPath, - ); + // Reuse (user unchecked) or update requested but the binary is missing — + // keep the existing asset and just repoint the reference, so it never + // lands as a dangling source uid in the entry. + assetsToDeleteFiles.push(assetId); + if (decision) { + writeLogEntry( + `Asset "${assetId}" marked for update but file missing at ${filePath}; reusing CS UID "${contentstackUid}"`, + "removeExistingAssets", + loggerPath, + ); + } else { + writeLogEntry( + `Asset "${assetId}" → reuse existing CS UID "${contentstackUid}"`, + "removeExistingAssets", + loggerPath, + ); + } } } - if (!assetsToReuse?.size) { + if (!assetUidReplacements.size) { writeLogEntry( - "No unchanged assets to deduplicate.", + "No matched assets to reuse or update.", "removeExistingAssets", loggerPath, ); - return; + return assetUpdates; } // 1. Replace asset references in entry JSON files @@ -382,7 +480,7 @@ export const removeExistingAssets = async ( const data = JSON.parse(raw); - const modified = replaceAssetRefsInObject(data, assetsToReuse); + const modified = replaceAssetRefsInObject(data, assetUidReplacements); if (modified) { fs.writeFileSync(filePath, JSON.stringify(data), "utf-8"); writeLogEntry( @@ -420,10 +518,10 @@ export const removeExistingAssets = async ( loggerPath, ); - // 3. Remove asset file folders - const filesDir = path.join(assetsDir, "files"); + // 3. Remove reused assets' file folders. Assets queued for an in-place update + // keep their folder so the replace step can still upload the binary. if (fs.existsSync(filesDir)) { - for (const assetId of assetsToRemoveFromIndex) { + for (const assetId of assetsToDeleteFiles) { const assetFolder = path.join(filesDir, assetId); if (fs.existsSync(assetFolder)) { fs.rmSync(assetFolder, { recursive: true, force: true }); @@ -442,9 +540,11 @@ export const removeExistingAssets = async ( } writeLogEntry( - `Asset dedup complete: ${assetsToReuse.size} reused, ` + + `Asset processing complete: ${assetsToDeleteFiles.length} reused, ` + + `${assetUpdates.length} to update in place, ` + `${Object?.keys(indexData)?.length} remaining for import.`, "removeExistingAssets", loggerPath, ); + return assetUpdates; }; diff --git a/api/src/utils/content-type-creator.utils.ts b/api/src/utils/content-type-creator.utils.ts index 0ef132703..fe39aea35 100644 --- a/api/src/utils/content-type-creator.utils.ts +++ b/api/src/utils/content-type-creator.utils.ts @@ -579,7 +579,9 @@ export const convertToSchemaFormate = ({ field, advanced = false, marketPlacePat } case 'json': { - if (["Object", "Array"].includes(field?.otherCmsType)) { + const isAemComponentFallback = + typeof field?.otherCmsType === 'string' && field.otherCmsType.includes('/components/'); + if (isAemComponentFallback || ["Object", "Array"].includes(field?.otherCmsType)) { return { data_type: "json", display_name: field?.title ?? cleanedUid, diff --git a/api/src/utils/entry-update-script.cjs b/api/src/utils/entry-update-script.cjs index fa4d36e3d..19ab3d280 100644 --- a/api/src/utils/entry-update-script.cjs +++ b/api/src/utils/entry-update-script.cjs @@ -101,13 +101,47 @@ module.exports = async ({ const assetMapping = config.__assetMapping__ || { old: {}, new: {} }; delete config.__assetMapping__; + // Assets the user chose to update in place (same UID, new file). + const assetUpdates = Array.isArray(config.__assetUpdates__) ? config.__assetUpdates__ : []; + delete config.__assetUpdates__; + const oldMapping = assetMapping.old || {}; const newMapping = assetMapping.new || {}; console.info(`Asset mappings loaded — old: ${Object.keys(oldMapping).length}, new: ${Object.keys(newMapping).length}`); + console.info(`Asset updates to replace in place: ${assetUpdates.length}`); const contentTypes = Object.keys(config); console.info('contentTypes', contentTypes); + /** + * Replaces each selected asset's binary on the existing Contentstack asset + * UID. A single asset failing is logged and skipped so it never aborts the + * remaining asset or entry updates. + */ + const updateAssetTask = () => { + return { + title: "Update Assets", + successMessage: 'Assets updated successfully', + failedMessage: "Failed to update assets", + task: async () => { + for (const asset of assetUpdates) { + if (!asset || !asset.uid || !asset.filePath) { + continue; + } + try { + await stackSDKInstance + .asset(asset.uid) + .replace({ upload: asset.filePath, title: asset.title }); + console.info(`Replaced asset in place: ${asset.uid} (${asset.filename})`); + } catch (error) { + console.error(`Failed to replace asset ${asset.uid} (${asset.filename}):`, error?.message || error); + } + } + console.info('All asset updates processed'); + }, + }; + }; + const updateEntryTask = () => { return { title: "Update Entries", @@ -179,5 +213,16 @@ module.exports = async ({ }; }; + if (assetUpdates.length) { + migration.addTask(updateAssetTask()); + } migration.addTask(updateEntryTask()); }; + +// Exposed for unit testing only. The CLI invokes the default function export +// above; these pure helpers are attached as properties on it so `require()` +// consumers keep calling the function directly while tests can exercise the +// helpers in isolation. +module.exports.isAssetField = isAssetField; +module.exports.resolveAssetField = resolveAssetField; +module.exports.mergeFlatPayloadIntoEntry = mergeFlatPayloadIntoEntry; diff --git a/api/src/utils/entry-update.utils.ts b/api/src/utils/entry-update.utils.ts index 7b5aecfcc..4d5be70f1 100644 --- a/api/src/utils/entry-update.utils.ts +++ b/api/src/utils/entry-update.utils.ts @@ -4,6 +4,7 @@ import path from "path"; import fs from "node:fs"; import { MIGRATION_DATA_CONFIG, DATABASE_FILES } from "../constants/index.js"; import { sanitizeStackId, assertResolvedPathUnderBase } from "./sanitize-path.utils.js"; +import type { AssetUpdate } from "./asset-update.utils.js"; /** * Helper function to write log entries to file @@ -245,7 +246,59 @@ export const enrichConfigWithAssetMapping = ( writeLogEntry(`No new asset mapping found for iteration ${iteration}`, "enrichConfigWithAssetMapping", loggerPath); } + try { + const config = JSON.parse(fs.readFileSync(configFilePath, "utf-8")); + config.__assetMapping__ = { old: oldAssetMapping, new: newAssetMapping }; + fs.writeFileSync(configFilePath, JSON.stringify(config), "utf-8"); + } catch (err) { + console.error("Failed to write asset mapping into update config:", err); + writeLogEntry(`Failed to write __assetMapping__ into ${configFilePath}: ${(err as Error)?.message}`, "enrichConfigWithAssetMapping", loggerPath); + return; + } + writeLogEntry(`Asset mapping enriched into config: old=${Object?.keys(oldAssetMapping)?.length} keys, new=${Object?.keys(newAssetMapping)?.length} keys`, "enrichConfigWithAssetMapping", loggerPath); writeLogEntry(`Asset mapping configuration has been enriched for iteration ${iteration}`, "enrichConfigWithAssetMapping", loggerPath); writeLogEntry(`Asset references will be resolved using combined old and new mappings`, "enrichConfigWithAssetMapping", loggerPath); +}; + +/** + * Ensures an update config file exists for this iteration and returns its path. + * Used when there are asset updates but no entry updates produced a config, so + * the update CLI still has a file to drive the asset-replace task. + */ +export const ensureUpdateConfigFile = ( + projectId: string, + iteration: number, +): string => { + const configDir = path.join(process.cwd(), DATABASE_FILES.DIRECTORY, projectId, iteration.toString()); + fs.mkdirSync(configDir, { recursive: true }); + const configPath = path.join(configDir, DATABASE_FILES.UPDATED_ENTRIES); + if (!fs.existsSync(configPath)) { + fs.writeFileSync(configPath, JSON.stringify({}), "utf-8"); + } + return configPath; +}; + +/** + * Injects the assets to replace in place into the update config under + * __assetUpdates__. The entry-update-script consumes this to call the + * "replace asset" API (same UID, new binary) before updating entries. + */ +export const enrichConfigWithAssetUpdates = ( + configFilePath: string, + assetUpdates: AssetUpdate[], + loggerPath?: string, +): void => { + if (!assetUpdates?.length) { + return; + } + try { + const config = JSON.parse(fs.readFileSync(configFilePath, "utf-8")); + config.__assetUpdates__ = assetUpdates; + fs.writeFileSync(configFilePath, JSON.stringify(config), "utf-8"); + writeLogEntry(`Asset updates enriched into config: ${assetUpdates.length} asset(s) to replace in place`, "enrichConfigWithAssetUpdates", loggerPath); + } catch (err) { + console.error("Failed to write asset updates into update config:", err); + writeLogEntry(`Failed to write __assetUpdates__ into ${configFilePath}: ${(err as Error)?.message}`, "enrichConfigWithAssetUpdates", loggerPath); + } }; \ No newline at end of file diff --git a/api/src/utils/uid-mapper.utils.ts b/api/src/utils/uid-mapper.utils.ts index 2b46ec6e3..492aa000f 100644 --- a/api/src/utils/uid-mapper.utils.ts +++ b/api/src/utils/uid-mapper.utils.ts @@ -5,6 +5,32 @@ import customLogger from "./custom-logger.utils"; import fs from "fs"; import projectModelLowdb from "../models/project-lowdb"; +/** + * Merges a previous iteration's uid map under the current run's map (current + * wins on conflict). Values can be plain strings (flat old→new maps) or + * one-level nested objects (per-content-type entry maps) — nested objects are + * merged key-wise so a partial current map doesn't clobber a content type's + * previously known uids. + */ +const mergeUidMaps = ( + prev: Record, + current: Record, +): Record => { + const merged: Record = { ...(prev || {}) }; + for (const [key, value] of Object.entries(current || {})) { + const existing = merged[key]; + if ( + value && typeof value === "object" && !Array.isArray(value) && + existing && typeof existing === "object" && !Array.isArray(existing) + ) { + merged[key] = { ...existing, ...value }; + } else { + merged[key] = value; + } + } + return merged; +}; + const writeUidMapping = async ( backupPath: string, projectId: string, @@ -41,31 +67,6 @@ const writeUidMapping = async ( ); } - // If no meaningful data found and we have previous iteration, use fallback - if (Object?.keys(assetJson)?.length === 0 && iteration > 1) { - const prevAssetMapperPath = path.join( - process.cwd(), - DATABASE_FILES.DIRECTORY, - projectId, - (iteration - 1).toString(), - DATABASE_FILES.UID_MAPPER, - ); - if (fs.existsSync(prevAssetMapperPath)) { - const prevData = JSON.parse( - fs.readFileSync(prevAssetMapperPath, "utf-8"), - ); - assetJson = prevData?.assets || {}; - } - await customLogger( - projectId, - destinationStackId, - "info", - `Using previous iteration data for assets from ${prevAssetMapperPath}: ${JSON.stringify( - assetJson, - )}`, - ); - } - const entryMapperPath = path.join( backupPath, "mapper", @@ -88,47 +89,47 @@ const writeUidMapping = async ( "info", `Entry UID mapping data read successfully from ${entryMapperPath}`, ); + } - // If no meaningful data found and we have previous iteration, use fallback - if (Object?.keys(entryJson)?.length === 0 && iteration > 1) { - const prevEntryMapperPath = path.join( - process.cwd(), - DATABASE_FILES.DIRECTORY, - projectId, - (iteration - 1).toString(), - DATABASE_FILES.UID_MAPPER, - ); - if (fs.existsSync(prevEntryMapperPath)) { - const prevData = JSON.parse( - fs.readFileSync(prevEntryMapperPath, "utf-8"), - ); - await customLogger( + // Carry the previous iteration's mappings forward under the current run's + // (current wins on conflict). The CLI only maps what it imported this run — + // deduped assets and updated entries are absent — so without this merge + // delta matching would only survive a single iteration. + if (iteration > 1) { + const prevMapperPath = path.join( + process.cwd(), + DATABASE_FILES.DIRECTORY, + projectId, + (iteration - 1).toString(), + DATABASE_FILES.UID_MAPPER, + ); + if (fs.existsSync(prevMapperPath)) { + const prevData = JSON.parse(fs.readFileSync(prevMapperPath, "utf-8")); + assetJson = mergeUidMaps(prevData?.assets || {}, assetJson); + entryJson = mergeUidMaps(prevData?.entry || {}, entryJson); + await customLogger( projectId, destinationStackId, "info", - `Using previous iteration data for entries from ${prevEntryMapperPath}: ${JSON.stringify( - entryJson, - )}`, + `Merged previous iteration uid mappings from ${prevMapperPath}`, ); - entryJson = prevData?.entry || {}; - } } - - const combinedMapping = { - assets: assetJson, - entry: entryJson, - }; - const UidMapperModelLowdb = getUidMapperDb(projectId, iteration); - await UidMapperModelLowdb.read(); - UidMapperModelLowdb.data = combinedMapping; - await UidMapperModelLowdb.write(); - await customLogger( - projectId, - destinationStackId, - "info", - "UID mapping data written successfully to Lowdb", - ); } + + const combinedMapping = { + assets: assetJson, + entry: entryJson, + }; + const UidMapperModelLowdb = getUidMapperDb(projectId, iteration); + await UidMapperModelLowdb.read(); + UidMapperModelLowdb.data = combinedMapping; + await UidMapperModelLowdb.write(); + await customLogger( + projectId, + destinationStackId, + "info", + "UID mapping data written successfully to Lowdb", + ); } catch (error) { console.error("Error writing UID mapping file:", error); } diff --git a/api/tests/unit/routes/contentMapper.routes.test.ts b/api/tests/unit/routes/contentMapper.routes.test.ts index b949cb1ab..734907599 100644 --- a/api/tests/unit/routes/contentMapper.routes.test.ts +++ b/api/tests/unit/routes/contentMapper.routes.test.ts @@ -14,6 +14,10 @@ vi.mock('../../../src/controllers/projects.contentMapper.controller.js', () => ( updateContentMapper: vi.fn((_req: any, res: any) => res.status(200).json({})), getEntryMapping: vi.fn((_req: any, res: any) => res.status(200).json({})), updateEntryStatus: vi.fn((_req: any, res: any) => res.status(200).json({})), + getSingleContentTypes: vi.fn((_req: any, res: any) => res.status(200).json({})), + getSingleGlobalField: vi.fn((_req: any, res: any) => res.status(200).json({})), + getAssetMapping: vi.fn((_req: any, res: any) => res.status(200).json({})), + updateAssetStatus: vi.fn((_req: any, res: any) => res.status(200).json({})), }, })); diff --git a/api/tests/unit/utils/asset-update.decisions.test.ts b/api/tests/unit/utils/asset-update.decisions.test.ts new file mode 100644 index 000000000..e909245f0 --- /dev/null +++ b/api/tests/unit/utils/asset-update.decisions.test.ts @@ -0,0 +1,292 @@ +import path from 'node:path'; +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// Covers the delta-AEM Asset Mapper decision branches in removeExistingAssets +// (isUpdate=true → replace-in-place, isUpdate=false → reuse, update-but-missing +// → reuse fallback) plus the saveAssetMetadata / loadPreviousAssetMetadata +// helpers directly. The existing asset-update.utils.test.ts deliberately leaves +// getAssetMapperDb unmocked, so it only exercises the no-decision automatic path. + +const { + mockProjectRead, + mockChainGet, + mockExistsSync, + mockReadFileSync, + mockWriteFileSync, + mockMkdirSync, + mockReaddirSync, + mockRmSync, + mockAppendFileSync, + mockGetAssetMapperDb, +} = vi.hoisted(() => ({ + mockProjectRead: vi.fn(), + mockChainGet: vi.fn(), + mockExistsSync: vi.fn(), + mockReadFileSync: vi.fn(), + mockWriteFileSync: vi.fn(), + mockMkdirSync: vi.fn(), + mockReaddirSync: vi.fn(), + mockRmSync: vi.fn(), + mockAppendFileSync: vi.fn(), + mockGetAssetMapperDb: vi.fn(), +})); + +vi.mock('../../../src/models/project-lowdb.js', () => ({ + default: { + read: mockProjectRead, + chain: { get: mockChainGet }, + }, +})); + +vi.mock('../../../src/models/assetMapper.js', () => ({ + default: mockGetAssetMapperDb, +})); + +vi.mock('node:fs', () => ({ + default: { + existsSync: mockExistsSync, + readFileSync: mockReadFileSync, + writeFileSync: mockWriteFileSync, + mkdirSync: mockMkdirSync, + readdirSync: mockReaddirSync, + rmSync: mockRmSync, + appendFileSync: mockAppendFileSync, + }, +})); + +const projectIter2 = () => ({ + id: 'p1', + iteration: 2, + destination_stack_id: 'stack1', +}); + +// Sets up a clean iteration-2 dedup scenario with a single previously-migrated +// asset "a1" (source uid → CS uid "cs-uid-1"). `decision` becomes the asset +// mapper row's isUpdate; `binaryExists` controls whether the asset's file is on +// disk. entriesDir is absent so reference-rewriting is skipped (covered +// elsewhere); filesDir + the asset folder exist so reuse can rmSync them. +const setupDedup = (opts: { + assetMapperRows: Array<{ otherCmsAssetUid: string; isUpdate: boolean }>; + filename?: string; + binaryExists?: boolean; + assetTitle?: string; +}) => { + const filename = opts.filename ?? 'photo.jpg'; + + mockChainGet.mockReturnValue({ + find: vi.fn().mockReturnValue({ + value: vi.fn().mockReturnValue(projectIter2()), + }), + }); + + mockGetAssetMapperDb.mockReturnValue({ + read: vi.fn().mockResolvedValue(undefined), + data: { asset_mapper: opts.assetMapperRows }, + }); + + mockExistsSync.mockImplementation((p: string) => { + const s = String(p); + if (s.endsWith('index.json')) return true; + if (s.includes('uid-mapper.json')) return true; + if (s.includes('asset-metadata.json')) return true; + // the asset binary: /files/a1/ + if (s.includes(`${path.sep}files${path.sep}a1${path.sep}${filename}`)) { + return opts.binaryExists ?? false; + } + // the asset folder: /files/a1 (used for rmSync on reuse) + if (s.endsWith(`${path.sep}files${path.sep}a1`)) return true; + // the files dir itself + if (s.endsWith(`${path.sep}files`)) return true; + // entriesDir absent → skip ref rewriting branch + return false; + }); + + mockReadFileSync.mockImplementation((p: string) => { + const s = String(p); + if (s.endsWith('index.json')) { + const asset: Record = { filename, file_size: '10', url: '' }; + if (opts.assetTitle) asset.title = opts.assetTitle; + return JSON.stringify({ a1: asset }); + } + if (s.includes('uid-mapper.json')) { + return JSON.stringify({ assets: { a1: 'cs-uid-1' } }); + } + if (s.includes('asset-metadata.json')) { + return JSON.stringify({ a1: { filename, file_size: '10', url: '' } }); + } + return '{}'; + }); +}; + +describe('asset-update.utils — Asset Mapper decisions', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockProjectRead.mockResolvedValue(undefined); + }); + + it('isUpdate=true with the binary present → queues an in-place replace and keeps the file', async () => { + setupDedup({ + assetMapperRows: [{ otherCmsAssetUid: 'a1', isUpdate: true }], + filename: 'photo.jpg', + binaryExists: true, + assetTitle: 'My Photo', // title comes from index.json, not the mapper row + }); + + const { removeExistingAssets } = await import('../../../src/utils/asset-update.utils.js'); + const updates = await removeExistingAssets('p1'); + + expect(updates).toHaveLength(1); + expect(updates[0]).toMatchObject({ + uid: 'cs-uid-1', + filename: 'photo.jpg', + title: 'My Photo', + }); + expect(updates[0].filePath).toContain(`files${path.sep}a1${path.sep}photo.jpg`); + // updated assets keep their folder so the replace step can upload the binary + expect(mockRmSync).not.toHaveBeenCalled(); + // a1 was dropped from index.json (it keeps its existing CS uid) + const indexWrite = mockWriteFileSync.mock.calls.find((c) => String(c[0]).endsWith('index.json')); + expect(indexWrite).toBeDefined(); + expect(String(indexWrite?.[1])).not.toContain('a1'); + }); + + it('isUpdate=false (reuse) → no replace queued and the local file folder is removed', async () => { + setupDedup({ + assetMapperRows: [{ otherCmsAssetUid: 'a1', isUpdate: false }], + }); + + const { removeExistingAssets } = await import('../../../src/utils/asset-update.utils.js'); + const updates = await removeExistingAssets('p1'); + + expect(updates).toEqual([]); + // reused asset's folder is deleted from migration data + expect(mockRmSync).toHaveBeenCalledWith( + expect.stringContaining(`files${path.sep}a1`), + { recursive: true, force: true }, + ); + }); + + it('isUpdate=true but the binary is missing → falls back to reuse (no replace, folder removed)', async () => { + setupDedup({ + assetMapperRows: [{ otherCmsAssetUid: 'a1', isUpdate: true }], + binaryExists: false, + }); + + const { removeExistingAssets } = await import('../../../src/utils/asset-update.utils.js'); + const updates = await removeExistingAssets('p1'); + + expect(updates).toEqual([]); + expect(mockRmSync).toHaveBeenCalled(); + }); + + it('ignores asset-mapper rows without otherCmsAssetUid', async () => { + setupDedup({ + // malformed row should be skipped; a1 then has no decision → automatic + // path. metadata matches (unchanged) → reused (folder removed), updates empty. + assetMapperRows: [{ otherCmsAssetUid: '', isUpdate: true } as any], + }); + + const { removeExistingAssets } = await import('../../../src/utils/asset-update.utils.js'); + const updates = await removeExistingAssets('p1'); + + expect(updates).toEqual([]); + }); + + it('repoints asset references in entry JSON files to the existing CS uid', async () => { + // Drives the entry-reference rewrite path: entriesDir exists, so the walk + // (content type -> locale -> chunk) runs and replaceAssetRefsInObject swaps + // the deduped source uid for the existing Contentstack uid in the entry. + mockChainGet.mockReturnValue({ + find: vi.fn().mockReturnValue({ + value: vi.fn().mockReturnValue(projectIter2()), + }), + }); + mockGetAssetMapperDb.mockReturnValue({ + read: vi.fn().mockResolvedValue(undefined), + data: { asset_mapper: [{ otherCmsAssetUid: 'a1', isUpdate: false }] }, // reuse + }); + mockExistsSync.mockImplementation((p: string) => { + const s = String(p); + if (s.endsWith('index.json')) return true; + if (s.includes('uid-mapper.json')) return true; + if (s.includes('asset-metadata.json')) return true; + if (s.includes('entries')) return true; // entriesDir + ct + locale dirs exist + if (s.includes('files')) return true; + return false; + }); + mockReadFileSync.mockImplementation((p: string) => { + const s = String(p); + if (s.endsWith('index.json')) return JSON.stringify({ a1: { filename: 'f.jpg', file_size: '1', url: '' } }); + if (s.includes('uid-mapper.json')) return JSON.stringify({ assets: { a1: 'cs-uid-1' } }); + if (s.includes('asset-metadata.json')) return JSON.stringify({ a1: { filename: 'f.jpg', file_size: '1', url: '' } }); + if (s.endsWith('entry1.json')) return JSON.stringify({ banner: { uid: 'a1' } }); + return '{}'; + }); + const dirent = (name: string, dir: boolean) => ({ name, isDirectory: () => dir }); + mockReaddirSync + .mockReturnValueOnce([dirent('page', true)]) // content type dirs + .mockReturnValueOnce([dirent('en-us', true)]) // locale dirs + .mockReturnValueOnce(['entry1.json', 'index.json']); // chunk files (index.json skipped) + + const { removeExistingAssets } = await import('../../../src/utils/asset-update.utils.js'); + await removeExistingAssets('p1'); + + // entry1.json rewritten with the source uid 'a1' replaced by the CS uid 'cs-uid-1' + const entryWrite = mockWriteFileSync.mock.calls.find((c) => String(c[0]).endsWith('entry1.json')); + expect(entryWrite).toBeDefined(); + expect(String(entryWrite?.[1])).toContain('cs-uid-1'); + expect(String(entryWrite?.[1])).not.toContain('a1'); + }); +}); + +describe('asset-update.utils — saveAssetMetadata / loadPreviousAssetMetadata', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('saveAssetMetadata writes only filename/file_size/url and creates the dir', async () => { + const { saveAssetMetadata } = await import('../../../src/utils/asset-update.utils.js'); + saveAssetMetadata( + { a1: { filename: 'f.jpg', file_size: '99', url: 'http://x/f.jpg', extra: 'drop-me' } }, + 'p1', + 3, + '/tmp/x.log', + ); + + expect(mockMkdirSync).toHaveBeenCalledWith(expect.any(String), { recursive: true }); + const call = mockWriteFileSync.mock.calls.find((c) => String(c[0]).includes('asset-metadata')); + expect(call).toBeDefined(); + const written = JSON.parse(String(call?.[1])); + expect(written).toEqual({ a1: { filename: 'f.jpg', file_size: '99', url: 'http://x/f.jpg' } }); + // logger path supplied → a log line is appended + expect(mockAppendFileSync).toHaveBeenCalled(); + }); + + it('saveAssetMetadata fills missing fields with empty strings', async () => { + const { saveAssetMetadata } = await import('../../../src/utils/asset-update.utils.js'); + saveAssetMetadata({ a1: {} }, 'p1', 1); + const call = mockWriteFileSync.mock.calls.find((c) => String(c[0]).includes('asset-metadata')); + const written = JSON.parse(String(call?.[1])); + expect(written).toEqual({ a1: { filename: '', file_size: '', url: '' } }); + }); + + it('loadPreviousAssetMetadata returns {} when the file is absent', async () => { + mockExistsSync.mockReturnValue(false); + const { loadPreviousAssetMetadata } = await import('../../../src/utils/asset-update.utils.js'); + expect(loadPreviousAssetMetadata('p1', 1)).toEqual({}); + }); + + it('loadPreviousAssetMetadata parses a valid metadata file', async () => { + mockExistsSync.mockReturnValue(true); + mockReadFileSync.mockReturnValue(JSON.stringify({ a1: { filename: 'f', file_size: '1', url: '' } })); + const { loadPreviousAssetMetadata } = await import('../../../src/utils/asset-update.utils.js'); + expect(loadPreviousAssetMetadata('p1', 2)).toEqual({ a1: { filename: 'f', file_size: '1', url: '' } }); + }); + + it('loadPreviousAssetMetadata returns {} on corrupt JSON', async () => { + mockExistsSync.mockReturnValue(true); + mockReadFileSync.mockReturnValue('{ not json'); + const { loadPreviousAssetMetadata } = await import('../../../src/utils/asset-update.utils.js'); + expect(loadPreviousAssetMetadata('p1', 2)).toEqual({}); + }); +}); diff --git a/api/tests/unit/utils/entry-update-script.test.ts b/api/tests/unit/utils/entry-update-script.test.ts new file mode 100644 index 000000000..ae0582a7a --- /dev/null +++ b/api/tests/unit/utils/entry-update-script.test.ts @@ -0,0 +1,149 @@ +import { createRequire } from 'node:module'; +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// The delta entry-update migration script (run by `cm:stacks:migration`). It is +// plain CommonJS executed by the CLI, so we load it with the real `require` +// rather than the vitest module system. The pure helpers are attached to the +// default function export for testing. +const require = createRequire(import.meta.url); +const script = require('../../../src/utils/entry-update-script.cjs'); +const { isAssetField, resolveAssetField, mergeFlatPayloadIntoEntry } = script; + +describe('entry-update-script — isAssetField', () => { + it('is true only for objects carrying urlPath + filename', () => { + expect(isAssetField({ urlPath: '/x', filename: 'f.jpg' })).toBe(true); + }); + + it('is false for non-asset shapes', () => { + expect(isAssetField(null)).toBeFalsy(); + expect(isAssetField('str')).toBeFalsy(); + expect(isAssetField(['/x'])).toBe(false); + expect(isAssetField({ urlPath: '/x' })).toBeFalsy(); // missing filename + expect(isAssetField({ filename: 'f' })).toBeFalsy(); // missing urlPath + }); +}); + +describe('entry-update-script — resolveAssetField (3-way resolution)', () => { + const field = 'pic'; + const uid = 'e1'; + + it('returns stackValue/updateValue when the update has no source uid', () => { + const stackValue = { uid: 'cs-stack' }; + expect(resolveAssetField(field, uid, { filename: 'f' }, stackValue, {}, {})).toBe(stackValue); + expect(resolveAssetField(field, uid, { filename: 'f' }, undefined, {}, {})).toEqual({ filename: 'f' }); + }); + + it('prefers a newly-imported asset (newMapping wins), merging onto the stack object', () => { + const out = resolveAssetField(field, uid, { uid: 'src-1' }, { uid: 'cs-old', title: 'keep' }, {}, { 'src-1': 'cs-new' }); + expect(out).toEqual({ uid: 'cs-new', title: 'keep' }); + }); + + it('newMapping with no stack object yields just the new uid', () => { + const out = resolveAssetField(field, uid, { uid: 'src-1' }, undefined, {}, { 'src-1': 'cs-new' }); + expect(out).toEqual({ uid: 'cs-new' }); + }); + + it('keeps a user-modified stack asset over the old mapping', () => { + const stackValue = { uid: 'cs-user-changed' }; + const out = resolveAssetField(field, uid, { uid: 'src-1' }, stackValue, { 'src-1': 'cs-old' }, {}); + expect(out).toBe(stackValue); + }); + + it('keeps the original mapped asset when the stack still matches', () => { + const stackValue = { uid: 'cs-old' }; + const out = resolveAssetField(field, uid, { uid: 'src-1' }, stackValue, { 'src-1': 'cs-old' }, {}); + expect(out).toBe(stackValue); + }); + + it('falls back to updateValue+oldUid when old mapping exists but no stack object', () => { + const out = resolveAssetField(field, uid, { uid: 'src-1', filename: 'f' }, null, { 'src-1': 'cs-old' }, {}); + expect(out).toEqual({ uid: 'cs-old', filename: 'f' }); + }); + + it('keeps the stack asset when there is no mapping at all', () => { + const stackValue = { uid: 'cs-stack' }; + expect(resolveAssetField(field, uid, { uid: 'src-1' }, stackValue, {}, {})).toBe(stackValue); + }); +}); + +describe('entry-update-script — mergeFlatPayloadIntoEntry', () => { + it('merges flat fields into entry.content, resolves assets, and skips reserved keys', async () => { + const update = vi.fn().mockResolvedValue(undefined); + const entry: any = { title: 'old', content: {}, update }; + + const updateData = { + uid: 'should-be-skipped', + _version: 9, + title: 'new title', + body: 'hello', + pic: { urlPath: '/p', filename: 'p.jpg', uid: 'src-1' }, + }; + + await mergeFlatPayloadIntoEntry(entry, 'e1', updateData, {}, { 'src-1': 'cs-new' }); + + expect(entry.title).toBe('new title'); + expect(entry.content.body).toBe('hello'); + expect(entry.content.pic).toEqual({ uid: 'cs-new' }); // asset resolved via newMapping + expect(entry.content.uid).toBeUndefined(); // reserved key skipped + expect(entry.content._version).toBeUndefined(); + expect(update).toHaveBeenCalledTimes(1); + }); +}); + +describe('entry-update-script — main task runner', () => { + let tasks: Array<{ title: string; task: () => Promise }>; + let migration: { addTask: (t: any) => void }; + + beforeEach(() => { + tasks = []; + migration = { addTask: (t: any) => tasks.push(t) }; + }); + + it('registers an asset-replace task and an entry-update task, and runs them', async () => { + const replace = vi.fn().mockResolvedValue(undefined); + const update = vi.fn().mockResolvedValue(undefined); + const fakeEntry = { content: { body: 'old' }, update }; + const fetch = vi.fn().mockResolvedValue(fakeEntry); + const entry = vi.fn(() => ({ fetch })); + const contentType = vi.fn(() => ({ entry })); + const asset = vi.fn(() => ({ replace })); + const stackSDKInstance = { contentType, asset }; + + const config = { + __assetUpdates__: [{ uid: 'cs-a', filePath: '/f/a1/p.jpg', filename: 'p.jpg', title: 'P' }], + __assetMapping__: { old: {}, new: {} }, + page: { 'cs-1': { content: { body: 'new' } } }, + }; + + await script({ migration, config, stackSDKInstance }); + + // asset-update task is added (because __assetUpdates__ is non-empty), then entries + expect(tasks).toHaveLength(2); + + await tasks[0].task(); + expect(asset).toHaveBeenCalledWith('cs-a'); + expect(replace).toHaveBeenCalledWith({ upload: '/f/a1/p.jpg', title: 'P' }); + + await tasks[1].task(); + expect(contentType).toHaveBeenCalledWith('page'); + expect(entry).toHaveBeenCalledWith('cs-1'); + expect(fetch).toHaveBeenCalled(); + expect(fakeEntry.content.body).toBe('new'); // nested content merged + expect(update).toHaveBeenCalledTimes(1); + }); + + it('registers only the entry-update task when there are no asset updates', async () => { + const update = vi.fn().mockResolvedValue(undefined); + const fetch = vi.fn().mockResolvedValue({ content: {}, update }); + const stackSDKInstance = { + contentType: vi.fn(() => ({ entry: vi.fn(() => ({ fetch })) })), + asset: vi.fn(), + }; + const config = { page: { 'cs-1': { content: { x: 1 } } } }; + + await script({ migration, config, stackSDKInstance }); + + expect(tasks).toHaveLength(1); + expect(tasks[0].title).toBe('Update Entries'); + }); +}); diff --git a/api/tests/unit/utils/entry-update.enrich.test.ts b/api/tests/unit/utils/entry-update.enrich.test.ts new file mode 100644 index 000000000..c4ee7e099 --- /dev/null +++ b/api/tests/unit/utils/entry-update.enrich.test.ts @@ -0,0 +1,122 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// Covers the delta-AEM config-enrichment helpers that the existing +// entry-update.utils.test.ts does not exercise: ensureUpdateConfigFile and +// enrichConfigWithAssetUpdates (plus an extra enrichConfigWithAssetUpdates +// failure branch). These functions only touch the filesystem. + +const { + mockExistsSync, + mockReadFileSync, + mockWriteFileSync, + mockMkdirSync, + mockAppendFileSync, +} = vi.hoisted(() => ({ + mockExistsSync: vi.fn(), + mockReadFileSync: vi.fn(), + mockWriteFileSync: vi.fn(), + mockMkdirSync: vi.fn(), + mockAppendFileSync: vi.fn(), +})); + +// entry-update.utils imports these at module scope; mock them so the module +// loads cleanly even though the enrich helpers don't use them. +vi.mock('../../../src/models/project-lowdb.js', () => ({ + default: { read: vi.fn(), chain: { get: vi.fn() } }, +})); +vi.mock('../../../src/models/EntryMapper.js', () => ({ + default: vi.fn(() => ({ read: vi.fn(), chain: { get: vi.fn() } })), +})); +vi.mock('../../../src/utils/sanitize-path.utils.js', () => ({ + sanitizeStackId: (id: string) => id, + assertResolvedPathUnderBase: vi.fn(), + getSafePath: (p: string) => p, +})); + +vi.mock('node:fs', () => ({ + default: { + existsSync: mockExistsSync, + readFileSync: mockReadFileSync, + writeFileSync: mockWriteFileSync, + mkdirSync: mockMkdirSync, + appendFileSync: mockAppendFileSync, + }, +})); + +describe('entry-update.utils — ensureUpdateConfigFile', () => { + beforeEach(() => vi.clearAllMocks()); + + it('creates the iteration dir and an empty config when none exists', async () => { + mockExistsSync.mockReturnValue(false); + const { ensureUpdateConfigFile } = await import('../../../src/utils/entry-update.utils.js'); + const p = ensureUpdateConfigFile('p1', 2); + + expect(mockMkdirSync).toHaveBeenCalledWith(expect.any(String), { recursive: true }); + const write = mockWriteFileSync.mock.calls.find((c) => c[0] === p); + expect(write).toBeDefined(); + expect(String(write?.[1])).toBe('{}'); + expect(p).toContain('p1'); + }); + + it('does not overwrite an existing config file', async () => { + mockExistsSync.mockReturnValue(true); + const { ensureUpdateConfigFile } = await import('../../../src/utils/entry-update.utils.js'); + const p = ensureUpdateConfigFile('p1', 2); + + expect(mockMkdirSync).toHaveBeenCalled(); + expect(mockWriteFileSync).not.toHaveBeenCalled(); + expect(typeof p).toBe('string'); + }); +}); + +describe('entry-update.utils — enrichConfigWithAssetUpdates', () => { + beforeEach(() => vi.clearAllMocks()); + + it('does nothing when there are no asset updates', async () => { + const { enrichConfigWithAssetUpdates } = await import('../../../src/utils/entry-update.utils.js'); + enrichConfigWithAssetUpdates('/tmp/config.json', []); + expect(mockReadFileSync).not.toHaveBeenCalled(); + expect(mockWriteFileSync).not.toHaveBeenCalled(); + }); + + it('injects __assetUpdates__ into the existing config', async () => { + mockReadFileSync.mockReturnValue(JSON.stringify({ page: { 'cs-1': { title: 'T' } } })); + const updates = [{ uid: 'cs-uid-1', filePath: '/f/a1/p.jpg', filename: 'p.jpg', title: 'P' }]; + + const { enrichConfigWithAssetUpdates } = await import('../../../src/utils/entry-update.utils.js'); + enrichConfigWithAssetUpdates('/tmp/config.json', updates, '/tmp/x.log'); + + const write = mockWriteFileSync.mock.calls.find((c) => c[0] === '/tmp/config.json'); + expect(write).toBeDefined(); + const written = JSON.parse(String(write?.[1])); + expect(written.__assetUpdates__).toEqual(updates); + // existing entry data is preserved + expect(written.page).toEqual({ 'cs-1': { title: 'T' } }); + expect(mockAppendFileSync).toHaveBeenCalled(); + }); + + it('swallows a read/parse error without writing', async () => { + mockReadFileSync.mockReturnValue('{ not json'); + const updates = [{ uid: 'cs-uid-1', filePath: '/f', filename: 'p.jpg', title: 'P' }]; + + const { enrichConfigWithAssetUpdates } = await import('../../../src/utils/entry-update.utils.js'); + expect(() => enrichConfigWithAssetUpdates('/tmp/config.json', updates)).not.toThrow(); + expect(mockWriteFileSync).not.toHaveBeenCalled(); + }); +}); + +describe('entry-update.utils — enrichConfigWithAssetMapping (extra branches)', () => { + beforeEach(() => vi.clearAllMocks()); + + it('writes empty old/new mappings when no uid-mapper files exist', async () => { + mockExistsSync.mockReturnValue(false); // no old or new uid-mapper + mockReadFileSync.mockReturnValue(JSON.stringify({ page: {} })); + + const { enrichConfigWithAssetMapping } = await import('../../../src/utils/entry-update.utils.js'); + enrichConfigWithAssetMapping('/tmp/config.json', 'p1', 2, '/tmp/x.log'); + + const write = mockWriteFileSync.mock.calls.find((c) => c[0] === '/tmp/config.json'); + const written = JSON.parse(String(write?.[1])); + expect(written.__assetMapping__).toEqual({ old: {}, new: {} }); + }); +}); diff --git a/api/tests/unit/utils/uid-mapper.utils.test.ts b/api/tests/unit/utils/uid-mapper.utils.test.ts index d4c2623e8..db0fc1682 100644 --- a/api/tests/unit/utils/uid-mapper.utils.test.ts +++ b/api/tests/unit/utils/uid-mapper.utils.test.ts @@ -85,14 +85,18 @@ describe('uid-mapper.utils - writeUidMapping', () => { expect(mockUidWrite).toHaveBeenCalled(); }); - it('does not write when entry mapper file is missing', async () => { - // asset file exists with data, entry file does not exist + it('still writes asset mappings when the entry mapper file is missing', async () => { + // asset file exists with data, entry file does not exist. writeUidMapping + // reads the two mappings independently and always persists the combined + // result, so asset mappings survive a run with no entry mapper file (entry + // stays empty) — required for delta carry-forward. mockExistsSync.mockImplementation((p: string) => p.includes('assets')); mockReadFileSync.mockReturnValue(JSON.stringify({ a1: 'asset-uid' })); await writeUidMapping('/backup', 'p1', 1); - expect(mockUidWrite).not.toHaveBeenCalled(); + expect(mockUidWrite).toHaveBeenCalled(); + expect(uidDb.data).toEqual({ assets: { a1: 'asset-uid' }, entry: {} }); }); it('falls back to previous iteration for assets when current asset data is empty', async () => { diff --git a/ui/src/components/ContentMapper/__tests__/assetMapper.utils.test.ts b/ui/src/components/ContentMapper/__tests__/assetMapper.utils.test.ts new file mode 100644 index 000000000..3c96b345a --- /dev/null +++ b/ui/src/components/ContentMapper/__tests__/assetMapper.utils.test.ts @@ -0,0 +1,124 @@ +import { describe, it, expect } from 'vitest'; +import { + formatFileSize, + mapAssetsToRows, + buildSelectedRowIds, + applySelectionToAssets, + toSelectedMap, + computeChangedUids, +} from '../assetMapper.utils'; +import { AssetMapperType } from '../contentMapper.interface'; + +// Minimal AssetMapperType factory for the pure-logic tests. +const asset = (over: Partial): AssetMapperType => ({ + id: 'a1', + projectId: 'p1', + otherCmsAssetUid: 'a1', + filename: 'f.jpg', + title: 'F', + file_size: 100, + assetPath: '/f.jpg', + isUpdate: false, + ...over, +}); + +describe('assetMapper.utils — formatFileSize', () => { + it("returns '-' for missing / non-positive / non-numeric sizes", () => { + expect(formatFileSize(undefined)).toBe('-'); + expect(formatFileSize(0)).toBe('-'); + expect(formatFileSize(-5)).toBe('-'); + expect(formatFileSize('abc')).toBe('-'); + }); + + it('formats bytes, KB and MB', () => { + expect(formatFileSize(512)).toBe('512 B'); + expect(formatFileSize('512')).toBe('512 B'); // numeric strings accepted + expect(formatFileSize(1024)).toBe('1.0 KB'); + expect(formatFileSize(1536)).toBe('1.5 KB'); + expect(formatFileSize(1024 * 1024)).toBe('1.0 MB'); + expect(formatFileSize(5 * 1024 * 1024)).toBe('5.0 MB'); + }); +}); + +describe('assetMapper.utils — mapAssetsToRows', () => { + it('marks rows selectable only when a Contentstack asset uid exists', () => { + const rows = mapAssetsToRows([ + asset({ id: 'a1', contentstackAssetUid: 'cs-1' }), + asset({ id: 'a2', contentstackAssetUid: undefined }), + ]); + expect(rows.map((r) => [r.id, r._canSelect])).toEqual([ + ['a1', true], + ['a2', false], + ]); + }); + + it('returns [] for undefined input', () => { + expect(mapAssetsToRows(undefined)).toEqual([]); + }); +}); + +describe('assetMapper.utils — buildSelectedRowIds', () => { + it('includes only rows that are selectable AND already isUpdate', () => { + const result = buildSelectedRowIds([ + asset({ id: 'a1', _canSelect: true, isUpdate: true }), + asset({ id: 'a2', _canSelect: true, isUpdate: false }), // not checked + asset({ id: 'a3', _canSelect: false, isUpdate: true }), // not selectable + ]); + expect(result).toEqual({ a1: true }); + }); + + it('returns {} for undefined input', () => { + expect(buildSelectedRowIds(undefined)).toEqual({}); + }); +}); + +describe('assetMapper.utils — applySelectionToAssets', () => { + it('sets isUpdate per the selection for selectable rows and leaves others untouched', () => { + const out = applySelectionToAssets( + [ + asset({ id: 'a1', _canSelect: true, isUpdate: false }), + asset({ id: 'a2', _canSelect: true, isUpdate: true }), + asset({ id: 'a3', _canSelect: false, isUpdate: true }), + ], + { a1: true }, // only a1 selected + ); + expect(out.map((r) => [r.id, r.isUpdate])).toEqual([ + ['a1', true], + ['a2', false], // deselected + ['a3', true], // non-selectable: unchanged + ]); + }); + + it('returns [] for undefined input', () => { + expect(applySelectionToAssets(undefined, {})).toEqual([]); + }); +}); + +describe('assetMapper.utils — toSelectedMap', () => { + it('turns a selected-id array into a { id: true } map', () => { + expect(toSelectedMap(['a1', 'a2'])).toEqual({ a1: true, a2: true }); + }); + + it('returns {} for undefined input', () => { + expect(toSelectedMap(undefined)).toEqual({}); + }); +}); + +describe('assetMapper.utils — computeChangedUids', () => { + it('returns only uids whose selected state flipped vs the persisted map', () => { + const changed = computeChangedUids( + { a1: true, a2: true, a3: false }, + { a1: true, a2: false }, // a2 newly checked, a3 absent (false) → no change + ); + expect(changed.sort()).toEqual(['a2']); + }); + + it('detects a newly-unchecked uid', () => { + expect(computeChangedUids({ a1: false }, { a1: true })).toEqual(['a1']); + }); + + it('returns [] when nothing changed', () => { + expect(computeChangedUids({ a1: true }, { a1: true })).toEqual([]); + expect(computeChangedUids(undefined, undefined)).toEqual([]); + }); +}); diff --git a/ui/src/components/ContentMapper/__tests__/entryMapper.utils.test.ts b/ui/src/components/ContentMapper/__tests__/entryMapper.utils.test.ts new file mode 100644 index 000000000..449d9dd6e --- /dev/null +++ b/ui/src/components/ContentMapper/__tests__/entryMapper.utils.test.ts @@ -0,0 +1,141 @@ +import { describe, it, expect } from 'vitest'; +import { + mapEntriesToRows, + buildSelectedEntryRowIds, + applySelectionToEntries, + selectableInitialRows, + filterContentTypesByStatus, + applyContentTypeStatus, +} from '../entryMapper.utils'; +import { ContentType, EntryMapperType } from '../contentMapper.interface'; + +const entry = (over: Partial): EntryMapperType => ({ + id: 'e1', + projectId: 'p1', + contentTypeId: 'ct1', + contentTypeUid: 'ct_1', + entryName: 'Entry 1', + otherCmsEntryUid: 'e1', + isUpdate: false, + ...over, +}); + +const ct = (over: Partial): ContentType => ({ + contentstackTitle: 'T', + contentstackUid: 'ct_1', + isUpdated: false, + otherCmsTitle: 'OT', + otherCmsUid: 'oct_1', + updateAt: '', + id: 'ct1', + status: '1', + type: 'content_type', + ...over, +}); + +describe('entryMapper.utils — mapEntriesToRows', () => { + it('marks rows selectable only when a Contentstack entry uid exists', () => { + const rows = mapEntriesToRows([ + entry({ id: 'e1', contentstackEntryUid: 'cs-1' }), + entry({ id: 'e2', contentstackEntryUid: undefined }), + ]); + expect(rows.map((r) => [r.id, r._canSelect])).toEqual([ + ['e1', true], + ['e2', false], + ]); + }); + + it('returns [] for undefined input', () => { + expect(mapEntriesToRows(undefined)).toEqual([]); + }); +}); + +describe('entryMapper.utils — buildSelectedEntryRowIds', () => { + it('includes only rows that are selectable AND already isUpdate', () => { + expect( + buildSelectedEntryRowIds([ + entry({ id: 'e1', _canSelect: true, isUpdate: true }), + entry({ id: 'e2', _canSelect: true, isUpdate: false }), + entry({ id: 'e3', _canSelect: false, isUpdate: true }), + ]), + ).toEqual({ e1: true }); + }); + + it('returns {} for undefined input', () => { + expect(buildSelectedEntryRowIds(undefined)).toEqual({}); + }); +}); + +describe('entryMapper.utils — applySelectionToEntries', () => { + it('sets isUpdate per the selection for selectable rows; non-selectable untouched', () => { + const out = applySelectionToEntries( + [ + entry({ id: 'e1', _canSelect: true, isUpdate: false }), + entry({ id: 'e2', _canSelect: true, isUpdate: true }), + entry({ id: 'e3', _canSelect: false, isUpdate: true }), + ], + { e1: true }, + ); + expect(out.map((r) => [r.id, r.isUpdate])).toEqual([ + ['e1', true], + ['e2', false], + ['e3', true], + ]); + }); + + it('returns [] for undefined input', () => { + expect(applySelectionToEntries(undefined, {})).toEqual([]); + }); +}); + +describe('entryMapper.utils — selectableInitialRows', () => { + it('keeps only rows that are not yet isUpdate', () => { + const rows = selectableInitialRows([ + entry({ id: 'e1', isUpdate: false }), + entry({ id: 'e2', isUpdate: true }), + ]); + expect(rows.map((r) => r.id)).toEqual(['e1']); + }); + + it('returns [] for undefined input', () => { + expect(selectableInitialRows(undefined)).toEqual([]); + }); +}); + +describe('entryMapper.utils — filterContentTypesByStatus', () => { + // CONTENT_MAPPING_STATUS: '1'->Mapped, '2'->Updated, '3'->Failed, '4'->All + it("filters by the status label (e.g. 'Updated' → status '2')", () => { + const list = [ + ct({ id: 'a', status: '1' }), // Mapped + ct({ id: 'b', status: '2' }), // Updated + ct({ id: 'c', status: '2' }), // Updated + ]; + expect(filterContentTypesByStatus(list, 'Updated').map((c) => c.id)).toEqual(['b', 'c']); + expect(filterContentTypesByStatus(list, 'Mapped').map((c) => c.id)).toEqual(['a']); + expect(filterContentTypesByStatus(list, 'Failed')).toEqual([]); + }); + + it('returns [] for undefined input', () => { + expect(filterContentTypesByStatus(undefined, 'Updated')).toEqual([]); + }); +}); + +describe('entryMapper.utils — applyContentTypeStatus', () => { + it("sets the target content type to '2' when it has a selection, others untouched", () => { + const list = [ct({ id: 'a', status: '1' }), ct({ id: 'b', status: '1' })]; + const out = applyContentTypeStatus(list, 'a', true); + expect(out.map((c) => [c.id, c.status])).toEqual([ + ['a', '2'], + ['b', '1'], + ]); + }); + + it("sets the target content type back to '1' when it has no selection", () => { + const list = [ct({ id: 'a', status: '2' })]; + expect(applyContentTypeStatus(list, 'a', false)[0].status).toBe('1'); + }); + + it('returns [] for undefined input', () => { + expect(applyContentTypeStatus(undefined, 'a', true)).toEqual([]); + }); +}); diff --git a/ui/src/components/ContentMapper/assetMapper.tsx b/ui/src/components/ContentMapper/assetMapper.tsx new file mode 100644 index 000000000..d51f38873 --- /dev/null +++ b/ui/src/components/ContentMapper/assetMapper.tsx @@ -0,0 +1,320 @@ +// Libraries +import { useEffect, useState } from 'react'; +import { useSelector } from 'react-redux'; +import { useParams } from 'react-router-dom'; +import { + Button, + InfiniteScrollTable, + Notification, +} from '@contentstack/venus-components'; + +// Services +import { + getAssetMapping, + updateAssetMapper, +} from '../../services/api/migration.service'; + +// Redux +import { RootState } from '../../store'; + +// Interface +import { AssetMapperType, TableTypes, UidMap } from './contentMapper.interface'; +import { ItemStatusMapProp } from '@contentstack/venus-components/build/components/Table/types'; + +// Pure logic (unit-tested in assetMapper.utils.test.ts) +import { + formatFileSize, + mapAssetsToRows, + buildSelectedRowIds, + applySelectionToAssets, + toSelectedMap, + computeChangedUids, +} from './assetMapper.utils'; + +// Styles and Assets +import './index.scss'; + +const AssetMapper = ({ + tableHeight, + onCountChange, +}: { + tableHeight: number; + onCountChange?: (count: number) => void; +}) => { + const { projectId = '' } = useParams<{ projectId: string }>(); + + const newMigrationData = useSelector((state: RootState) => state?.migration?.newMigrationData); + + // Component State + const [tableData, setTableData] = useState([]); + const [loading, setLoading] = useState(false); + const [totalCounts, setTotalCounts] = useState(0); + const [itemStatusMap, setItemStatusMap] = useState({}); + const [rowIds, setRowIds] = useState>({}); + const [persistedRowIds, setPersistedRowIds] = useState>({}); + const [isLoadingSaveButton, setisLoadingSaveButton] = useState(false); + + useEffect(() => { + fetchAssets(''); + }, []); + + const fetchAssets = async (searchText: string) => { + try { + const statusMap: ItemStatusMapProp = {}; + for (let index = 0; index <= 1000; index++) { + statusMap[index] = 'loading'; + } + setItemStatusMap(statusMap); + setLoading(true); + + const { data } = await getAssetMapping(0, 1000, searchText ?? '', projectId); + + for (let index = 0; index <= 1000; index++) { + statusMap[index] = 'loaded'; + } + setItemStatusMap({ ...statusMap }); + setLoading(false); + + const validTableData: AssetMapperType[] = mapAssetsToRows(data?.assetMapping); + + // The API returns the full (filtered) total separately from the page it + // sends back. Use that so the count and pagination are correct when the + // result set is larger than the requested page size. + const total = data?.count ?? validTableData?.length ?? 0; + + const initialSelected = buildSelectedRowIds(validTableData ?? []); + setTableData(validTableData ?? []); + setRowIds(initialSelected); + setPersistedRowIds(initialSelected); + setTotalCounts(total); + onCountChange?.(total); + } catch (error) { + console.error('fetchAssets -> error', error); + } + }; + + // Fetch table data + const fetchData = async ({ searchText }: TableTypes) => { + fetchAssets(searchText ?? ''); + }; + + // Method for Load more table data + const loadMoreItems = async ({ searchText, skip, limit, startIndex, stopIndex }: TableTypes) => { + try { + const itemStatusMapCopy: ItemStatusMapProp = { ...itemStatusMap }; + for (let index = startIndex; index <= stopIndex; index++) { + itemStatusMapCopy[index] = 'loading'; + } + setItemStatusMap({ ...itemStatusMapCopy }); + setLoading(true); + + const { data } = await getAssetMapping(skip, limit, searchText ?? '', projectId); + + const updateditemStatusMapCopy: ItemStatusMapProp = { ...itemStatusMap }; + for (let index = startIndex; index <= stopIndex; index++) { + updateditemStatusMapCopy[index] = 'loaded'; + } + setItemStatusMap({ ...updateditemStatusMapCopy }); + setLoading(false); + + const validTableData: AssetMapperType[] = mapAssetsToRows(data?.assetMapping); + const newRows = applySelectionToAssets(validTableData ?? [], rowIds); + + // Merge the fetched page into the existing rows at its offset so the + // virtualized table keeps previously loaded rows instead of dropping + // them when the next range is requested. + setTableData((prev) => { + const merged = [...(prev ?? [])]; + newRows.forEach((row, index) => { + merged[Number(skip) + index] = row; + }); + return merged; + }); + } catch (error) { + console.error('loadMoreItems -> error', error); + } + }; + + /** + * Handle the selected assets. A selected (checked) asset is updated in place + * on the existing Contentstack asset; an unselected matched asset is kept + * as-is (reused from the previous migration). + */ + const handleSelectedAssets = (singleSelectedRowIds: string[]) => { + const selectedObj: UidMap = toSelectedMap(singleSelectedRowIds); + + setRowIds(selectedObj); + setTableData((prev) => applySelectionToAssets(prev ?? [], selectedObj)); + }; + + const handleSaveAssets = async () => { + setisLoadingSaveButton(true); + const changedUids = computeChangedUids(rowIds, persistedRowIds); + + try { + if (changedUids.length === 0) { + setisLoadingSaveButton(false); + return Notification({ + notificationContent: { text: 'No changes to save' }, + notificationProps: { + position: 'bottom-center', + hideProgressBar: true + }, + type: 'info' + }); + } + const { status } = await updateAssetMapper(projectId, { ids: changedUids }); + + setisLoadingSaveButton(false); + if (status === 200) { + setPersistedRowIds({ ...(rowIds ?? {}) }); + return Notification({ + notificationContent: { text: 'Assets saved successfully' }, + notificationProps: { + position: 'bottom-center', + hideProgressBar: true + }, + type: 'success' + }); + } else { + return Notification({ + notificationContent: { text: 'Failed to save assets' }, + notificationProps: { + position: 'bottom-center', + hideProgressBar: true + }, + type: 'error' + }); + } + } catch (error) { + console.error(error); + setisLoadingSaveButton(false); + return error; + } + }; + + const accessorAssetName = (data: AssetMapperType) => { + return ( +
+
+
+ {data?.filename || data?.title || '-'} +
+
+
+ ); + }; + + const accessorAssetPath = (data: AssetMapperType) => { + return ( +
+
+
+ {data?.assetPath || '-'} +
+
+
+ ); + }; + + const accessorFileSize = (data: AssetMapperType) => { + return ( +
+
+
+ {formatFileSize(data?.file_size)} +
+
+
+ ); + }; + + const accessorContentstackUid = (data: AssetMapperType) => { + return ( +
+
+
+ {data?.contentstackAssetUid ? data?.contentstackAssetUid : '-'} +
+
+
+ ); + }; + + const columns = [ + { + disableSortBy: true, + Header: ( + + {`${newMigrationData?.legacy_cms?.selectedCms?.title}: Assets`} + + ), + accessor: accessorAssetName, + id: 'uuid', + width: '220px', + }, + { + disableSortBy: true, + Header: ({'Path:'}), + accessor: accessorAssetPath, + id: '1' + }, + { + disableSortBy: true, + Header: ({'Size:'}), + accessor: accessorFileSize, + id: '2', + width: '100px', + }, + { + disableSortBy: true, + Header: ({'Contentstack UIDs:'}), + accessor: accessorContentstackUid, + id: '3' + } + ]; + + return ( +
+ +
+
Total Assets: {totalCounts}
+ +
+
+ ); +}; + +export default AssetMapper; diff --git a/ui/src/components/ContentMapper/assetMapper.utils.ts b/ui/src/components/ContentMapper/assetMapper.utils.ts new file mode 100644 index 000000000..0c7f53463 --- /dev/null +++ b/ui/src/components/ContentMapper/assetMapper.utils.ts @@ -0,0 +1,76 @@ +// Pure helpers for the delta-AEM Asset Mapper. Extracted from assetMapper.tsx so +// the selection / save / formatting logic can be unit-tested without rendering +// the component (which needs redux, the router and the Venus table). +import { AssetMapperType, UidMap } from './contentMapper.interface'; + +/** Human-readable file size; returns '-' for missing / non-positive sizes. */ +export const formatFileSize = (size: number | string | undefined): string => { + const bytes = Number(size); + if (!Number.isFinite(bytes) || bytes <= 0) return '-'; + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +}; + +/** + * API asset records → table rows. A row is selectable only when the asset + * already has a Contentstack asset uid (i.e. it was matched to an existing + * asset and can be updated/reused). + */ +export const mapAssetsToRows = ( + assetMapping: AssetMapperType[] | undefined, +): AssetMapperType[] => + (assetMapping ?? []).map((asset) => ({ + ...asset, + _canSelect: !!asset?.contentstackAssetUid, + })); + +/** Initial checked set: selectable rows whose isUpdate flag is already on. */ +export const buildSelectedRowIds = ( + assets: AssetMapperType[] | undefined, +): UidMap => + (assets ?? []).reduce((acc, item) => { + if (item?._canSelect && item?.isUpdate) { + acc[item.id] = true; + } + return acc; + }, {}); + +/** Reflect the current selection back onto each selectable row's isUpdate flag. */ +export const applySelectionToAssets = ( + assets: AssetMapperType[] | undefined, + selected: Record, +): AssetMapperType[] => + (assets ?? []).map((item) => { + if (!item?._canSelect) return item; + return { + ...item, + isUpdate: !!selected?.[item.id], + }; + }); + +/** Selected row-id array → `{ id: true }` lookup map. */ +export const toSelectedMap = (ids: string[] | undefined): UidMap => { + const selected: UidMap = {}; + ids?.forEach((uid) => { + selected[uid] = true; + }); + return selected; +}; + +/** + * Uids whose selected state differs between the current and persisted maps — + * i.e. the set that actually needs to be saved. + */ +export const computeChangedUids = ( + rowIds: Record | undefined, + persistedRowIds: Record | undefined, +): string[] => { + const allKeys = new Set([ + ...Object.keys(rowIds ?? {}), + ...Object.keys(persistedRowIds ?? {}), + ]); + return Array.from(allKeys).filter( + (uid) => !!rowIds?.[uid] !== !!persistedRowIds?.[uid], + ); +}; diff --git a/ui/src/components/ContentMapper/contentMapper.interface.ts b/ui/src/components/ContentMapper/contentMapper.interface.ts index 23a52ac58..1d7e65e49 100644 --- a/ui/src/components/ContentMapper/contentMapper.interface.ts +++ b/ui/src/components/ContentMapper/contentMapper.interface.ts @@ -236,4 +236,18 @@ export interface EntryMapperType { contentstackEntryUid?: string; _canSelect?: boolean; isDuplicateEntry?: boolean; +} + +export interface AssetMapperType { + id: string; + projectId: string; + otherCmsAssetUid: string; + filename: string; + title: string; + file_size: number | string; + assetPath: string; + isUpdate: boolean; + isChanged?: boolean; + contentstackAssetUid?: string; + _canSelect?: boolean; } \ No newline at end of file diff --git a/ui/src/components/ContentMapper/entryMapper.tsx b/ui/src/components/ContentMapper/entryMapper.tsx index 462237d0b..64ae29b78 100644 --- a/ui/src/components/ContentMapper/entryMapper.tsx +++ b/ui/src/components/ContentMapper/entryMapper.tsx @@ -46,6 +46,17 @@ import { ModalObj } from '../Modal/modal.interface'; // Components import SchemaModal from '../SchemaModal'; +// Pure logic (unit-tested in __tests__/entryMapper.utils.test.ts) +import { + mapEntriesToRows, + buildSelectedEntryRowIds, + applySelectionToEntries, + selectableInitialRows, + filterContentTypesByStatus, + applyContentTypeStatus, +} from './entryMapper.utils'; +import { toSelectedMap, computeChangedUids } from './assetMapper.utils'; + // Styles and Assets import './index.scss'; import { NoDataFound, SCHEMA_PREVIEW } from '../../common/assets'; @@ -131,28 +142,6 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { }, []); /********** HELPERS *************/ - const buildSelectedRowIds = (entries: EntryMapperType[]) => { - return (entries ?? []).reduce((acc, item) => { - if (item?._canSelect && item?.isUpdate) { - acc[item.id] = true; - } - return acc; - }, {}); - }; - - const applySelectionToEntries = ( - entries: EntryMapperType[], - selected: Record, - ) => { - return (entries ?? []).map((item) => { - if (!item?._canSelect) return item; - return { - ...item, - isUpdate: !!selected?.[item.id], - }; - }); - }; - /********** CONTENT TYPE LIST (left panel) *************/ // Fetch ALREADY-MIGRATED content types only (filter='old') — these are the ones whose entries // exist and can be mapped in this delta iteration. @@ -262,7 +251,7 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { li_list?.forEach((ele) => ele?.classList?.remove('active-filter')); (e?.target as HTMLElement)?.closest('li')?.classList?.add('active-filter'); - const filteredCT = contentTypes?.filter((ct) => CONTENT_MAPPING_STATUS[ct?.status] === value); + const filteredCT = filterContentTypesByStatus(contentTypes, value); if (value !== 'All') { setFilteredContentTypes(filteredCT); setCount(filteredCT?.length); @@ -306,17 +295,14 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { setItemStatusMap({ ...itemStatusMapLocal }); setLoading(false); - const validTableData: EntryMapperType[] = (data?.entryMapping ?? []).map((entry: EntryMapperType) => ({ - ...entry, - _canSelect: !!entry?.contentstackEntryUid, - })); + const validTableData: EntryMapperType[] = mapEntriesToRows(data?.entryMapping); - const initialSelected = buildSelectedRowIds(validTableData ?? []); + const initialSelected = buildSelectedEntryRowIds(validTableData ?? []); setTableData(validTableData ?? []); setRowIds(initialSelected); setPersistedRowIds(initialSelected); setTotalCounts(validTableData?.length); - setInitialRowSelectedData(validTableData?.filter((item: EntryMapperType) => !item?.isUpdate)); + setInitialRowSelectedData(selectableInitialRows(validTableData)); // Reflect any pre-existing entry selections on the content type icon (green when present). updateContentTypeStatus(ctId, Object.keys(initialSelected ?? {}).length > 0); } catch (error) { @@ -349,10 +335,7 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { setItemStatusMap({ ...updated }); setLoading(false); - const validTableData: EntryMapperType[] = (data?.entryMapping ?? []).map((entry: EntryMapperType) => ({ - ...entry, - _canSelect: !!entry?.contentstackEntryUid, - })); + const validTableData: EntryMapperType[] = mapEntriesToRows(data?.entryMapping); setTableData(applySelectionToEntries(validTableData ?? [], rowIds)); } catch (error) { @@ -366,32 +349,20 @@ const EntryMapper = ({ handleStepChange }: entryMapperProps) => { */ const updateContentTypeStatus = (contentTypeId: string, hasSelection: boolean) => { if (!contentTypeId) return; - const nextStatus = hasSelection ? '2' : '1'; - const applyStatus = (list: ContentType[]) => - list?.map?.((ct) => (ct?.id === contentTypeId ? { ...ct, status: nextStatus } : ct)); - setContentTypes((prev) => applyStatus(prev)); - setFilteredContentTypes((prev) => applyStatus(prev)); + setContentTypes((prev) => applyContentTypeStatus(prev, contentTypeId, hasSelection)); + setFilteredContentTypes((prev) => applyContentTypeStatus(prev, contentTypeId, hasSelection)); }; // Handle selected entries const handleSelectedEntries = (singleSelectedRowIds: string[]) => { - const selectedObj: UidMap = {}; - singleSelectedRowIds?.forEach((uid: string) => { - selectedObj[uid] = true; - }); + const selectedObj: UidMap = toSelectedMap(singleSelectedRowIds); setRowIds(selectedObj); setTableData((prev) => applySelectionToEntries(prev ?? [], selectedObj)); }; const handleSaveContentType = async () => { setisLoadingSaveButton(true); - const allKeys = new Set([ - ...Object.keys(rowIds ?? {}), - ...Object.keys(persistedRowIds ?? {}), - ]); - const changedUids = Array.from(allKeys).filter( - (uid) => !!rowIds?.[uid] !== !!persistedRowIds?.[uid], - ); + const changedUids = computeChangedUids(rowIds, persistedRowIds); const orgId = selectedOrganisation?.uid; if (orgId && contentTypeUid) { diff --git a/ui/src/components/ContentMapper/entryMapper.utils.ts b/ui/src/components/ContentMapper/entryMapper.utils.ts new file mode 100644 index 000000000..72bd458b5 --- /dev/null +++ b/ui/src/components/ContentMapper/entryMapper.utils.ts @@ -0,0 +1,72 @@ +// Pure helpers for the delta-AEM Entry Mapper. Extracted from entryMapper.tsx so +// the row-mapping / selection / content-type-status logic can be unit-tested +// without rendering the component (which needs redux, the router and the Venus +// table). Generic selection helpers (toSelectedMap, computeChangedUids) are +// shared from assetMapper.utils. +import { EntryMapperType, ContentType, UidMap } from './contentMapper.interface'; +import { CONTENT_MAPPING_STATUS } from '../../utilities/constants'; + +/** + * API entry records → table rows. A row is selectable only when the entry is + * already mapped to an existing Contentstack entry uid (i.e. can be updated). + */ +export const mapEntriesToRows = ( + entryMapping: EntryMapperType[] | undefined, +): EntryMapperType[] => + (entryMapping ?? []).map((entry) => ({ + ...entry, + _canSelect: !!entry?.contentstackEntryUid, + })); + +/** Initial checked set: selectable rows whose isUpdate flag is already on. */ +export const buildSelectedEntryRowIds = ( + entries: EntryMapperType[] | undefined, +): UidMap => + (entries ?? []).reduce((acc, item) => { + if (item?._canSelect && item?.isUpdate) { + acc[item.id] = true; + } + return acc; + }, {}); + +/** Reflect the current selection back onto each selectable row's isUpdate flag. */ +export const applySelectionToEntries = ( + entries: EntryMapperType[] | undefined, + selected: Record, +): EntryMapperType[] => + (entries ?? []).map((item) => { + if (!item?._canSelect) return item; + return { + ...item, + isUpdate: !!selected?.[item.id], + }; + }); + +/** Rows that start unselected (isUpdate=false) — the table's initial selectable set. */ +export const selectableInitialRows = ( + rows: EntryMapperType[] | undefined, +): EntryMapperType[] => (rows ?? []).filter((item) => !item?.isUpdate); + +/** Filter the content-type list by a human-readable status label (e.g. 'Updated'). */ +export const filterContentTypesByStatus = ( + contentTypes: ContentType[] | undefined, + value: string, +): ContentType[] => + (contentTypes ?? []).filter( + (ct) => CONTENT_MAPPING_STATUS[ct?.status] === value, + ); + +/** + * Set a single content type's status to '2' (has selected entries → "Updated") + * or '1' (none → "Mapped"), leaving every other content type unchanged. + */ +export const applyContentTypeStatus = ( + list: ContentType[] | undefined, + contentTypeId: string, + hasSelection: boolean, +): ContentType[] => { + const nextStatus = hasSelection ? '2' : '1'; + return (list ?? []).map((ct) => + ct?.id === contentTypeId ? { ...ct, status: nextStatus } : ct, + ); +}; diff --git a/ui/src/components/ContentMapper/index.scss b/ui/src/components/ContentMapper/index.scss index db598688c..f1843cf22 100644 --- a/ui/src/components/ContentMapper/index.scss +++ b/ui/src/components/ContentMapper/index.scss @@ -463,6 +463,15 @@ div .table-row { padding-bottom: 70px; } +// Entries / Assets switch shown on delta iterations +.mapper-view-toggle { + display: flex; + gap: 8px; + padding: 10px 18px; + border-bottom: 1px solid #e5e5e5; + background: #fff; +} + // Sticky footer for Save button .mapper-footer { position: sticky; diff --git a/ui/src/components/ContentMapper/index.tsx b/ui/src/components/ContentMapper/index.tsx index bf7cc1ab1..fef43cbb3 100644 --- a/ui/src/components/ContentMapper/index.tsx +++ b/ui/src/components/ContentMapper/index.tsx @@ -85,6 +85,8 @@ import { // Styles and Assets import './index.scss'; import { NoDataFound, SCHEMA_PREVIEW } from '../../common/assets'; +import EntryMapper from './entryMapper'; +import AssetMapper from './assetMapper'; const FIELD_MAP_MENU_VIEW_MARGIN = 8; const FIELD_MAP_MENU_HYSTERESIS = 36; @@ -566,6 +568,8 @@ const ContentMapper = forwardRef(({ handleStepChange }: contentMapperProps, ref: const [isAllCheck, setIsAllCheck] = useState(false); const [isResetFetch, setIsResetFetch] = useState(false); const [iterationCount, setIterationCount] = useState(newMigrationData?.iteration); + const [mapperView, setMapperView] = useState<'entries' | 'assets'>('entries'); + const [assetCount, setAssetCount] = useState(0); /** ALL HOOKS Here */ const { projectId = '' } = useParams(); @@ -3294,9 +3298,31 @@ const ContentMapper = forwardRef(({ handleStepChange }: contentMapperProps, ref: {/* Content Types List */}
- {contentTypesHeading &&

{`${contentTypesHeading} (${contentTypes && count})`}

} + {mapperView === 'assets' + ?

{`Assets (${assetCount})`}

+ : (contentTypesHeading &&

{`${contentTypesHeading} (${contentTypes && count})`}

)}
+
+ + +
+ + {mapperView !== 'assets' && (<>
:
No Content Types Found.
} + )}
{/* Content Type Fields */}
+
+ {mapperView === 'assets' ? ( + + ) : isDeltaIteration ? ( + + ) : (
)} -
+
+ )} +
: diff --git a/ui/src/services/api/migration.service.ts b/ui/src/services/api/migration.service.ts index 2e50ca9d0..0cbdb9c66 100644 --- a/ui/src/services/api/migration.service.ts +++ b/ui/src/services/api/migration.service.ts @@ -422,3 +422,43 @@ export const updateEntryMapper = async ( } }; +export const getAssetMapping = async ( + skip: number, + limit: number, + searchText: string, + projectId: string +) => { + try { + const encodedSearchText = encodeURIComponent(searchText); + return await getCall( + `${API_VERSION}/mapper/assetMapping/${projectId}/${skip}/${limit}/${encodedSearchText}?`, + options() + ); + } catch (error) { + if (error instanceof Error) { + throw new Error(`${error.message}`); + } else { + throw new Error('Unknown error'); + } + } +}; + +export const updateAssetMapper = async ( + projectId: string, + data: ObjectType +) => { + try { + return await putCall( + `${API_VERSION}/mapper/updateAssetStatus/${projectId}`, + data, + options() + ); + } catch (error) { + if (error instanceof Error) { + throw new Error(`${error.message}`); + } else { + throw new Error('Unknown error'); + } + } +}; + diff --git a/upload-api/migration-aem/index.ts b/upload-api/migration-aem/index.ts index f5ccc9d59..3e6574432 100644 --- a/upload-api/migration-aem/index.ts +++ b/upload-api/migration-aem/index.ts @@ -1,9 +1,13 @@ import contentTypes from './libs/contentType'; import locales from './libs/locales'; import validator from './libs/validate' +import extractEntries from './libs/entries'; +import extractAssets from './libs/assets'; export { contentTypes, locales, - validator + validator, + extractEntries, + extractAssets } \ No newline at end of file diff --git a/upload-api/migration-aem/libs/assets/index.ts b/upload-api/migration-aem/libs/assets/index.ts new file mode 100644 index 000000000..f94687c3c --- /dev/null +++ b/upload-api/migration-aem/libs/assets/index.ts @@ -0,0 +1,139 @@ +import path from 'path'; +import fs from 'fs'; +import { createHash } from 'crypto'; +import read from 'fs-readdir-recursive'; +import { CONSTANTS } from '../../constant'; +import { readFiles } from '../../helper'; + +export interface AssetMappingRow { + id: string; + otherCmsAssetUid: string; + filename: string; + title: string; + file_size: number | string; + assetPath: string; + isUpdate: boolean; +} + +/** + * Discovery and uid derivation here intentionally mirror createAssets in + * api/src/services/aem.service.ts: only assets actually referenced by page + * model JSONs are imported, deduped by filename, with a stable uid cascade of + * jcr:uuid → sha256(asset.path) → none. Rows whose uid cannot be derived + * stably are skipped — they get a random uid at migration time and cannot be + * tracked across delta iterations. + */ +const isImageType = (value: string): boolean => + /\.(jpeg|jpg|png|gif|webp|svg)$/i.test(value); + +const deepFlattenObject = (obj: any, prefix = '', res: any = {}) => { + if (Array.isArray(obj) || (obj && typeof obj === 'object')) { + const entries = Array.isArray(obj) + ? obj.map((v, i) => [i, v]) + : Object.entries(obj); + for (const [key, value] of entries) { + const newKey = prefix ? `${prefix}.${key}` : `${key}`; + if (value && typeof value === 'object') { + deepFlattenObject(value, newKey, res); + } else { + res[newKey] = value; + } + } + } else { + res[prefix] = obj; + } + return res; +}; + +const deriveStableAssetUid = (metadata: any, usedUids: Set): string => { + const jcrUuid = metadata?._raw?.assetNode?.['jcr:uuid']; + let uid = + typeof jcrUuid === 'string' && jcrUuid.trim() !== '' + ? jcrUuid.replace(/-/g, '').toLowerCase() + : ''; + if (!uid || usedUids.has(uid)) { + const assetPath = metadata?.asset?.path; + uid = + typeof assetPath === 'string' && assetPath.trim() !== '' + ? createHash('sha256').update(assetPath).digest('hex').slice(0, 32) + : ''; + } + if (!uid || usedUids.has(uid)) { + return ''; + } + return uid; +}; + +const extractAssets = async (dirPath: string): Promise => { + const templatesDir = path.resolve(dirPath); + const damPath = path.resolve(path.join(templatesDir, CONSTANTS.AEM_DAM_DIR)); + const rows: AssetMappingRow[] = []; + + if (!fs.existsSync(damPath)) { + return rows; + } + + const damFiles = read(damPath).map((f) => path.join(damPath, f)); + const seenFilenames = new Set(); + const usedUids = new Set(); + + for (const fileName of read(templatesDir)) { + const filePath = path.join(templatesDir, fileName); + if (filePath?.startsWith?.(damPath)) { + continue; + } + if (!fileName?.endsWith?.('.json')) { + continue; + } + try { + const parseData: any = await readFiles(filePath); + if (!parseData || typeof parseData !== 'object') { + continue; + } + const flatData = deepFlattenObject(parseData); + + for (const value of Object.values(flatData)) { + if (typeof value !== 'string' || !isImageType(value)) { + continue; + } + const lastSegment = value?.split?.('/')?.pop?.(); + if (typeof lastSegment !== 'string') { + continue; + } + const firstJson = damFiles.find( + (fp) => fp.includes(lastSegment) && fp.endsWith('.json') + ); + if (!firstJson) { + continue; + } + const metadata: any = await readFiles(firstJson); + const filename = metadata?.asset?.name; + if (typeof filename !== 'string' || seenFilenames.has(filename)) { + continue; + } + seenFilenames.add(filename); + + const uid = deriveStableAssetUid(metadata, usedUids); + if (!uid) { + continue; + } + usedUids.add(uid); + + rows.push({ + id: uid, + otherCmsAssetUid: uid, + filename, + title: filename.split('.').slice(0, -1).join('.') || filename, + file_size: metadata?.download?.downloadedSize ?? '', + assetPath: metadata?.asset?.path ?? '', + isUpdate: false, + }); + } + } catch (err) { + console.error(`🚀 ~ extractAssets ~ failed to process ${filePath}:`, err); + } + } + return rows; +}; + +export default extractAssets; diff --git a/upload-api/migration-aem/libs/contentType/createContentTypes.ts b/upload-api/migration-aem/libs/contentType/createContentTypes.ts index 6cd859e5c..ff9ccfe1e 100644 --- a/upload-api/migration-aem/libs/contentType/createContentTypes.ts +++ b/upload-api/migration-aem/libs/contentType/createContentTypes.ts @@ -4,13 +4,14 @@ import { createContentTypeObject, ensureField, findComponentByType, writeJsonFil import { isContainerComponent, parseXFPath } from "../../helper/component.identifier"; import { createFragmentComponent } from "./fragment"; import { IContentTypeMaker } from "./types/createContentTypes.interface"; -import { ModularBlocksField } from "./fields/contentstackFields"; +import { JsonField, ModularBlocksField } from "./fields/contentstackFields"; import { processContentModels } from "../../helper/fieldMappings.merge"; import { flattenContentTypes } from "../../helper/contentType.flatten"; async function processTemplateItems(itemsOrder: string[], items: any, contentstackComponents: any) { - const schema = []; + const schema: any[] = []; + if (!Array.isArray(itemsOrder)) return schema; for (const element of itemsOrder) { const item = items?.[element]; const type = item?.[':type']; @@ -20,17 +21,26 @@ async function processTemplateItems(itemsOrder: string[], items: any, contentsta const keyElement = element?.split('-'); const segmentData = keys?.filter((segment: string) => keyElement?.includes(segment)); const referenceField = await createFragmentComponent(segmentData, item, contentstackComponents); - schema?.push(referenceField); + if (referenceField) { + schema?.push(referenceField); + } else if (element) { + const fallbackField = new JsonField({ + uid: element, + displayName: element, + }).toContentstack(); + if (type) fallbackField.otherCmsType = type; + schema?.push(fallbackField); + } } else if (isContainerCheck?.isContainer) { const itemsOrder = item?.[':itemsOrder']; const items = item?.[':items']; const conatinerSchema: any = await processTemplateItems(itemsOrder, items, contentstackComponents); + const modularData = new ModularBlocksField({ + uid: element, + displayName: element, + blocks: [], + }).toContentstack(); if (conatinerSchema?.length) { - const modularData = new ModularBlocksField({ - uid: element, - displayName: element, - blocks: [], - }).toContentstack(); for (const object of conatinerSchema) { if (object?.contentstackFieldType === 'group' || object?.contentstackFieldType === 'modular_blocks') { @@ -61,14 +71,19 @@ async function processTemplateItems(itemsOrder: string[], items: any, contentsta modularData.blocks?.push(block); } } - schema?.push(modularData); } + schema?.push(modularData); } else { const [, csValue] = findComponentByType(contentstackComponents, type) ?? []; if (csValue && typeof csValue === "object" && "type" in csValue) { schema?.push(csValue) - } else { - console.info("🚀 ~ processTemplateItems ~ type:", type); + } else if (element) { + const fallbackField = new JsonField({ + uid: element, + displayName: element, + }).toContentstack(); + if (type) fallbackField.otherCmsType = type; + schema?.push(fallbackField); } } } diff --git a/upload-api/migration-aem/libs/entries/index.ts b/upload-api/migration-aem/libs/entries/index.ts new file mode 100644 index 000000000..e6b477881 --- /dev/null +++ b/upload-api/migration-aem/libs/entries/index.ts @@ -0,0 +1,122 @@ +import path from 'path'; +import read from 'fs-readdir-recursive'; +import { CONSTANTS } from '../../constant'; +import { readFiles } from '../../helper'; + +interface EntryMappingRow { + id: string; + contentTypeUid: string; + entryName: string; + language?: string; + otherCmsEntryUid: string; + otherCmsCTName: string; + isUpdate: boolean; +} + +/** + * Mirrors uidCorrector in api/src/services/aem.service.ts. The otherCmsEntryUid + * stored here must exactly equal the entry uid written into the import data at + * migration time, otherwise the delta flow (entry-mapper ↔ uid-mapper lookup and + * removeEntriesFromDatabase key matching) can never associate the two. + */ +const entryUidCorrector = (str: string): string => + str?.replace(/[^a-zA-Z0-9_]/g, '_')?.toLowerCase(); + +const isExperienceFragment = (data: any): boolean => { + if (data?.templateType && data?.[':type']) { + return ( + data?.templateType?.startsWith?.('xf-') || + data?.[':type']?.includes?.('components/xfpage') + ); + } + return false; +}; + +const getCurrentLocale = (parseData: any): string | undefined => { + if (parseData?.language) { + return parseData.language; + } else if (parseData?.[':path']) { + const segments = parseData[':path'].split('/'); + return segments[segments.length - 1]; + } + return undefined; +}; + +/** + * Walks the AEM page model JSON files and attaches an entryMapping array to each + * matching content type, so the backend can populate entry-mapper.json and the + * Entry Mapper UI can offer create-vs-update selection on delta iterations. + * + * Template resolution intentionally matches createEntry in the api's aem.service: + * experience fragments map by title, pages by templateName then templateType. + */ +const extractEntries = async (dirPath: string, contentTypes: any[]): Promise => { + const templatesDir = path.resolve(dirPath); + const damPath = path.resolve(path.join(templatesDir, CONSTANTS.AEM_DAM_DIR)); + const seenEntryUids = new Set(); + + for (const fileName of read(templatesDir)) { + const filePath = path.join(templatesDir, fileName); + if (filePath?.startsWith?.(damPath)) { + continue; + } + try { + const parseData: any = await readFiles(filePath); + if (!parseData || typeof parseData !== 'object') { + continue; + } + const templateUid = isExperienceFragment(parseData) + ? parseData?.title + : parseData?.templateName ?? parseData?.templateType; + if (!templateUid) { + continue; + } + const contentType = contentTypes?.find?.( + (element: any) => element?.otherCmsUid === templateUid + ); + if (!contentType) { + continue; + } + let modelId = + typeof parseData?.id === 'string' && parseData.id.trim() !== '' + ? entryUidCorrector(parseData.id) + : ''; + // Template-based entries (experience fragments like xf-web-variation, and + // pages like content-page) carry no stable page "id"; derive a stable uid + // from title + templateType (or just templateType when there's no title) + // so they appear in the mapper and track across iterations (must match + // createEntry in the api's aem.service). + if (!modelId && parseData?.templateType) { + modelId = parseData?.title + ? entryUidCorrector(`${parseData.title}_${parseData.templateType}`) + : entryUidCorrector(parseData.templateType); + } + // Entries without a stable page model id receive a random uid at migration + // time and cannot be tracked across iterations, so they get no mapping row. + // Duplicate ids likewise fall back to random uids on the migration side. + if (!modelId || seenEntryUids.has(modelId)) { + continue; + } + seenEntryUids.add(modelId); + + const row: EntryMappingRow = { + id: modelId, + contentTypeUid: contentType?.otherCmsUid, + entryName: parseData?.title ?? parseData?.templateType, + language: getCurrentLocale(parseData), + otherCmsEntryUid: modelId, + otherCmsCTName: templateUid, + isUpdate: false, + }; + if (!Array.isArray(contentType.entryMapping)) { + contentType.entryMapping = []; + } + contentType.entryMapping.push(row); + } catch (err) { + console.error(`🚀 ~ extractEntries ~ failed to process ${filePath}:`, err); + } + } + return contentTypes; +}; + +export default extractEntries; diff --git a/upload-api/src/config/index.json b/upload-api/src/config/index.json index aec40668a..789a93bf4 100644 --- a/upload-api/src/config/index.json +++ b/upload-api/src/config/index.json @@ -1,8 +1,10 @@ { "plan": { - "dropdown": { "optionLimit": 100 } + "dropdown": { + "optionLimit": 100 + } }, - "cmsType": "cmsType", + "cmsType": "", "isLocalPath": true, "awsData": { "awsRegion": "us-east-2", @@ -23,5 +25,5 @@ "base_url": "drupal_assets_base_url", "public_path": "drupal_assets_public_path" }, - "localPath": "your_local_cms_data_path" + "localPath": "" } \ No newline at end of file diff --git a/upload-api/src/config/index.ts b/upload-api/src/config/index.ts new file mode 100644 index 000000000..4bde1a7e6 --- /dev/null +++ b/upload-api/src/config/index.ts @@ -0,0 +1,27 @@ +export default { + plan: { + dropdown: { optionLimit: 100 } + }, + cmsType: process.env.CMS_TYPE || 'cmsType', + isLocalPath: true, + awsData: { + awsRegion: 'us-east-2', + awsAccessKeyId: '', + awsSecretAccessKey: '', + awsSessionToken: '', + bucketName: '', + bucketKey: '' + }, + mysql: { + host: process.env.MYSQL_HOST || 'host_name', + user: process.env.MYSQL_USER || 'user_name', + password: process.env.MYSQL_PASSWORD || '', + database: process.env.MYSQL_DATABASE || 'database_name', + port: process.env.MYSQL_PORT || 'port_number' + }, + assetsConfig: { + base_url: process.env.DRUPAL_ASSETS_BASE_URL || 'drupal_assets_base_url', + public_path: process.env.DRUPAL_ASSETS_PUBLIC_PATH || 'drupal_assets_public_path' + }, + localPath: process.env.CMS_LOCAL_PATH || process.env.CONTAINER_PATH || 'your_local_cms_data_path', +}; diff --git a/upload-api/src/controllers/aem/index.ts b/upload-api/src/controllers/aem/index.ts index 9a3f01697..827856d22 100644 --- a/upload-api/src/controllers/aem/index.ts +++ b/upload-api/src/controllers/aem/index.ts @@ -3,7 +3,7 @@ import axios, { AxiosResponse, AxiosError } from "axios"; import http from 'http'; import logger from "../../utils/logger"; import { HTTP_CODES, HTTP_TEXTS } from "../../constants"; -import { contentTypes, locales } from 'migration-aem'; +import { contentTypes, locales, extractEntries, extractAssets } from 'migration-aem'; interface RequestParams { payload: any; @@ -109,7 +109,9 @@ const createAemMapper = async (filePath: string, projectId: string | string[], a const localeData = await locales().processAndSave(filePath); await createLocaleSource({ app_token, projectId, localeData }); const ctData = await ct.convertAndCreate(filePath); - const fieldMapping: any = { contentTypes: ctData, extractPath: filePath }; + await extractEntries(filePath, ctData as any[]); + const assetMapping = await extractAssets(filePath); + const fieldMapping: any = { contentTypes: ctData, extractPath: filePath, assetMapping }; const { data } = await sendRequestWithRetry({ payload: fieldMapping, projectId, diff --git a/upload-api/tests/unit/controllers/aem.controller.test.ts b/upload-api/tests/unit/controllers/aem.controller.test.ts index 578797d1b..34b0cf1ba 100644 --- a/upload-api/tests/unit/controllers/aem.controller.test.ts +++ b/upload-api/tests/unit/controllers/aem.controller.test.ts @@ -10,6 +10,8 @@ vi.mock('migration-aem', () => ({ contentTypes: mockContentTypes, locales: mockLocales, validator: vi.fn(), + extractEntries: vi.fn().mockResolvedValue(undefined), + extractAssets: vi.fn().mockResolvedValue([]), })); vi.mock('axios', () => ({ default: { request: mockAxiosRequest } }));