diff --git a/cli/backup.js b/cli/backup.js new file mode 100644 index 00000000..565c727e --- /dev/null +++ b/cli/backup.js @@ -0,0 +1,49 @@ +'use strict'; +const chalk = require('chalk'), + tools = require('../lib/cmd/dev-tools'); + +/** + * Configure `clay backup` CLI arguments. + * @param {object} yargs + * @returns {object} + */ +function builder(yargs) { + return yargs + .usage('Usage: $0 backup ') + .example('$0 backup https://domain.com/_pages/homepage --output homepage-backup.clay', 'Create page snapshot') + .option('output', { + alias: 'o', + describe: 'output snapshot file path', + type: 'string' + }) + .option('json', { + describe: 'output machine-readable json', + type: 'boolean' + }); +} + +/** + * Create a page snapshot and render output. + * @param {object} argv + * @returns {Promise} + */ +async function handler(argv) { + const result = await tools.backupPage(argv.url, argv.output); + + if (argv.json) { + console.log(JSON.stringify(result, null, 2)); // eslint-disable-line no-console + return; + } + + console.log(chalk.green(`Backup saved: ${result.filePath}`)); // eslint-disable-line no-console + console.log(`Dispatches: ${result.dispatchCount}`); // eslint-disable-line no-console + console.log(`Page: ${result.resolved.pageUri}`); // eslint-disable-line no-console +} + +module.exports = { + command: 'backup ', + describe: 'Snapshot a page and its layout dispatches', + aliases: ['snap'], + builder, + handler +}; diff --git a/cli/cli-options.js b/cli/cli-options.js index e8a59d1f..e375c7da 100644 --- a/cli/cli-options.js +++ b/cli/cli-options.js @@ -43,6 +43,10 @@ module.exports = { describe: 'export layout when exporting page(s)', type: 'boolean' }, + yesLayout: { + describe: 'confirm that layout refs may be modified', + type: 'boolean' + }, yaml: { alias: 'yaml', // -y, --yaml describe: 'parse bootstrap format', diff --git a/cli/doctor.js b/cli/doctor.js new file mode 100644 index 00000000..b5969cc4 --- /dev/null +++ b/cli/doctor.js @@ -0,0 +1,102 @@ +'use strict'; +const chalk = require('chalk'), + options = require('./cli-options'), + { ensureLayoutConfirmation } = require('./layout-confirmation'), + doctor = require('../lib/cmd/doctor'); + +/** + * Configure `clay doctor` CLI arguments. + * @param {object} yargs + * @returns {object} + */ +function builder(yargs) { + return yargs + .usage('Usage: $0 doctor ') + .example('$0 doctor https://domain.com/_pages/homepage -k qa', 'Diagnose page refs') + .example('$0 doctor https://domain.com/_pages/homepage --fix --apply -k qa', 'Prune missing refs and apply') + .option('k', options.key) + .option('c', options.concurrency) + .option('fix', { + describe: 'run safe auto-fix plan for missing refs', + type: 'boolean' + }) + .option('apply', { + describe: 'apply mutations (default is dry-run)', + type: 'boolean' + }) + .option('publish', { + describe: 'publish page after apply', + type: 'boolean' + }) + .option('layout', { + alias: 'l', + describe: 'include layout refs in checks and mutations', + type: 'boolean' + }) + .option('yes-layout', options.yesLayout) + .option('json', { + describe: 'output machine-readable json', + type: 'boolean' + }); +} + +/** + * Run diagnosis or safe-fix mode and render output. + * @param {object} argv + * @returns {Promise} + */ +async function handler(argv) { + await ensureLayoutConfirmation(argv, 'doctor'); + + if (argv.fix) { + const result = await doctor.safeFix(argv.url, { + key: argv.key, + apply: argv.apply, + publish: argv.publish, + concurrency: argv.concurrency, + layout: argv.layout + }); + + if (argv.json) { + console.log(JSON.stringify(result, null, 2)); // eslint-disable-line no-console + return; + } + + console.log(chalk.cyan(`Doctor ${result.dryRun ? 'plan' : 'apply'} for ${result.resolved.pageUri}`)); // eslint-disable-line no-console + console.log(`Missing refs: ${result.missingRefs.length}`); // eslint-disable-line no-console + console.log(`Changes: ${result.changes.length}`); // eslint-disable-line no-console + if (result.changes.length) { + result.changes.forEach((change) => console.log(`- ${change.action} ${change.path}`)); // eslint-disable-line no-console + } + return; + } + + const diagnosis = await doctor.diagnose(argv.url, { + key: argv.key, + concurrency: argv.concurrency, + layout: argv.layout + }); + + if (argv.json) { + console.log(JSON.stringify(diagnosis, null, 2)); // eslint-disable-line no-console + return; + } + + console.log(chalk.cyan(`Doctor report for ${diagnosis.resolved.pageUri}`)); // eslint-disable-line no-console + console.log(`Refs scanned: ${diagnosis.refsCount}`); // eslint-disable-line no-console + console.log(`Missing refs: ${diagnosis.missingRefs.length}`); // eslint-disable-line no-console + diagnosis.missingRefs.forEach((ref) => console.log(`- ${ref}`)); // eslint-disable-line no-console + + if (diagnosis.lintErrors.length) { + console.log(chalk.yellow('\nLint errors:')); // eslint-disable-line no-console + diagnosis.lintErrors.forEach((msg) => console.log(`- ${msg}`)); // eslint-disable-line no-console + } +} + +module.exports = { + command: 'doctor ', + describe: 'Diagnose and safely repair broken refs on pages', + aliases: ['doc'], + builder, + handler +}; diff --git a/cli/index.js b/cli/index.js index 54e203e1..4e4f3478 100755 --- a/cli/index.js +++ b/cli/index.js @@ -15,13 +15,24 @@ const yargs = require('yargs'), notifier = updateNotifier({ pkg }), + // Map short aliases and full command names to command modules. commands = { c: 'compile', cfg: 'config', + d: 'doctor', e: 'export', i: 'import', l: 'lint', - p: 'pack' + p: 'pack', + rf: 'refs', + b: 'backup', + rs: 'restore', + rsc: 'rescue', + doctor: 'doctor', + refs: 'refs', + backup: 'backup', + restore: 'restore', + rescue: 'rescue' }, listCommands = Object.keys(commands).concat(Object.values(commands)); diff --git a/cli/layout-confirmation.js b/cli/layout-confirmation.js new file mode 100644 index 00000000..ddd8531b --- /dev/null +++ b/cli/layout-confirmation.js @@ -0,0 +1,48 @@ +'use strict'; +const readline = require('readline'); + +const LAYOUT_CONFIRM_TOKEN = 'LAYOUT'; + +/** + * Require an explicit confirmation whenever --layout is enabled. + * In non-interactive environments, users must pass --yes-layout. + * + * @param {object} argv + * @param {string} commandName + * @returns {Promise} + */ +async function ensureLayoutConfirmation(argv, commandName) { + if (!argv.layout) return; + if (argv.yesLayout) return; + + if (!process.stdin.isTTY || !process.stdout.isTTY) { + throw new Error(`--layout requires confirmation for ${commandName}. Re-run with --yes-layout.`); + } + + const answer = await askQuestion(`--layout can modify layout references for ${commandName}. Type ${LAYOUT_CONFIRM_TOKEN} to continue: `); + + if (answer !== LAYOUT_CONFIRM_TOKEN) { + throw new Error('Layout confirmation did not match. Aborting without changes.'); + } +} + +/** + * Ask a single stdin question and resolve with the response string. + * @param {string} prompt + * @returns {Promise} + */ +function askQuestion(prompt) { + return new Promise((resolve) => { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout + }); + + rl.question(prompt, (answer) => { + rl.close(); + resolve(answer.trim()); + }); + }); +} + +module.exports.ensureLayoutConfirmation = ensureLayoutConfirmation; diff --git a/cli/refs.js b/cli/refs.js new file mode 100644 index 00000000..036224e1 --- /dev/null +++ b/cli/refs.js @@ -0,0 +1,126 @@ +'use strict'; +const _ = require('lodash'), + chalk = require('chalk'), + options = require('./cli-options'), + { ensureLayoutConfirmation } = require('./layout-confirmation'), + refs = require('../lib/cmd/refs'); + +/** + * Configure `clay refs` CLI arguments. + * @param {object} yargs + * @returns {object} + */ +function builder(yargs) { + return yargs + .usage('Usage: $0 refs ') + .example('$0 refs https://domain.com/_pages/homepage --action prune -k qa', 'Prune missing refs') + .example('$0 refs https://domain.com/_pages/homepage --action replace --ref /_components/a --to /_components/b --apply -k qa', 'Replace refs in page') + .example('$0 refs stg --action where-used --ref /_components/foo/instances/bar -k qa', 'Find pages using ref') + .example('$0 refs stg --action reset --ref /_components/foo/instances/bar --apply -k qa', 'Reset a broken component instance') + .option('k', options.key) + .option('c', options.concurrency) + .option('action', { + describe: 'refs operation', + choices: ['prune', 'replace', 'reset', 'where-used'], + demandOption: true + }) + .option('ref', { + describe: 'reference uri for replace/reset/where-used', + type: 'string' + }) + .option('to', { + describe: 'replacement ref uri, or {} behavior via literal "{}"', + type: 'string' + }) + .option('apply', { + describe: 'apply mutation operations (default dry-run)', + type: 'boolean' + }) + .option('publish', { + describe: 'publish page after apply', + type: 'boolean' + }) + .option('layout', { + alias: 'l', + describe: 'include layout refs in checks and mutations', + type: 'boolean' + }) + .option('yes-layout', options.yesLayout) + .option('where-used', { + describe: 'when resetting, also return pages that reference this ref', + type: 'boolean' + }) + .option('size', { + describe: 'max search hits for where-used', + type: 'number', + default: 1000 + }) + .option('json', { + describe: 'output machine-readable json', + type: 'boolean' + }); +} + +/** + * Dispatch refs action and print results. + * @param {object} argv + * @returns {Promise} + */ +async function handler(argv) { + await ensureLayoutConfirmation(argv, 'refs'); + + const result = await runAction(argv); + + if (argv.json) { + console.log(JSON.stringify(result, null, 2)); // eslint-disable-line no-console + return; + } + + console.log(chalk.cyan(`refs:${result.action}`)); // eslint-disable-line no-console + if (result.missingRefs) console.log(`Missing refs: ${result.missingRefs.length}`); // eslint-disable-line no-console + if (result.changes) console.log(`Changes: ${result.changes.length}`); // eslint-disable-line no-console + if (result.pages) { + console.log(`Pages: ${result.pages.length}`); // eslint-disable-line no-console + result.pages.forEach((page) => console.log(`- ${page}`)); // eslint-disable-line no-console + } + if (result.applied) console.log(chalk.green('Changes applied')); // eslint-disable-line no-console + else if (_.has(result, 'dryRun') && result.dryRun) console.log(chalk.yellow('Dry-run only. Re-run with --apply to mutate data.')); // eslint-disable-line no-console +} + +/** + * Validate action-specific required args before execution. + * @param {object} argv + */ +function validateArgs(argv) { + if (argv.action === 'replace' && (!argv.ref || !argv.to)) { + throw new Error('--ref and --to are required for --action replace'); + } + + if ((argv.action === 'reset' || argv.action === 'where-used') && !argv.ref) { + throw new Error('--ref is required for this action'); + } +} + +/** + * Route action string to refs command implementation. + * @param {object} argv + * @returns {Promise} + */ +function runAction(argv) { + validateArgs(argv); + + switch (argv.action) { + case 'prune': return refs.prune(argv.url, argv); + case 'replace': return refs.replace(argv.url, argv.ref, argv.to, argv); + case 'reset': return refs.reset(argv.ref, argv.url, argv); + default: return refs.whereUsed(argv.url, argv.ref, argv); + } +} + +module.exports = { + command: 'refs ', + describe: 'Prune, replace, reset, and locate refs', + aliases: ['ref'], + builder, + handler +}; diff --git a/cli/rescue.js b/cli/rescue.js new file mode 100644 index 00000000..60f3d509 --- /dev/null +++ b/cli/rescue.js @@ -0,0 +1,77 @@ +'use strict'; +const chalk = require('chalk'), + options = require('./cli-options'), + { ensureLayoutConfirmation } = require('./layout-confirmation'), + rescue = require('../lib/cmd/rescue'); + +/** + * Configure `clay rescue` CLI arguments. + * @param {object} yargs + * @returns {object} + */ +function builder(yargs) { + return yargs + .usage('Usage: $0 rescue ') + .example('$0 rescue https://domain.com/_pages/homepage -k qa', 'Backup + diagnose + fix plan') + .example('$0 rescue https://domain.com/_pages/homepage -k qa --apply --publish', 'Backup + diagnose + apply safe fix + publish') + .option('k', options.key) + .option('c', options.concurrency) + .option('output', { + alias: 'o', + describe: 'backup output file path', + type: 'string' + }) + .option('apply', { + describe: 'apply safe fix (default dry-run)', + type: 'boolean' + }) + .option('publish', { + describe: 'publish page after apply', + type: 'boolean' + }) + .option('layout', { + alias: 'l', + describe: 'include layout refs in checks and mutations', + type: 'boolean' + }) + .option('yes-layout', options.yesLayout) + .option('json', { + describe: 'output machine-readable json', + type: 'boolean' + }); +} + +/** + * Run backup + diagnose + safe-fix workflow and render output. + * @param {object} argv + * @returns {Promise} + */ +async function handler(argv) { + await ensureLayoutConfirmation(argv, 'rescue'); + + const result = await rescue.run(argv.url, argv); + + if (argv.json) { + console.log(JSON.stringify(result, null, 2)); // eslint-disable-line no-console + return; + } + + console.log(chalk.cyan(`Rescue report for ${result.backup.resolved.pageUri}`)); // eslint-disable-line no-console + console.log(`Backup: ${result.backup.filePath}`); // eslint-disable-line no-console + console.log(`Refs scanned: ${result.diagnosis.refsCount}`); // eslint-disable-line no-console + console.log(`Missing refs: ${result.diagnosis.missingRefs.length}`); // eslint-disable-line no-console + console.log(`Fix changes: ${result.fixResult.changes.length}`); // eslint-disable-line no-console + if (result.fixResult.dryRun) { + console.log(chalk.yellow('Dry-run only. Re-run with --apply to mutate data.')); // eslint-disable-line no-console + } else { + console.log(chalk.green('Applied safe fixes.')); // eslint-disable-line no-console + } +} + +module.exports = { + command: 'rescue ', + describe: 'Backup + diagnose + safe-fix workflow for broken pages', + aliases: ['heal'], + builder, + handler +}; diff --git a/cli/restore.js b/cli/restore.js new file mode 100644 index 00000000..43bc09c8 --- /dev/null +++ b/cli/restore.js @@ -0,0 +1,59 @@ +'use strict'; +const chalk = require('chalk'), + options = require('./cli-options'), + tools = require('../lib/cmd/dev-tools'); + +/** + * Configure `clay restore` CLI arguments. + * @param {object} yargs + * @returns {object} + */ +function builder(yargs) { + return yargs + .usage('Usage: $0 restore ') + .example('$0 restore https://domain.com --file homepage-backup.clay -k qa', 'Restore snapshot to target site') + .option('k', options.key) + .option('file', { + alias: 'f', + describe: 'snapshot file created by clay backup', + type: 'string', + demandOption: true + }) + .option('publish', { + describe: 'publish restored items', + type: 'boolean' + }) + .option('json', { + describe: 'output machine-readable json', + type: 'boolean' + }); +} + +/** + * Restore a snapshot into a target environment. + * @param {object} argv + * @returns {Promise} + */ +async function handler(argv) { + const key = tools.getKey(argv.key), + result = await tools.restoreSnapshot(argv.file, argv.url, key, argv.publish); + + if (argv.json) { + console.log(JSON.stringify(result, null, 2)); // eslint-disable-line no-console + return; + } + + console.log(chalk.green(`Restored ${result.successes} item(s)`)); // eslint-disable-line no-console + if (result.errors.length) { + console.log(chalk.red(`Errors: ${result.errors.length}`)); // eslint-disable-line no-console + result.errors.forEach((err) => console.log(`- ${err.message}`)); // eslint-disable-line no-console + } +} + +module.exports = { + command: 'restore ', + describe: 'Restore a dispatch snapshot to a target site', + aliases: ['load'], + builder, + handler +}; diff --git a/docs/cli.md b/docs/cli.md index 8ef3ee1e..913998ac 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -40,6 +40,11 @@ For smaller `Clay` installations (or, ironically, for very large teams where dev * [`config`](#config) * [`lint`](#lint) +* [`doctor`](#doctor) +* [`refs`](#refs) +* [`backup`](#backup) +* [`restore`](#restore) +* [`rescue`](#rescue) * [`import`](#import) * [`export`](#export) * [`compile`](#compile) @@ -210,6 +215,173 @@ $ clay lint my-cool-article $ clay lint < components/article/schema.yml ``` +## Doctor + +Diagnose broken references on a page and optionally run a safe auto-fix. `doctor` combines recursive lint checks with direct reference existence checks so developers can quickly identify and remediate corrupted page data. + +```bash +clay doctor [--key ] [--concurrency ] [--fix] [--apply] [--publish] [--json] +``` + +### Arguments + +* `-k, --key` specifies an api key or key alias for the target environment +* `-c, --concurrency` controls concurrent reference checks +* `--fix` switches from diagnostics-only mode to safe fix mode +* `--apply` applies safe fix mutations (default is dry-run) +* `--publish` publishes the page after apply +* `-l, --layout` include layout refs in checks and mutations (default excludes layout refs) +* `--yes-layout` bypasses interactive layout confirmation (required in non-interactive shells) +* `--json` returns structured output + +### Safe Fix Behavior + +When `--fix` is enabled, `doctor` applies deterministic repair operations: + +* remove missing refs from arrays +* reset objects with missing `_ref` values to `{}` + +If `--apply` is omitted, no writes are made and the command prints a dry-run mutation plan. + +### Examples + +```bash +# Diagnose a page in qa +$ clay doctor https://domain.com/_pages/homepage -k qa + +# Generate a dry-run repair plan +$ clay doctor https://domain.com/_pages/homepage -k qa --fix + +# Apply safe repairs and publish +$ clay doctor https://domain.com/_pages/homepage -k qa --fix --apply --publish +``` + +## Refs + +Perform targeted reference operations for incident response and cleanup workflows. + +```bash +clay refs [--key ] [--concurrency ] [--action ] [options] +``` + +Supported actions: + +* `prune` removes/resets missing refs from a page +* `replace` swaps one ref for another in a page payload +* `reset` overwrites a single ref URI with an empty object +* `where-used` returns pages that contain a given ref + +### Arguments + +* `-k, --key` specifies an api key or key alias for the target environment +* `-c, --concurrency` controls concurrent reference checks for page-scoped actions +* `--action` required operation (`prune`, `replace`, `reset`, `where-used`) +* `--ref` required for `replace`, `reset`, and `where-used` +* `--to` required for `replace` +* `--apply` applies mutations (default is dry-run for mutating actions) +* `--publish` publishes page after `prune` / `replace` when `--apply` is set +* `-l, --layout` include layout refs in checks and mutations (default excludes layout refs) +* `--yes-layout` bypasses interactive layout confirmation (required in non-interactive shells) +* `--where-used` (with `reset`) also returns pages containing the ref +* `--size` max hits for where-used search (default `1000`) +* `--json` returns structured output + +### Examples + +```bash +# Dry-run prune missing refs in qa +$ clay refs https://domain.com/_pages/homepage --action prune -k qa + +# Apply prune + publish +$ clay refs https://domain.com/_pages/homepage --action prune -k qa --apply --publish + +# Replace a broken ref with a valid instance +$ clay refs https://domain.com/_pages/homepage --action replace --ref /_components/a/instances/1 --to /_components/a/instances/2 -k qa --apply + +# Reset a broken instance and also show pages using it +$ clay refs stg --action reset --ref /_components/foo/instances/bar -k qa --apply --where-used + +# Find usage only +$ clay refs stg --action where-used --ref /_components/foo/instances/bar -k qa +``` + +## Backup + +Create a snapshot file of a page and associated dispatches before repairs. + +```bash +clay backup [--output ] [--json] +``` + +### Arguments + +* `-o, --output` custom snapshot output path +* `--json` returns structured output + +### Examples + +```bash +# Generate timestamped snapshot in current directory +$ clay backup https://domain.com/_pages/homepage + +# Write snapshot to explicit file +$ clay backup https://domain.com/_pages/homepage --output homepage-before-fix.clay +``` + +## Restore + +Restore a previously created dispatch snapshot into a target environment. + +```bash +clay restore [--key ] --file [--publish] [--json] +``` + +### Arguments + +* `-k, --key` specifies an api key or key alias for the target environment +* `-f, --file` snapshot file created by `clay backup` +* `--publish` publishes imported items +* `--json` returns structured output + +### Examples + +```bash +# Restore to qa site prefix +$ clay restore https://qa.domain.com --file homepage-before-fix.clay -k qa + +# Restore and publish +$ clay restore https://qa.domain.com --file homepage-before-fix.clay -k qa --publish +``` + +## Rescue + +Run a full remediation workflow in one command: backup -> diagnose -> safe-fix (dry-run by default). + +```bash +clay rescue [--key ] [--concurrency ] [--output ] [--apply] [--publish] [--json] +``` + +### Arguments + +* `-k, --key` specifies an api key or key alias for the target environment +* `-c, --concurrency` controls concurrent reference checks +* `-o, --output` custom backup output path +* `--apply` applies safe fix mutations (default is dry-run) +* `--publish` publishes the page after apply +* `-l, --layout` include layout refs in checks and mutations (default excludes layout refs) +* `--yes-layout` bypasses interactive layout confirmation (required in non-interactive shells) +* `--json` returns structured output + +### Examples + +```bash +# Full rescue plan (no writes) +$ clay rescue https://domain.com/_pages/homepage -k qa + +# Full rescue apply + publish +$ clay rescue https://domain.com/_pages/homepage -k qa --apply --publish +``` + ## Import Imports data into Clay from `stdin`. Data may be in _dispatch_ or _bootstrap_ format. Site prefix must be a raw url, an alias specified via `clay config`, or omitted in favor of `CLAYCLI_DEFAULT_URL`. Key must be an alias specified via `clay config`, or omitted in favor of `CLAYCLI_DEFAULT_KEY`. diff --git a/lib/cmd/dev-tools.js b/lib/cmd/dev-tools.js new file mode 100644 index 00000000..93f366e1 --- /dev/null +++ b/lib/cmd/dev-tools.js @@ -0,0 +1,410 @@ +'use strict'; +const _ = require('lodash'), + pLimit = require('p-limit'), + fs = require('fs-extra'), + path = require('path'), + config = require('./config'), + prefixes = require('../prefixes'), + rest = require('../rest'), + exportCmd = require('./export'), + importCmd = require('./import'); + +/** + * Resolve an API key from a raw key or configured alias. + * @param {string} keyAlias + * @returns {string} + */ +function getKey(keyAlias) { + return config.get('key', keyAlias); +} + +/** + * Resolve a URL/site prefix from a raw URL or configured alias. + * @param {string} urlAlias + * @returns {string} + */ +function getUrl(urlAlias) { + return config.get('url', urlAlias); +} + +/** + * Resolve a page input (public URL or _pages URL) into normalized page metadata. + * @param {string} rawUrl + * @returns {Promise} + */ +async function resolvePage(rawUrl) { + const url = getUrl(rawUrl); + + if (!url) { + throw new Error('URL is not defined! Please specify a page url or alias'); + } + + if (_.includes(url, '/_pages/')) { + return { + rawUrl: rawUrl, + inputUrl: url, + pageUrl: url.replace(/\.html$/, ''), + pageUri: prefixes.urlToUri(url.replace(/\.html$/, '')), + prefix: prefixes.getFromUrl(url.replace(/\.html$/, '')) + }; + } + + const found = await rest.findURI(url).toPromise(Promise); + + if (found instanceof Error) { + throw found; + } + + return { + rawUrl: rawUrl, + inputUrl: url, + pageUrl: prefixes.uriToUrl(found.prefix, found.uri), + pageUri: found.uri, + prefix: found.prefix + }; +} + +/** + * Build authorization headers for GET calls that require API auth. + * @param {string} key + * @returns {object|undefined} + */ +function authHeaders(key) { + return key ? { Authorization: `Token ${key}` } : undefined; +} + +/** + * Fetch page JSON data and convert request errors into thrown exceptions. + * @param {string} pageUrl + * @param {string} key + * @returns {Promise} + */ +async function getPageData(pageUrl, key) { + const res = await rest.get(pageUrl, { headers: authHeaders(key) }).toPromise(Promise); + + if (res instanceof Error) { + throw res; + } + + return res; +} + +/** + * Determine if a URI points at a layout record. + * @param {string} value + * @returns {boolean} + */ +function isLayoutRef(value) { + return _.isString(value) && _.startsWith(value, '/_layouts/'); +} + +/** + * Recursively collect `_ref` values and direct URI strings from page data. + * @param {*} value + * @param {Set} [refs] + * @param {object} [state] + * @returns {Set} + */ +/* eslint-disable complexity */ +function listRefs(value, refs = new Set(), state = {}) { + const currentPath = state.currentPath || '$', + includeLayouts = state.includeLayouts === true; + + if (_.isString(value) && _.startsWith(value, '/_')) { + if (!includeLayouts && (currentPath === '$.layout' || isLayoutRef(value))) { + return refs; + } + + refs.add(value); + return refs; + } + + if (_.isArray(value)) { + _.forEach(value, (item, index) => listRefs(item, refs, { currentPath: `${currentPath}[${index}]`, includeLayouts })); + return refs; + } + + if (_.isPlainObject(value)) { + if (_.isString(value._ref) && _.startsWith(value._ref, '/_')) { + if (!includeLayouts && isLayoutRef(value._ref)) { + return refs; + } + + refs.add(value._ref); + } + + _.forEach(value, (item, key) => listRefs(item, refs, { currentPath: `${currentPath}.${key}`, includeLayouts })); + } + + return refs; +} +/* eslint-enable complexity */ + +/** + * Check a set of refs for existence and return only the missing refs. + * @param {string} prefix + * @param {string[]} refs + * @param {string} key + * @param {number} [concurrency=10] + * @returns {Promise} + */ +async function getMissingRefs(prefix, refs, key, concurrency = 10) { + const limit = pLimit(concurrency), + headers = authHeaders(key), + checks = _.map(refs, (ref) => limit(async () => { + const url = `${prefixes.uriToUrl(prefix, ref)}.json`, + res = await rest.get(url, { headers }).toPromise(Promise); + + return { ref, missing: res instanceof Error }; + })), + results = await Promise.all(checks); + + return _.map(_.filter(results, { missing: true }), 'ref'); +} + +/** + * Prune or reset references that point to missing items. + * + * Rules: + * - Missing ref strings in arrays are removed. + * - Objects with missing `_ref` values are reset to `{}`. + * - Every mutation is recorded in `changes`. + * + * @param {*} value + * @param {Set} missingSet + * @param {object[]} changes + * @param {object} [state] + * @returns {*} + */ +/* eslint-disable complexity */ +function pruneMissingRefs(value, missingSet, changes, state = {}) { + const currentPath = state.currentPath || '$', + includeLayouts = state.includeLayouts === true; + + if (_.isArray(value)) { + const next = []; + + _.forEach(value, (item, index) => { + const itemPath = `${currentPath}[${index}]`; + + if (_.isString(item) && missingSet.has(item) && (includeLayouts || !isLayoutRef(item))) { + changes.push({ action: 'remove-array-ref', path: itemPath, ref: item }); + return; + } + + if (_.isPlainObject(item) && _.isString(item._ref) && missingSet.has(item._ref) && (includeLayouts || !isLayoutRef(item._ref))) { + changes.push({ action: 'remove-array-ref-object', path: itemPath, ref: item._ref }); + return; + } + + next.push(pruneMissingRefs(item, missingSet, changes, { currentPath: itemPath, includeLayouts })); + }); + + return next; + } + + if (_.isPlainObject(value)) { + if (_.isString(value._ref) && missingSet.has(value._ref) && (includeLayouts || !isLayoutRef(value._ref))) { + changes.push({ action: 'reset-object-ref', path: currentPath, ref: value._ref }); + return {}; + } + + const out = {}; + + _.forEach(value, (item, key) => { + out[key] = pruneMissingRefs(item, missingSet, changes, { currentPath: `${currentPath}.${key}`, includeLayouts }); + }); + + return out; + } + + return value; +} +/* eslint-enable complexity */ + +/** + * Recursively replace one ref with another (or reset to `{}`). + * @param {*} value + * @param {string} fromRef + * @param {string} toRef + * @param {object} [state] + * @returns {*} + */ +/* eslint-disable complexity */ +function replaceRef(value, fromRef, toRef, state = {}) { + const changes = state.changes || [], + currentPath = state.currentPath || '$', + includeLayouts = state.includeLayouts === true; + + if (_.isArray(value)) { + return _.map(value, (item, index) => { + const itemPath = `${currentPath}[${index}]`; + + if (_.isString(item) && item === fromRef && (includeLayouts || !isLayoutRef(item))) { + changes.push({ action: 'replace-array-ref', path: itemPath, from: fromRef, to: toRef }); + return toRef; + } + + return replaceRef(item, fromRef, toRef, { changes, currentPath: itemPath, includeLayouts }); + }); + } + + if (_.isPlainObject(value)) { + if (_.isString(value._ref) && value._ref === fromRef && (includeLayouts || !isLayoutRef(value._ref))) { + if (toRef === '{}') { + changes.push({ action: 'reset-object-ref', path: currentPath, from: fromRef, to: '{}' }); + return {}; + } + + changes.push({ action: 'replace-object-ref', path: currentPath, from: fromRef, to: toRef }); + return _.assign({}, value, { _ref: toRef }); + } + + const out = {}; + + _.forEach(value, (item, key) => { + out[key] = replaceRef(item, fromRef, toRef, { changes, currentPath: `${currentPath}.${key}`, includeLayouts }); + }); + + return out; + } + + return value; +} +/* eslint-enable complexity */ + +/** + * PUT the updated page payload to latest. + * @param {string} pageUrl + * @param {object} pageData + * @param {string} key + * @returns {Promise} + */ +async function putPage(pageUrl, pageData, key) { + const res = await rest.put(pageUrl, pageData, { key }).toPromise(Promise); + + if (res.type === 'error') { + throw new Error(res.message || `Failed to update ${pageUrl}`); + } + + return res; +} + +/** + * Publish a page after latest data has been updated. + * @param {string} pageUrl + * @param {string} key + * @returns {Promise} + */ +async function publishPage(pageUrl, key) { + const res = await rest.put(`${pageUrl}@published`, undefined, { key }).toPromise(Promise); + + if (res.type === 'error') { + throw new Error(res.message || `Failed to publish ${pageUrl}`); + } + + return res; +} + +/** + * Find pages that include a given ref using the pages index. + * @param {string} prefixOrAlias + * @param {string} ref + * @param {string} key + * @param {number} [size=1000] + * @returns {Promise} + */ +async function whereUsed(prefixOrAlias, ref, key, size = 1000) { + const prefix = getUrl(prefixOrAlias); + + if (!prefix) { + throw new Error('URL is not defined! Please specify a site prefix to query'); + } + + const query = { + index: 'pages', + size: size, + body: { + query: { + query_string: { + query: `"${ref}"` + } + } + } + }, + res = await rest.query(`${prefix}/_search`, query, { key }).toPromise(Promise); + + if (res.type === 'error') { + throw new Error(res.message || `Search failed for ${ref}`); + } + + return _.map( + _.filter(res.data, (item) => JSON.stringify(item).includes(ref)), + (item) => item._id + ); +} + +/** + * Generate a timestamped snapshot file path in the current directory. + * @param {string} pageUri + * @returns {string} + */ +function defaultSnapshotPath(pageUri) { + const stamp = new Date().toISOString().replace(/[:.]/g, '-'), + fileName = `${pageUri.replace(/[\/@]/g, '_')}-${stamp}.clay`; + + return path.join(process.cwd(), fileName); +} + +/** + * Export a page (with layout dispatches) and write it to a snapshot file. + * @param {string} rawUrl + * @param {string} [outputPath] + * @returns {Promise} + */ +async function backupPage(rawUrl, outputPath) { + const resolved = await resolvePage(rawUrl), + dispatches = await exportCmd.fromURL(resolved.pageUrl, { layout: true }).toArray(Promise), + filePath = outputPath || defaultSnapshotPath(resolved.pageUri), + payload = _.map(dispatches, (dispatch) => JSON.stringify(dispatch)).join('\n'); + + await fs.outputFile(filePath, payload); + + return { + filePath, + dispatchCount: dispatches.length, + resolved + }; +} + +/** + * Restore a dispatch snapshot into a target site prefix. + * @param {string} filePath + * @param {string} targetUrl + * @param {string} key + * @param {boolean} publish + * @returns {Promise} + */ +async function restoreSnapshot(filePath, targetUrl, key, publish) { + const input = await fs.readFile(filePath, 'utf8'), + results = await importCmd(input, targetUrl, { key, publish }).toArray(Promise), + successes = _.filter(results, { type: 'success' }).length, + errors = _.filter(results, { type: 'error' }); + + return { results, successes, errors }; +} + +module.exports.getKey = getKey; +module.exports.getUrl = getUrl; +module.exports.resolvePage = resolvePage; +module.exports.getPageData = getPageData; +module.exports.listRefs = listRefs; +module.exports.getMissingRefs = getMissingRefs; +module.exports.pruneMissingRefs = pruneMissingRefs; +module.exports.replaceRef = replaceRef; +module.exports.putPage = putPage; +module.exports.publishPage = publishPage; +module.exports.whereUsed = whereUsed; +module.exports.backupPage = backupPage; +module.exports.restoreSnapshot = restoreSnapshot; +module.exports.defaultSnapshotPath = defaultSnapshotPath; diff --git a/lib/cmd/doctor.js b/lib/cmd/doctor.js new file mode 100644 index 00000000..087df043 --- /dev/null +++ b/lib/cmd/doctor.js @@ -0,0 +1,73 @@ +'use strict'; +const _ = require('lodash'), + lint = require('./lint'), + tools = require('./dev-tools'); + +/** + * Produce a page diagnosis report: + * - lint errors from recursive linting + * - total refs scanned from page JSON + * - refs missing in the target environment + * + * @param {string} url + * @param {object} [options={}] + * @returns {Promise} + */ +async function diagnose(url, options = {}) { + const key = tools.getKey(options.key), + resolved = await tools.resolvePage(url), + lintResults = await lint.lintUrl(resolved.pageUrl, { concurrency: options.concurrency }).toArray(Promise), + lintErrors = _.uniq(_.map(_.filter(lintResults, { type: 'error' }), 'message')), + pageData = await tools.getPageData(resolved.pageUrl, key), + refs = [...tools.listRefs(pageData, new Set(), { includeLayouts: options.layout === true })], + missingRefs = await tools.getMissingRefs(resolved.prefix, refs, key, options.concurrency || 10); + + return { + resolved, + lintErrors, + refsCount: refs.length, + missingRefs + }; +} + +/** + * Plan or apply safe reference repairs on a page. + * + * Safe fix behavior: + * - remove missing refs from arrays + * - reset objects with missing `_ref` to `{}` + * + * @param {string} url + * @param {object} [options={}] + * @returns {Promise} + */ +async function safeFix(url, options = {}) { + const key = tools.getKey(options.key), + dryRun = options.apply !== true, + resolved = await tools.resolvePage(url), + pageData = await tools.getPageData(resolved.pageUrl, key), + refs = [...tools.listRefs(pageData, new Set(), { includeLayouts: options.layout === true })], + missingRefs = await tools.getMissingRefs(resolved.prefix, refs, key, options.concurrency || 10), + missingSet = new Set(missingRefs), + changes = [], + next = tools.pruneMissingRefs(pageData, missingSet, changes, { includeLayouts: options.layout === true }); + + if (!dryRun && changes.length) { + await tools.putPage(resolved.pageUrl, next, key); + + if (options.publish) { + await tools.publishPage(resolved.pageUrl, key); + } + } + + return { + resolved, + dryRun, + missingRefs, + changes, + applied: !dryRun && changes.length > 0 + }; +} + +module.exports.diagnose = diagnose; +module.exports.safeFix = safeFix; diff --git a/lib/cmd/refs.js b/lib/cmd/refs.js new file mode 100644 index 00000000..859ad327 --- /dev/null +++ b/lib/cmd/refs.js @@ -0,0 +1,109 @@ +'use strict'; +const rest = require('../rest'), + tools = require('./dev-tools'); + +/** + * Remove/reset missing refs within a page payload. + * @param {string} url + * @param {object} [options={}] + * @returns {Promise} + */ +async function prune(url, options = {}) { + const key = tools.getKey(options.key), + dryRun = options.apply !== true, + resolved = await tools.resolvePage(url), + pageData = await tools.getPageData(resolved.pageUrl, key), + refs = [...tools.listRefs(pageData, new Set(), { includeLayouts: options.layout === true })], + missingRefs = await tools.getMissingRefs(resolved.prefix, refs, key, options.concurrency || 10), + changes = [], + next = tools.pruneMissingRefs(pageData, new Set(missingRefs), changes, { includeLayouts: options.layout === true }); + + if (!dryRun && changes.length) { + await tools.putPage(resolved.pageUrl, next, key); + if (options.publish) { + await tools.publishPage(resolved.pageUrl, key); + } + } + + return { action: 'prune', resolved, dryRun, missingRefs, changes, applied: !dryRun && changes.length > 0 }; +} + +/** + * Replace a ref with another ref (or `{}` via literal "{}") in a page payload. + * @param {string} url + * @param {string} fromRef + * @param {string} toRef + * @param {object} [options={}] + * @returns {Promise} + */ +async function replace(url, fromRef, toRef, options = {}) { + const key = tools.getKey(options.key), + dryRun = options.apply !== true, + resolved = await tools.resolvePage(url), + pageData = await tools.getPageData(resolved.pageUrl, key), + changes = [], + next = tools.replaceRef(pageData, fromRef, toRef, { changes, includeLayouts: options.layout === true }); + + if (!dryRun && changes.length) { + await tools.putPage(resolved.pageUrl, next, key); + if (options.publish) { + await tools.publishPage(resolved.pageUrl, key); + } + } + + return { action: 'replace', resolved, dryRun, fromRef, toRef, changes, applied: !dryRun && changes.length > 0 }; +} + +/** + * Reset a single ref URI to an empty object. + * Can optionally list pages that use the ref. + * @param {string} ref + * @param {string} prefixOrAlias + * @param {object} [options={}] + * @returns {Promise} + */ +async function reset(ref, prefixOrAlias, options = {}) { + const key = tools.getKey(options.key), + prefix = tools.getUrl(prefixOrAlias), + refUrl = `${prefix}${ref}`, + dryRun = options.apply !== true; + + if (!prefix) { + throw new Error('URL is not defined! Please specify a site prefix to reset refs'); + } + + if (!dryRun) { + const res = await rest.put(refUrl, {}, { key }).toPromise(Promise); + + if (res.type === 'error') { + throw new Error(res.message || `Unable to reset ${ref}`); + } + } + + let pages = []; + + if (options.whereUsed) { + pages = await tools.whereUsed(prefix, ref, key, options.size || 1000); + } + + return { action: 'reset', dryRun, ref, refUrl, pages, applied: !dryRun }; +} + +/** + * Lookup page URIs that contain the provided ref. + * @param {string} prefixOrAlias + * @param {string} ref + * @param {object} [options={}] + * @returns {Promise} + */ +async function whereUsed(prefixOrAlias, ref, options = {}) { + const key = tools.getKey(options.key), + pages = await tools.whereUsed(prefixOrAlias, ref, key, options.size || 1000); + + return { action: 'where-used', ref, pages }; +} + +module.exports.prune = prune; +module.exports.replace = replace; +module.exports.reset = reset; +module.exports.whereUsed = whereUsed; diff --git a/lib/cmd/rescue.js b/lib/cmd/rescue.js new file mode 100644 index 00000000..57f89f79 --- /dev/null +++ b/lib/cmd/rescue.js @@ -0,0 +1,32 @@ +'use strict'; +const doctor = require('./doctor'), + tools = require('./dev-tools'); + +/** + * Run full rescue workflow: + * 1) backup page + * 2) diagnose + * 3) safe-fix (dry-run by default) + * + * @param {string} url + * @param {object} [options={}] + * @returns {Promise} + */ +async function run(url, options = {}) { + const backup = await tools.backupPage(url, options.output), + diagnosis = await doctor.diagnose(url, options), + fixResult = await doctor.safeFix(url, { + key: options.key, + apply: options.apply, + publish: options.publish, + concurrency: options.concurrency + }); + + return { + backup, + diagnosis, + fixResult + }; +} + +module.exports.run = run;