From 5a062248b347a52c5ed27d621d9d0cc9f78119c1 Mon Sep 17 00:00:00 2001 From: Ariel Caplan Date: Thu, 2 Jul 2026 01:05:56 +0300 Subject: [PATCH] Suggest installing the Shopify AI Toolkit for detected AI coding agents Detects when Shopify CLI is being run by an AI coding agent (Pi, Claude Code, Codex) using a combination of a non-TTY stdout and a known harness environment variable (PI_CODING_AGENT, CLAUDE_CODE, CODEX_THREAD_ID). If the Shopify AI Toolkit isn't already installed for that harness, prints a line at the start of command execution telling the agent how to install it, per https://shopify.dev/docs/apps/build/ai-toolkit. Assisted-By: devx/02722c6b-83ec-413e-9eb9-236c0943393c --- .changeset/suggest-ai-toolkit-install.md | 5 + .../src/public/node/ai-agent-toolkit.test.ts | 212 ++++++++++++++++++ .../src/public/node/ai-agent-toolkit.ts | 196 ++++++++++++++++ .../cli-kit/src/public/node/base-command.ts | 2 + 4 files changed, 415 insertions(+) create mode 100644 .changeset/suggest-ai-toolkit-install.md create mode 100644 packages/cli-kit/src/public/node/ai-agent-toolkit.test.ts create mode 100644 packages/cli-kit/src/public/node/ai-agent-toolkit.ts diff --git a/.changeset/suggest-ai-toolkit-install.md b/.changeset/suggest-ai-toolkit-install.md new file mode 100644 index 00000000000..e3534bc3b5e --- /dev/null +++ b/.changeset/suggest-ai-toolkit-install.md @@ -0,0 +1,5 @@ +--- +'@shopify/cli-kit': patch +--- + +Suggest installing the Shopify AI Toolkit when Shopify CLI is run by an AI coding agent (Pi, Claude Code, Codex) that doesn't have it installed diff --git a/packages/cli-kit/src/public/node/ai-agent-toolkit.test.ts b/packages/cli-kit/src/public/node/ai-agent-toolkit.test.ts new file mode 100644 index 00000000000..fb85f386997 --- /dev/null +++ b/packages/cli-kit/src/public/node/ai-agent-toolkit.test.ts @@ -0,0 +1,212 @@ +import { + aiAgentHarnessName, + aiToolkitInstallCommand, + detectAIAgentHarness, + isAIToolkitInstalled, + isRunningInsideAIAgent, + suggestAIToolkitInstallIfNeeded, +} from './ai-agent-toolkit.js' +import {homeDirectory} from './context/local.js' +import {inTemporaryDirectory, mkdir, writeFile} from './fs.js' +import {joinPath} from './path.js' +import {outputInfo} from './output.js' +import {describe, expect, test, vi, beforeEach, afterEach} from 'vitest' + +vi.mock('./context/local.js', async () => { + const actual = await vi.importActual('./context/local.js') + return {...actual, homeDirectory: vi.fn()} +}) +vi.mock('./output.js', async () => { + const actual = await vi.importActual('./output.js') + return {...actual, outputInfo: vi.fn()} +}) + +let originalIsTTY: boolean | undefined + +beforeEach(() => { + originalIsTTY = process.stdout.isTTY + vi.unstubAllEnvs() +}) + +afterEach(() => { + Object.defineProperty(process.stdout, 'isTTY', {value: originalIsTTY, configurable: true, writable: true}) +}) + +describe('detectAIAgentHarness', () => { + test('detects pi via PI_CODING_AGENT', () => { + expect(detectAIAgentHarness({PI_CODING_AGENT: '1'})).toBe('pi') + }) + + test('detects claude code via CLAUDE_CODE', () => { + expect(detectAIAgentHarness({CLAUDE_CODE: '1'})).toBe('claude-code') + }) + + test('detects codex via CODEX_THREAD_ID', () => { + expect(detectAIAgentHarness({CODEX_THREAD_ID: 'thread-123'})).toBe('codex') + }) + + test('returns undefined when no known env var is present', () => { + expect(detectAIAgentHarness({})).toBeUndefined() + }) + + test('ignores empty string values', () => { + expect(detectAIAgentHarness({CLAUDE_CODE: ''})).toBeUndefined() + }) +}) + +describe('isRunningInsideAIAgent', () => { + test('returns true when non-tty and a harness env var is present', () => { + Object.defineProperty(process.stdout, 'isTTY', {value: false, configurable: true, writable: true}) + expect(isRunningInsideAIAgent({CLAUDE_CODE: '1'})).toBe(true) + }) + + test('returns false when tty even if a harness env var is present', () => { + Object.defineProperty(process.stdout, 'isTTY', {value: true, configurable: true, writable: true}) + expect(isRunningInsideAIAgent({CLAUDE_CODE: '1'})).toBe(false) + }) + + test('returns false when non-tty but no harness env var is present', () => { + Object.defineProperty(process.stdout, 'isTTY', {value: false, configurable: true, writable: true}) + expect(isRunningInsideAIAgent({})).toBe(false) + }) +}) + +describe('aiToolkitInstallCommand / aiAgentHarnessName', () => { + test('returns the expected install command and name per harness', () => { + expect(aiToolkitInstallCommand('pi')).toContain('npx skills add') + expect(aiToolkitInstallCommand('claude-code')).toContain('claude plugin install') + expect(aiToolkitInstallCommand('codex')).toContain('codex plugin add') + + expect(aiAgentHarnessName('pi')).toBe('Pi') + expect(aiAgentHarnessName('claude-code')).toBe('Claude Code') + expect(aiAgentHarnessName('codex')).toBe('Codex') + }) +}) + +describe('isAIToolkitInstalled', () => { + test('returns true for pi when a shopify- skill directory exists', async () => { + await inTemporaryDirectory(async (tmpDir) => { + vi.mocked(homeDirectory).mockReturnValue(tmpDir) + await mkdir(joinPath(tmpDir, '.pi', 'agent', 'skills', 'shopify-admin')) + + await expect(isAIToolkitInstalled('pi')).resolves.toBe(true) + }) + }) + + test('returns false for pi when no shopify- skill directory exists', async () => { + await inTemporaryDirectory(async (tmpDir) => { + vi.mocked(homeDirectory).mockReturnValue(tmpDir) + await mkdir(joinPath(tmpDir, '.pi', 'agent', 'skills', 'some-other-skill')) + + await expect(isAIToolkitInstalled('pi')).resolves.toBe(false) + }) + }) + + test('returns false for pi when the skills directory does not exist', async () => { + await inTemporaryDirectory(async (tmpDir) => { + vi.mocked(homeDirectory).mockReturnValue(tmpDir) + + await expect(isAIToolkitInstalled('pi')).resolves.toBe(false) + }) + }) + + test('returns true for claude-code when installed_plugins.json references shopify', async () => { + await inTemporaryDirectory(async (tmpDir) => { + vi.mocked(homeDirectory).mockReturnValue(tmpDir) + await mkdir(joinPath(tmpDir, '.claude', 'plugins')) + await writeFile( + joinPath(tmpDir, '.claude', 'plugins', 'installed_plugins.json'), + JSON.stringify({plugins: ['shopify-ai-toolkit@claude-plugins-official']}), + ) + + await expect(isAIToolkitInstalled('claude-code')).resolves.toBe(true) + }) + }) + + test('returns true for claude-code via the plugin cache fallback', async () => { + await inTemporaryDirectory(async (tmpDir) => { + vi.mocked(homeDirectory).mockReturnValue(tmpDir) + await mkdir(joinPath(tmpDir, '.claude', 'plugins', 'cache', 'claude-plugins-official', 'shopify-ai-toolkit')) + + await expect(isAIToolkitInstalled('claude-code')).resolves.toBe(true) + }) + }) + + test('returns false for claude-code when nothing is installed', async () => { + await inTemporaryDirectory(async (tmpDir) => { + vi.mocked(homeDirectory).mockReturnValue(tmpDir) + + await expect(isAIToolkitInstalled('claude-code')).resolves.toBe(false) + }) + }) + + test('returns true for codex when config.toml references shopify', async () => { + await inTemporaryDirectory(async (tmpDir) => { + vi.mocked(homeDirectory).mockReturnValue(tmpDir) + await mkdir(joinPath(tmpDir, '.codex')) + await writeFile(joinPath(tmpDir, '.codex', 'config.toml'), '[plugins.shopify]\nenabled = true\n') + + await expect(isAIToolkitInstalled('codex')).resolves.toBe(true) + }) + }) + + test('returns false for codex when nothing is installed', async () => { + await inTemporaryDirectory(async (tmpDir) => { + vi.mocked(homeDirectory).mockReturnValue(tmpDir) + + await expect(isAIToolkitInstalled('codex')).resolves.toBe(false) + }) + }) +}) + +describe('suggestAIToolkitInstallIfNeeded', () => { + beforeEach(() => { + vi.mocked(outputInfo).mockClear() + }) + + test('prints a suggestion when running inside an agent without the toolkit installed', async () => { + await inTemporaryDirectory(async (tmpDir) => { + Object.defineProperty(process.stdout, 'isTTY', {value: false, configurable: true, writable: true}) + vi.mocked(homeDirectory).mockReturnValue(tmpDir) + + await suggestAIToolkitInstallIfNeeded({CLAUDE_CODE: '1'}) + + expect(outputInfo).toHaveBeenCalledOnce() + expect(vi.mocked(outputInfo).mock.calls[0]?.[0]).toContain('claude plugin install') + }) + }) + + test('does not print when running interactively', async () => { + await inTemporaryDirectory(async (tmpDir) => { + Object.defineProperty(process.stdout, 'isTTY', {value: true, configurable: true, writable: true}) + vi.mocked(homeDirectory).mockReturnValue(tmpDir) + + await suggestAIToolkitInstallIfNeeded({CLAUDE_CODE: '1'}) + + expect(outputInfo).not.toHaveBeenCalled() + }) + }) + + test('does not print when no harness is detected', async () => { + await inTemporaryDirectory(async (tmpDir) => { + Object.defineProperty(process.stdout, 'isTTY', {value: false, configurable: true, writable: true}) + vi.mocked(homeDirectory).mockReturnValue(tmpDir) + + await suggestAIToolkitInstallIfNeeded({}) + + expect(outputInfo).not.toHaveBeenCalled() + }) + }) + + test('does not print when the toolkit is already installed', async () => { + await inTemporaryDirectory(async (tmpDir) => { + Object.defineProperty(process.stdout, 'isTTY', {value: false, configurable: true, writable: true}) + vi.mocked(homeDirectory).mockReturnValue(tmpDir) + await mkdir(joinPath(tmpDir, '.pi', 'agent', 'skills', 'shopify-admin')) + + await suggestAIToolkitInstallIfNeeded({PI_CODING_AGENT: '1'}) + + expect(outputInfo).not.toHaveBeenCalled() + }) + }) +}) diff --git a/packages/cli-kit/src/public/node/ai-agent-toolkit.ts b/packages/cli-kit/src/public/node/ai-agent-toolkit.ts new file mode 100644 index 00000000000..9e4e1b0e3be --- /dev/null +++ b/packages/cli-kit/src/public/node/ai-agent-toolkit.ts @@ -0,0 +1,196 @@ +import {homeDirectory} from './context/local.js' +import {fileExists, readFile, readdir} from './fs.js' +import {joinPath} from './path.js' +import {outputInfo} from './output.js' + +/** + * The AI coding agent harnesses that the CLI knows how to detect. + */ +export type AIAgentHarness = 'pi' | 'claude-code' | 'codex' + +interface HarnessDefinition { + /** + * Human friendly name of the harness, used in messaging. + */ + name: string + /** + * The environment variable that, when present, indicates the harness is running the command. + */ + envVar: string + /** + * The shell command that installs the Shopify AI Toolkit for this harness. + */ + installCommand: string + /** + * Returns true if the Shopify AI Toolkit is already installed for this harness. + */ + isInstalled: () => Promise +} + +async function directoryEntries(path: string): Promise { + if (!(await fileExists(path))) return [] + try { + return await readdir(path) + // eslint-disable-next-line no-catch-all/no-catch-all + } catch { + return [] + } +} + +async function fileContains(path: string, needle: string): Promise { + if (!(await fileExists(path))) return false + try { + const content = await readFile(path) + return content.includes(needle) + // eslint-disable-next-line no-catch-all/no-catch-all + } catch { + return false + } +} + +async function isPiToolkitInstalled(): Promise { + // The AI Toolkit isn't distributed as a Pi plugin. For Pi, it's installed as a set of + // agent skills (e.g. via `npx skills add Shopify/shopify-ai-toolkit`), which are symlinked + // or copied into ~/.pi/agent/skills using the `shopify-` prefix. + const skillsDirectory = joinPath(homeDirectory(), '.pi', 'agent', 'skills') + const entries = await directoryEntries(skillsDirectory) + return entries.some((entry) => entry.startsWith('shopify-')) +} + +async function isClaudeCodeToolkitInstalled(): Promise { + const installedPluginsFile = joinPath(homeDirectory(), '.claude', 'plugins', 'installed_plugins.json') + if (await fileContains(installedPluginsFile, 'shopify')) return true + + // Fall back to checking the plugin download cache in case the installed plugins manifest + // doesn't exist or uses a different shape than expected. + const cacheDirectory = joinPath(homeDirectory(), '.claude', 'plugins', 'cache') + const marketplaces = await directoryEntries(cacheDirectory) + for (const marketplace of marketplaces) { + // eslint-disable-next-line no-await-in-loop + const plugins = await directoryEntries(joinPath(cacheDirectory, marketplace)) + if (plugins.some((plugin) => plugin.includes('shopify'))) return true + } + return false +} + +async function isCodexToolkitInstalled(): Promise { + const configFile = joinPath(homeDirectory(), '.codex', 'config.toml') + if (await fileContains(configFile, 'shopify')) return true + + // Fall back to checking the plugin download cache in case config.toml doesn't reference + // the plugin directly. + const cacheDirectory = joinPath(homeDirectory(), '.codex', 'plugins', 'cache') + const marketplaces = await directoryEntries(cacheDirectory) + for (const marketplace of marketplaces) { + // eslint-disable-next-line no-await-in-loop + const plugins = await directoryEntries(joinPath(cacheDirectory, marketplace)) + if (plugins.some((plugin) => plugin.includes('shopify'))) return true + } + return false +} + +const harnessDefinitions: {[harness in AIAgentHarness]: HarnessDefinition} = { + pi: { + name: 'Pi', + envVar: 'PI_CODING_AGENT', + installCommand: 'npx skills add Shopify/shopify-ai-toolkit', + isInstalled: isPiToolkitInstalled, + }, + 'claude-code': { + name: 'Claude Code', + envVar: 'CLAUDE_CODE', + installCommand: 'claude plugin install shopify-ai-toolkit@claude-plugins-official', + isInstalled: isClaudeCodeToolkitInstalled, + }, + codex: { + name: 'Codex', + envVar: 'CODEX_THREAD_ID', + installCommand: 'codex plugin add shopify@openai-curated', + isInstalled: isCodexToolkitInstalled, + }, +} + +/** + * Detects which AI coding agent harness (if any) is running the current process, based on + * well-known environment variables set by each harness. + * + * @param env - The environment variables from the environment of the current process. + * @returns The detected harness, or undefined if none was detected. + */ +export function detectAIAgentHarness(env = process.env): AIAgentHarness | undefined { + return (Object.keys(harnessDefinitions) as AIAgentHarness[]).find((harness) => { + const value = env[harnessDefinitions[harness].envVar] + return value !== undefined && value !== '' + }) +} + +/** + * Returns true if the CLI is likely being driven by an AI coding agent rather than a human in + * an interactive terminal. This is a best-effort heuristic based on the absence of a TTY + * combined with the presence of a known agent harness environment variable. + * + * @param env - The environment variables from the environment of the current process. + * @returns True if the current process appears to be running inside an AI coding agent. + */ +export function isRunningInsideAIAgent(env = process.env): boolean { + return !process.stdout.isTTY && detectAIAgentHarness(env) !== undefined +} + +/** + * Checks whether the Shopify AI Toolkit is installed for the given harness. + * + * @param harness - The AI agent harness to check. + * @returns True if the toolkit appears to be installed. + */ +export async function isAIToolkitInstalled(harness: AIAgentHarness): Promise { + try { + return await harnessDefinitions[harness].isInstalled() + // eslint-disable-next-line no-catch-all/no-catch-all + } catch { + return false + } +} + +/** + * Returns the shell command an AI agent should run to install the Shopify AI Toolkit for the + * given harness. + * + * @param harness - The AI agent harness. + * @returns The install command for the harness. + */ +export function aiToolkitInstallCommand(harness: AIAgentHarness): string { + return harnessDefinitions[harness].installCommand +} + +/** + * Returns the human-friendly display name for a harness. + * + * @param harness - The AI agent harness. + * @returns The display name for the harness. + */ +export function aiAgentHarnessName(harness: AIAgentHarness): string { + return harnessDefinitions[harness].name +} + +/** + * If the current process appears to be running inside a known AI coding agent and the Shopify + * AI Toolkit isn't installed for that agent, prints a message telling the agent how to install + * it. This is a no-op when running interactively, when no known harness is detected, or when the + * toolkit is already installed. + * + * @param env - The environment variables from the environment of the current process. + */ +export async function suggestAIToolkitInstallIfNeeded(env = process.env): Promise { + if (!isRunningInsideAIAgent(env)) return + + const harness = detectAIAgentHarness(env) + if (!harness) return + + if (await isAIToolkitInstalled(harness)) return + + outputInfo( + `This command is running inside ${aiAgentHarnessName(harness)}. For better results when working with Shopify APIs, install the Shopify AI Toolkit by running \`${aiToolkitInstallCommand( + harness, + )}\`. Learn more: https://shopify.dev/docs/apps/build/ai-toolkit`, + ) +} diff --git a/packages/cli-kit/src/public/node/base-command.ts b/packages/cli-kit/src/public/node/base-command.ts index 2cb2cf4b5a2..3b7e8025a74 100644 --- a/packages/cli-kit/src/public/node/base-command.ts +++ b/packages/cli-kit/src/public/node/base-command.ts @@ -47,6 +47,8 @@ abstract class BaseCommand extends Command { protected async init(): Promise { this.exitWithTimestampWhenEnvVariablePresent() + const {suggestAIToolkitInstallIfNeeded} = await import('./ai-agent-toolkit.js') + await suggestAIToolkitInstallIfNeeded() setCurrentCommandId(this.id ?? '') if (!isDevelopment()) { // This function runs just prior to `run`