From 49c5484f12017317a3d39a41fceb9514c042385e Mon Sep 17 00:00:00 2001 From: Developers Digest <124798203+developersdigest@users.noreply.github.com> Date: Wed, 25 Mar 2026 16:20:59 -0400 Subject: [PATCH] add native skill installer and post-install setup flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - new skills-native.ts: replicates `npx skills add` without requiring Node.js — clones repo, copies skills to ~/.agents/skills/, creates relative symlinks to detected agent dirs (claude, cursor, etc.) - setup.ts + init.ts: try npx first, fall back to native installer - install.sh: after binary installs, prompt to continue with `firecrawl init --skip-install` for login + skills + integrations --- scripts/install.sh | 30 ++- src/commands/init.ts | 77 ++++-- src/commands/setup.ts | 33 ++- src/commands/skills-native.ts | 471 ++++++++++++++++++++++++++++++++++ 4 files changed, 572 insertions(+), 39 deletions(-) create mode 100644 src/commands/skills-native.ts diff --git a/scripts/install.sh b/scripts/install.sh index 5cc1e45..046e29d 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -204,9 +204,33 @@ main() { echo "" success "Firecrawl CLI v${version} installed successfully!" echo "" - echo " Run 'firecrawl --help' to get started." - echo " Run 'firecrawl login' to authenticate with your API key." - echo "" + + # Resolve the binary path (may need updated PATH) + local firecrawl_bin="$install_dir/firecrawl" + + # Offer to continue with setup (login, skills, integrations) + if [ -t 0 ] && [ -t 1 ]; then + # Interactive terminal — prompt + echo " Next: authenticate and install AI coding skills." + echo "" + printf " Continue with setup? [Y/n] " + read -r answer { switch (integration) { case 'skills': { console.log(`\n Setting up skills...`); - const args = buildSkillsInstallArgs({ - agent: options.agent, - yes: options.yes || options.all, - global: true, - includeNpxYes: true, - }); - try { - execSync(args.join(' '), { stdio: 'inherit' }); - console.log(` ${green}✓${reset} Skills installed`); - } catch { - console.error( - ' Failed to install skills. Run "firecrawl setup skills" later.' - ); + if (hasNpx()) { + const args = buildSkillsInstallArgs({ + agent: options.agent, + yes: options.yes || options.all, + global: true, + includeNpxYes: true, + }); + try { + execSync(args.join(' '), { stdio: 'inherit' }); + console.log(` ${green}✓${reset} Skills installed`); + } catch { + console.error( + ' Failed to install skills. Run "firecrawl setup skills" later.' + ); + } + } else { + try { + await installSkillsNative(); + } catch { + console.error( + ' Failed to install skills. Run "firecrawl setup skills" later.' + ); + } } break; } @@ -616,20 +627,32 @@ async function runNonInteractive(options: InitOptions): Promise { console.log( `${stepLabel()} Installing firecrawl skill for AI coding agents...` ); - const args = buildSkillsInstallArgs({ - agent: options.agent, - yes: true, - global: true, - includeNpxYes: true, - }); - try { - execSync(args.join(' '), { stdio: 'inherit' }); - console.log(`${green}✓${reset} Skills installed\n`); - } catch { - console.error( - '\nFailed to install skills. You can retry with: firecrawl setup skills' - ); - process.exit(1); + if (hasNpx()) { + const args = buildSkillsInstallArgs({ + agent: options.agent, + yes: true, + global: true, + includeNpxYes: true, + }); + try { + execSync(args.join(' '), { stdio: 'inherit' }); + console.log(`${green}✓${reset} Skills installed\n`); + } catch { + console.error( + '\nFailed to install skills. You can retry with: firecrawl setup skills' + ); + process.exit(1); + } + } else { + try { + await installSkillsNative(); + console.log(''); + } catch { + console.error( + '\nFailed to install skills. You can retry with: firecrawl setup skills' + ); + process.exit(1); + } } } diff --git a/src/commands/setup.ts b/src/commands/setup.ts index a9c0379..820fb28 100644 --- a/src/commands/setup.ts +++ b/src/commands/setup.ts @@ -6,6 +6,7 @@ import { execSync } from 'child_process'; import { getApiKey } from '../utils/config'; import { buildSkillsInstallArgs } from './skills-install'; +import { hasNpx, installSkillsNative } from './skills-native'; export type SetupSubcommand = 'skills' | 'mcp'; @@ -40,18 +41,32 @@ export async function handleSetupCommand( } async function installSkills(options: SetupOptions): Promise { - const args = buildSkillsInstallArgs({ - agent: options.agent, - global: true, - includeNpxYes: true, - }); + if (hasNpx()) { + const args = buildSkillsInstallArgs({ + agent: options.agent, + global: true, + includeNpxYes: true, + }); - const cmd = args.join(' '); - console.log(`Running: ${cmd}\n`); + const cmd = args.join(' '); + console.log(`Running: ${cmd}\n`); + try { + execSync(cmd, { stdio: 'inherit' }); + return; + } catch { + process.exit(1); + } + } + + // Fallback: native install (no npx/Node required) try { - execSync(cmd, { stdio: 'inherit' }); - } catch { + await installSkillsNative(); + } catch (error) { + console.error( + 'Failed to install skills:', + error instanceof Error ? error.message : 'Unknown error' + ); process.exit(1); } } diff --git a/src/commands/skills-native.ts b/src/commands/skills-native.ts new file mode 100644 index 0000000..a9b03af --- /dev/null +++ b/src/commands/skills-native.ts @@ -0,0 +1,471 @@ +/** + * Native skill installation — replicates `npx skills add firecrawl/cli --full-depth --global --all` + * without requiring Node.js or npx. Used as a fallback for binary installs. + */ + +import { execSync } from 'child_process'; +import fs from 'fs'; +import path from 'path'; +import os from 'os'; + +const green = '\x1b[32m'; +const dim = '\x1b[2m'; +const bold = '\x1b[1m'; +const reset = '\x1b[0m'; + +const REPO = 'firecrawl/cli'; +const REPO_URL = `https://github.com/${REPO}.git`; +const SKILLS_SUBDIR = 'skills'; + +/** Where each agent stores global skills */ +interface AgentConfig { + name: string; + globalSkillsDir: string; + /** Directory to check for existence (relative to HOME) */ + detectDir: string; +} + +const AGENTS: AgentConfig[] = [ + { + name: 'claude-code', + globalSkillsDir: '.claude/skills', + detectDir: '.claude', + }, + { + name: 'cursor', + globalSkillsDir: '.cursor/skills', + detectDir: '.cursor', + }, + { + name: 'windsurf', + globalSkillsDir: '.windsurf/skills', + detectDir: '.windsurf', + }, + { + name: 'codex', + globalSkillsDir: '.codex/skills', + detectDir: '.codex', + }, + { + name: 'continue', + globalSkillsDir: '.continue/skills', + detectDir: '.continue', + }, + { + name: 'augment', + globalSkillsDir: '.augment/skills', + detectDir: '.augment', + }, + { + name: 'roo', + globalSkillsDir: '.roo/skills', + detectDir: '.roo', + }, + { + name: 'gemini-cli', + globalSkillsDir: '.gemini/skills', + detectDir: '.gemini', + }, + { + name: 'copilot', + globalSkillsDir: '.copilot/skills', + detectDir: '.copilot', + }, + { + name: 'droid', + globalSkillsDir: '.factory/skills', + detectDir: '.factory', + }, +]; + +/** Canonical directory for skill files — single source of truth */ +const CANONICAL_DIR = '.agents/skills'; +const LOCK_FILE = '.agents/.skill-lock.json'; + +interface SkillEntry { + /** Skill name from SKILL.md frontmatter */ + name: string; + /** Path to the skill directory (in temp clone) */ + srcDir: string; + /** Relative path of SKILL.md within the repo */ + skillPath: string; +} + +interface LockEntry { + source: string; + sourceType: string; + sourceUrl: string; + skillPath: string; + skillFolderHash: string; + installedAt: string; + updatedAt: string; +} + +interface LockFile { + version: number; + skills: Record; +} + +/** + * Parse SKILL.md frontmatter to extract name and description. + * Minimal parser — handles `---` delimited YAML frontmatter. + */ +function parseFrontmatter(content: string): { + name?: string; + description?: string; +} { + const match = content.match(/^---\s*\n([\s\S]*?)\n---/); + if (!match) return {}; + + const yaml = match[1]; + const result: Record = {}; + + for (const line of yaml.split('\n')) { + // Handle single-line key: value + const kv = line.match(/^(\w[\w-]*):\s*(.+)/); + if (kv) { + let val = kv[2].trim(); + // Strip surrounding quotes + if ( + (val.startsWith('"') && val.endsWith('"')) || + (val.startsWith("'") && val.endsWith("'")) + ) { + val = val.slice(1, -1); + } + result[kv[1]] = val; + } + // Handle multi-line description with | + const multiline = line.match(/^(\w[\w-]*):\s*\|/); + if (multiline) { + result[multiline[1]] = '(multiline)'; // just mark as present + } + } + + return { name: result.name, description: result.description }; +} + +/** + * Discover all skills in a directory tree by finding SKILL.md files. + */ +function discoverSkills(baseDir: string): SkillEntry[] { + const skills: SkillEntry[] = []; + const seen = new Set(); + + function walk(dir: string, depth: number) { + if (depth > 5) return; + + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + return; + } + + for (const entry of entries) { + if (entry.name.startsWith('.') || entry.name === 'node_modules') continue; + + const fullPath = path.join(dir, entry.name); + + if (entry.isDirectory()) { + const skillMd = path.join(fullPath, 'SKILL.md'); + if (fs.existsSync(skillMd)) { + const content = fs.readFileSync(skillMd, 'utf-8'); + const fm = parseFrontmatter(content); + if (fm.name && fm.description && !seen.has(fm.name)) { + seen.add(fm.name); + skills.push({ + name: sanitizeName(fm.name), + srcDir: fullPath, + skillPath: path.relative(baseDir, skillMd), + }); + } + } + walk(fullPath, depth + 1); + } + } + } + + walk(baseDir, 0); + return skills; +} + +/** Sanitize skill name: lowercase, replace non-alnum with hyphens */ +function sanitizeName(name: string): string { + return name + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^[-.]|[-.]$/g, '') + .slice(0, 255); +} + +/** Recursively copy a directory, filtering out dotfiles and metadata */ +function copyDir(src: string, dest: string) { + fs.mkdirSync(dest, { recursive: true }); + + for (const entry of fs.readdirSync(src, { withFileTypes: true })) { + if ( + entry.name.startsWith('.') || + entry.name === 'metadata.json' || + entry.name === '__pycache__' + ) { + continue; + } + + const srcPath = path.join(src, entry.name); + const destPath = path.join(dest, entry.name); + + if (entry.isDirectory()) { + copyDir(srcPath, destPath); + } else { + fs.copyFileSync(srcPath, destPath); + } + } +} + +/** Compute a simple hash of a directory's contents for the lock file */ +function hashDir(dir: string): string { + const crypto = require('crypto'); + const hash = crypto.createHash('sha1'); + + function walk(d: string) { + for (const entry of fs + .readdirSync(d, { withFileTypes: true }) + .sort((a, b) => a.name.localeCompare(b.name))) { + if (entry.name.startsWith('.')) continue; + const p = path.join(d, entry.name); + if (entry.isDirectory()) { + walk(p); + } else { + hash.update(entry.name); + hash.update(fs.readFileSync(p)); + } + } + } + + walk(dir); + return hash.digest('hex'); +} + +/** Detect which agents are installed by checking for their config directories */ +function detectInstalledAgents(): AgentConfig[] { + const home = os.homedir(); + return AGENTS.filter((agent) => { + const dir = path.join(home, agent.detectDir); + try { + return fs.statSync(dir).isDirectory(); + } catch { + return false; + } + }); +} + +/** Check if git is available */ +function hasGit(): boolean { + try { + execSync('git --version', { stdio: 'pipe' }); + return true; + } catch { + return false; + } +} + +/** Check if curl or wget is available */ +function hasDownloader(): 'curl' | 'wget' | null { + try { + execSync('curl --version', { stdio: 'pipe' }); + return 'curl'; + } catch { + try { + execSync('wget --version', { stdio: 'pipe' }); + return 'wget'; + } catch { + return null; + } + } +} + +/** Clone repo to temp directory */ +function cloneRepo(tmpDir: string): void { + if (hasGit()) { + execSync(`git clone --depth 1 "${REPO_URL}" "${tmpDir}"`, { + stdio: 'pipe', + }); + return; + } + + // Fallback: download tarball + const downloader = hasDownloader(); + if (!downloader) { + throw new Error('Neither git nor curl/wget found. Cannot download skills.'); + } + + fs.mkdirSync(tmpDir, { recursive: true }); + const tarball = path.join(tmpDir, 'repo.tar.gz'); + const tarballUrl = `https://api.github.com/repos/${REPO}/tarball`; + + if (downloader === 'curl') { + execSync( + `curl -fsSL -o "${tarball}" -H "Accept: application/vnd.github+json" -L "${tarballUrl}"`, + { stdio: 'pipe' } + ); + } else { + execSync( + `wget -q -O "${tarball}" --header="Accept: application/vnd.github+json" "${tarballUrl}"`, + { stdio: 'pipe' } + ); + } + + execSync(`tar -xzf "${tarball}" -C "${tmpDir}" --strip-components=1`, { + stdio: 'pipe', + }); + fs.unlinkSync(tarball); +} + +/** + * Install skills natively — no npx required. + * + * Replicates: npx skills add firecrawl/cli --full-depth --global --all + */ +export async function installSkillsNative(): Promise { + const home = os.homedir(); + const canonicalBase = path.join(home, CANONICAL_DIR); + const lockFilePath = path.join(home, LOCK_FILE); + + // Clone repo + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'firecrawl-skills-')); + + try { + console.log( + ` ${dim}Downloading skills from github.com/${REPO}...${reset}` + ); + cloneRepo(tmpDir); + + // Discover skills + const skillsDir = path.join(tmpDir, SKILLS_SUBDIR); + if (!fs.existsSync(skillsDir)) { + throw new Error(`No ${SKILLS_SUBDIR}/ directory found in repository`); + } + + const skills = discoverSkills(skillsDir); + if (skills.length === 0) { + throw new Error('No skills found in repository'); + } + + console.log(` ${dim}Found ${skills.length} skills${reset}`); + + // Copy skills to canonical directory + fs.mkdirSync(canonicalBase, { recursive: true }); + + for (const skill of skills) { + const destDir = path.join(canonicalBase, skill.name); + + // Remove existing and copy fresh + if (fs.existsSync(destDir)) { + fs.rmSync(destDir, { recursive: true, force: true }); + } + copyDir(skill.srcDir, destDir); + } + + // Detect installed agents and create symlinks + const agents = detectInstalledAgents(); + const linkedAgents: string[] = []; + + for (const agent of agents) { + const agentSkillsDir = path.join(home, agent.globalSkillsDir); + fs.mkdirSync(agentSkillsDir, { recursive: true }); + + for (const skill of skills) { + const linkPath = path.join(agentSkillsDir, skill.name); + const canonicalPath = path.join(canonicalBase, skill.name); + + // Skip if already a correct symlink + try { + const existing = fs.readlinkSync(linkPath); + const expectedTarget = path.relative( + path.dirname(linkPath), + canonicalPath + ); + if (existing === expectedTarget) continue; + } catch { + // Not a symlink — remove if exists + } + + // Remove existing (file, dir, or broken symlink) + try { + const stat = fs.lstatSync(linkPath); + if (stat.isSymbolicLink() || stat.isFile()) { + fs.unlinkSync(linkPath); + } else if (stat.isDirectory()) { + fs.rmSync(linkPath, { recursive: true, force: true }); + } + } catch { + // Doesn't exist — fine + } + + // Create relative symlink + const relTarget = path.relative(agentSkillsDir, canonicalPath); + try { + fs.symlinkSync(relTarget, linkPath); + } catch { + // Symlink failed — fall back to copy + copyDir(canonicalPath, linkPath); + } + } + + linkedAgents.push(agent.name); + } + + // Update lock file + let lock: LockFile = { version: 3, skills: {} }; + try { + if (fs.existsSync(lockFilePath)) { + lock = JSON.parse(fs.readFileSync(lockFilePath, 'utf-8')); + } + } catch { + // Corrupted lock file — start fresh + } + + const now = new Date().toISOString(); + for (const skill of skills) { + const canonicalPath = path.join(canonicalBase, skill.name); + const existing = lock.skills[skill.name]; + lock.skills[skill.name] = { + source: REPO, + sourceType: 'github', + sourceUrl: REPO_URL, + skillPath: skill.skillPath, + skillFolderHash: hashDir(canonicalPath), + installedAt: existing?.installedAt ?? now, + updatedAt: now, + }; + } + + fs.mkdirSync(path.dirname(lockFilePath), { recursive: true }); + fs.writeFileSync(lockFilePath, JSON.stringify(lock, null, 2) + '\n'); + + // Summary + console.log( + ` ${green}✓${reset} ${skills.length} skills installed to ${dim}~/${CANONICAL_DIR}/${reset}` + ); + if (linkedAgents.length > 0) { + console.log(` ${green}✓${reset} Linked to: ${linkedAgents.join(', ')}`); + } + } finally { + // Clean up temp directory + try { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } catch { + // Best effort cleanup + } + } +} + +/** Check if npx is available */ +export function hasNpx(): boolean { + try { + execSync('npx --version', { stdio: 'pipe' }); + return true; + } catch { + return false; + } +}