Skip to content

API_SERVICE_AND_SDK_PLAN.md

Latest commit

 

History

History
620 lines (516 loc) · 16.4 KB

API_SERVICE_AND_SDK_PLAN.md

File metadata and controls

620 lines (516 loc) · 16.4 KB

API Service & SDK Development Plan

Executive Summary

Transform the CLI automation tool into a headless API service with a TypeScript/JavaScript SDK for easy integration into any frontend application.

🎯 Architecture Overview

Simplified Architecture (No Frontend)

┌─────────────────────────────────────────────────────────────┐
│                    Client Applications                       │
│         (Your Frontend, Mobile Apps, Other Services)         │
└─────────────────────────────────────────────────────────────┘
                               │
                               ▼
┌─────────────────────────────────────────────────────────────┐
│                         SDK Layer                            │
│      TypeScript/JavaScript SDK with full type safety         │
└─────────────────────────────────────────────────────────────┘
                               │
                               ▼
┌─────────────────────────────────────────────────────────────┐
│                    API Service (Node.js)                     │
│     REST API • WebSockets • Authentication • Rate Limiting   │
└─────────────────────────────────────────────────────────────┘
                               │
                    ┌──────────┴──────────┐
                    ▼                      ▼
┌─────────────────────────┐    ┌─────────────────────────────┐
│   Queue System (Redis)  │    │   Database (Supabase)       │
│  Job Queue • Scheduling │    │  Profiles • Campaigns • Logs │
└─────────────────────────┘    └─────────────────────────────┘
                    │
                    ▼
┌─────────────────────────────────────────────────────────────┐
│              Worker Pool (Separate Instances)                │
│     Playwright Workers • Browser Sessions • Automation       │
└─────────────────────────────────────────────────────────────┘

📦 SDK Features

Core SDK Structure

// SDK Package: @your-org/automation-sdk

export class AutomationClient {
  constructor(config: ClientConfig)

  // Profile Management
  profiles: ProfileManager

  // Campaign Management
  campaigns: CampaignManager

  // Submission Management
  submissions: SubmissionManager

  // Real-time Updates
  realtime: RealtimeManager

  // Analytics
  analytics: AnalyticsManager
}

SDK Implementation

// src/sdk/index.ts
import axios, { AxiosInstance } from 'axios';
import { io, Socket } from 'socket.io-client';
import { EventEmitter } from 'events';

export interface ClientConfig {
  apiKey: string;
  apiUrl?: string;
  websocketUrl?: string;
  timeout?: number;
  retryAttempts?: number;
}

export class AutomationSDK extends EventEmitter {
  private api: AxiosInstance;
  private socket: Socket | null = null;
  private config: ClientConfig;

  constructor(config: ClientConfig) {
    super();
    this.config = {
      apiUrl: 'https://api.yourdomain.com',
      websocketUrl: 'wss://api.yourdomain.com',
      timeout: 30000,
      retryAttempts: 3,
      ...config
    };

    this.api = axios.create({
      baseURL: this.config.apiUrl,
      timeout: this.config.timeout,
      headers: {
        'X-API-Key': this.config.apiKey,
        'Content-Type': 'application/json'
      }
    });

    this.setupInterceptors();
  }

  // Profile Management
  profiles = {
    list: (params?: ProfileListParams) =>
      this.request<Profile[]>('/profiles', { params }),

    get: (id: string) =>
      this.request<Profile>(`/profiles/${id}`),

    create: (data: CreateProfileDto) =>
      this.request<Profile>('/profiles', { method: 'POST', data }),

    update: (id: string, data: UpdateProfileDto) =>
      this.request<Profile>(`/profiles/${id}`, { method: 'PUT', data }),

    delete: (id: string) =>
      this.request<void>(`/profiles/${id}`, { method: 'DELETE' }),

    sync: (id: string) =>
      this.request<SyncResult>(`/profiles/${id}/sync`, { method: 'POST' })
  };

  // Campaign Management
  campaigns = {
    list: (params?: CampaignListParams) =>
      this.request<Campaign[]>('/campaigns', { params }),

    create: (data: CreateCampaignDto) =>
      this.request<Campaign>('/campaigns', { method: 'POST', data }),

    start: (id: string) =>
      this.request<CampaignStatus>(`/campaigns/${id}/start`, { method: 'POST' }),

    pause: (id: string) =>
      this.request<CampaignStatus>(`/campaigns/${id}/pause`, { method: 'POST' }),

    status: (id: string) =>
      this.request<CampaignStatus>(`/campaigns/${id}/status`)
  };

  // Submission Management
  submissions = {
    create: (data: CreateSubmissionDto) =>
      this.request<Submission>('/submissions', { method: 'POST', data }),

    get: (id: string) =>
      this.request<Submission>(`/submissions/${id}`),

    list: (params?: SubmissionListParams) =>
      this.request<Submission[]>('/submissions', { params }),

    logs: (id: string) =>
      this.request<SubmissionLog[]>(`/submissions/${id}/logs`),

    retry: (id: string) =>
      this.request<Submission>(`/submissions/${id}/retry`, { method: 'POST' }),

    // Real-time submission tracking
    track: (submissionId: string, callback: (update: SubmissionUpdate) => void) => {
      this.connectWebSocket();
      this.socket?.emit('subscribe-submission', submissionId);
      this.on(`submission-${submissionId}`, callback);

      return () => {
        this.socket?.emit('unsubscribe-submission', submissionId);
        this.off(`submission-${submissionId}`, callback);
      };
    }
  };

  // WebSocket Connection
  private connectWebSocket() {
    if (this.socket?.connected) return;

    this.socket = io(this.config.websocketUrl!, {
      auth: { apiKey: this.config.apiKey }
    });

    this.socket.on('connect', () => {
      this.emit('connected');
    });

    this.socket.on('disconnect', () => {
      this.emit('disconnected');
    });

    this.socket.on('submission-update', (data) => {
      this.emit(`submission-${data.submissionId}`, data);
    });
  }

  // Batch Operations
  batch = {
    submitMultiple: async (submissions: CreateSubmissionDto[]) => {
      return this.request<BatchResult>('/batch/submissions', {
        method: 'POST',
        data: { submissions }
      });
    },

    createCampaign: async (campaign: CreateCampaignDto) => {
      const result = await this.campaigns.create(campaign);
      await this.campaigns.start(result.id);
      return this.submissions.track(result.id, (update) => {
        this.emit('campaign-progress', { campaignId: result.id, ...update });
      });
    }
  };

  // Helper method for API requests
  private async request<T>(url: string, options?: any): Promise<T> {
    const response = await this.api.request<ApiResponse<T>>({
      url,
      ...options
    });
    return response.data.data;
  }

  // Cleanup
  destroy() {
    this.socket?.disconnect();
    this.removeAllListeners();
  }
}

🚀 Implementation Phases (Simplified)

Phase 1: API Service (2 weeks)

Directory Structure

src/
├── api/
│   ├── routes/
│   │   ├── auth.routes.ts
│   │   ├── profiles.routes.ts
│   │   ├── campaigns.routes.ts
│   │   ├── submissions.routes.ts
│   │   └── webhooks.routes.ts
│   ├── middleware/
│   │   ├── apiKey.middleware.ts
│   │   ├── rateLimiter.ts
│   │   └── validation.ts
│   ├── services/
│   │   └── queue.service.ts
│   └── server.ts
├── workers/
│   ├── automation.worker.ts
│   └── pool.manager.ts
├── sdk/
│   ├── index.ts
│   ├── types/
│   └── package.json
└── shared/
    └── types/

API Authentication (API Keys)

// Simplified API key authentication
interface ApiKey {
  id: string;
  key: string;
  userId: string;
  name: string;
  permissions: string[];
  rateLimit: number;
  createdAt: Date;
  lastUsed: Date;
}

// Middleware
export const authenticateApiKey = async (req, res, next) => {
  const apiKey = req.headers['x-api-key'] || req.query.api_key;

  if (!apiKey) {
    return res.status(401).json({ error: 'API key required' });
  }

  const keyData = await validateApiKey(apiKey);
  if (!keyData) {
    return res.status(401).json({ error: 'Invalid API key' });
  }

  req.user = keyData;
  next();
};

Phase 2: Worker Pool (1 week)

Worker Management

// Worker pool configuration
const workerPool = {
  minWorkers: 2,
  maxWorkers: 10,
  workerIdleTimeout: 300000, // 5 minutes
  jobTimeout: 600000, // 10 minutes
  autoScale: true
};

// Worker deployment via Docker

Phase 3: SDK Development (1 week)

SDK Package Setup

// sdk/package.json
{
  "name": "@your-org/automation-sdk",
  "version": "1.0.0",
  "main": "dist/index.js",
  "types": "dist/index.d.ts",
  "files": ["dist"],
  "dependencies": {
    "axios": "^1.6.0",
    "socket.io-client": "^4.5.0"
  },
  "scripts": {
    "build": "tsc",
    "prepublishOnly": "npm run build"
  }
}

SDK Usage Example

// In your frontend application
import { AutomationSDK } from '@your-org/automation-sdk';

const client = new AutomationSDK({
  apiKey: process.env.AUTOMATION_API_KEY
});

// Create a profile
const profile = await client.profiles.create({
  name: 'Marketing Account',
  platform: 'twitter',
  credentials: { /* ... */ }
});

// Submit content
const submission = await client.submissions.create({
  profileId: profile.id,
  content: {
    title: 'Check out our new product!',
    description: 'Amazing features...',
    url: 'https://example.com'
  }
});

// Track progress in real-time
const unsubscribe = client.submissions.track(submission.id, (update) => {
  console.log('Progress:', update.progress);
  console.log('Status:', update.status);
});

// Cleanup when done
unsubscribe();

🔑 API Endpoints

RESTful API

# Authentication
POST   /api/keys/generate    # Generate new API key
GET    /api/keys             # List API keys
DELETE /api/keys/:id         # Revoke API key

# Profiles
GET    /api/profiles
POST   /api/profiles
GET    /api/profiles/:id
PUT    /api/profiles/:id
DELETE /api/profiles/:id

# Campaigns
GET    /api/campaigns
POST   /api/campaigns
GET    /api/campaigns/:id
PUT    /api/campaigns/:id
DELETE /api/campaigns/:id
POST   /api/campaigns/:id/start
POST   /api/campaigns/:id/pause

# Submissions
GET    /api/submissions
POST   /api/submissions
GET    /api/submissions/:id
GET    /api/submissions/:id/logs
POST   /api/submissions/:id/retry

# Webhooks
POST   /api/webhooks
GET    /api/webhooks
PUT    /api/webhooks/:id
DELETE /api/webhooks/:id

# Analytics
GET    /api/analytics/usage
GET    /api/analytics/success-rate
GET    /api/analytics/platforms

WebSocket Events

// Client -> Server
'subscribe-submission'    // Subscribe to submission updates
'subscribe-campaign'      // Subscribe to campaign updates

// Server -> Client
'submission-progress'     // Progress updates (0-100)
'submission-completed'    // Submission successful
'submission-failed'       // Submission failed
'campaign-update'         // Campaign status change

🚢 Deployment Strategy

API Service Deployment

# Option 1: Railway.app (Recommended)
- Easy deployment from GitHub
- Built-in Redis support
- WebSocket support
- Auto-scaling
- $5/month start

# Option 2: Fly.io
- Global edge deployment
- Built-in Redis
- WebSocket support
- Pay-per-use pricing

# Option 3: Digital Ocean App Platform
- Simple deployment
- Managed databases
- $5/month minimum

Worker Deployment

# Docker Compose for Workers
version: '3.8'
services:
  worker:
    image: your-org/automation-worker
    environment:
      - REDIS_URL=${REDIS_URL}
      - DATABASE_URL=${DATABASE_URL}
    deploy:
      replicas: 3
    restart: unless-stopped

📊 SDK Features Comparison

Feature Basic SDK Pro SDK Enterprise SDK
REST API Access ✅ ✅ ✅
WebSocket Updates ❌ ✅ ✅
Batch Operations ❌ ✅ ✅
Retry Logic ❌ ✅ ✅
Offline Queue ❌ ❌ ✅
Custom Endpoints ❌ ❌ ✅
SLA ❌ ❌ ✅

💰 Pricing Model

API Pricing Tiers

const pricingTiers = {
  free: {
    price: 0,
    requests: 1000,      // per month
    submissions: 10,     // per month
    rateLimit: 10        // requests per minute
  },
  developer: {
    price: 29,
    requests: 10000,
    submissions: 100,
    rateLimit: 60,
    features: ['webhooks', 'batch_api']
  },
  business: {
    price: 99,
    requests: 100000,
    submissions: 1000,
    rateLimit: 300,
    features: ['webhooks', 'batch_api', 'priority_queue']
  },
  enterprise: {
    price: 'custom',
    requests: 'unlimited',
    submissions: 'unlimited',
    rateLimit: 'custom',
    features: ['all', 'sla', 'dedicated_support']
  }
};

🔒 Security

API Security Measures

  1. API Key Authentication: Secure key generation and management
  2. Rate Limiting: Per-key and per-IP limits
  3. Request Signing: Optional HMAC signing for sensitive operations
  4. IP Whitelisting: Enterprise feature
  5. Audit Logging: All API calls logged
  6. Encryption: TLS for transit, AES for sensitive data at rest

📈 Monitoring & Analytics

Built-in Analytics

// API provides analytics endpoints
GET /api/analytics/usage
{
  "period": "30d",
  "total_requests": 15234,
  "total_submissions": 432,
  "success_rate": 0.94,
  "by_platform": {
    "twitter": 150,
    "linkedin": 120,
    "producthunt": 162
  }
}

🎯 Quick Start

1. Deploy API Service

# Clone and setup
git clone <your-repo>
cd auto-crawler

# Install dependencies
npm install

# Configure environment
cp .env.example .env.api
# Edit .env.api with your settings

# Build
npm run build:api

# Deploy to Railway
railway up

2. Deploy Workers

# Build worker image
docker build -t automation-worker -f Dockerfile.worker .

# Deploy to your VPS or container service
docker run -d \
  --name worker-1 \
  --env-file .env.worker \
  automation-worker

3. Publish SDK

cd sdk
npm run build
npm publish

4. Use in Frontend

// Install SDK
npm install @your-org/automation-sdk

// Use in your app
import { AutomationSDK } from '@your-org/automation-sdk';

const client = new AutomationSDK({
  apiKey: 'your-api-key'
});

// Start automating!

📅 Timeline (Reduced)

Phase Duration Description
API Development 2 weeks Core API with all endpoints
Worker Setup 1 week Docker-based worker pool
SDK Development 1 week TypeScript SDK with full types
Testing 1 week Integration testing
Documentation 3 days API docs, SDK docs
Deployment 2 days Production deployment

Total: 5-6 weeks (vs 10-11 weeks with frontend)

🚀 Advantages of API + SDK Approach

  1. Faster Development: No frontend = 50% less work
  2. Greater Flexibility: Any frontend can integrate
  3. Better Scalability: API-first design
  4. Easier Maintenance: Single codebase for backend
  5. Multiple Integrations: Mobile apps, CLI tools, other services
  6. Revenue Opportunity: Sell API access as SaaS