diff --git a/.env.example b/.env.example index fd605b32..a7e7bba2 100644 --- a/.env.example +++ b/.env.example @@ -1,29 +1,8 @@ -# ============================================ -# OpenMemory - Environment Configuration -# ============================================ +# OpenMemory JS server environment -# -------------------------------------------- -# Backend Server Settings -# -------------------------------------------- OM_PORT=8080 - -# API Authentication (IMPORTANT: Set a strong API key for production!) -# Generate a secure key: openssl rand -base64 32 -# Leave empty to disable authentication (development only) -OM_API_KEY=your-secret-api-key-here - -# Rate Limiting -# Enable rate limiting to prevent abuse -OM_RATE_LIMIT_ENABLED=true -# Time window in milliseconds (default: 60000 = 1 minute) -OM_RATE_LIMIT_WINDOW_MS=60000 -# Maximum requests per window (default: 100 requests per minute) -OM_RATE_LIMIT_MAX_REQUESTS=100 - -# Optional: Log all authenticated requests (set to 'true' for debugging) +OM_API_KEY= OM_LOG_AUTH=false - -# Telemetry (true by default, set to false to opt out of anonymous ping) OM_TELEMETRY=true # Server Mode @@ -44,198 +23,58 @@ OM_PG_USER=postgres OM_PG_PASSWORD=postgres OM_PG_SCHEMA=public OM_PG_TABLE=openmemory_memories -OM_PG_SSL=disable # disable | require +OM_PG_SSL=disable -# -------------------------------------------- -# Vector Store Backend -# -------------------------------------------- -# Vector storage follows OM_METADATA_BACKEND (sqlite/postgres) unless set to 'valkey' -# Options: valkey (Redis-compatible), or leave unset to follow OM_METADATA_BACKEND -# Note: When using postgres metadata backend, vectors are stored in the same database -OM_VECTOR_BACKEND=sqlite -# Table name for vectors (configurable, will be created if it doesn't exist) -OM_VECTOR_TABLE=vectors -OM_WEAVIATE_URL= -OM_WEAVIATE_API_KEY= -OM_WEAVIATE_CLASS=OpenMemory +# Vector storage +OM_VECTOR_BACKEND=postgres +OM_VECTOR_TABLE=openmemory_vectors -# -------------------------------------------- -# Embeddings Configuration -# -------------------------------------------- -# Available providers: openai, gemini, aws, ollama, local, synthetic -# Embedding models per sector can be configured in models.yaml -# -# NOTE: Your selected TIER (fast/smart/deep) affects how embeddings work: -# • FAST tier: Uses synthetic embeddings regardless of OM_EMBEDDINGS setting -# • SMART tier: Combines synthetic + compressed semantic from your chosen provider -# • DEEP tier: Uses full embeddings from your chosen provider -# -# For SMART/DEEP tiers, set your preferred provider: +# Embeddings OM_EMBEDDINGS=openai - -# Fallback chain: comma-separated list of providers to try if primary fails -# Each provider exhausts its own retry logic before moving to the next -# Example: OM_EMBEDDING_FALLBACK=ollama,synthetic -# Default: synthetic (always works as final fallback) OM_EMBEDDING_FALLBACK=synthetic - -# Vector dimension (auto-adjusted by tier, but can be overridden) -# • FAST: 256-dim • SMART: 384-dim • DEEP: 1536-dim -# OM_VEC_DIM=1536 - -# Embedding Mode -# simple = 1 unified batch call for all sectors (faster, rate-limit safe, recommended) -# advanced = 5 separate calls, one per sector (higher precision, more API calls) OM_EMBED_MODE=simple - -# Advanced Mode Options (only used when OM_EMBED_MODE=advanced) -# Enable parallel embedding (not recommended for Gemini due to rate limits) OM_ADV_EMBED_PARALLEL=false -# Delay between embeddings in milliseconds OM_EMBED_DELAY_MS=200 +OM_VEC_DIM=1536 -# OpenAI-compatible Embeddings Provider -# OM_OPENAI_BASE_URL=https://api.openai.com/v1 -# Model override for all sector embeddings (leave empty to use defaults) -# OM_OPENAI_MODEL=text-embedding-qwen3-embedding-4b - -# API Configuration -# Max request body size in bytes (default: 1MB) -OM_MAX_PAYLOAD_SIZE=1000000 - -# -------------------------------------------- -# Embedding Provider API Keys -# -------------------------------------------- -# OpenAI Embeddings -OPENAI_API_KEY=your-openai-api-key-here - -# Google Gemini Embeddings -GEMINI_API_KEY=your-gemini-api-key-here - -# AWS Titan Text Embeddings V2 -AWS_ACCESS_KEY_ID=YOUR_ACCESS_KEY -AWS_SECRET_ACCESS_KEY=YOUR_SECRET_KEY +OPENAI_API_KEY= +GEMINI_API_KEY= +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= AWS_REGION=us-east-1 - -# Ollama Local Embeddings OLLAMA_URL=http://localhost:11434 +LOCAL_MODEL_PATH= -# Local Model Path (for custom embedding models) -LOCAL_MODEL_PATH=/path/to/your/local/model - -# -------------------------------------------- -# Memory System Settings -# -------------------------------------------- - -# ============================================ -# PERFORMANCE TIER (Manual Configuration Required) -# ============================================ -# OpenMemory requires you to manually set the performance tier. -# Set OM_TIER to one of: hybrid, fast, smart, or deep -# -# Available Tiers: -# -# HYBRID - Keyword + Synthetic embeddings (256-dim) with BM25 ranking -# • Recall: ~100% (exact keyword matching) • QPS: 800-1000 • RAM: 0.5GB/10k memories -# • Best for: Exact searches, documentation, code search, personal knowledge -# • Features: Exact phrase matching, BM25 scoring, n-gram matching, 100% accuracy -# • Use when: You need guaranteed exact matches and keyword-based retrieval -# -# FAST - Synthetic embeddings only (256-dim) -# • Recall: ~70-75% • QPS: 700-850 • RAM: 0.6GB/10k memories -# • Best for: Local apps, VS Code extensions, low-end hardware -# • Use when: < 4 CPU cores or < 8GB RAM -# -# SMART - Hybrid embeddings (256-dim synthetic + 128-dim compressed semantic = 384-dim) -# • Recall: ~85% • QPS: 500-600 • RAM: 0.9GB/10k memories -# • Best for: Production servers, AI copilots, mid-range hardware -# • Use when: 4-7 CPU cores and 8-15GB RAM -# -# DEEP - Full AI embeddings (1536-dim OpenAI/Gemini) -# • Recall: ~95-100% • QPS: 350-400 • RAM: 1.6GB/10k memories -# • Best for: Cloud deployments, high-accuracy systems, semantic research -# • Use when: 8+ CPU cores and 16+ GB RAM -# -# REQUIRED: Set your tier (no auto-detection): -OM_TIER=hybrid - -# Keyword Matching Settings (HYBRID tier only) -# Boost multiplier for keyword matches (default: 2.5) -OM_KEYWORD_BOOST=2.5 -# Minimum keyword length for matching (default: 3) -OM_KEYWORD_MIN_LENGTH=3 +# Server limits +OM_MAX_PAYLOAD_SIZE=1000000 +OM_RATE_LIMIT_ENABLED=false +OM_RATE_LIMIT_WINDOW_MS=60000 +OM_RATE_LIMIT_MAX_REQUESTS=100 +# Memory settings OM_MIN_SCORE=0.3 - -# ============================================ -# Smart Decay Settings (Time-Based Algorithm) -# ============================================ -# Decay interval in minutes - how often the decay cycle runs -# The new algorithm uses time-based decay with daily lambda rates (hot=0.005/day, warm=0.02/day, cold=0.05/day) -# Unlike batch-based systems, running more frequently doesn't increase decay speed -# Decay is calculated from: decay_factor = exp(-lambda * days_since_access / (salience + 0.1)) -# -# Recommended intervals: -# • Testing: 30 minutes (for rapid validation) -# • Development: 60-120 minutes (balanced testing) -# • Production: 120-180 minutes (optimal - captures meaningful decay deltas while minimizing overhead) -# -# At 2-3 hours: hot tier decays ~0.04-0.06%, warm ~0.16-0.24%, cold ~0.4-0.6% per cycle OM_DECAY_INTERVAL_MINUTES=120 - -# Number of parallel decay worker threads (default: 3) OM_DECAY_THREADS=3 -# Cold tier threshold - memories below this salience get fingerprinted (default: 0.25) OM_DECAY_COLD_THRESHOLD=0.25 -# Reinforce memory salience when queried (default: true) OM_DECAY_REINFORCE_ON_QUERY=true -# Enable regeneration of cold memories on query hits (default: true) OM_REGENERATION_ENABLED=true -# Maximum vector dimensions (default: 1536) OM_MAX_VECTOR_DIM=1536 -# Minimum vector dimensions for compression (default: 64) OM_MIN_VECTOR_DIM=64 -# Number of summary compression layers 1-3 (default: 3) OM_SUMMARY_LAYERS=3 - -# Full Semantic Graph MVP Settings -# Use summary-only storage (≤300 chars, intelligent extraction) OM_USE_SUMMARY_ONLY=true -# Maximum summary length - smart extraction preserves dates, names, numbers, actions OM_SUMMARY_MAX_LENGTH=300 -# Memories per segment (10k recommended for optimal cache performance) OM_SEG_SIZE=10000 +OM_KEYWORD_BOOST=2.5 +OM_KEYWORD_MIN_LENGTH=3 -# Cache segments (auto-tuned by tier, but can be overridden) -# • FAST: 2 segments • SMART: 3 segments • DEEP: 5 segments -# OM_CACHE_SEGMENTS=3 - -# Max active queries (auto-tuned by tier, but can be overridden) -# • FAST: 32 queries • SMART: 64 queries • DEEP: 128 queries -# OM_MAX_ACTIVE=64 - -# Brain Sector Configuration (auto-classified, but you can override) -# Sectors: episodic, semantic, procedural, emotional, reflective - -# Auto-Reflection System -# Automatically creates reflective memories by clustering similar memories +# Optional background jobs OM_AUTO_REFLECT=false -# Reflection interval in minutes (default: 10) OM_REFLECT_INTERVAL=10 -# Minimum memories required before reflection runs (default: 20) OM_REFLECT_MIN_MEMORIES=20 +OM_USER_SUMMARY_INTERVAL=30 -# Compression -# Enable automatic content compression for large memories -OM_COMPRESSION_ENABLED=false -# Minimum content length (characters) to trigger compression (default: 100) -OM_COMPRESSION_MIN_LENGTH=100 -# Compression algorithm: semantic, syntactic, aggressive, auto (default: auto) -OM_COMPRESSION_ALGORITHM=auto - -# -------------------------------------------- -# LangGraph Integration Mode (LGM) -# -------------------------------------------- +# Optional LangGraph mode OM_LG_NAMESPACE=default OM_LG_MAX_CONTEXT=50 OM_LG_REFLECTIVE=true + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87c047de..3e21bb2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,7 @@ on: workflow_dispatch: jobs: +<<<<<<< Updated upstream test-python: name: Test Python SDK runs-on: ubuntu-latest @@ -35,25 +36,28 @@ jobs: test-node: name: Test Node.js SDK +======= + test-js: + name: Test JavaScript package +>>>>>>> Stashed changes runs-on: ubuntu-latest env: - OM_DB_URL: "sqlite:///:memory:" - OM_TIER: "fast" + OM_TIER: fast OM_VEC_DIM: "1536" + OM_EMBEDDINGS: synthetic steps: - uses: actions/checkout@v4 - - - name: Set up Node.js - uses: actions/setup-node@v4 + + - uses: actions/setup-node@v4 with: - node-version: '22' - - - name: Install Dependencies - run: | - cd packages/openmemory-js - npm ci - - - name: Run Verification Test - run: | - cd packages/openmemory-js - npx tsx tests/test_omnibus.ts + node-version: "22" + + - name: Install dependencies + run: cd packages/openmemory-js && npm ci + + - name: Build + run: cd packages/openmemory-js && npm run build + + - name: Run verification test + run: cd packages/openmemory-js && npx tsx tests/test_omnibus.ts + diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 54f76715..4e20921b 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -2,70 +2,27 @@ name: Docker Build Test on: push: - branches: - - main - - develop + branches: [main, develop] pull_request: - branches: - - main - - develop + branches: [main, develop] workflow_dispatch: jobs: docker-build-test: runs-on: ubuntu-latest - name: Test Docker Build - steps: - - name: Checkout code - uses: actions/checkout@v4 + - uses: actions/checkout@v4 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + - uses: docker/setup-buildx-action@v3 - - name: Build Backend Docker Image + - name: Build OpenMemory Docker image uses: docker/build-push-action@v5 with: context: ./packages/openmemory-js file: ./packages/openmemory-js/Dockerfile push: false - tags: openmemory-backend:test - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Test Docker Compose Build - run: | - docker compose build - docker compose config - - - name: Start Services - run: | - docker compose up -d - sleep 10 - - - name: Check Health Endpoint - run: | - max_attempts=30 - attempt=0 - until curl -f http://localhost:8080/health || [ $attempt -eq $max_attempts ]; do - echo "Waiting for service to be healthy... (attempt $((attempt+1))/$max_attempts)" - sleep 2 - attempt=$((attempt+1)) - done - - if [ $attempt -eq $max_attempts ]; then - echo "Service failed to become healthy" - docker compose logs - exit 1 - fi - - echo "✅ Service is healthy!" - curl -v http://localhost:8080/health + tags: openmemory-js:test - - name: Show Container Logs - if: failure() - run: docker compose logs + - name: Validate compose config + run: docker compose config - - name: Cleanup - if: always() - run: docker compose down -v diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index a889c4e9..00000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,17 +0,0 @@ -on: - push: - branches: - - main - -jobs: - contrib-readme-job: - runs-on: ubuntu-latest - name: A job to automate contrib in readme - permissions: - contents: write - pull-requests: write - steps: - - name: Contribute List - uses: akhilmhdh/contributors-readme-action@v2.3.11 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish-sdks.yml b/.github/workflows/publish-sdks.yml index bdc0c792..2d49c487 100644 --- a/.github/workflows/publish-sdks.yml +++ b/.github/workflows/publish-sdks.yml @@ -1,4 +1,4 @@ -name: publish sdks +name: Publish JavaScript package permissions: contents: read @@ -6,93 +6,28 @@ on: push: branches: [main] paths: - - 'packages/openmemory-js/**' - - 'packages/openmemory-py/**' + - "packages/openmemory-js/**" workflow_dispatch: jobs: - detect-changes: - runs-on: ubuntu-latest - outputs: - js_changed: ${{ steps.filter.outputs.js }} - py_changed: ${{ steps.filter.outputs.py }} - steps: - - uses: actions/checkout@v4 - - uses: dorny/paths-filter@v3 - id: filter - with: - filters: | - js: - - 'packages/openmemory-js/**' - py: - - 'packages/openmemory-py/**' - publish-js: - needs: detect-changes runs-on: ubuntu-latest - if: needs.detect-changes.outputs.js_changed == 'true' || github.event_name == 'workflow_dispatch' steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: '22' - registry-url: 'https://registry.npmjs.org' + node-version: "22" + registry-url: "https://registry.npmjs.org" - - name: get version - id: version - run: | - cd packages/openmemory-js - VERSION=$(node -p "require('./package.json').version") - echo "version=$VERSION" >> $GITHUB_OUTPUT - echo "Publishing openmemory-js@$VERSION" - - - name: install deps + - name: Install dependencies run: cd packages/openmemory-js && npm ci - - name: build + - name: Build run: cd packages/openmemory-js && npm run build - - name: publish + - name: Publish run: cd packages/openmemory-js && npm publish --access public env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - - name: notify - run: | - echo "✅ Published openmemory-js@${{ steps.version.outputs.version }}" - - publish-py: - needs: detect-changes - runs-on: ubuntu-latest - if: needs.detect-changes.outputs.py_changed == 'true' || github.event_name == 'workflow_dispatch' - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: get version - id: version - run: | - cd packages/openmemory-py - VERSION=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])") - echo "version=$VERSION" >> $GITHUB_OUTPUT - echo "Publishing openmemory-py@$VERSION" - - - name: install build tools - run: pip install build twine - - - name: build - run: cd packages/openmemory-py && python -m build - - - name: publish - run: cd packages/openmemory-py && python -m twine upload dist/* --skip-existing - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - - - name: notify - run: | - echo "✅ Published openmemory-py@${{ steps.version.outputs.version }}" diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 00000000..9df91080 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,13 @@ +# OpenMemory Architecture + +The previous mixed architecture has been retired from the active tree. + +Current direction: + +- JavaScript/Node-first server package in `packages/openmemory-js`. +- Durable cognitive graph architecture. +- Postgres + pgvector as the production storage target. +- Small public API centered on remember, recall, explain, consolidate, and contradiction resolution. + +See `docs/architecture-rewrite-plan.md` for the current rewrite plan. + diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..9f7b2257 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,8 @@ +# Changelog + +## 2026-05-13 + +- Started aggressive cleanup toward a JavaScript-only server/package repository. +- Removed deferred product surfaces from the active tree. +- Reduced top-level documentation to the current JS server rewrite direction. + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..e89b2130 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,23 @@ +# Contributing + +OpenMemory is currently in an architectural cleanup phase. + +## Active Area + +Contributions should target `packages/openmemory-js` unless a maintainer asks otherwise. + +## Local Checks + +```bash +cd packages/openmemory-js +npm install +npm run build +npx tsx tests/test_omnibus.ts +``` + +## Guidelines + +- Keep changes scoped to the JavaScript server/package path. +- Do not reintroduce deferred app, dashboard, or secondary SDK surfaces during the cleanup. +- Update `docs/ai-context.md`, `docs/ai-rules.md`, or `docs/decisions.md` when reusable architecture context changes. + diff --git a/GOVERNANCE.md b/GOVERNANCE.md new file mode 100644 index 00000000..4fe85059 --- /dev/null +++ b/GOVERNANCE.md @@ -0,0 +1,12 @@ +# Governance + +OpenMemory is being simplified around one active implementation path. + +## Active Ownership + +- `packages/openmemory-js`: JavaScript server/package implementation. +- `docs`: rewrite plan, API notes, and persistent AI context. +- `.github`: repository process automation. + +Deferred surfaces should not be reintroduced without an explicit decision recorded in `docs/decisions.md`. + diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..465e6ecf --- /dev/null +++ b/Makefile @@ -0,0 +1,31 @@ +.PHONY: help install dev build start test clean + +PKG=packages/openmemory-js + +help: + @echo "OpenMemory JS cleanup commands" + @echo " make install - install JS package dependencies" + @echo " make dev - start JS server in development mode" + @echo " make build - build JS package" + @echo " make start - start built JS server" + @echo " make test - run JS verification test" + @echo " make clean - remove JS build output" + +install: + cd $(PKG) && npm install + +dev: + cd $(PKG) && npm run dev + +build: + cd $(PKG) && npm run build + +start: + cd $(PKG) && npm run start + +test: + cd $(PKG) && npx tsx tests/test_omnibus.ts + +clean: + cd $(PKG) && rm -rf dist + diff --git a/README.md b/README.md new file mode 100644 index 00000000..05eb05c1 --- /dev/null +++ b/README.md @@ -0,0 +1,43 @@ +# OpenMemory + +OpenMemory is being cleaned up into a JavaScript/Node-first durable memory server. + +The active product path is currently: + +- `packages/openmemory-js` +- Node/TypeScript server runtime +- Postgres + pgvector as the production storage target +- npm-based development and release workflow + +Deferred surfaces such as editor extensions, dashboard UI, secondary SDKs, old examples, and hosted deploy templates have been removed from the active tree for now. + +## Current Setup + +```bash +cd packages/openmemory-js +npm install +npm run build +npm run start +``` + +The default server port is `8080`. + +## Development + +```bash +cd packages/openmemory-js +npm run dev +npm run build +npx tsx tests/test_omnibus.ts +``` + +## Documentation + +- Rewrite plan: `docs/architecture-rewrite-plan.md` +- Persistent AI context: `docs/ai-context.md`, `docs/ai-rules.md`, `docs/decisions.md` +- Package docs: `packages/openmemory-js/README.md` + +## Status + +This repository is in an architectural cleanup phase. Keep new work focused on the JS package and server path until the durable core rewrite is complete. + diff --git a/Why.md b/Why.md new file mode 100644 index 00000000..4722fd77 --- /dev/null +++ b/Why.md @@ -0,0 +1,12 @@ +# Why OpenMemory + +OpenMemory exists to give AI applications durable memory that is more structured than a plain vector lookup. + +The current rewrite focuses on: + +- Durable records with provenance. +- Temporal correctness. +- Explainable recall. +- Contract-aware memory usage. +- A small JavaScript server package that can be installed and run through npm. + diff --git a/docker-compose.yml b/docker-compose.yml index 67eafe8a..919eaff2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,28 @@ services: + postgres: + image: pgvector/pgvector:pg16 + environment: + POSTGRES_DB: ${OM_PG_DB:-openmemory} + POSTGRES_USER: ${OM_PG_USER:-postgres} + POSTGRES_PASSWORD: ${OM_PG_PASSWORD:-postgres} + ports: + - "5432:5432" + volumes: + - openmemory_pg:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${OM_PG_USER:-postgres} -d ${OM_PG_DB:-openmemory}"] + interval: 10s + timeout: 5s + retries: 5 + openmemory: build: context: ./packages/openmemory-js dockerfile: Dockerfile ports: - - '8080:8080' + - "8080:8080" environment: +<<<<<<< Updated upstream # Core Configuration - OM_PORT=${OM_PORT:-8080} - OM_MODE=${OM_MODE:-standard} @@ -147,37 +164,39 @@ services: environment: - NEXT_PUBLIC_API_URL=${NEXT_PUBLIC_API_URL:-http://localhost:8080} - NEXT_PUBLIC_API_KEY=${NEXT_PUBLIC_API_KEY:-} +======= + OM_PORT: ${OM_PORT:-8080} + OM_MODE: ${OM_MODE:-standard} + OM_TIER: ${OM_TIER:-hybrid} + OM_METADATA_BACKEND: postgres + OM_VECTOR_BACKEND: postgres + OM_PG_HOST: postgres + OM_PG_PORT: 5432 + OM_PG_DB: ${OM_PG_DB:-openmemory} + OM_PG_USER: ${OM_PG_USER:-postgres} + OM_PG_PASSWORD: ${OM_PG_PASSWORD:-postgres} + OM_PG_SCHEMA: ${OM_PG_SCHEMA:-public} + OM_PG_TABLE: ${OM_PG_TABLE:-openmemory_memories} + OM_PG_SSL: disable + OM_VECTOR_TABLE: ${OM_VECTOR_TABLE:-openmemory_vectors} + OM_API_KEY: ${OM_API_KEY:-} + OM_EMBEDDINGS: ${OM_EMBEDDINGS:-synthetic} + OM_EMBEDDING_FALLBACK: ${OM_EMBEDDING_FALLBACK:-synthetic} + OM_EMBED_MODE: ${OM_EMBED_MODE:-simple} + OM_VEC_DIM: ${OM_VEC_DIM:-1536} + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + GEMINI_API_KEY: ${GEMINI_API_KEY:-} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-} + AWS_REGION: ${AWS_REGION:-} + OLLAMA_URL: ${OLLAMA_URL:-http://localhost:11434} + OM_MAX_PAYLOAD_SIZE: ${OM_MAX_PAYLOAD_SIZE:-1000000} +>>>>>>> Stashed changes depends_on: - openmemory: + postgres: condition: service_healthy restart: unless-stopped - healthcheck: - test: ['CMD', 'wget', '--no-verbose', '--tries=1', '--spider', 'http://localhost:3000'] - interval: 30s - timeout: 10s - retries: 3 - start_period: 30s - - # Optional: Valkey for high-performance vector storage - # Uncomment to use Valkey instead of SQLite/PostgreSQL for vectors - # Don't forget to set OM_VECTOR_BACKEND=valkey in your .env - # valkey: - # image: valkey/valkey:8.0 - # ports: - # - '6379:6379' - # volumes: - # - valkey_data:/data - # restart: unless-stopped - # healthcheck: - # test: ['CMD', 'valkey-cli', 'ping'] - # interval: 30s - # timeout: 10s - # retries: 3 - # start_period: 30s volumes: - openmemory_data: - driver: local - # Uncomment if using Valkey - # valkey_data: - # driver: local + openmemory_pg: + diff --git a/docs/ai-context.md b/docs/ai-context.md new file mode 100644 index 00000000..be519f14 --- /dev/null +++ b/docs/ai-context.md @@ -0,0 +1,18 @@ +# AI Context + +## Project +- OpenMemory repository. +- Current focus: architectural rewrite and code improvement. +- User priority: remove Python surfaces; make the project JavaScript-only for now. + +## Current Goal +- Clean the repository down to the JavaScript server/package path before implementing the durable core rewrite. + +## Architecture Inputs +- Target architecture from the attached architecture document: bitemporal, append-only cognitive graph with working memory, facets, provenance, contracts, contradictions, consolidation, three recall modes, explain API, and Postgres + pgvector first. +- FigJam board `cHvNZ1CU304RAH4ccuADBv`: input event -> working memory buffer -> ingestion pipeline -> durable cognitive graph -> executable edge runtime -> consolidation -> memory tiers -> recall engine -> explain API -> agent/app response. + +## Repo Snapshot +- Active implementation package is `packages/openmemory-js`. +- Deferred surfaces were removed from the active tree during cleanup: secondary SDKs, old examples, ops tools, editor extension, dashboard shell, local DB/temp artifacts, and hosted deploy configs. +- Current JS package still contains legacy internals that will be handled in later rewrite phases: custom `server.js`, SQLite/Postgres branches, sector-based HSG memory, waypoints, and route groups beyond the future `/v1` API. diff --git a/docs/ai-rules.md b/docs/ai-rules.md new file mode 100644 index 00000000..f4d3e45d --- /dev/null +++ b/docs/ai-rules.md @@ -0,0 +1,13 @@ +# AI Rules + +## Persistent Memory +- Read `docs/ai-context.md`, `docs/ai-rules.md`, and `docs/decisions.md` at the start of each task. +- Store reusable architecture, constraints, patterns, and decisions in these files. +- Treat these files as the source of truth instead of chat memory. + +## Current Constraints +- Prioritize JavaScript-only implementation. +- Remove Python components from the main product path. +- Main near-term runtime target: package installable via npm or forkable from GitHub, with `npm run start` launching a server. +- Defer VS Code extension, dashboard rebuild, Python SDK compatibility, and non-core connectors until the JS server path is stable. +- Interpret "JS-only" as Node/TypeScript runtime and tooling unless explicitly changed to literal `.js` source only. diff --git a/docs/api-server.md b/docs/api-server.md new file mode 100644 index 00000000..9edb906f --- /dev/null +++ b/docs/api-server.md @@ -0,0 +1,26 @@ +# API Server + +The active server lives in `packages/openmemory-js`. + +## Run + +```bash +cd packages/openmemory-js +npm install +npm run build +npm run start +``` + +## Current Endpoints + +- `GET /health` +- `POST /memory/add` +- `POST /memory/query` +- `GET /memory/all` +- `GET /memory/:id` +- `PATCH /memory/:id` +- `DELETE /memory/:id` +- `POST /memory/reinforce` + +The durable rewrite will introduce the smaller `/v1` API described in `docs/architecture-rewrite-plan.md`. + diff --git a/docs/architecture-rewrite-plan.md b/docs/architecture-rewrite-plan.md new file mode 100644 index 00000000..d0896904 --- /dev/null +++ b/docs/architecture-rewrite-plan.md @@ -0,0 +1,261 @@ +# OpenMemory JS-Only Architecture Rewrite Plan + +## Goal + +Make OpenMemory a durable, JavaScript/Node-first memory server that can be installed from npm or forked from GitHub, then started with `npm run start`. + +Near-term scope is the server and JS package. Python, VS Code, dashboards, and secondary integrations are deferred unless needed to keep the server usable. + +## Planning Inputs + +- Attached architecture document: target architecture is a bitemporal, append-only cognitive graph with facets, provenance, contracts, contradictions, recall modes, explainability, consolidation, and Postgres + pgvector first. +- FigJam board `cHvNZ1CU304RAH4ccuADBv`: input event -> working memory -> ingestion pipeline -> durable cognitive graph -> executable edge runtime -> consolidation -> memory tiers -> recall engine -> explain API -> app response, with append-only audit log. +- Initial repo scan found multiple deferred surfaces. The cleanup pass removed them from the active tree so the JS package is the primary implementation path. + +## Recommended Approach + +Use a strangler rewrite inside the JS package: + +1. Keep `packages/openmemory-js` as the product package. +2. Create a clean `src/durable/*` core beside the current HSG implementation. +3. Put the new server API on the target architecture. +4. Keep legacy endpoints only as thin compatibility adapters during transition. +5. Delete Python and stale surfaces once the JS server has parity for core remember/recall/explain flows. + +This avoids a blank rewrite while still forcing clean boundaries. + +## Step-By-Step Plan + +### Phase 0: Freeze Scope and Define the Product Shape + +1. Declare `packages/openmemory-js` the canonical product. +2. Add a root `package.json` workspace so a fresh fork can run: + - `npm install` + - `npm run start` +3. Make root `npm run start` delegate to the JS package server. +4. Define the supported near-term commands: + - `npm run start`: build if needed and start server. + - `npm run dev`: run server from TypeScript for local work. + - `npm run test`: run JS tests only. + - `npm run migrate`: run JS/Postgres migrations. +5. Treat TypeScript as acceptable JS ecosystem code unless the product requirement becomes literal `.js` only. + +### Phase 1: Remove Python From the Product Path + +1. Remove or archive secondary SDKs, old examples, migration utilities, related docs, and non-JS CI/publish workflows. +2. Replace Python migration providers with JS-only migration modules where still needed. +3. Remove Python references from README, docs, Makefile, Docker, deploy templates, and CI. +4. Remove build-time Python dependencies by making Postgres + pgvector the default and eliminating `sqlite3` from the first supported server path. +5. Keep any removed behavior documented in a short migration note, not as live code. + +### Phase 2: Make the JS Server Start Reliably + +1. Split package imports from server startup: + - `src/index.ts` exports SDK/client APIs only. + - `src/server/entry.ts` starts the HTTP server. + - Importing `openmemory-js` must not start a server. +2. Replace or isolate the custom `server.js` wrapper. + - Preferred: use a boring HTTP framework with schema validation and predictable middleware. + - Minimum: keep the wrapper temporarily but hide it behind a small app adapter. +3. Add `/health` as the first contract. +4. Add clear startup validation: + - Node version + - required Postgres settings + - pgvector availability + - embedding provider configuration +5. Ensure fresh fork path works without generated artifacts checked in: + - `npm run start` runs build then `node dist/server/entry.js`. + +### Phase 3: Build the Durable Storage Core + +1. Replace inline schema creation in `core/db.ts` with versioned JS SQL migrations. +2. Make Postgres + pgvector the default production store. +3. Create the target tables: + - `memories` + - `memory_versions` + - `entities` + - `memory_entities` + - `edges` + - `contradictions` + - `provenance` + - `inferences` + - `working_memory` + - `consolidations` + - `audit_log` +4. Store facets and contracts as typed JSONB with database-level defaults. +5. Add bitemporal columns: + - `valid_from` + - `valid_to` + - `observed_at` + - `recorded_at` + - `superseded_at` +6. Require every durable write to emit an audit row in the same transaction. + +### Phase 4: Replace Sectors With Facets + +1. Keep old sector names only as compatibility input. +2. Introduce `memory_facets`: + - episodic + - semantic + - procedural + - emotional + - reflective +3. Implement a deterministic facet extractor first. +4. Add optional LLM-assisted extraction later, behind a provider interface. +5. Convert `primary_sector` scoring to facet-aware scoring. +6. Store multiple embeddings or embedding metadata only where it improves recall quality. + +### Phase 5: Create the Ingestion Pipeline + +1. Make all durable writes pass through one pipeline: + - validate input event + - update bounded working memory + - resolve entities + - extract facets + - score provenance + - assign memory contract + - detect contradictions + - write memory, entities, edges, provenance, audit +2. Block raw direct database writes from routes. +3. Normalize source records: + - source kind + - source URI or ID + - extraction method + - trust score + - observed time +4. Keep document connectors out of the first rewrite unless they feed this same pipeline cleanly. + +### Phase 6: Implement Executable Edge Runtime + +1. Add transaction handlers for key edge types: + - `supersedes`: close old validity, mark superseded, transfer partial salience, audit. + - `contradicts`: create contradiction record, keep both memories, lower strict confidence, audit. + - `derived_from`: store inference path, inherit provenance confidence with damping, audit. + - `same_as`: merge entity references without deleting mentions, audit. +2. Do not allow edge writes that bypass handlers. +3. Keep graph traversal bounded and explainable. + +### Phase 7: Build Recall Modes + +1. Replace generic internal recall with explicit modes: + - `strict` + - `historical` + - `associative` +2. Public API target: + - `memory.remember({ content, source, metadata })` + - `memory.recall({ query, mode, at_time, limit })` + - `memory.explain({ memory_id })` + - `memory.consolidate({ scope })` + - `memory.resolve_contradiction({ contradiction_id, resolution })` +3. HTTP API target: + - `POST /v1/memories` + - `POST /v1/recall` + - `GET /v1/memories/:id/explain` + - `POST /v1/consolidations` + - `POST /v1/contradictions/:id/resolve` +4. Keep old `/memory/add` and `/memory/query` as adapters until docs and examples are migrated. +5. Enforce contracts at recall time. + +### Phase 8: Add Explainability and Trust Controls + +1. Return score components: + - embedding similarity + - graph support + - temporal relevance + - salience + - confidence + - provenance + - contract penalty + - contradiction penalty +2. Add an explain API that returns: + - source trail + - bitemporal state + - confidence components + - contradictions + - inference path +3. Require strict recall to either produce sourced current memory or abstain. + +### Phase 9: Consolidation and Memory Tiers + +1. Add tiers: + - active + - warm + - cold + - archived +2. Implement tier movement as accessibility changes, not deletion. +3. Add conservative consolidation jobs: + - episodes -> patterns + - patterns -> preferences + - failures -> procedures + - conflicts -> reflections +4. Log every consolidation trace. +5. Keep consolidation disabled by default until evals show it reduces noise. + +### Phase 10: JS-Only Testing and Evals + +1. Use Node test tooling only. +2. Add unit tests for: + - bitemporal visibility + - contract filtering + - contradiction handling + - edge handlers + - recall mode behavior +3. Add integration tests against Postgres + pgvector. +4. Add an eval harness for: + - information extraction + - multi-session reasoning + - temporal reasoning + - knowledge updates + - abstention + - contradiction handling + - long-run degradation +5. CI should run JS build, tests, migrations, and package smoke start. + +### Phase 11: Migration From Current Data + +1. Write a JS migration from existing schema to durable schema: + - `primary_sector` -> facet hints + - `tags` and `meta` -> metadata/facets/contracts + - `waypoints` -> typed `relates_to` edges + - `temporal_facts` -> semantic memories with bitemporal fields +2. Do not destructively mutate old databases. +3. Output a migration report with counts, skipped rows, and warnings. + +### Phase 12: Documentation and Release Cleanup + +1. Rewrite README around one path: + - `npm install` + - configure Postgres + - `npm run start` + - call remember/recall/explain +2. Remove Python badges, examples, docs, CI, and publish flow. +3. Update Docker image to Node/Postgres-only. +4. Publish `openmemory-js` as the npm package. +5. Add a release smoke test: + - install package + - start server + - hit `/health` + - remember one memory + - recall it in strict mode + - explain it + +## Deferred + +- VS Code extension rewrite. +- Dashboard rebuild. +- Python SDK compatibility. +- SQLite local mode. +- Deep source connectors. +- Custom graph database. +- Merkle/content-addressed storage. +- Recursive memory worlds. + +## First Implementation Milestone + +The first useful milestone is not the full architecture. It is: + +1. Root `npm run start` works. +2. Python package/docs/workflows are removed from the active product path. +3. Postgres + pgvector migration creates the durable core tables. +4. `POST /v1/memories`, `POST /v1/recall`, and `/health` work. +5. Strict recall respects provenance, contracts, current validity, and audit logging. diff --git a/docs/concepts.md b/docs/concepts.md new file mode 100644 index 00000000..61b5542c --- /dev/null +++ b/docs/concepts.md @@ -0,0 +1,15 @@ +# Core Concepts + +OpenMemory is being rewritten toward a durable cognitive graph. + +## Current Direction + +- JavaScript/Node-first server package. +- Postgres + pgvector production storage target. +- Append-friendly memory records with provenance and auditability. +- Recall modes that separate strict, historical, and associative use cases. + +## Rewrite Reference + +Use `docs/architecture-rewrite-plan.md` as the current architecture planning source. + diff --git a/docs/decisions.md b/docs/decisions.md new file mode 100644 index 00000000..87d07b35 --- /dev/null +++ b/docs/decisions.md @@ -0,0 +1,11 @@ +# Decisions + +## 2026-05-13 +- Initialize persistent AI memory files because they were missing. +- Near-term rewrite direction: JavaScript-only server-first package; defer VS Code and adjacent integrations. +- Use `packages/openmemory-js` as the canonical product package for the rewrite. +- Prefer a strangler rewrite: build a clean durable core inside the JS package while temporarily adapting legacy endpoints. +- Production storage target is Postgres + pgvector first; SQLite local mode is deferred. +- Public API should stay small: remember, recall, explain, consolidate, resolve contradiction. +- Aggressive cleanup removes deferred product surfaces from the active tree instead of parking them. +- Cleanup pass may edit docs/config/workflows/package metadata, but should avoid changing JS implementation logic until the setup task is complete. diff --git a/docs/faq.md b/docs/faq.md new file mode 100644 index 00000000..399bc76c --- /dev/null +++ b/docs/faq.md @@ -0,0 +1,18 @@ +# FAQ + +## What is the active product path? + +`packages/openmemory-js` is the active product package. + +## What runtime is supported right now? + +Node.js with TypeScript source and npm scripts. + +## What storage target should new work assume? + +Postgres with pgvector. + +## Are editor extensions, dashboards, and secondary SDKs supported right now? + +No. They are deferred until the JS server path and durable core are stable. + diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 00000000..a7b7e9fc --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,31 @@ +# Getting Started + +OpenMemory is currently focused on the JavaScript server package. + +## Install + +```bash +cd packages/openmemory-js +npm install +``` + +## Configure + +Copy the root environment template and set the Postgres and embedding values you need: + +```bash +cp .env.example .env +``` + +Production storage target is Postgres with pgvector. + +## Run + +```bash +cd packages/openmemory-js +npm run build +npm run start +``` + +The server listens on `http://localhost:8080` by default. + diff --git a/docs/mcp.md b/docs/mcp.md new file mode 100644 index 00000000..3188b93b --- /dev/null +++ b/docs/mcp.md @@ -0,0 +1,8 @@ +# MCP + +MCP support remains inside `packages/openmemory-js`, but it is not the focus of the current cleanup. + +For now, keep MCP changes limited to the JavaScript package and avoid adding separate app or editor-extension surfaces. + +The durable rewrite should preserve memory tools through the JS server once the core remember, recall, and explain flows are stable. + diff --git a/docs/node-sdk.md b/docs/node-sdk.md new file mode 100644 index 00000000..f7244335 --- /dev/null +++ b/docs/node-sdk.md @@ -0,0 +1,31 @@ +# JavaScript Package + +The active package is `openmemory-js` in `packages/openmemory-js`. + +## Install + +```bash +npm install openmemory-js +``` + +## Local Development + +```bash +cd packages/openmemory-js +npm install +npm run build +npm run start +``` + +## SDK Usage + +```ts +import { Memory } from "openmemory-js"; + +const memory = new Memory("user_1"); +await memory.add("User prefers concise responses"); +const results = await memory.search("response preference"); +``` + +The current SDK is kept while the durable JS server rewrite is staged. +