feat(nodejs): root entry, prebuilt, install script, package.json files for tarball

.gitignore

@@ -1,5 +1,9 @@
+tree-sitter/
+tree-sitter-cypher/
+
 .idea/
 .vscode
+.cursor
 .vs
 bazel-*
 .clwb

Makefile

@@ -187,7 +187,7 @@ nodejs-deps:
 	cd tools/nodejs_api && npm install --include=dev
 
 nodejstest: nodejs
-	cd tools/nodejs_api && npm test
+	cd tools/nodejs_api && node copy_src_to_build.js && npm test
 
 nodejstest-deps: nodejs-deps nodejstest
 

tools/nodejs_api/README.md

@@ -7,10 +7,24 @@ A high-performance graph database for knowledge-intensive applications. This Nod
 
 ## 📦 Installation
 
+**From npm (if published):**
+
 ```bash
 npm install lbug
 ```
 
+**From GitHub** (monorepo; the Node package lives in `tools/nodejs_api`):
+
+- **pnpm** (v9+), subdirectory is supported:
+
+  ```bash
+  pnpm add lbug@github:LadybugDB/ladybug#path:tools/nodejs_api
+  ```
+
+  On install, the package will build the native addon from source (needs CMake and a C++20 compiler).
+
+- **npm**: no built-in subdirectory install. Either use a **local path** after cloning and building (see [Build and use in other projects](#-build-and-use-in-other-projects-local)), or a tarball from [GitPkg](https://gitpkg.vercel.app/) (e.g. `https://gitpkg.vercel.app/LadybugDB/ladybug/tools/nodejs_api?main`).
+
 ---
 
 ## 🚀 Quick Start
@@ -49,10 +63,8 @@ const main = async () => {
   // Run a query
   const result = await conn.query("MATCH (u:User) RETURN u.name, u.age;");
 
-  // Fetch all results
+  // Consume results (choose one style)
   const rows = await result.getAll();
-
-  // Output results
   for (const row of rows) {
     console.log(row);
   }
@@ -68,12 +80,182 @@ main().catch(console.error);
 
 The `lbug` package exposes the following primary classes:
 
-* `Database` – Initializes a database from a file path.
-* `Connection` – Executes queries on a connected database.
-* `QueryResult` – Provides methods like `getAll()` to retrieve results.
+* **Database** – `new Database(path, bufferPoolSize?, ...)`. Initialize with `init()` / `initSync()` (optional; done on first use). When the file is locked, **async init() retries for up to 5s** (configurable: last ctor arg `openLockRetryMs`; set `0` to fail immediately). Close with `close()`.
+* **Connection** – `new Connection(database, numThreads?)`. Run Cypher with `query(statement)` or `prepare(statement)` then `execute(preparedStatement, params)`. Use `transaction(fn)` for a single write transaction, `ping()` for liveness checks. **`getNumNodes(nodeName)`** and **`getNumRels(relName)`** return row counts for node/rel tables. Use `registerStream(name, source, { columns })` to load data from an AsyncIterable via `LOAD FROM name`; `unregisterStream(name)` when done. Configure with `setQueryTimeout(ms)`, `setMaxNumThreadForExec(n)`.
+* **QueryResult** – Returned by `query()` / `execute()`. Consume with `getAll()`, `getNext()` / `hasNext()`, **async iteration** (`for await...of`), or **`toStream()`** (Node.js `Readable`). Use **`toString()`** for a string representation (header + rows; useful for debugging). Metadata: `getColumnNames()`, `getColumnDataTypes()`, `getQuerySummary()`. Call `close()` when done (optional if fully consumed).
+* **PreparedStatement** – Created by `conn.prepare(statement)`. Execute with `conn.execute(preparedStatement, params)`. Reuse for parameterized queries.
+* **Pool** – `createPool({ databasePath, maxSize, ... })` returns a connection pool. Use **`pool.run(conn => ...)`** (recommended) or `acquire()` / `release(conn)`; call **`pool.close()`** when done.
 
 Both CommonJS (`require`) and ES Modules (`import`) are fully supported.
 
+### Consuming query results
+
+```js
+const result = await conn.query("MATCH (n:User) RETURN n.name LIMIT 1000");
+
+// Option 1: get all rows (loads into memory)
+const rows = await result.getAll();
+
+// Option 2: row by row (async)
+while (result.hasNext()) {
+  const row = await result.getNext();
+  console.log(row);
+}
+
+// Option 3: async iterator (streaming, no full materialization)
+for await (const row of result) {
+  console.log(row);
+}
+
+// Option 4: Node.js Readable stream (e.g. for .pipe())
+const stream = result.toStream();
+stream.on("data", (row) => console.log(row));
+
+// Option 5: string representation (e.g. for debugging)
+console.log(result.toString());
+```
+
+### Table counts
+
+After creating node/rel tables and loading data, you can get row counts:
+
+```js
+conn.initSync(); // or await conn.init()
+const numUsers = conn.getNumNodes("User");
+const numFollows = conn.getNumRels("Follows");
+```
+
+### Connection pool
+
+Use **`createPool(options)`** to get a pool of connections (one shared `Database`, up to `maxSize` connections). Prefer **`pool.run(fn)`**: it acquires a connection, runs `fn(conn)`, and releases in `finally` (on success or throw), so you never leak a connection.
+
+**Options:** `maxSize` (required), `databasePath`, `databaseOptions` (same shape as `Database` constructor), `minSize` (default 0), `acquireTimeoutMillis` (default 0 = wait forever), `validateOnAcquire` (default false; if true, `conn.ping()` before hand-out).
+
+**Example (recommended: `run`):**
+
+```js
+import { createPool } from "lbug";
+
+const pool = createPool({ databasePath: "./mydb", maxSize: 10 });
+
+const rows = await pool.run(async (conn) => {
+  const result = await conn.query("MATCH (u:User) RETURN u.name LIMIT 5");
+  const rows = await result.getAll();
+  result.close();
+  return rows;
+});
+console.log(rows);
+
+await pool.close();
+```
+
+**Manual acquire/release:** If you need the same connection for multiple operations, use `acquire()` and always call `release(conn)` in a `finally` block so the connection is returned even on throw.
+
+```js
+const conn = await pool.acquire();
+try {
+  await conn.query("...");
+  // ...
+} finally {
+  pool.release(conn);
+}
+```
+
+When shutting down, call **`pool.close()`**: it rejects new and pending `acquire()`, then closes all connections and the database.
+
+### Transactions
+
+**Manual:** Run `BEGIN TRANSACTION`, then your queries, then `COMMIT` or `ROLLBACK`. On error, call `ROLLBACK` before continuing.
+
+```js
+await conn.query("BEGIN TRANSACTION");
+await conn.query("CREATE NODE TABLE Nodes(id INT64, PRIMARY KEY(id))");
+await conn.query('COPY Nodes FROM "data.csv"');
+await conn.query("COMMIT");
+// or on error: await conn.query("ROLLBACK");
+```
+
+**Read-only transaction:** `BEGIN TRANSACTION READ ONLY` then queries, then `COMMIT` / `ROLLBACK`.
+
+**Wrapper:** One write transaction with automatic commit on success and rollback on throw:
+
+```js
+await conn.transaction(async () => {
+  await conn.query("CREATE NODE TABLE Nodes(id INT64, PRIMARY KEY(id))");
+  await conn.query('COPY Nodes FROM "data.csv"');
+  // commit happens automatically; on throw, rollback then rethrow
+});
+```
+
+### Loading data from a Node.js stream
+
+You can feed data from an **AsyncIterable** (generator, async generator, or any `Symbol.asyncIterator`) into Cypher using **scan replacement**: register a stream by name, then use `LOAD FROM name` in your query. Rows are pulled from JavaScript on demand during execution.
+
+**API:**
+
+* **`conn.registerStream(name, source, options)`** (async)  
+  * `name` – string used in Cypher: `LOAD FROM name RETURN ...`  
+  * `source` – AsyncIterable of rows. Each row is an **array** of column values (same order as `options.columns`) or an **object** keyed by column name.  
+  * `options.columns` – **required**. Schema: array of `{ name: string, type: string }`. Supported types: `INT64`, `INT32`, `INT16`, `INT8`, `UINT64`, `UINT32`, `DOUBLE`, `FLOAT`, `STRING`, `BOOL`, `DATE`, `TIMESTAMP`.
+
+* **`conn.unregisterStream(name)`**  
+  Unregisters the source so the name can be reused or to avoid leaving stale entries. Call after the query (or when done with the stream).
+
+**Example:**
+
+```js
+async function* generateRows() {
+  yield [1, "Alice"];
+  yield [2, "Bob"];
+  yield [3, "Carol"];
+}
+
+await conn.registerStream("users", generateRows(), {
+  columns: [
+    { name: "id", type: "INT64" },
+    { name: "name", type: "STRING" },
+  ],
+});
+
+const result = await conn.query("LOAD FROM users RETURN *");
+for await (const row of result) {
+  console.log(row); // { id: 1, name: "Alice" }, ...
+}
+
+conn.unregisterStream("users");
+```
+
+You can combine the stream with other Cypher: e.g. `LOAD FROM stream RETURN * WHERE col > 0`, or `COPY MyTable FROM (LOAD FROM stream RETURN *)`.
+
+### Database locked
+
+Only one process can open the same database path for writing. If the file is already locked, **async `init()` retries for up to 5 seconds** by default (grace period), then throws. You can tune or disable this:
+
+- **Default**: `new Database("./my.db")` — last ctor arg `openLockRetryMs` defaults to `5000` (retry for up to 5s on lock).
+- **No retry**: `new Database("./my.db", 0, true, false, 0, true, -1, true, true, 0)` or pass `openLockRetryMs = 0` as the 10th argument to fail immediately.
+- **Longer grace**: e.g. `openLockRetryMs = 3000` to wait up to 3s.
+
+The error has **`code === 'LBUG_DATABASE_LOCKED'`** so you can catch and handle it if the grace period wasn’t enough:
+
+```js
+import { Database, Connection, LBUG_DATABASE_LOCKED } from "lbug";
+
+const db = new Database("./my.db"); // already retries ~5s on lock
+try {
+  await db.init();
+} catch (err) {
+  if (err.code === LBUG_DATABASE_LOCKED) {
+    console.error("Database still locked after grace period.");
+  }
+  throw err;
+}
+const conn = new Connection(db);
+```
+
+Use **read-only** mode for concurrent readers: `new Database(path, undefined, undefined, true)` so multiple processes can open the same DB for read.
+
+See [docs/database_locked.md](docs/database_locked.md) for how other systems handle this and best practices.
+
 ---
 
 ## 🛠️ Local Development (for Contributors)
@@ -96,6 +278,94 @@ npm run build
 npm test
 ```
 
+When developing from the **monorepo root**, build the native addon first so tests see the latest C++ code:
+
+```bash
+# From repo root (D:\prj\ladybug or similar)
+make nodejs
+# Or: cmake --build build/release --target lbugjs
+# Then from tools/nodejs_api:
+cd tools/nodejs_api && npm test
+```
+
+---
+
+## 🔧 Build and use in other projects (local)
+
+To use the Node.js API from the Ladybug repo in another project without publishing to npm:
+
+1. **Build the addon** (from the Ladybug repo root):
+
+   ```bash
+   make nodejs
+   ```
+
+   Or from this directory:
+
+   ```bash
+   npm run build
+   ```
+
+   This compiles the native addon into `build/lbugjs.node` and copies JS and types.
+
+2. **In your other project**, add a file dependency in `package.json`:
+
+   ```json
+   "dependencies": {
+     "lbug": "file:../path/to/ladybug/tools/nodejs_api"
+   }
+   ```
+
+   Then run `npm install`. After that, `require("lbug")` or `import ... from "lbug"` will use your local build.
+
+3. **Optional:** to pack and install a tarball instead:
+
+   ```bash
+   cd /path/to/ladybug/tools/nodejs_api
+   npm run build
+   npm pack
+   ```
+
+   In the other project: `npm install /path/to/ladybug/tools/nodejs_api/lbug-0.0.1.tgz`.
+
+### Prebuilt in your fork (install from GitHub without building)
+
+If you install from GitHub (e.g. `pnpm add lbug@github:user/ladybug#path:tools/nodejs_api`), the package runs `install.js`: if it finds a prebuilt binary, it uses it and does not build from source. To ship a prebuilt in your fork:
+
+1. **Build once** in your clone (from repo root):
+
+   ```bash
+   make nodejs
+   ```
+
+2. **Create the prebuilt file** (name = `lbugjs-<platform>-<arch>.node`):
+
+   - Windows x64: copy `tools/nodejs_api/build/lbugjs.node` → `tools/nodejs_api/prebuilt/lbugjs-win32-x64.node`
+   - Linux x64: `lbugjs-linux-x64.node`
+   - macOS x64: `lbugjs-darwin-x64.node`, arm64: `lbugjs-darwin-arm64.node`
+
+   Example (from repo root). **Windows (PowerShell):**
+
+   ```powershell
+   New-Item -ItemType Directory -Force -Path tools/nodejs_api/prebuilt
+   Copy-Item tools/nodejs_api/build/lbugjs.node tools/nodejs_api/prebuilt/lbugjs-win32-x64.node
+   ```
+
+   **Linux/macOS:**
+
+   ```bash
+   mkdir -p tools/nodejs_api/prebuilt
+   cp tools/nodejs_api/build/lbugjs.node tools/nodejs_api/prebuilt/lbugjs-$(node -p "process.platform")-$(node -p "process.arch").node
+   ```
+
+3. **Commit and push** the `prebuilt/` folder. Then anyone (or you in another project) can do:
+
+   ```bash
+   pnpm add lbug@github:YOUR_USERNAME/ladybug#path:tools/nodejs_api
+   ```
+
+   and the addon will be used from prebuilt without a local build.
+
 ---
 
 ## 📦 Packaging and Binary Distribution

tools/nodejs_api/build.js

@@ -3,6 +3,9 @@ const path = require("path");
 const { execSync } = require("child_process");
 
 const SRC_PATH = path.resolve(__dirname, "../..");
+const NODEJS_API = path.resolve(__dirname, ".");
+const BUILD_DIR = path.join(NODEJS_API, "build");
+const SRC_JS_DIR = path.join(NODEJS_API, "src_js");
 const THREADS = require("os").cpus().length;
 
 console.log(`Using ${THREADS} threads to build Lbug.`);
@@ -12,3 +15,19 @@ execSync(`make nodejs NUM_THREADS=${THREADS}`, {
   cwd: SRC_PATH,
   stdio: "inherit",
 });
+
+// Ensure build/ has latest JS from src_js (CMake copies at configure time only)
+if (fs.existsSync(SRC_JS_DIR) && fs.existsSync(BUILD_DIR)) {
+  const files = fs.readdirSync(SRC_JS_DIR);
+  for (const name of files) {
+    if (name.endsWith(".js") || name.endsWith(".mjs") || name.endsWith(".d.ts")) {
+      fs.copyFileSync(path.join(SRC_JS_DIR, name), path.join(BUILD_DIR, name));
+    }
+  }
+  // So package root has types when used as file: dependency
+  const dts = path.join(BUILD_DIR, "lbug.d.ts");
+  if (fs.existsSync(dts)) {
+    fs.copyFileSync(dts, path.join(NODEJS_API, "lbug.d.ts"));
+  }
+  console.log("Copied src_js to build.");
+}

tools/nodejs_api/copy_src_to_build.js

@@ -0,0 +1,22 @@
+/**
+ * Copies src_js/*.js, *.mjs, *.d.ts into build/ so tests run with the latest JS
+ * after "make nodejs" (which only copies at cmake configure time).
+ * Run from tools/nodejs_api.
+ */
+const fs = require("fs");
+const path = require("path");
+
+const srcDir = path.join(__dirname, "src_js");
+const buildDir = path.join(__dirname, "build");
+
+if (!fs.existsSync(buildDir)) {
+  console.warn("copy_src_to_build: build/ missing, run make nodejs first.");
+  process.exit(0);
+}
+
+const re = /\.(js|mjs|d\.ts)$/;
+const files = fs.readdirSync(srcDir).filter((n) => re.test(n));
+for (const name of files) {
+  fs.copyFileSync(path.join(srcDir, name), path.join(buildDir, name));
+}
+console.log("Copied", files.length, "files from src_js to build.");