diff --git a/.github/workflows/full_test.yml b/.github/workflows/full_test.yml new file mode 100644 index 0000000..2b0744e --- /dev/null +++ b/.github/workflows/full_test.yml @@ -0,0 +1,66 @@ +name: Full Test +on: + pull_request: + types: [opened, synchronize] + +env: + INTEGRATION_TEST_SPREADSHEET_ID: ${{ secrets.INTEGRATION_TEST_SPREADSHEET_ID }} + INTEGRATION_TEST_AUTH_JSON: ${{ secrets.INTEGRATION_TEST_AUTH_JSON }} + +jobs: + full_test: + runs-on: ubuntu-latest + strategy: + matrix: + node-version: ['22.x'] # Can expand if needed + + if: github.event.review.state == 'approved' || github.event.pull_request.user.login == 'edocsss' + + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: echo "$GITHUB_CONTEXT" + + - name: Setup Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Compile TypeScript + run: npx tsc --noEmit + + - name: Run Unit Tests with Coverage + run: | + npm run test -- --coverage + cp coverage/lcov.info coverage.out + + - name: Generate Coverage Badge + uses: tj-actions/coverage-badge-js@v1 + with: + green: 80 + coverage-summary-path: coverage/coverage-summary.json + + - name: Add Coverage Badge + uses: stefanzweifel/git-auto-commit-action@v4 + id: auto-commit-action + with: + commit_message: Apply Code Coverage Badge + skip_fetch: true + skip_checkout: true + file_pattern: ./README.md + + - name: Push Changes + if: steps.auto-commit-action.outputs.changes_detected == 'true' + uses: ad-m/github-push-action@master + with: + github_token: ${{ github.token }} + branch: ${{ github.head_ref }} \ No newline at end of file diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml new file mode 100644 index 0000000..a4281a6 --- /dev/null +++ b/.github/workflows/unit_test.yml @@ -0,0 +1,27 @@ +name: Unit Test +on: push + +jobs: + unit_test: + runs-on: ubuntu-latest + strategy: + matrix: + node-version: ['18.x', '20.x', '22.x'] + + steps: + - uses: actions/checkout@v3 + + - name: Setup Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: TypeScript compile check (optional) + run: npx tsc --noEmit + + - name: Run unit tests + run: npm test \ No newline at end of file diff --git a/README.md b/README.md index 09ad6ad..75a732b 100644 --- a/README.md +++ b/README.md @@ -1 +1,413 @@ -# JSFreeDB \ No newline at end of file +# GoFreeDB +
+ +
+ + + + +

Ship Faster with Google Sheets as a Database!

+
+ +

+ JSFreeDB is a JavaScript library that provides common and simple database abstractions on top of Google Sheets. +

+ +
+ +
+ + ![Unit Test](https://github.com/FreeLeh/JSFreeDB/actions/workflows/unit_test.yml/badge.svg) + ![Integration Test](https://github.com/FreeLeh/JSFreeDB/actions/workflows/full_test.yml/badge.svg) + ![Coverage](https://img.shields.io/badge/Coverage-82.8%25-brightgreen) + +
+ +## Features + +1. Provide a straightforward **key-value** and **row based database** interfaces on top of Google Sheets. +2. Serve your data **without any server setup** (by leveraging Google Sheets infrastructure). +3. Support **flexible enough query language** to perform various data queries. +4. **Manually manipulate data** via the familiar Google Sheets UI (no admin page required). + +> For more details, please read [our analysis](https://github.com/FreeLeh/docs/blob/main/freedb/alternatives.md#why-should-you-choose-freedb) +> on other alternatives and how it compares with `FreeDB`. + +## Table of Contents + +* [Protocols](#protocols) +* [Getting Started](#getting-started) + * [Installation](#installation) + * [Pre-requisites](#pre-requisites) +* [Row Store](#row-store) + * [Querying Rows](#querying-rows) + * [Counting Rows](#counting-rows) + * [Inserting Rows](#inserting-rows) + * [Updating Rows](#updating-rows) + * [Deleting Rows](#deleting-rows) + * [Struct Field to Column Mapping](#struct-field-to-column-mapping) +* [KV Store](#kv-store) + * [Get Value](#get-value) + * [Set Key](#set-key) + * [Delete Key](#delete-key) + * [Supported Modes](#supported-modes) +* [KV Store V2](#kv-store-v2) + * [Get Value](#get-value-v2) + * [Set Key](#set-key-v2) + * [Delete Key](#delete-key-v2) + * [Supported Modes](#supported-modes-v2) + +## Protocols + +Clients are strongly encouraged to read through the **[protocols document](https://github.com/FreeLeh/docs/blob/main/freedb/protocols.md)** to see how things work +under the hood and **the limitations**. + +## Getting Started + +### Installation + +``` +go get github.com/FreeLeh/GoFreeDB +``` + +### Pre-requisites + +1. Obtain a Google [OAuth2](https://github.com/FreeLeh/docs/blob/main/google/authentication.md#oauth2-flow) or [Service Account](https://github.com/FreeLeh/docs/blob/main/google/authentication.md#service-account-flow) credentials. +2. Prepare a Google Sheets spreadsheet where the data will be stored. + +## Row Store + +Let's assume each row in the table is represented by the `Person` struct. + +```go +type Person struct { + Name string `db:"name"` + Age int `db:"age"` +} +``` + +Please read the [struct field to column mapping](#struct-field-to-column-mapping) section +to understand the purpose of the `db` struct field tag. + +```go +import ( + "github.com/FreeLeh/GoFreeDB" + "github.com/FreeLeh/GoFreeDB/google/auth" +) + +// If using Google Service Account. +auth, err := auth.NewServiceFromFile( + "", + freedb.FreeDBGoogleAuthScopes, + auth.ServiceConfig{}, +) + +// If using Google OAuth2 Flow. +auth, err := auth.NewOAuth2FromFile( + "", + "", + freedb.FreeDBGoogleAuthScopes, + auth.OAuth2Config{}, +) + +store := freedb.NewGoogleSheetRowStore( + auth, + "", + "", + freedb.GoogleSheetRowStoreConfig{Columns: []string{"name", "age"}}, +) +defer store.Close(context.Background()) +``` + +### Querying Rows + +```go +// Output variable +var output []Person + +// Select all columns for all rows +err := store. + Select(&output). + Exec(context.Background()) + +// Select a few columns for all rows (non-selected struct fields will have default value) +err := store. + Select(&output, "name"). + Exec(context.Background()) + +// Select rows with conditions +err := store. + Select(&output). + Where("name = ? OR age >= ?", "freedb", 10). + Exec(context.Background()) + +// Select rows with sorting/order by +ordering := []freedb.ColumnOrderBy{ + {Column: "name", OrderBy: freedb.OrderByAsc}, + {Column: "age", OrderBy: freedb.OrderByDesc}, +} +err := store. + Select(&output). + OrderBy(ordering). + Exec(context.Background()) + +// Select rows with offset and limit +err := store. + Select(&output). + Offset(10). + Limit(20). + Exec(context.Background()) +``` + +### Counting Rows + +```go +// Count all rows +count, err := store. + Count(). + Exec(context.Background()) + +// Count rows with conditions +count, err := store. + Count(). + Where("name = ? OR age >= ?", "freedb", 10). + Exec(context.Background()) +``` + +### Inserting Rows + +```go +err := store.Insert( + Person{Name: "no_pointer", Age: 10}, + &Person{Name: "with_pointer", Age: 20}, +).Exec(context.Background()) +``` + +### Updating Rows + +```go +colToUpdate := make(map[string]interface{}) +colToUpdate["name"] = "new_name" +colToUpdate["age"] = 12 + +// Update all rows +err := store. + Update(colToUpdate). + Exec(context.Background()) + +// Update rows with conditions +err := store. + Update(colToUpdate). + Where("name = ? OR age >= ?", "freedb", 10). + Exec(context.Background()) +``` + +### Deleting Rows + +```go +// Delete all rows +err := store. + Delete(). + Exec(context.Background()) + +// Delete rows with conditions +err := store. + Delete(). + Where("name = ? OR age >= ?", "freedb", 10). + Exec(context.Background()) +``` + +### Struct Field to Column Mapping + +The struct field tag `db` can be used for defining the mapping between the struct field and the column name. +This works just like the `json` tag from [`encoding/json`](https://pkg.go.dev/encoding/json). + +Without `db` tag, the library will use the field name directly (case-sensitive). + +```go +// This will map to the exact column name of "Name" and "Age". +type NoTagPerson struct { + Name string + Age int +} + +// This will map to the exact column name of "name" and "age" +type WithTagPerson struct { + Name string `db:"name"` + Age int `db:"age"` +} +``` + +## KV Store + +> Please use `KV Store V2` as much as possible, especially if you are creating a new storage. + +```go +import ( + "github.com/FreeLeh/GoFreeDB" + "github.com/FreeLeh/GoFreeDB/google/auth" +) + +// If using Google Service Account. +auth, err := auth.NewServiceFromFile( + "", + freedb.FreeDBGoogleAuthScopes, + auth.ServiceConfig{}, +) + +// If using Google OAuth2 Flow. +auth, err := auth.NewOAuth2FromFile( + "", + "", + freedb.FreeDBGoogleAuthScopes, + auth.OAuth2Config{}, +) + +kv := freedb.NewGoogleSheetKVStore( + auth, + "", + "", + freedb.GoogleSheetKVStoreConfig{Mode: freedb.KVSetModeAppendOnly}, +) +defer kv.Close(context.Background()) +``` + +### Get Value + +If the key is not found, `freedb.ErrKeyNotFound` will be returned. + +```go +value, err := kv.Get(context.Background(), "k1") +``` + +### Set Key + +```go +err := kv.Set(context.Background(), "k1", []byte("some_value")) +``` + +### Delete Key + +```go +err := kv.Delete(context.Background(), "k1") +``` + +### Supported Modes + +> For more details on how the two modes are different, please read the [protocol document](https://github.com/FreeLeh/docs/blob/main/freedb/protocols.md). + +There are 2 different modes supported: + +1. Default mode. +2. Append only mode. + +```go +// Default mode +kv := freedb.NewGoogleSheetKVStore( + auth, + "", + "", + freedb.GoogleSheetKVStoreConfig{Mode: freedb.KVModeDefault}, +) + +// Append only mode +kv := freedb.NewGoogleSheetKVStore( + auth, + "", + "", + freedb.GoogleSheetKVStoreConfig{Mode: freedb.KVModeAppendOnly}, +) +``` + +## KV Store V2 + +The KV Store V2 is implemented internally using the row store. + +> The original `KV Store` was created using more complicated formulas, making it less maintainable. +> You can still use the original `KV Store` implementation, but we strongly suggest using this new `KV Store V2`. + +You cannot use an existing sheet based on `KV Store` with `KV Store V2` as the sheet structure is different. +- If you want to convert an existing sheet, just add an `_rid` column and insert the first key-value row with `1` + and increase it by 1 until the last row. +- Remove the timestamp column as `KV Store V2` does not depend on it anymore. + +```go +import ( + "github.com/FreeLeh/GoFreeDB" + "github.com/FreeLeh/GoFreeDB/google/auth" +) + +// If using Google Service Account. +auth, err := auth.NewServiceFromFile( + "", + freedb.FreeDBGoogleAuthScopes, + auth.ServiceConfig{}, +) + +// If using Google OAuth2 Flow. +auth, err := auth.NewOAuth2FromFile( + "", + "", + freedb.FreeDBGoogleAuthScopes, + auth.OAuth2Config{}, +) + +kv := freedb.NewGoogleSheetKVStoreV2( + auth, + "", + "", + freedb.GoogleSheetKVStoreV2Config{Mode: freedb.KVSetModeAppendOnly}, +) +defer kv.Close(context.Background()) +``` + +### Get Value V2 + +If the key is not found, `freedb.ErrKeyNotFound` will be returned. + +```go +value, err := kv.Get(context.Background(), "k1") +``` + +### Set Key V2 + +```go +err := kv.Set(context.Background(), "k1", []byte("some_value")) +``` + +### Delete Key V2 + +```go +err := kv.Delete(context.Background(), "k1") +``` + +### Supported Modes V2 + +> For more details on how the two modes are different, please read the [protocol document](https://github.com/FreeLeh/docs/blob/main/freedb/protocols.md). + +There are 2 different modes supported: + +1. Default mode. +2. Append only mode. + +```go +// Default mode +kv := freedb.NewGoogleSheetKVStoreV2( + auth, + "", + "", + freedb.GoogleSheetKVStoreV2Config{Mode: freedb.KVModeDefault}, +) + +// Append only mode +kv := freedb.NewGoogleSheetKVStoreV2( + auth, + "", + "", + freedb.GoogleSheetKVStoreV2Config{Mode: freedb.KVModeAppendOnly}, +) +``` + +## License + +This project is [MIT licensed](https://github.com/FreeLeh/GoFreeDB/blob/main/LICENSE). diff --git a/jest.config.js b/jest.config.js index 2b0a1d3..1f02ea3 100644 --- a/jest.config.js +++ b/jest.config.js @@ -6,5 +6,8 @@ module.exports = { } }, testEnvironment: 'node', - testMatch: ['**/tests/**/*.[jt]s?(x)', '**/?(*.)+(spec|test).[tj]s?(x)'], + testMatch: ['**/tests/**/*.(spec|test).[jt]s?(x)', '**/?(*.)+(spec|test).[tj]s?(x)'], + collectCoverage: true, + coverageDirectory: 'coverage', + coverageReporters: ['text', 'lcov', 'json-summary'], }; \ No newline at end of file diff --git a/package.json b/package.json index b0d9bfe..eaba04b 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "description": "JSFreeDB is a JavaScript library that provides common and simple database abstractions on top of Google Sheets.", "main": "dist/index.js", "scripts": { - "test": "jest" + "test": "jest", + "integration-test": "npm run test -- --testPathPattern=integration" }, "repository": { "type": "git", @@ -27,4 +28,4 @@ "axios": "^1.8.1", "googleapis": "^105.0.0" } -} +} \ No newline at end of file diff --git a/src/google/auth/models.ts b/src/google/auth/models.ts new file mode 100644 index 0000000..e48c159 --- /dev/null +++ b/src/google/auth/models.ts @@ -0,0 +1,3 @@ +export const GOOGLE_SHEETS_READ_ONLY: string[] = ['https://www.googleapis.com/auth/spreadsheets.readonly'] +export const GOOGLE_SHEETS_WRITE_ONLY: string[] = ['https://www.googleapis.com/auth/spreadsheets'] +export const GOOGLE_SHEETS_READ_WRITE: string[] = ['https://www.googleapis.com/auth/spreadsheets'] diff --git a/src/google/auth/service_account.ts b/src/google/auth/service_account.ts index d543533..cac84b8 100644 --- a/src/google/auth/service_account.ts +++ b/src/google/auth/service_account.ts @@ -2,7 +2,7 @@ import * as google from 'googleapis'; import { AuthClient } from './base'; -export default class ServiceAccountGoogleAuthClient implements AuthClient { +export class ServiceAccountGoogleAuthClient implements AuthClient { private auth!: google.Auth.GoogleAuth; private constructor(auth: google.Auth.GoogleAuth) { @@ -10,9 +10,8 @@ export default class ServiceAccountGoogleAuthClient implements AuthClient { } public static fromServiceAccountInfo(serviceAccountInfo: google.Auth.JWTInput, scopes: string[]): ServiceAccountGoogleAuthClient { - const jsonAuthClient = new google.Auth.GoogleAuth().fromJSON(serviceAccountInfo); const authClient = new google.Auth.GoogleAuth({ - authClient: jsonAuthClient, + credentials: serviceAccountInfo, scopes: scopes, }); return new ServiceAccountGoogleAuthClient(authClient); diff --git a/src/google/sheets/models.ts b/src/google/sheets/models.ts index 9b95771..c4c662b 100644 --- a/src/google/sheets/models.ts +++ b/src/google/sheets/models.ts @@ -1,5 +1,3 @@ -import { OAuth2Client } from 'google-auth-library'; - export const MAJOR_DIMENSION_ROWS = "ROWS"; export const VALUE_INPUT_USER_ENTERED = "USER_ENTERED"; export const RESPONSE_VALUE_RENDER_FORMATTED = "FORMATTED_VALUE"; diff --git a/src/google/sheets/wrapper.ts b/src/google/sheets/wrapper.ts index 1a4bf48..60cffa0 100644 --- a/src/google/sheets/wrapper.ts +++ b/src/google/sheets/wrapper.ts @@ -2,6 +2,7 @@ import { google, sheets_v4 } from 'googleapis'; import axios, { AxiosInstance } from 'axios'; import { GoogleAuth } from 'google-auth-library'; +import { AuthClient } from '../auth/base'; import { AppendMode, A1Range, @@ -19,13 +20,13 @@ import { } from './models'; export class Wrapper { - private auth: GoogleAuth; + private googleAuth: GoogleAuth; private service: sheets_v4.Sheets; private rawClient: AxiosInstance; - constructor(auth: GoogleAuth) { - this.auth = auth; - this.service = google.sheets({ version: 'v4', auth }); + constructor(auth: AuthClient) { + this.googleAuth = auth.getAuth(); + this.service = google.sheets({ version: 'v4', auth: this.googleAuth }); this.rawClient = axios.create({ validateStatus: () => true }); } @@ -67,7 +68,7 @@ export class Wrapper { /** * Gets a mapping of sheet names to sheet IDs */ - async getSheetNameToID(spreadsheetId: string): Promise> { + async getSheetNameToID(spreadsheetId: string): Promise> { const response = await this.service.spreadsheets.get({ spreadsheetId, }); @@ -76,7 +77,7 @@ export class Wrapper { throw new Error('Failed to get sheet information'); } - const result: Map = new Map(); + const result: Record = {}; for (const sheet of response.data.sheets) { if (!sheet.properties) { throw new Error('Failed getSheetIDByName due to empty sheet properties'); @@ -86,7 +87,7 @@ export class Wrapper { const id = sheet.properties.sheetId; if (title && id) { - result.set(title, id); + result[title] = id } } @@ -283,7 +284,7 @@ export class Wrapper { // This ensures the latest access token is used (and refreshed if needed). const response = await this.rawClient.get( url, - { headers: await this.auth.getRequestHeaders() }, + { headers: await this.googleAuth.getRequestHeaders() }, ); if (response.status !== 200) { throw new Error(`Failed to query rows, status: ${response.status}`); diff --git a/src/google/store/kv.ts b/src/google/store/kv.ts index e69de29..c811514 100644 --- a/src/google/store/kv.ts +++ b/src/google/store/kv.ts @@ -0,0 +1,86 @@ +import { AuthClient } from '../auth/base' +import { GoogleSheetRowStore, GoogleSheetRowStoreConfig } from './row'; +import { KVMode, KeyNotFoundError } from '../utils/kv'; +import { Codec } from '../codec/base'; +import { BasicCodec } from '../codec/basic'; +import { OrderBy } from '../utils/row'; + +interface GoogleSheetKVStoreConfig { + mode: KVMode; +} + +interface GoogleSheetKVStoreRow { + key: string + value: string +} + +export class GoogleSheetKVStore { + private rowStore: GoogleSheetRowStore + private mode: KVMode + private codec: Codec + + constructor(rowStore: GoogleSheetRowStore, mode: KVMode, codec: Codec) { + this.rowStore = rowStore + this.mode = mode + this.codec = codec + } + + async get(key: string): Promise { + let rawRows: Record[] + if (this.mode === KVMode.Default) { + rawRows = await this.rowStore.select('value') + .where('key = ?', key) + .limit(1) + .exec(); + } else { + rawRows = await this.rowStore.select('value') + .where('key = ?', key) + .orderBy([{ column: '_rid', orderBy: OrderBy.DESC }]) + .limit(1) + .exec(); + } + + if (rawRows.length === 0) { + throw new KeyNotFoundError(); + } + + const value = rawRows[0]!.value; + if (value === '' || value === null || value === undefined) { + throw new KeyNotFoundError(); + } + return this.codec.decode(rawRows[0]!.value); + } + + async set(key: string, value: any): Promise { + const encoded = await this.codec.encode(value); + + if (this.mode === KVMode.Default) { + await this.rowStore.delete() + .where('key =?', key) + .exec(); + } + await this.rowStore.insert({ key, value: encoded }).exec(); + } + + async delete(key: string): Promise { + if (this.mode === KVMode.Default) { + await this.rowStore.delete() + .where('key = ?', key) + .exec(); + } else { + const row: GoogleSheetKVStoreRow = { key, value: '' }; + await this.rowStore.insert(row).exec(); + } + } + + static async create( + auth: AuthClient, + spreadsheetId: string, + sheetName: string, + config: GoogleSheetKVStoreConfig, + ): Promise { + const rowStoreConfig = new GoogleSheetRowStoreConfig(['key', 'value']); + const rowStore = await GoogleSheetRowStore.create(auth, spreadsheetId, sheetName, rowStoreConfig); + return new GoogleSheetKVStore(rowStore, config.mode, new BasicCodec()); + } +} \ No newline at end of file diff --git a/src/google/store/models.ts b/src/google/store/models.ts new file mode 100644 index 0000000..ec5a4ea --- /dev/null +++ b/src/google/store/models.ts @@ -0,0 +1,53 @@ +import { + BatchUpdateRowsRequest, + BatchUpdateRowsResult, + InsertRowsResult, + QueryRowsResult, + UpdateRowsResult, +} from "../sheets/models"; + +export const MAX_COLUMN = 26; + +export const SCRATCHPAD_BOOKED = "BOOKED"; +export const SCRATCHPAD_SHEET_NAME_SUFFIX = "_scratch"; + +export const DEFAULT_KV_TABLE_RANGE = "A1:C5000000"; +export const DEFAULT_KV_KEY_COL_RANGE = "A1:A5000000"; +export const DEFAULT_KV_FIRST_ROW_RANGE = "A1:C1"; + +export const KV_GET_APPEND_QUERY_TEMPLATE = "=VLOOKUP(\"%s\", SORT(%s, 3, FALSE), 2, FALSE)"; +export const KV_GET_DEFAULT_QUERY_TEMPLATE = "=VLOOKUP(\"%s\", %s, 2, FALSE)"; +export const KV_FIND_KEY_A1_RANGE_QUERY_TEMPLATE = "=MATCH(\"%s\", %s, 0)"; + +export const ROW_IDX_COL = "_rid"; +export const ROW_IDX_FORMULA = "=ROW()"; + +// Helper function to generate column name (similar to common.GenerateColumnName) +function generateColumnName(index: number): string { + return String.fromCharCode(65 + index); // 'A' is 65 in ASCII +} + +// Variables +export const DEFAULT_ROW_HEADER_RANGE = `A1:${generateColumnName(MAX_COLUMN - 1)}1`; +export const DEFAULT_ROW_FULL_TABLE_RANGE = `A2:${generateColumnName(MAX_COLUMN - 1)}`; +export const ROW_DELETE_RANGE_TEMPLATE_FUNC = (from: number, to: number) => `A${from}:${generateColumnName(MAX_COLUMN - 1)}${to}`; + +// The first condition `_rid IS NOT NULL` is necessary to ensure we are just updating rows that are non-empty. +// This is required for UPDATE without WHERE clause (otherwise it will see every row as update target). +export const ROW_WHERE_NON_EMPTY_CONDITION_TEMPLATE_FUNC = (where: string) => `${ROW_IDX_COL} is not null AND ${where}`; +export const ROW_WHERE_EMPTY_CONDITION_TEMPLATE = `${ROW_IDX_COL} is not null`; + +export const GOOGLE_SHEET_SELECT_STMT_STRING_KEYWORD = /^(date|datetime|timeofday)/; + +export interface SheetsWrapper { + createSpreadsheet(title: string): Promise; + getSheetNameToID(spreadsheetId: string): Promise>; + createSheet(spreadsheetId: string, sheetName: string): Promise; + deleteSheets(spreadsheetId: string, sheetIDs: number[]): Promise; + insertRows(spreadsheetId: string, a1Range: string, values: any[][]): Promise; + overwriteRows(spreadsheetId: string, a1Range: string, values: any[][]): Promise; + updateRows(spreadsheetId: string, a1Range: string, values: any[][]): Promise; + batchUpdateRows(spreadsheetId: string, requests: BatchUpdateRowsRequest[]): Promise; + queryRows(spreadsheetId: string, sheetName: string, query: string, skipHeader: boolean): Promise; + clear(spreadsheetId: string, ranges: string[]): Promise; +} diff --git a/src/google/store/query.ts b/src/google/store/query.ts new file mode 100644 index 0000000..024a54b --- /dev/null +++ b/src/google/store/query.ts @@ -0,0 +1,246 @@ +/** + * Defines a function type for intercepting and modifying WHERE clauses. + */ +type WhereInterceptorFunc = (where: string) => string; + +import { ColumnOrderBy } from '../utils/row'; +import { GOOGLE_SHEET_SELECT_STMT_STRING_KEYWORD } from './models'; + +/** + * QueryBuilder is responsible for building SQL-like queries for Google Sheets. + */ +export class QueryBuilder { + private _replacer: Map; + private _columns: string[]; + private _where: string = ''; + private _whereArgs: any[] = []; + private _whereInterceptor: WhereInterceptorFunc | null = null; + private _orderBy: string[] = []; + private _limit: number = 0; + private _offset: number = 0; + + /** + * Creates a new QueryBuilder instance. + * + * @param colReplacements Map of column name replacements + * @param whereInterceptor Optional function to intercept and modify WHERE clauses + * @param colSelected Columns to select + */ + constructor( + colReplacements: Map, + whereInterceptor: WhereInterceptorFunc | null, + colSelected: string[] + ) { + this._replacer = colReplacements; + this._columns = colSelected; + this._whereInterceptor = whereInterceptor; + } + + /** + * Sets the WHERE condition for the query. + * + * @param condition The WHERE condition with ? placeholders + * @param args Arguments to replace the ? placeholders + * @returns This QueryBuilder instance for chaining + */ + where(condition: string, ...args: any[]): QueryBuilder { + this._where = condition; + this._whereArgs = args; + return this; + } + + /** + * Sets the ORDER BY clause for the query. + * + * @param ordering Array of column ordering specifications + * @returns This QueryBuilder instance for chaining + */ + orderBy(ordering: ColumnOrderBy[]): QueryBuilder { + const orderByStrings: string[] = []; + for (const o of ordering) { + orderByStrings.push(`${o.column} ${o.orderBy}`); + } + + this._orderBy = orderByStrings; + return this; + } + + /** + * Sets the LIMIT clause for the query. + * + * @param limit Maximum number of rows to return + * @returns This QueryBuilder instance for chaining + */ + limit(limit: number): QueryBuilder { + this._limit = limit; + return this; + } + + /** + * Sets the OFFSET clause for the query. + * + * @param offset Number of rows to skip + * @returns This QueryBuilder instance for chaining + */ + offset(offset: number): QueryBuilder { + this._offset = offset; + return this; + } + + /** + * Generates the SQL query string. + * + * @returns The generated SQL query string or an error + */ + generate(): string { + const stmt: string[] = ['select']; + + this.writeCols(stmt); + this.writeWhere(stmt); + this.writeOrderBy(stmt); + this.writeOffset(stmt); + this.writeLimit(stmt); + + return stmt.join(' '); + } + + /** + * Writes the column selection part of the query. + * + * @param stmt The statement builder array + */ + private writeCols(stmt: string[]): void { + const translated: string[] = []; + for (const col of this._columns) { + translated.push(this.replaceColumn(col)); + } + + stmt.push(translated.join(', ')); + } + + /** + * Writes the WHERE clause of the query. + * + * @param stmt The statement builder array + */ + private writeWhere(stmt: string[]): void { + let whereClause = this._where; + if (this._whereInterceptor) { + whereClause = this._whereInterceptor(this._where); + } + + const nArgs = (whereClause.match(/\?/g) || []).length; + if (nArgs !== this._whereArgs.length) { + throw new Error(`Number of arguments required in the 'where' clause (${nArgs}) is not the same as the number of provided arguments (${this._whereArgs.length})`); + } + + whereClause = this.replaceColumn(whereClause); + const tokens = whereClause.split('?'); + + const result: string[] = []; + result.push(tokens[0]?.trim() ?? ''); + + for (let i = 0; i < this._whereArgs.length; i++) { + const arg = this.convertArg(this._whereArgs[i]); + result.push(arg, tokens[i + 1]?.trim() ?? ''); + } + + stmt.push('where', result.join(' ')); + } + + /** + * Converts an argument to its string representation for the query. + * + * @param arg The argument to convert + * @returns String representation of the argument + */ + private convertArg(arg: any): string { + if (typeof arg === 'number') { + return String(arg); + } else if (typeof arg === 'string' || arg instanceof Uint8Array) { + return this.convertString(arg); + } else if (typeof arg === 'boolean') { + return String(arg); + } else { + throw new Error('Unsupported argument type'); + } + } + + /** + * Converts a string or byte array to its string representation. + * + * @param arg The string or byte array to convert + * @returns String representation of the string or byte array + */ + private convertString(arg: string | Uint8Array): string { + if (arg instanceof Uint8Array) { + return JSON.stringify(new TextDecoder().decode(arg)); + } + + const cleaned = arg.toLowerCase().trim(); + // Check if the string is a Google Sheets keyword + if (GOOGLE_SHEET_SELECT_STMT_STRING_KEYWORD.test(cleaned)) { + return arg; + } + + return JSON.stringify(arg); + } + + /** + * Writes the ORDER BY clause of the query. + * + * @param stmt The statement builder array + */ + private writeOrderBy(stmt: string[]): void { + if (this._orderBy.length === 0) { + return; + } + + const result: string[] = []; + for (const o of this._orderBy) { + result.push(this.replaceColumn(o)); + } + + stmt.push('order by', result.join(', ')); + } + + /** + * Writes the OFFSET clause of the query. + * + * @param stmt The statement builder array + */ + private writeOffset(stmt: string[]): void { + if (this._offset === 0) { + return; + } + + stmt.push('offset', String(this._offset)); + } + + /** + * Writes the LIMIT clause of the query. + * + * @param stmt The statement builder array + */ + private writeLimit(stmt: string[]): void { + if (this._limit === 0) { + return; + } + + stmt.push('limit', String(this._limit)); + } + + /** + * Replaces column names in a string according to the replacer map. + * + * @param str The string containing column names to replace + * @returns The string with column names replaced + */ + private replaceColumn(str: string): string { + let result = str; + for (const [col, repl] of this._replacer.entries()) { + result = result.replace(new RegExp(col, 'g'), repl); + } + return result; + } +} diff --git a/src/google/store/row.ts b/src/google/store/row.ts index e69de29..876e18d 100644 --- a/src/google/store/row.ts +++ b/src/google/store/row.ts @@ -0,0 +1,222 @@ +import { AuthClient } from '../auth/base' + +import { Wrapper } from '../sheets/wrapper' +import { + GoogleSheetSelectStmt, + GoogleSheetInsertStmt, + GoogleSheetUpdateStmt, + GoogleSheetDeleteStmt, + GoogleSheetCountStmt +} from './stmt' +import { DEFAULT_ROW_HEADER_RANGE } from './models' +import { + ColsMapping, + generateColumnMapping +} from '../utils/range' +import { getA1Range } from '../utils/range' +import { ROW_IDX_COL } from './models' + +/** + * GoogleSheetRowStoreConfig defines a list of configurations that can be used to customise how the GoogleSheetRowStore works. + */ +export class GoogleSheetRowStoreConfig { + /** + * Defines the list of column names. + * Note that the column ordering matters and will be used for arranging the real columns in Google Sheet. + * Changing the column ordering in this config but not in the sheet will result in unexpected behaviour. + */ + columns: string[]; + /** + * Defines the list of column names containing a Google Sheet formula. + * Note that only string fields can have a formula. + */ + columnsWithFormula: string[]; + + constructor( + columns: string[], + columnsWithFormula: string[] = [] + ) { + this.columns = columns; + this.columnsWithFormula = columnsWithFormula; + } + + /** + * Validates the configuration. + * Throws an error if no columns are defined or if the number of columns exceeds the maximum allowed. + */ + validate(): void { + if (this.columns.length === 0) { + throw new Error('columns must have at least one column'); + } + const maxColumn = 26; // adjust as needed + if (this.columns.length > maxColumn) { + throw new Error(`you can only have up to ${maxColumn} columns`); + } + } +} + +/** Custom error thrown for invalid GoogleSheetRowStoreConfig operations */ +export class GoogleSheetRowStoreConfigError extends Error { } + +/** + * GoogleSheetRowStore encapsulates row store functionality on top of a Google Sheet. + */ +export class GoogleSheetRowStore { + private wrapper: Wrapper; + private spreadsheetId: string; + private sheetName: string; + private config: GoogleSheetRowStoreConfig; + private colsMapping: ColsMapping; + private colsWithFormula: Set; + + getColsMapping: () => ColsMapping; + getColsWithFormula: () => Set; + getWrapper: () => Wrapper; + getSpreadsheetId: () => string; + getSheetName: () => string; + getConfig: () => { columns: string[]; columnsWithFormula: string[] }; + + private constructor( + wrapper: Wrapper, + spreadsheetId: string, + sheetName: string, + config: GoogleSheetRowStoreConfig + ) { + this.wrapper = wrapper; + this.spreadsheetId = spreadsheetId; + this.sheetName = sheetName; + this.config = config; + this.colsMapping = generateColumnMapping(config.columns); + this.colsWithFormula = new Set(config.columnsWithFormula); + + this.getColsMapping = () => this.colsMapping; + this.getColsWithFormula = () => this.colsWithFormula; + this.getWrapper = () => this.wrapper; + this.getSpreadsheetId = () => this.spreadsheetId; + this.getSheetName = () => this.sheetName; + this.getConfig = () => this.config; + } + + /** + * Creates an instance of the row-based store with the given configuration. + * It will also try to create the sheet if it does not exist yet. + */ + public static async create( + auth: AuthClient, + spreadsheetId: string, + sheetName: string, + config: GoogleSheetRowStoreConfig + ): Promise { + config.validate(); + config = injectTimestampCol(config); + + const wrapper = new Wrapper(auth); + await wrapper.createSheet(spreadsheetId, sheetName); + + const store = new GoogleSheetRowStore( + wrapper, + spreadsheetId, + sheetName, + config + ); + + // Clears existing headers and writes configured column names to the header row + await store.ensureHeaders(); + return store; + } + + /** + * Specifies which columns to return from the Google Sheet when querying. + * You can think of this operation like a SQL SELECT statement (with limitations). + * + * If columns is empty, then all columns will be returned. + * If a column is not found in config.columns, it will be ignored. + * + * Note: calling select() does not execute the query yet. + * Call GoogleSheetSelectStmt.exec() to execute. + */ + public select( + ...columns: string[] + ): GoogleSheetSelectStmt { + return new GoogleSheetSelectStmt(this, columns); + } + + /** + * Specifies the rows to be inserted into the Google Sheet. + * + * The underlying data type of each row must be a plain object. + * Providing other data types will result in an error. + * + * By default, the column name follows the object key name (case-sensitive). + * + * Note: calling insert() does not execute the insertion yet. + * Call GoogleSheetInsertStmt.exec() to execute. + */ + public insert(...rows: any[]): GoogleSheetInsertStmt { + return new GoogleSheetInsertStmt(this, rows); + } + + /** + * Specifies the new value for each of the targeted columns. + * + * The colToValue parameter specifies what value should be updated for which column. + * Each value will be JSON serialized before sending. + * If colToValue is empty, an error will be thrown when exec() is called. + */ + public update(colToValue: Record): GoogleSheetUpdateStmt { + return new GoogleSheetUpdateStmt(this, colToValue); + } + + /** + * Prepares a rows deletion operation. + * + * Note: calling delete() does not execute the deletion yet. + * Call GoogleSheetDeleteStmt.exec() to execute. + */ + public delete(): GoogleSheetDeleteStmt { + return new GoogleSheetDeleteStmt(this); + } + + /** + * Prepares a rows counting operation. + * + * Note: calling count() does not execute the query yet. + * Call GoogleSheetCountStmt.exec() to execute. + */ + public count(): GoogleSheetCountStmt { + return new GoogleSheetCountStmt(this); + } + + /** + * Ensures that the header row is set up correctly. + * Clears existing header and writes the configured column names. + */ + private async ensureHeaders(): Promise { + await this.wrapper.clear( + this.spreadsheetId, + [getA1Range(this.sheetName, DEFAULT_ROW_HEADER_RANGE)], + ); + + const headerValues = this.config.columns.map(col => col); + await this.wrapper.updateRows( + this.spreadsheetId, + getA1Range(this.sheetName, DEFAULT_ROW_HEADER_RANGE), + [headerValues], + ); + } +} + +/** + * The additional ROW_IDX_COL column is needed to differentiate which row is truly empty and which one is not. + * We use this for detecting empty rows for UPDATE without a WHERE clause. + * Otherwise, updates would affect all rows instead of non-empty ones only. + */ +export function injectTimestampCol( + config: GoogleSheetRowStoreConfig +): GoogleSheetRowStoreConfig { + const newColumns = [ROW_IDX_COL, ...config.columns]; + return new GoogleSheetRowStoreConfig( + newColumns, + config.columnsWithFormula + ); +} \ No newline at end of file diff --git a/src/google/store/stmt.ts b/src/google/store/stmt.ts index e69de29..afdb073 100644 --- a/src/google/store/stmt.ts +++ b/src/google/store/stmt.ts @@ -0,0 +1,361 @@ +import { + DEFAULT_ROW_FULL_TABLE_RANGE, + ROW_WHERE_NON_EMPTY_CONDITION_TEMPLATE_FUNC, + ROW_DELETE_RANGE_TEMPLATE_FUNC, + ROW_WHERE_EMPTY_CONDITION_TEMPLATE, + ROW_IDX_COL, + ROW_IDX_FORMULA, +} from './models'; + +import { QueryBuilder } from './query'; +import { QueryRowsResult } from '../sheets/models'; +import { ColumnOrderBy } from '../utils/row'; +import { ColsMapping } from '../utils/range'; +import { Wrapper } from '../sheets/wrapper'; +import { escapeValue as commonEscapeValue, checkIEEE754SafeInteger } from '../utils/values'; +import { getA1Range } from '../utils/range'; +import { + BatchUpdateRowsRequest, +} from '../sheets/models'; + +interface RowStore { + getColsMapping(): ColsMapping + getColsWithFormula(): Set + getWrapper(): Wrapper + getSpreadsheetId(): string + getSheetName(): string + getConfig(): { + columns: string[], + columnsWithFormula: string[], + } +} + +export function ridWhereClauseInterceptor(where: string) { + if (where && where.length > 0) { + return ROW_WHERE_NON_EMPTY_CONDITION_TEMPLATE_FUNC(where); + } + return ROW_WHERE_EMPTY_CONDITION_TEMPLATE; +} + +export class GoogleSheetSelectStmt { + private store: RowStore; + private columns: string[]; + private queryBuilder: QueryBuilder; + + constructor(store: RowStore, columns: string[]) { + if (columns.length === 0) { + columns = store.getConfig().columns + } + this.store = store; + this.columns = columns; + this.queryBuilder = new QueryBuilder( + this.store.getColsMapping().getNameMap(), + ridWhereClauseInterceptor, + columns, + ); + } + + /** + * Specifies the WHERE clause condition with placeholders (`?`) and args + */ + where(condition: string, ...args: any[]): this { + this.queryBuilder.where(condition, ...args); + return this; + } + + /** + * Specifies the ORDER BY clause + */ + orderBy(ordering: ColumnOrderBy[]): this { + this.queryBuilder.orderBy(ordering); + return this; + } + + /** + * Limits the number of rows to retrieve + */ + limit(limit: number): this { + this.queryBuilder.limit(limit) + return this + } + + /** + * Skips the first `offset` rows + */ + offset(offset: number): this { + this.queryBuilder.offset(offset) + return this + } + + /** + * Executes the query and returns an array of row objects + */ + async exec(): Promise>> { + const stmt = this.queryBuilder.generate(); + const result: QueryRowsResult = await this.store.getWrapper().queryRows( + this.store.getSpreadsheetId(), + this.store.getSheetName(), + stmt, + true, + ); + return this.buildQueryResultMap(result); + } + + /** + * Maps raw row data into an array of objects keyed by column name + */ + private buildQueryResultMap( + original: QueryRowsResult + ): Array> { + return original.rows.map(row => { + const obj: Record = {}; + this.columns.forEach((col, idx) => { + if (col === '_rid') { + return + } + obj[col] = row[idx] + }); + return obj; + }); + } +} + +export class GoogleSheetInsertStmt { + constructor( + private store: RowStore, + private rows: object[], + ) { + this.store = store; + this.rows = rows; + } + + private convertObjectToFieldMap(row: any): Map { + const fieldMap = new Map(); + + while (row && row !== Object.prototype) { + Object.getOwnPropertyNames(row) + .filter(key => typeof row[key] !== "function" && !(key in fieldMap)) + .forEach(key => { + fieldMap.set(key, row[key]); + }); + + row = Object.getPrototypeOf(row); + } + + return fieldMap; + } + + private convertRowToArray(row: object): any[] { + if (row === null) { + throw new Error('row type must not be null') + } + if (typeof row !== 'object' || Array.isArray(row)) { + throw new Error('row type must be an object') + } + + const output: Map = this.convertObjectToFieldMap(row) + const result: any[] = new Array(this.store.getColsMapping().getNameMap().size).fill(undefined) + result[0] = ROW_IDX_FORMULA + + output.forEach((val, col) => { + if (!this.store.getColsMapping().hasCol(col)) { + return + } + + const colIdx = this.store.getColsMapping().getColIdx(col) + const escapedValue = escapeValue(col, val, this.store.getColsWithFormula()) + checkIEEE754SafeInteger(escapedValue) + result[colIdx.idx] = escapedValue + }) + + return result + } + + async exec(): Promise { + if (this.rows.length === 0) { + return + } + + const convertedRows: any[][] = this.rows.map(row => this.convertRowToArray(row)) + await this.store.getWrapper().overwriteRows( + this.store.getSpreadsheetId(), + getA1Range(this.store.getSheetName(), DEFAULT_ROW_FULL_TABLE_RANGE), + convertedRows, + ) + } +} + +export class GoogleSheetUpdateStmt { + private queryBuilder: QueryBuilder + + constructor( + private store: RowStore, + private colToValue: Record + ) { + this.queryBuilder = new QueryBuilder( + this.store.getColsMapping().getNameMap(), + ridWhereClauseInterceptor, + [ROW_IDX_COL], + ) + this.colToValue = colToValue + } + + where(condition: string, ...args: any[]): this { + this.queryBuilder.where(condition, ...args) + return this + } + + async exec(): Promise { + if (Object.keys(this.colToValue).length === 0) { + throw new Error('empty colToValue, at least one column must be updated') + } + + const indices = await getRowIndices(this.store, this.queryBuilder.generate()) + if (indices.length === 0) { + return + } + + const requests = this.generateBatchUpdateRequests(indices) + await this.store.getWrapper().batchUpdateRows(this.store.getSpreadsheetId(), requests) + } + + private generateBatchUpdateRequests(rowIndices: number[]): BatchUpdateRowsRequest[] { + const reqs: BatchUpdateRowsRequest[] = [] + + Object.entries(this.colToValue).forEach(([col, val]) => { + if (!this.store.getColsMapping().hasCol(col)) { + throw new Error(`failed to update, unknown column name provided: ${col}`) + } + + const colIdx = this.store.getColsMapping().getColIdx(col) + const escaped = escapeValue(col, val, this.store.getColsWithFormula()) + checkIEEE754SafeInteger(escaped) + + for (const rowIdx of rowIndices) { + const a1Range = colIdx.name + rowIdx + reqs.push({ + a1Range: getA1Range(this.store.getSheetName(), a1Range), + values: [[escaped]], + }) + } + }) + + return reqs + } +} + +export class GoogleSheetDeleteStmt { + private queryBuilder: QueryBuilder + + constructor(private store: RowStore) { + this.queryBuilder = new QueryBuilder( + this.store.getColsMapping().getNameMap(), + ridWhereClauseInterceptor, + [ROW_IDX_COL], + ) + } + + where(condition: string, ...args: any[]): this { + this.queryBuilder.where(condition, ...args) + return this + } + + async exec(): Promise { + const selectSql = this.queryBuilder.generate() + const indices = await getRowIndices(this.store, selectSql) + + if (indices.length === 0) { + return + } + + const ranges = generateRowA1Ranges(this.store.getSheetName(), indices) + await this.store.getWrapper().clear( + this.store.getSpreadsheetId(), + ranges, + ) + } +} + +export class GoogleSheetCountStmt { + private queryBuilder: QueryBuilder + + constructor(private store: RowStore) { + const countExpr = `COUNT(${ROW_IDX_COL})` + this.queryBuilder = new QueryBuilder( + this.store.getColsMapping().getNameMap(), + ridWhereClauseInterceptor, + [countExpr], + ) + } + + where(condition: string, ...args: any[]): this { + this.queryBuilder.where(condition, ...args) + return this + } + + async exec(): Promise { + const selectSql = this.queryBuilder.generate() + const result: QueryRowsResult = await this.store.getWrapper().queryRows( + this.store.getSpreadsheetId(), + this.store.getSheetName(), + selectSql, + true, + ) + + if (result.rows.length !== 1 || result.rows[0]!.length !== 1) { + throw new Error(`unexpected count result: ${JSON.stringify(result.rows)}`) + } + + const raw = result.rows[0]![0] + if (typeof raw !== 'number') { + throw new Error(`invalid count type: ${typeof raw}`) + } + return Math.trunc(raw) + } +} + +async function getRowIndices(store: RowStore, selectSql: string): Promise { + const res: QueryRowsResult = await store.getWrapper().queryRows( + store.getSpreadsheetId(), + store.getSheetName(), + selectSql, + true, + ) + if (res.rows.length === 0) { + return [] + } + + const indices: number[] = [] + for (const row of res.rows) { + if (row.length !== 1) { + throw new Error(`error retrieving row indices: ${JSON.stringify(res)}`) + } + + const v = row[0] + if (typeof v !== 'number') { + throw new Error(`error converting row index, value: ${JSON.stringify(v)}`) + } + indices.push(Math.trunc(v)) + } + return indices +} + +function generateRowA1Ranges( + sheetName: string, + indices: number[] +): string[] { + return indices.map(idx => { + const rawRange = ROW_DELETE_RANGE_TEMPLATE_FUNC(idx, idx) + return getA1Range(sheetName, rawRange) + }) +} + +export function escapeValue(col: string, value: any, colsWithFormula: Set): any { + if (!colsWithFormula.has(col)) { + return commonEscapeValue(value) + } + if (typeof value !== 'string') { + throw new Error(`value of column ${col} is not a string, but expected to contain formula`) + } + return value +} \ No newline at end of file diff --git a/src/google/utils/kv.ts b/src/google/utils/kv.ts new file mode 100644 index 0000000..2c3a9ec --- /dev/null +++ b/src/google/utils/kv.ts @@ -0,0 +1,24 @@ +/** + * Defines the mode of the key value store. + * For more details, please read the README file. + */ +export enum KVMode { + Default = 0, + AppendOnly = 1 +} + +/** + * Constants for special values in the key-value store + */ +export const NAValue = "#N/A"; +export const ErrorValue = "#ERROR!"; + +/** + * Error class for when a key is not found in the key-value store + */ +export class KeyNotFoundError extends Error { + constructor() { + super("error key not found"); + this.name = "KeyNotFoundError"; + } +} diff --git a/src/google/utils/range.ts b/src/google/utils/range.ts index d24521a..60d8547 100644 --- a/src/google/utils/range.ts +++ b/src/google/utils/range.ts @@ -27,12 +27,20 @@ export class ColsMapping { this.mapping = mapping; } - public getNameMap(): Map { + getNameMap(): Map { const entries = this.mapping.entries(); return new Map( Array.from(entries).map(([key, value]) => [key, value.name]) ); } + + hasCol(col: string): boolean { + return this.mapping.has(col); + } + + getColIdx(col: string): ColIdx { + return this.mapping.get(col)!; + } } const ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'; @@ -43,8 +51,8 @@ const ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'; * @param columns - Array of column identifiers * @returns Mapping of column identifiers to their A1 notation and index */ -export function generateColumnMapping(columns: string[]): Map { - return new Map( +export function generateColumnMapping(columns: string[]): ColsMapping { + const mapping = new Map( columns.map((col, idx) => [ col, { @@ -53,6 +61,8 @@ export function generateColumnMapping(columns: string[]): Map { }, ]) ); + + return new ColsMapping(mapping); } /** diff --git a/src/google/utils/row.ts b/src/google/utils/row.ts new file mode 100644 index 0000000..13ec2e1 --- /dev/null +++ b/src/google/utils/row.ts @@ -0,0 +1,16 @@ +/** + * OrderBy defines the type of column ordering used for selecting rows. + */ +export enum OrderBy { + ASC = 'ASC', + DESC = 'DESC', +} + +/** + * ColumnOrderBy defines what ordering is required for a particular column + * when selecting rows. + */ +export interface ColumnOrderBy { + column: string; + orderBy: OrderBy; +} \ No newline at end of file diff --git a/tests/google/sheets/models.test.ts b/tests/google/sheets/models.test.ts index 6c07739..f275678 100644 --- a/tests/google/sheets/models.test.ts +++ b/tests/google/sheets/models.test.ts @@ -1,4 +1,3 @@ -import { GoogleAuth } from 'google-auth-library'; import { Wrapper } from '../../../src/google/sheets/wrapper'; import { A1Range, RawQueryRowsResult, QueryRowsResult } from '../../../src/google/sheets/models'; @@ -74,7 +73,8 @@ describe('RawQueryRowsResult', () => { }; const expected: QueryRowsResult = { rows: [] }; - const wrapper = new Wrapper(new GoogleAuth()); + const mockAuth = { getAuth: jest.fn() }; + const wrapper = new Wrapper(mockAuth); const result = wrapper['toQueryRowsResult'](rawResult); expect(result).toEqual(expected); }); @@ -121,7 +121,8 @@ describe('RawQueryRowsResult', () => { ], }; - const wrapper = new Wrapper(new GoogleAuth()); + const mockAuth = { getAuth: jest.fn() }; + const wrapper = new Wrapper(mockAuth); const result = wrapper['toQueryRowsResult'](rawResult); expect(result).toEqual(expected); }); @@ -146,7 +147,8 @@ describe('RawQueryRowsResult', () => { }, }; - const wrapper = new Wrapper(new GoogleAuth()); + const mockAuth = { getAuth: jest.fn() }; + const wrapper = new Wrapper(mockAuth); expect(() => wrapper['toQueryRowsResult'](rawResult)).toThrow('Unsupported cell value type: something'); }); }); diff --git a/tests/google/sheets/wrapper.test.ts b/tests/google/sheets/wrapper.test.ts index 5178f52..ac596d7 100644 --- a/tests/google/sheets/wrapper.test.ts +++ b/tests/google/sheets/wrapper.test.ts @@ -1,4 +1,4 @@ -import { GoogleAuth } from 'google-auth-library'; +import { AuthClient } from '../../../src/google/auth/base' import { Wrapper } from '../../../src/google/sheets/wrapper'; import { sheets_v4 } from 'googleapis'; import { AppendMode, A1Range } from '../../../src/google/sheets/models'; @@ -22,11 +22,11 @@ jest.mock('googleapis', () => ({ describe('Wrapper', () => { let wrapper: Wrapper; - let mockAuth: jest.Mocked; + let mockAuth: AuthClient; let mockSheetsService: jest.Mocked; beforeEach(() => { - mockAuth = new GoogleAuth() as jest.Mocked; + mockAuth = { getAuth: jest.fn() }; wrapper = new Wrapper(mockAuth); mockSheetsService = wrapper['service'] as unknown as jest.Mocked; }); diff --git a/tests/google/store/kv.integration.test.ts b/tests/google/store/kv.integration.test.ts new file mode 100644 index 0000000..7d9a6a8 --- /dev/null +++ b/tests/google/store/kv.integration.test.ts @@ -0,0 +1,97 @@ +import { AuthClient } from '../../../src/google/auth/base' +import { ServiceAccountGoogleAuthClient } from '../../../src/google/auth/service_account' +import { GOOGLE_SHEETS_READ_WRITE } from '../../../src/google/auth/models' +import { GoogleSheetKVStore } from '../../../src/google/store/kv' +import { KVMode, KeyNotFoundError } from '../../../src/google/utils/kv' +import { getIntegrationTestInfo, deleteSheets } from './utils' + +function sleep(ms: number) { + return new Promise(resolve => setTimeout(resolve, ms)) +} + +describe('GoogleSheetKVStore integration', () => { + const { spreadsheetId, authJson, runIntegration } = getIntegrationTestInfo() + + if (!runIntegration) { + it.skip('skipping integration tests – not in CI or missing env', () => { }) + return + } + + let googleAuth: AuthClient + + beforeAll(async () => { + googleAuth = ServiceAccountGoogleAuthClient.fromServiceAccountInfo( + JSON.parse(authJson), + GOOGLE_SHEETS_READ_WRITE, + ) + }) + + it('append-only mode', async () => { + const sheetName = `integration_kv_append_only_${Date.now()}` + + let store = await GoogleSheetKVStore.create( + googleAuth, + spreadsheetId, + sheetName, + { mode: KVMode.AppendOnly } + ) + + try { + await sleep(1000) + await expect(store.get('k1')).rejects.toBeInstanceOf(KeyNotFoundError) + + await sleep(1000) + await store.set('k1', 'test') + + await sleep(1000) + const v = await store.get('k1') + expect(v).toEqual('test') + + await sleep(1000) + await store.delete('k1') + + await sleep(1000) + await expect(store.get('k1')).rejects.toBeInstanceOf(KeyNotFoundError) + } finally { + await sleep(1000) + await deleteSheets(store['rowStore'].getWrapper(), spreadsheetId, [sheetName]) + } + }, 60000) + + it('default mode', async () => { + const sheetName = `integration_kv_default_${Date.now()}` + + let store = await GoogleSheetKVStore.create( + googleAuth, + spreadsheetId, + sheetName, + { mode: KVMode.Default } + ) + + try { + await sleep(1000) + await expect(store.get('k1')).rejects.toBeInstanceOf(KeyNotFoundError) + + await sleep(1000) + await store.set('k1', 'test') + + await sleep(1000) + expect(await store.get('k1')).toEqual('test') + + await sleep(1000) + await store.set('k1', 'test2') + + await sleep(1000) + expect(await store.get('k1')).toEqual('test2') + + await sleep(1000) + await store.delete('k1') + + await sleep(1000) + await expect(store.get('k1')).rejects.toBeInstanceOf(KeyNotFoundError) + } finally { + await sleep(1000) + await deleteSheets(store['rowStore'].getWrapper(), spreadsheetId, [sheetName]) + } + }, 60000) +}) \ No newline at end of file diff --git a/tests/google/store/query.test.ts b/tests/google/store/query.test.ts new file mode 100644 index 0000000..d23f060 --- /dev/null +++ b/tests/google/store/query.test.ts @@ -0,0 +1,94 @@ +import { QueryBuilder } from '../../../src/google/store/query'; +import { ridWhereClauseInterceptor } from '../../../src/google/store/stmt'; +import { ROW_IDX_COL } from '../../../src/google/store/models'; +import { OrderBy } from '../../../src/google/utils/row'; +import { ColsMapping } from '../../../src/google/utils/range'; + +describe('QueryBuilder', () => { + const colsMapping = new ColsMapping(new Map(Object.entries({ + [ROW_IDX_COL]: { name: 'A', idx: 0 }, + col1: { name: 'B', idx: 1 }, + col2: { name: 'C', idx: 2 }, + }))); + + describe('basic generation', () => { + it('should generate a basic select statement', () => { + const builder = new QueryBuilder(colsMapping.getNameMap(), ridWhereClauseInterceptor, ['col1', 'col2']); + const result = builder.generate(); + expect(result).toBe('select B, C where A is not null'); + }); + + it('should handle unknown columns gracefully', () => { + const builder = new QueryBuilder(colsMapping.getNameMap(), ridWhereClauseInterceptor, ['col1', 'col2', 'col3']); + const result = builder.generate(); + expect(result).toBe('select B, C, col3 where A is not null'); + }); + }); + + describe('where clause', () => { + it('should generate with where and correct args', () => { + const builder = new QueryBuilder(colsMapping.getNameMap(), ridWhereClauseInterceptor, ['col1', 'col2']); + builder.where('(col1 > ? AND col2 <= ?) OR (col1 != ? AND col2 == ?)', 100, true, 'value', 3.14); + const result = builder.generate(); + expect(result).toBe('select B, C where A is not null AND (B > 100 AND C <= true ) OR (B != "value" AND C == 3.14 )'); + }); + + it('should throw if arg count does not match', () => { + const builder = new QueryBuilder(colsMapping.getNameMap(), ridWhereClauseInterceptor, ['col1', 'col2']); + builder.where('(col1 > ? AND col2 <= ?) OR (col1 != ? AND col2 == ?)', 100, true); + expect(() => builder.generate()).toThrow(); + }); + + it('should throw if unsupported arg type', () => { + const builder = new QueryBuilder(colsMapping.getNameMap(), ridWhereClauseInterceptor, ['col1', 'col2']); + builder.where('(col1 > ? AND col2 <= ?) OR (col1 != ? AND col2 == ?)', 100, true, null, []); + expect(() => builder.generate()).toThrow(); + }); + }); + + describe('limit and offset', () => { + it('should generate with limit and offset', () => { + const builder = new QueryBuilder(colsMapping.getNameMap(), ridWhereClauseInterceptor, ['col1', 'col2']); + builder.limit(10).offset(100); + const result = builder.generate(); + expect(result).toBe('select B, C where A is not null offset 100 limit 10'); + }); + }); + + describe('order by', () => { + it('should generate with order by clause', () => { + const orderBy = [ + { column: 'col2', orderBy: OrderBy.DESC }, + { column: 'col1', orderBy: OrderBy.ASC }, + ]; + const builder = new QueryBuilder(colsMapping.getNameMap(), ridWhereClauseInterceptor, ['col1', 'col2']); + builder.orderBy(orderBy); + const result = builder.generate(); + expect(result).toBe('select B, C where A is not null order by C DESC, B ASC'); + }); + }); + + describe('argument conversion', () => { + it('should convert various argument types', () => { + const builder = new QueryBuilder(colsMapping.getNameMap(), ridWhereClauseInterceptor, ['col1', 'col2']); + const testCases = [ + { input: 1, output: '1' }, + { input: 1.5, output: '1.5' }, + { input: 'something', output: '"something"' }, + { input: 'date', output: 'date' }, + { input: 'datetime', output: 'datetime' }, + { input: 'timeofday', output: 'timeofday' }, + { input: true, output: 'true' }, + { input: new Uint8Array([115, 111, 109, 101, 116, 104, 105, 110, 103]), output: '"something"' }, + ]; + + for (const c of testCases) { + // @ts-ignore + const result = builder['convertArg'](c.input); + expect(result).toBe(c.output); + } + + expect(() => builder['convertArg']({})).toThrow(); + }); + }); +}); \ No newline at end of file diff --git a/tests/google/store/row.integration.test.ts b/tests/google/store/row.integration.test.ts new file mode 100644 index 0000000..3928182 --- /dev/null +++ b/tests/google/store/row.integration.test.ts @@ -0,0 +1,201 @@ +import { AuthClient } from '../../../src/google/auth/base' +import { ServiceAccountGoogleAuthClient } from '../../../src/google/auth/service_account' +import { GOOGLE_SHEETS_READ_WRITE } from '../../../src/google/auth/models' +import { GoogleSheetRowStore, GoogleSheetRowStoreConfig } from '../../../src/google/store/row' +import { + ROW_IDX_COL +} from '../../../src/google/store/models' +import { OrderBy } from '../../../src/google/utils/row' +import { injectTimestampCol } from '../../../src/google/store/row' +import { getIntegrationTestInfo, deleteSheets } from './utils' + +function sleep(ms: number) { + return new Promise(resolve => setTimeout(resolve, ms)) +} + +describe('GoogleSheetRowStore integration', () => { + const { spreadsheetId, authJson, runIntegration } = getIntegrationTestInfo() + if (!runIntegration) { + it.skip('skipping integration tests – missing CI env', () => { }) + return + } + + let googleAuth: AuthClient + + beforeAll(async () => { + googleAuth = ServiceAccountGoogleAuthClient.fromServiceAccountInfo( + JSON.parse(authJson), + GOOGLE_SHEETS_READ_WRITE, + ) + }, 30000) + + it('basic CRUD and query', async () => { + const sheetName = `integration_row_${Date.now()}` + let store = await GoogleSheetRowStore.create( + googleAuth, + spreadsheetId, + sheetName, + new GoogleSheetRowStoreConfig(['name', 'age', 'dob']) + ) + + try { + await sleep(1000) + await expect( + store + .select('name', 'age') + .offset(10) + .limit(10) + .exec() + ).resolves.toEqual([]) + + await sleep(1000) + await store.insert( + { name: 'name1', age: 10, dob: '1999-01-01' }, + { name: 'name2', age: 11, dob: '2000-01-01' } + ).exec() + + await sleep(1000) + await expect(store.insert(null!).exec()).rejects.toThrow() + + await sleep(1000) + await store.insert({ + name: 'name3', + age: Number.MAX_SAFE_INTEGER, + dob: '2001-01-01' + }).exec() + + await sleep(1000) + await store.update({ name: 'name4' }) + .where('age = ?', 10) + .exec() + + const expected = [ + { name: 'name2', age: 11, dob: '2000-01-01' }, + { name: 'name3', age: Number.MAX_SAFE_INTEGER, dob: '2001-01-01' } + ] + + await sleep(1000) + const out = await store + .select('name', 'age', 'dob') + .where('name = ? OR name = ?', 'name2', 'name3') + .orderBy([{ column: 'name', orderBy: OrderBy.ASC }]) + .limit(2) + .exec() + + expect(out).toEqual(expected) + + await sleep(1000) + const cnt = await store + .count() + .where('name = ? OR name = ?', 'name2', 'name3') + .exec() + expect(cnt).toBe(2) + + await sleep(1000) + await store.delete() + .where('name = ?', 'name4') + .exec() + } finally { + await sleep(1000) + await deleteSheets((store as any).wrapper, spreadsheetId, [sheetName]) + } + }, 60000) + + it('edge cases', async () => { + const sheetName = `integration_edge_${Date.now()}` + let store = await GoogleSheetRowStore.create( + googleAuth, + spreadsheetId, + sheetName, + new GoogleSheetRowStoreConfig(['name', 'age', 'dob']) + ) + + try { + await sleep(1000) + await expect(store.insert(['name3', 12, '2001-01-01']).exec()) + .rejects.toThrow() + + await sleep(1000) + await expect(store.insert({ + name: 'name3', + age: Number.MAX_SAFE_INTEGER + 1, + dob: '2001-01-01' + }).exec()).rejects.toThrow() + + await sleep(1000) + await store.insert( + { name: 'name1', age: 10, dob: '1999-01-01' }, + { name: 'name2', age: 11, dob: '2000-01-01' } + ).exec() + await sleep(1000) + + const map = { + name: 'name4', + age: Number.MAX_SAFE_INTEGER + 1 + } + await expect(store.update(map).exec()).rejects.toThrow() + } finally { + await sleep(1000) + await deleteSheets((store as any).wrapper, spreadsheetId, [sheetName]) + } + }, 60000) + + it('formula support', async () => { + const sheetName = `integration_formula_${Date.now()}` + let store = await GoogleSheetRowStore.create( + googleAuth, + spreadsheetId, + sheetName, + new GoogleSheetRowStoreConfig( + ['value'], + ['value'] + ) + ) + + try { + await sleep(1000) + await store.insert({ value: '=ROW()-1' }).exec() + + await sleep(1000) + const out = await store.select().exec() + expect(out).toEqual([{ value: 1 }]) + + await sleep(1000) + await store.update({ value: '=ROW()' }).exec() + + await sleep(1000) + const out2 = await store.select().exec() + expect(out2).toEqual([{ value: 2 }]) + } finally { + await sleep(1000) + await deleteSheets((store as any).wrapper, spreadsheetId, [sheetName]) + } + }, 60000) +}) + +describe('GoogleSheetRowStoreConfig', () => { + it('empty columns', () => { + expect(() => new GoogleSheetRowStoreConfig([]).validate()) + .toThrow() + }) + + it('too many columns', () => { + expect(() => new GoogleSheetRowStoreConfig( + Array.from({ length: 27 }, (_, i) => String(i)) + ).validate()).toThrow() + }) + + it('no error', () => { + expect(() => new GoogleSheetRowStoreConfig( + Array.from({ length: 10 }, (_, i) => String(i)) + ).validate()).not.toThrow() + }) +}) + +describe('injectTimestampCol', () => { + it('injectTimestampCol prepends ROW_IDX_COL', () => { + const cfg = new GoogleSheetRowStoreConfig(['col1', 'col2']) + const result = injectTimestampCol(cfg) + expect(result.columns).toEqual([ROW_IDX_COL, 'col1', 'col2']) + }) +}) \ No newline at end of file diff --git a/tests/google/store/stmt.test.ts b/tests/google/store/stmt.test.ts new file mode 100644 index 0000000..5041ee6 --- /dev/null +++ b/tests/google/store/stmt.test.ts @@ -0,0 +1,596 @@ +import { + GoogleSheetSelectStmt, + GoogleSheetInsertStmt, + GoogleSheetUpdateStmt, + GoogleSheetDeleteStmt, + GoogleSheetCountStmt, + escapeValue, +} from '../../../src/google/store/stmt'; +import { DEFAULT_ROW_FULL_TABLE_RANGE, ROW_IDX_COL, ROW_IDX_FORMULA } from '../../../src/google/store/models'; +import { ColsMapping } from '../../../src/google/utils/range'; +import { Wrapper } from '../../../src/google/sheets/wrapper'; +import { QueryRowsResult, BatchUpdateRowsRequest } from '../../../src/google/sheets/models'; +import { getA1Range } from '../../../src/google/utils/range'; + +describe('GoogleSheetSelectStmt', () => { + const spreadsheetId = 'sheet_id'; + const sheetName = 'sheet_name'; + const colsWithFormula = new Set(); + const config = { + columns: [], + columnsWithFormula: [], + } + + describe('SQL generation', () => { + it('should generate a basic select statement for two columns', () => { + const cols = new Map(Object.entries({ + [ROW_IDX_COL]: { name: 'A', idx: 0 }, + col1: { name: 'B', idx: 1 }, + col2: { name: 'C', idx: 2 }, + })); + const colsMapping = new ColsMapping(cols); + const mockWrapper = { queryRows: jest.fn() } as unknown as Wrapper; + + const store = { + getColsMapping: () => colsMapping, + getWrapper: () => mockWrapper, + getSpreadsheetId: () => spreadsheetId, + getSheetName: () => sheetName, + getColsWithFormula: () => colsWithFormula, + getConfig: () => config, + }; + const stmt = new GoogleSheetSelectStmt(store, ['col1', 'col2']); + + // Access the private QueryBuilder to verify .generate() + const sql = (stmt as any).queryBuilder.generate(); + expect(sql).toBe('select B, C where A is not null'); + }); + }); + + describe('exec()', () => { + it('should propagate errors from the wrapper', async () => { + const cols = new Map(Object.entries({ + [ROW_IDX_COL]: { name: 'A', idx: 0 }, + col1: { name: 'B', idx: 1 }, + col2: { name: 'C', idx: 2 }, + })); + const colsMapping = new ColsMapping(cols); + const mockWrapper = { queryRows: jest.fn().mockRejectedValue(new Error('some error')) } as unknown as Wrapper; + + const store = { + getColsMapping: () => colsMapping, + getWrapper: () => mockWrapper, + getSpreadsheetId: () => spreadsheetId, + getSheetName: () => sheetName, + getColsWithFormula: () => colsWithFormula, + getConfig: () => config, + }; + const stmt = new GoogleSheetSelectStmt(store, ['col1', 'col2']); + + await expect(stmt.exec()).rejects.toThrow('some error'); + expect(mockWrapper.queryRows).toHaveBeenCalledWith( + spreadsheetId, + sheetName, + 'select B, C where A is not null', + true + ); + }); + + it('successful select all', async () => { + // Prepare a mapping for age & dob + const cols = new Map(Object.entries({ + [ROW_IDX_COL]: { name: 'A', idx: 0 }, + age: { name: 'B', idx: 1 }, + dob: { name: 'C', idx: 2 }, + })); + const colsMapping = new ColsMapping(cols); + const rows = [ + [10, '17-01-2001'], + [11, '18-01-2000'], + ]; + const mockWrapper = { + queryRows: jest.fn().mockResolvedValue({ rows } as QueryRowsResult) + } as unknown as Wrapper; + + const store = { + getColsMapping: () => colsMapping, + getWrapper: () => mockWrapper, + getSpreadsheetId: () => spreadsheetId, + getSheetName: () => sheetName, + getColsWithFormula: () => colsWithFormula, + getConfig: () => config, + }; + const stmt = new GoogleSheetSelectStmt(store, ['age', 'dob']); + const result = await stmt.exec(); + + expect(result).toEqual([ + { age: 10, dob: '17-01-2001' }, + { age: 11, dob: '18-01-2000' }, + ]); + expect(mockWrapper.queryRows).toHaveBeenCalledWith( + spreadsheetId, + sheetName, + 'select B, C where A is not null', + true + ); + }); + }); +}); + +describe('GoogleSheetInsertStmt', () => { + let wrapper: jest.Mocked; + let store: any; + let stmt: GoogleSheetInsertStmt; + + beforeEach(() => { + wrapper = { + overwriteRows: jest.fn().mockResolvedValue(undefined), + } as unknown as jest.Mocked; + + const cols = new Map(Object.entries({ + [ROW_IDX_COL]: { name: 'A', idx: 0 }, + name: { name: 'B', idx: 1 }, + age: { name: 'C', idx: 2 }, + dob: { name: 'D', idx: 3 }, + })); + const config = { + columns: ["name", "age", "dob"], + columnsWithFormula: ["name"], + }; + + store = { + getWrapper: () => wrapper, + getColsMapping: () => new ColsMapping(cols), + getColsWithFormula: () => new Set(['name']), + getSpreadsheetId: () => 'sheet-123', + getSheetName: () => 'MySheet', + getConfig: () => config, + }; + stmt = new GoogleSheetInsertStmt(store, []); + }); + + describe('convertRowToArray', () => { + it('throws on null or undefined', () => { + expect(() => (stmt as any).convertRowToArray(null)).toThrow( + 'row type must not be null' + ); + expect(() => (stmt as any).convertRowToArray(undefined)).toThrow( + 'row type must be an object' + ); + }); + + it('throws on non‐object values', () => { + expect(() => + (stmt as any).convertRowToArray(123 as any) + ).toThrow('row type must be an object'); + expect(() => + (stmt as any).convertRowToArray('foo' as any) + ).toThrow('row type must be an object'); + expect(() => + (stmt as any).convertRowToArray(['foo'] as any) + ).toThrow('row type must be an object'); + }); + + it('converts a plain object correctly', () => { + const row = { name: 'blah', age: 10, dob: '2021' }; + const result = (stmt as any).convertRowToArray(row); + expect(result).toEqual([ + ROW_IDX_FORMULA, + 'blah', // name (no formula) + 10, // age (no formula) + "'2021", // dob (string → prefixed) + ]); + }); + + it('converts a class instance correctly', () => { + class Person { + constructor( + public name: string, + public age: number, + public dob: string + ) { } + } + const person = new Person('blah', 10, '2021'); + const result = (stmt as any).convertRowToArray(person); + expect(result).toEqual([ + ROW_IDX_FORMULA, + 'blah', + 10, + "'2021", + ]); + }); + + it('fills missing fields as undefined', () => { + const partial = { name: 'blah', dob: '2021' } as any; + const result = (stmt as any).convertRowToArray(partial); + expect(result).toEqual([ + ROW_IDX_FORMULA, + 'blah', + undefined, // age missing + "'2021", + ]); + }); + + it('handles an object with only name', () => { + const onlyName = { name: 'blah' }; + const result = (stmt as any).convertRowToArray(onlyName); + expect(result).toEqual([ + ROW_IDX_FORMULA, + 'blah', + undefined, + undefined, + ]); + }); + + it('enforces IEEE-754 safe integers', () => { + const maxSafe = Number.MAX_SAFE_INTEGER; // 2^53-1 + const safeRow = { name: 'x', age: maxSafe, dob: '2021' }; + expect(() => + (stmt as any).convertRowToArray(safeRow) + ).not.toThrow(); + + const overSafe = { name: 'x', age: maxSafe + 1, dob: '2021' }; + expect(() => + (stmt as any).convertRowToArray(overSafe) + ).toThrow(); + }); + }); + + describe('exec()', () => { + it('does nothing when there are no rows', async () => { + const stmt = new GoogleSheetInsertStmt(store, []); + await expect(stmt.exec()).resolves.toBeUndefined(); + expect(wrapper.overwriteRows).not.toHaveBeenCalled(); + }); + + it('calls overwriteRows once with converted rows and correct args', async () => { + const rows = [{ name: 'blah', age: 10, dob: '2021' }]; + const stmt = new GoogleSheetInsertStmt(store, rows); + + await stmt.exec(); + + expect(wrapper.overwriteRows).toHaveBeenCalledTimes(1); + const [sheetId, range, convertedRows] = wrapper.overwriteRows.mock.calls[0]!; + + expect(sheetId).toBe(store.getSpreadsheetId()); + expect(range).toBe( + getA1Range(store.getSheetName(), DEFAULT_ROW_FULL_TABLE_RANGE) + ); + expect(convertedRows).toEqual([ + [ + ROW_IDX_FORMULA, + 'blah', + 10, + "'2021", + ], + ]); + }); + + it('propagates errors from the sheets wrapper', async () => { + wrapper.overwriteRows.mockRejectedValueOnce(new Error('API down')); + const stmt = new GoogleSheetInsertStmt(store, [{ name: 'x' }]); + await expect(stmt.exec()).rejects.toThrow('API down'); + }); + }); +}); + +describe('GoogleSheetUpdateStmt', () => { + let wrapper: jest.Mocked; + let store: any; + + beforeEach(() => { + wrapper = { + queryRows: jest.fn().mockResolvedValue({ rows: [] }), + batchUpdateRows: jest.fn().mockResolvedValue(undefined), + } as unknown as jest.Mocked; + + const cols = new Map(Object.entries({ + [ROW_IDX_COL]: { name: 'A', idx: 0 }, + name: { name: 'B', idx: 1 }, + age: { name: 'C', idx: 2 }, + dob: { name: 'D', idx: 3 }, + })); + const config = { + columns: ["name", "age", "dob"], + columnsWithFormula: ["name"], + }; + + store = { + getWrapper: () => wrapper, + getColsMapping: () => new ColsMapping(cols), + getColsWithFormula: () => new Set(['name']), + getSpreadsheetId: () => 'sheet-123', + getSheetName: () => 'MySheet', + getConfig: () => config, + }; + }); + + describe('generateBatchUpdateRequests()', () => { + it('generates requests for each (col, row) pair and escapes strings', () => { + const stmt = new GoogleSheetUpdateStmt( + store, + { + name: 'name1', + age: 100, + dob: 'hello', + }, + ); + + const requests = (stmt as any).generateBatchUpdateRequests([1, 2]) as BatchUpdateRowsRequest[]; + const expected: BatchUpdateRowsRequest[] = [ + { a1Range: getA1Range('MySheet', 'B1'), values: [['name1']] }, + { a1Range: getA1Range('MySheet', 'B2'), values: [['name1']] }, + { a1Range: getA1Range('MySheet', 'C1'), values: [[100]] }, + { a1Range: getA1Range('MySheet', 'C2'), values: [[100]] }, + { a1Range: getA1Range('MySheet', 'D1'), values: [["'hello"]] }, + { a1Range: getA1Range('MySheet', 'D2'), values: [["'hello"]] }, + ]; + + expect(requests).toEqual(expect.arrayContaining(expected)); + expect(requests).toHaveLength(expected.length); + }); + + it('allows up to Number.MAX_SAFE_INTEGER but rejects larger integers', () => { + const maxSafe = Number.MAX_SAFE_INTEGER; // 2^53-1 + const safeStmt = new GoogleSheetUpdateStmt( + store, + { + name: 'x', + age: maxSafe, + }, + ); + expect(() => (safeStmt as any).generateBatchUpdateRequests([1])).not.toThrow(); + + const unsafeStmt = new GoogleSheetUpdateStmt( + store, + { + name: 'x', + age: maxSafe + 1, + }, + ); + expect(() => (unsafeStmt as any).generateBatchUpdateRequests([1])).toThrow(); + }); + }); + + describe('exec()', () => { + it('throws if no columns to update', async () => { + const stmt = new GoogleSheetUpdateStmt(store, {}); + await expect(stmt.exec()) + .rejects + .toThrow('empty colToValue, at least one column must be updated'); + expect(wrapper.queryRows).not.toHaveBeenCalled(); + expect(wrapper.batchUpdateRows).not.toHaveBeenCalled(); + }); + + it('does nothing when queryRows returns no indices', async () => { + wrapper.queryRows.mockResolvedValueOnce({ rows: [] }); + const stmt = new GoogleSheetUpdateStmt( + store, + { + name: 'foo', + }, + ); + await expect(stmt.exec()).resolves.toBeUndefined(); + expect(wrapper.queryRows).toHaveBeenCalledTimes(1); + expect(wrapper.batchUpdateRows).not.toHaveBeenCalled(); + }); + + it('calls batchUpdateRows with correct requests when rows exist', async () => { + // simulate two matching rows, indices [1,2] + wrapper.queryRows.mockResolvedValueOnce({ rows: [[1], [2]] }); + const stmt = new GoogleSheetUpdateStmt( + store, + { + name: 'foo', + }, + ); + + await stmt.exec(); + + expect(wrapper.queryRows).toHaveBeenCalledTimes(1); + expect(wrapper.batchUpdateRows).toHaveBeenCalledTimes(1); + + const [sheetId, requests] = wrapper.batchUpdateRows.mock.calls[0]!; + expect(sheetId).toBe(store.getSpreadsheetId()); + const expectedRequests: BatchUpdateRowsRequest[] = [ + { a1Range: getA1Range('MySheet', 'B1'), values: [['foo']] }, + { a1Range: getA1Range('MySheet', 'B2'), values: [['foo']] }, + ]; + expect(requests).toEqual(expectedRequests); + }); + + it('propagates errors from queryRows', async () => { + wrapper.queryRows.mockRejectedValueOnce(new Error('query failed')); + const stmt = new GoogleSheetUpdateStmt( + store, + { + name: 'foo', + }, + ); + await expect(stmt.exec()).rejects.toThrow('query failed'); + expect(wrapper.batchUpdateRows).not.toHaveBeenCalled(); + }); + + it('propagates errors from batchUpdateRows', async () => { + wrapper.queryRows.mockResolvedValueOnce({ rows: [[1]] }); + wrapper.batchUpdateRows.mockRejectedValueOnce(new Error('API down')); + const stmt = new GoogleSheetUpdateStmt( + store, + { + name: 'foo', + }, + ); + await expect(stmt.exec()).rejects.toThrow('API down'); + }); + }); +}); + +describe('GoogleSheetDeleteStmt', () => { + let wrapper: jest.Mocked + let store: any; + + beforeEach(() => { + wrapper = { + queryRows: jest.fn().mockResolvedValue({ rows: [] } as QueryRowsResult), + clear: jest.fn().mockResolvedValue(undefined), + } as unknown as jest.Mocked + + const cols = new Map(Object.entries({ + [ROW_IDX_COL]: { name: 'A', idx: 0 }, + name: { name: 'B', idx: 1 }, + })); + const config = { + columns: ["name"], + columnsWithFormula: ["name"], + }; + + store = { + getWrapper: () => wrapper, + getColsMapping: () => new ColsMapping(cols), + getColsWithFormula: () => new Set(['name']), + getSpreadsheetId: () => 'sheet-123', + getSheetName: () => 'MySheet', + getConfig: () => config, + }; + }) + + it('is chainable via where()', () => { + const stmt = new GoogleSheetDeleteStmt(store) + expect(stmt.where('foo = ?', 1)).toBe(stmt) + }) + + it('does nothing when no rows match', async () => { + wrapper.queryRows.mockResolvedValueOnce({ rows: [] }) + const stmt = new GoogleSheetDeleteStmt(store) + await expect(stmt.exec()).resolves.toBeUndefined() + expect(wrapper.clear).not.toHaveBeenCalled() + }) + + it('clears the correct ranges when rows exist', async () => { + // simulate matching row indices 1 and 2 + wrapper.queryRows.mockResolvedValueOnce({ rows: [[1], [2]] } as QueryRowsResult) + const stmt = new GoogleSheetDeleteStmt(store) + await stmt.exec() + + const expectedRanges = [1, 2].map(i => + getA1Range(store.getSheetName(), `A${i}:Z${i}`) + ) + expect(wrapper.clear).toHaveBeenCalledTimes(1) + expect(wrapper.clear).toHaveBeenCalledWith(store.getSpreadsheetId(), expectedRanges) + }) + + it('propagates errors from queryRows()', async () => { + wrapper.queryRows.mockRejectedValueOnce(new Error('query failed')) + const stmt = new GoogleSheetDeleteStmt(store) + await expect(stmt.exec()).rejects.toThrow('query failed') + expect(wrapper.clear).not.toHaveBeenCalled() + }) + + it('propagates errors from clear()', async () => { + wrapper.queryRows.mockResolvedValueOnce({ rows: [[5]] } as QueryRowsResult) + wrapper.clear.mockRejectedValueOnce(new Error('clear failed')) + const stmt = new GoogleSheetDeleteStmt(store) + await expect(stmt.exec()).rejects.toThrow('clear failed') + }) +}) + +describe('GoogleSheetCountStmt', () => { + let wrapper: jest.Mocked; + let store: any; + + beforeEach(() => { + wrapper = { + queryRows: jest.fn().mockResolvedValue({ rows: [] } as QueryRowsResult), + clear: jest.fn().mockResolvedValue(undefined), + } as unknown as jest.Mocked + + const cols = new Map(Object.entries({ + [ROW_IDX_COL]: { name: 'A', idx: 0 }, + name: { name: 'B', idx: 1 }, + })); + const config = { + columns: ["name"], + columnsWithFormula: ["name"], + }; + + store = { + getWrapper: () => wrapper, + getColsMapping: () => new ColsMapping(cols), + getColsWithFormula: () => new Set(['name']), + getSpreadsheetId: () => 'sheet-123', + getSheetName: () => 'MySheet', + getConfig: () => config, + }; + }) + + it('is chainable via where()', () => { + const stmt = new GoogleSheetCountStmt(store) + expect(stmt.where('bar > ?', 10)).toBe(stmt) + }) + + it('returns 0 when COUNT is 0', async () => { + wrapper.queryRows.mockResolvedValueOnce({ rows: [[0]] } as QueryRowsResult) + const stmt = new GoogleSheetCountStmt(store) + await expect(stmt.exec()).resolves.toBe(0) + }) + + it('truncates non‐integer counts', async () => { + wrapper.queryRows.mockResolvedValueOnce({ rows: [[5.7]] } as QueryRowsResult) + const stmt = new GoogleSheetCountStmt(store) + await expect(stmt.exec()).resolves.toBe(5) + }) + + it('throws if result shape is unexpected (no rows)', async () => { + wrapper.queryRows.mockResolvedValueOnce({ rows: [] } as QueryRowsResult) + const stmt = new GoogleSheetCountStmt(store) + await expect(stmt.exec()) + .rejects + .toThrow(/unexpected count result/) + }) + + it('throws if result shape is unexpected (too many cols)', async () => { + wrapper.queryRows.mockResolvedValueOnce({ rows: [[1, 2]] } as QueryRowsResult) + const stmt = new GoogleSheetCountStmt(store) + await expect(stmt.exec()) + .rejects + .toThrow(/unexpected count result/) + }) + + it('throws if raw count is not a number', async () => { + wrapper.queryRows.mockResolvedValueOnce({ rows: [['foo']] } as QueryRowsResult) + const stmt = new GoogleSheetCountStmt(store) + await expect(stmt.exec()) + .rejects + .toThrow(/invalid count type: string/) + }) + + it('propagates errors from queryRows()', async () => { + wrapper.queryRows.mockRejectedValueOnce(new Error('db down')) + const stmt = new GoogleSheetCountStmt(store) + await expect(stmt.exec()).rejects.toThrow('db down') + }) +}) + +describe('escapeValue', () => { + describe('when the column is NOT in colsWithFormula', () => { + it('returns numbers unchanged', () => { + expect(escapeValue('A', 123, new Set(['B']))).toBe(123); + }); + + it('prefixes strings with a single quote', () => { + expect(escapeValue('A', '123', new Set(['B']))).toBe("'123"); + }); + }); + + describe('when the column IS in colsWithFormula', () => { + it('throws if the value is not a string', () => { + expect(() => escapeValue('A', 123, new Set(['A']))).toThrowError( + 'value of column A is not a string, but expected to contain formula' + ); + }); + + it('returns the raw string when it is a string', () => { + expect(escapeValue('A', '123', new Set(['A']))).toBe('123'); + }); + }); +}); \ No newline at end of file diff --git a/tests/google/store/utils.ts b/tests/google/store/utils.ts new file mode 100644 index 0000000..6a5a119 --- /dev/null +++ b/tests/google/store/utils.ts @@ -0,0 +1,36 @@ +import { Wrapper } from '../../../src/google/sheets/wrapper' + +export function getIntegrationTestInfo(): { + spreadsheetId: string; + authJson: string; + runIntegration: boolean; +} { + const spreadsheetId = process.env.INTEGRATION_TEST_SPREADSHEET_ID || ''; + const authJson = process.env.INTEGRATION_TEST_AUTH_JSON || ''; + const isGithubActions = !!process.env.GITHUB_ACTIONS; + const runIntegration = isGithubActions && Boolean(spreadsheetId) && Boolean(authJson); + + return { spreadsheetId, authJson, runIntegration }; +} + +export async function deleteSheets( + wrapper: Wrapper, + spreadsheetId: string, + sheetNames: string[] +): Promise { + const sheetNameToId = await wrapper.getSheetNameToID(spreadsheetId); + + const sheetIds = sheetNames.map(name => { + const id = sheetNameToId[name]; + if (id === undefined) { + throw new Error(`Sheet ID for name "${name}" not found`); + } + return id; + }); + + try { + await wrapper.deleteSheets(spreadsheetId, sheetIds); + } catch (err: any) { + console.warn(`Failed deleting sheets: ${err.message || err}`); + } +} \ No newline at end of file diff --git a/tests/google/utils/range.test.ts b/tests/google/utils/range.test.ts index 72c16e3..ded33a6 100644 --- a/tests/google/utils/range.test.ts +++ b/tests/google/utils/range.test.ts @@ -1,3 +1,4 @@ +import { ColsMapping } from '../../../src/google/utils/range'; import { getA1Range, generateColumnName, @@ -123,7 +124,7 @@ describe('range', () => { testCases.forEach(tc => { it(tc.name, () => { - expect(generateColumnMapping(tc.input)).toEqual(tc.expected); + expect(generateColumnMapping(tc.input)).toEqual(new ColsMapping(tc.expected)); }); }); }); diff --git a/tsconfig.json b/tsconfig.json index 788e63f..f3c3f00 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -25,7 +25,7 @@ "moduleDetection": "force", /* Control what method is used to detect module-format JS files. */ /* Modules */ "module": "NodeNext", /* Specify what module code is generated. */ - "rootDir": "./src", /* Specify the root folder within your source files. */ + "rootDir": ".", /* Specify the root folder within your source files. */ // "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */ // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */