From 336cc2db1a4606a853ae3d0d9d4fcd034a73899d Mon Sep 17 00:00:00 2001 From: Ganesh Kumar Date: Sat, 21 Feb 2026 21:25:22 +0530 Subject: [PATCH 1/5] added-initial-implementation-doc LiveReview Pre-Commit Check: skipped --- b2-manager/Changeset.md | 1197 +++++++++++++++++ b2-manager/docs/fdtdb.md | 1079 +++++++++++++++ b2-manager/docs/workflow/migrations.md | 103 ++ frontend/changeset/changeset.py | 34 + .../20260214162434583026694_descroption.py | 62 + frontend/package-lock.json | 16 - 6 files changed, 2475 insertions(+), 16 deletions(-) create mode 100644 b2-manager/Changeset.md create mode 100644 b2-manager/docs/fdtdb.md create mode 100644 b2-manager/docs/workflow/migrations.md create mode 100644 frontend/changeset/changeset.py create mode 100644 frontend/changeset/changeset_scripts/20260214162434583026694_descroption.py diff --git a/b2-manager/Changeset.md b/b2-manager/Changeset.md new file mode 100644 index 0000000000..12663b8168 --- /dev/null +++ b/b2-manager/Changeset.md @@ -0,0 +1,1197 @@ +# 17th Feb 2026 + +# DB Changeset Script Policy + +To Upload server db / local db to b2 without any data loss. + +## States of DB + +Db will be present in these 3 locations mainly: + +1. b2: +- It is the source of truth for the database version. +- Any db updated and inserted, should be done via changeset script integrated with `b2m`. + +2. Server: +- This is the production db. +- Data will be inserted regularly from API and inserted/updated data should reflect in live server. +- Inserted/Updated Data should be exportable during changeset phase which I will be further defining on how it is done in changeset script. +3. Local +- These are the db which are present in the local machine of the developer. +- These db are used for the development purposes. +- Any new feature or changes in the db can be done with only changeset script. + + +## Changeset Script + +I have divided into two types of changeset scripts based on where it is triggered from: + +1. Server -> B2: No version confict (happy flow) +2. Server -> B2: Version conflict (Changeset required) +4. Team Member -> B2: No version confict (happy flow) +5. Team Member -> B2: Version conflict (Changeset required) + +Both of these should be well defined by the developer in the changeset script. +### Automated Changeset Script +In this changeset script, It should automatically applies changeset of data from server to b2. Both b2 and New data should be present before uploading to b2. + + +#### Scenario 1: Server -> B2: No version confict (happy flow) + +Assume These Constraints : +1. Current IPM DB Version States at 2PM. + +| DB Location | version | +| ----------- | ------- | +| b2 | v1 | +| server | v1 | + +**At 3PM:** +1. DB version in same state. +2. User runs ipm and generates IPM json for a new repo which didn't exist earlier. +3. The db gets updated in server db immediately. + +| DB Location | Initial Version | +| ----------- | --------------- | +| b2 | v1 | +| server | v2* | + +Using `*` to say db is updated and not present in b2. + +**At 6PM:** +Changeset script is triggered at 6pm daily. +Step 1: Check Status of DB using `b2m`. +Result:`b2m` will result **Ready to upload** as both DB. +Step 2: Trigger `b2m` to upload DB to b2. +DB Version After Upload to b2 + +| DB Location | Initial Version | +| ----------- | --------------- | +| b2 db | v2 | +| server db | v2 | + +#### Scenario 2: Server -> B2: Version conflict (Changeset required) +1. Current IPM DB Version States at 2PM. + +| DB Location | DB Version | +| ----------- | ---------- | +| b2 | v1 | +| server | v1 | + +**At 3PM:** +1. DB version in same state. +2. User runs ipm and generates IPM json for a new repo which didn't exist earlier. +3. The db gets updated in server db immediately. + +| DB Location | Initial Version | +| ----------- | --------------- | +| b2 | v1 | +| server | v2* | + +Using `*` to say db is updated and not present in b2. +**At 4PM:** + 1. IPM db was updated with bulk json insertion. + 2. This db is uploaded to b2 + +| DB Location | Initial Version | +| ----------- | --------------- | +| b2 | v2 | +| server | v2* | + +Using `*` to say db is updated and not present in b2. + +**At 6PM:** +Changeset script is triggered at 6pm. +Step 1: Check Status of DB using `b2m`. +Result:`b2m` will result **Outdated DB** as both DB. +Step 2: Changeset should export/backup db version `v2*` +Step 3: Download `v2` from `b2` via`b2m` +Step 4: Trigger builk insertion of json from `v2*` to `v2` hence creating `v3` +Step 4: Trigger `b2m` to upload `v3` db to b2. +DB Version After Upload to b2 + +| DB Location | Initial Version | +| ----------- | --------------- | +| b2 db | v3 | +| server db | v3 | + + + + + +#### Scenario 3: Team Member -> B2: No version confict (happy flow) + + +Assume These Constraints : +1. Current emoji DB Version States at 2PM. + +| DB Location | version | +| ----------- | ------- | +| b2 | v1 | +| Athreya | v1 | + +**At 2:10PM:** +1. DB version in same state. +2. Athreya uses Changeset Script to update emoji db + + +| DB Location | Initial Version | +| ----------- | --------------- | +| b2 | v1 | +| Athreya | v2* | + +Using `*` to say db is updated and not present in b2. + +3. Changeset script will be triggered by Athreya and it will continue with these steps. +Step 1: Check Status of DB using `b2m`. +Result:`b2m` will result **Ready to upload** as both DB. +Step 2: Trigger `b2m` to upload DB to b2. +DB Version After Upload to b2 + +| DB Location | Initial Version | +| ----------- | --------------- | +| b2 db | v2 | +| Athreya db | v2 | + + +#### Scenario 4: Team Member -> B2: Version conflict (Changeset required) + + +Assume These Constraints : +1. Current emoji DB Version States at 2PM. + +| DB Location | version | +| ----------- | ------- | +| b2 | v1 | +| Athreya | v1 | +| Lince | v1 | + +**At 2:10PM:** +1. DB version in same state. +2. Athreya uses Changeset Script to update emoji db + + +| DB Location | Initial Version | +| ----------- | --------------- | +| b2 | v1 | +| Athreya | v2* | +| Lince | v1 | + +Using `*` to say db is updated and not present in b2. + +Also Currently Athreya's db is updating DB + + +**At 2:11PM:** + +1. Lince uses Changeset Script to update emoji db + + +| DB Location | Initial Version | +| ----------- | --------------- | +| b2 | v1 | +| Athreya | v2* | +| Lince | v2* | + +Using `*` to say db is updated and not present in b2. + +3. Changeset script will be triggered by Lince and it will continue with these steps. +Step 1: Check Status of DB using `b2m`. +Result:`b2m` will result **Ready to upload** as both DB. +Step 2: Trigger `b2m` to upload DB to b2. +DB Version After Upload to b2 + +| DB Location | Initial Version | +| ----------- | --------------- | +| b2 db | v2 | +| Athreya db | v2* | +| Lince db | v2 | + + +**At 2:12PM:** + +4. Changeset script will be triggered by Athreya and it will continue with these steps. +Step 1: Check Status of DB using `b2m`. +Result:`b2m` will result **Outdated DB** as both DB. +Step 2: Changeset should export/backup db version `v2*` +Step 3: Download `v2` from `b2` via`b2m` +Step 4: Trigger builk insertion of json from `v2*` to `v2` hence creating `v3` +Step 5: Trigger `b2m` to upload `v3` db to b2. +DB Version After Upload to b2 + +| DB Location | Initial Version | +| ----------- | --------------- | +| b2 | v3 | +| Athreya | v3 | +| Lince db | v2 | + + + + +### Default Action Performed by Changeset Script. + +These rules should be followed on start of changeset script. + +This is my assumption, you can correct me if I am wrong. +1. `*-wal` file and the `*-shm` file should not be present. There should be no active connection to the db. +2. `fdt-templ` should have cli to disconnect to slected changeset db's. +3. Make sure Server should still be serving other pages even the db which is being migrated via recent cache. +4. Trigger `b2m` to check status of db. +5. If `b2m` returns **Ready to upload** then trigger `b2m` to upload db to b2. +6. If `b2m` returns **Outdated DB** then + 1. trigger `b2m` to export new data from db version `v2*` or create cp of `v2*` to predifned directory. + 2. download `v2` from `b2` via `b2m` + 3. Insertion of new data from `v2*` to `v2` hence creating `v3` + 4. trigger `b2m` to upload `v3` db to b2. +7. `fdt-templ` should have cli to connect to slected changeset db's. + + + +### How To Use `b2m` cli in Changeset Script. + +> This is proposal of `b2m` cli integration. + +`b2m` cli should have + + +1. `--status` flag to check status of db. + Return `ready_to_upload`, `outdated_db` and `up_to_date`. +Multiple DBs can be checked at once. +```shell +./b2m --status +``` +This Will Check DB Version Status. + 1. `ready_to_upload` - Local DB is up to date and ready to upload to `b2`. + 2. `outdated_db` - Local DB is outdated and needs to download from `b2` and proceed with changeset and then upload to `b2`. + 3. `up_to_date` - Local DB is up to date with `b2`. + +Single DB Check +```shell +./b2m --status +``` +This will be for single DB amd Result will be same. + +2. `--upload` flag to upload db to b2. + return `success` or `failed`. + + +```shell +./b2m --upload +``` +This will upload selected DBs to b2. + + + +3. `--download` flag to download db from b2. + return `success` or `failed`. + +```shell +./b2m --download +``` +This will download selected DBs from b2. +> Note: This will override local DBs. Have a backup of local DBs before downloading. + + + + +# 18th Feb 2026 + +## Goal + +Implementation of Changeset Script Where Team can use it for db version changes, Db Upload to b2 and Db Download from b2 without any data loss/Server Downtime. + + +## Current State + +Identified 4 Scenarios where db version changes can happen. +Many Comments on Proposal + +I have added ✅ and ❌ to the comments to indicate that the comment has been addressed or not. + + +1. How To execute changeset script? ✅ +2. Is there any command for it? ✅ +3. What is the naming convention for that script? ✅ +4. How does it look like? ✅ +5. Download : Isnt this destructive? Some safeguards can be put right? ❌ +7. even in scenario 2,3and 4 changeset should be used to create the v2* db ❌ +1. when git pull happens? cuz changesets *.py are commited to git right for every conflict case ❌ +2. after changing versions how are you handing in code? we have paths hardcoded in code man-pages-db-v2.db ❌ + + +## Problem + +Team Coudn't Understand the Proposal especially in implementation of the changeset script. +1. No Clear Explaination of the the changeset script, How it works. +2. How ipm-db-v2.db -> ipm-db-v3.db is handled? +3. How is changescript will handle hardcoded values in `fdt-templ` server? +4. Does git pull play major role in this? + +## Expectations + +In this iteration on defineing changeset script and it's deps. + +1. Defining Changeset Script Template and how to create changeset script. +2. Explaination of how to execute changeset script. +3. How Change Set script handles ipm-db-v2.db -> ipm-db-v3.db? +4. Changeset script should have these main criteria. + 1. Script will be generated using `b2m` cli. + 2. File will be placed in `changeset` with structure mentioned below by `b2m`. + 3. `changeset_cron`: cron job script (ex: 6:00PM db changeset). + `changeset`: Will be common changeset script + ```shell + changeset/ + ├── changeset_scripts/ + │ ├── _.py or .py + │ └── ... + │ changeset.py # common functions. + └── README.md + ``` + 3. Script should have same template with version tag inside it. (#Template Version: v1) + 4. All the migration (sql scripts), Data extraction (sql scripts), Data insertion (sql scripts) should be in one file. + 5. Script should be executable using `python ` +5. Final Expectation Full template to write code for changeset script. + + +## Solution +### 1. Changeset Script Generation + +1. This will be done by `b2m` cli. +```shell +./b2m --changeset +``` +2. This will create a changeset script in `changeset` directory. +3. It will be generated based on predifned template. + + +### 2. Defining Changeset Script Template + + +Template Proposal: + +Template should look like this. + +1. Predifned Imports and Functions +2. These can be done by creating custom common library where changeset script can import it. +```py +# Template Version: v1 +# : _ +# is a short description of the change. + +## Predifned Imports and Functions +import sqlite3 +import urllib.parse +import os +import time + +def upload(db_name): + # Define cli command to upload db to b2. + # This will be added once cli defined + # Example: + # os.system(f"./b2m --upload {db_name}") + pass + +def download(db_name): + # Define cli command to download db from b2. + # This will be added once cli defined + # Example: + # os.system(f"./b2m --download {db_name}") + pass + +def status(db_name): + # Define cli command to export db to json. + # This will be added once cli defined + # Example: + # os.system(f"./b2m --status {db_name}") + pass + +def update(db_name): + # Use migration code. + # This consist of sql script to update db. + # + pass +### There will be still more need to define those gradualy. + +def main(): + # Check status of db. + # If status is outdated_db, then download db from b2. + # If status is ready_to_upload, then upload db to b2. + # If status is up_to_date, then do nothing. + pass + +if __name__ == "__main__": + main() +``` + + +### 3. Exectuing Changeset Script + +1. This will be done by `b2m` cli. +```shell +./b2m --execute +``` +2. This will execute the changeset script. +3. It will update the db and upload it to b2. + + +## Result + +1. Defined script template, execution methods. + + +## Difference + +1. Main comments not addressed. + + + +# 19th Feb 2026 + + +## Goal 2 + +I have added ✅ and ❌ to the comments to indicate that the comment has been addressed or not. + +1. Address lot of ambiguity in the proposal.(Did 2 iteration before posting for review)✅ +2. How ipm-db-v2.db -> ipm-db-v3.db is handled? (Trying to automate this also) so, not yet solved ❌ +3. How changescript will handle hardcoded values in `fdt-templ` server?✅ +4. Does git pull play major role in this?❌ As, of now I don't think we need push from server side. + +## Expectation +Defineing these points. +1. How to create, execute changeset script? +2. Template of changeset script. +3. Common functions used. +4. Give Example of how changeset script will be handling ipm db changeset. +5. Steps involved in changeset script. with proper description. +6. `db.toml` file will be used to define the db version. + + +## Proposal + + +In this Proposal first I will define all the structure's which include any cli and steps involved in defining changeset script. + + +**Structure** + +This Consist of 4 main parts. +1. `changeset` directory +2. `b2m` cli +3. `db.toml` file +4. `fdt-templ` server +5. `changeset_script` template +6. `changeset.py` common function + +### Create changeset script + +Folder Structure: +``` +changeset/ +├── scripts/ +│ ├── _.py +│ └── ... +├── dbs/ +| ├── _/ +| | ├── _b2.db +| | ├── _server.db +| | └── ... +| └── ... +├── logs/ +| ├── _.log +| └── ... +├── changeset.py # common functions. +└── README.md +``` + + +1. `_.py` will be generated by `b2m` cli. +```shell +./b2m --create-changeset +``` +2. Create a changeset script in `changeset/scripts` directory. +3. It consist of 2 subdirectories. + 1. `scripts`: This will contain changeset script. + 2. `dbs`: (Details Explaination Below) + 1. Temporary dbs until changeset is executed. + 2. This is for safety purpose. + 3. Any Db download, upload should be done in this file. +3. `logs`: + 1. This will contain logs of changeset script. + 2. This is for logging purpose. +4. `changeset.py`: Common functions used in changeset script. + 1. Such as `b2m --upload `, `b2m --download `, `b2m --status `, etc. + + +Reason For dbs Directory: + + +Goal: +1. Move new data added to ipm db in master to b2. + +Understanding Situation with IPM Db: +1. B2 has new data and Master has new data. +2. To Keep Both data we need to download latest db from b2 and insert new data to b2 db. + + +Options to do this: + +1. Use same `db/all_dbs` directory for changeset operations. (Can be done but need to be careful) +2. Use changeset directory for operation. (safe) + + +Option 1: + +1. Export New data to prediffned json file to `db/all_dbs` directory. +2. Download latest db from b2 to `db/all_dbs` directory. +3. Insert new data to b2 db. +4. Upload b2 db to b2. + +Option 2: + +1. Create a copy of original ipm db to `changeset/dbs/_` directory name `ipm-v3.master.db`. +2. Download latest db from b2 to `changeset/dbs/_` directory name `ipm-v3.b2.db` (it can be `ipm-v4.b2.db` depends on version present in b2). +3. Insert new data from `ipm-v3.master.db` to `ipm-v3.b2.db`. +4. Upload `ipm-v3.b2.db` to b2. +5. copy `ipm-v3.b2.db` to `db/all_dbs` directory. +6. Remove `changeset/dbs/_` directory if all the operations are successfully done. + +I choosed option 2 to define any db changeset operations should be done in changeset directory. + + + + +### B2M CLI + +1. `b2m` cli will be used to create, execute, and manage changeset script. +2. `b2m` cli will be placed in `frontend` directory. +3. `b2m` cli will be having following commands. There are 2 types of commands. + 1. User specific commands: These commands are used regurly by us. + 1. `b2m --create-changeset `: Create a changeset script. + 1. This will create a changeset script in `changeset/scripts` directory. (Added Detailed Description Above) + 2. `b2m --execute `: Execute a changeset script. + 1. This will execute the changeset script. It can also be done by `python ` just adding for making it easy to execute. + 2. Db specific commands: These commands are used and predifned in `changeset.py`. (Added Detailed Descript Above) + 1. `b2m --status `: Check status of db. + 1. This will check the status of db. + 2. It will check the status of db. + 2. `b2m --upload `: Upload db to b2. + 1. This will upload the db to b2 form `changeset/dbs//_b2.db`. (Added Detailed Description Above) + 3. `b2m --download `: Download db from b2. + 1. This will download the db from b2 to `changeset/dbs//_b2.db`. (Added Detailed Description Above) + +Reasons: + +1. Potentially use only 2 commands `b2m --create-changeset ` and `b2m --execute `. +2. Other commands will defined under `changeset.py`. + + +### Db.toml + +This is added to remove any hardcoded values in `fdt-templ` server. + +```toml +[db] +ipmdb = "ipm-db-v2.db" +emojidb = "emoji-db-v2.db" +path = "/frontend/db/alldbs/" +``` + +This can also be defined in `fdt-dev.toml`, `fdt-prod.toml`, `fdt-staging.toml` but +for defining I have choosed `db.toml` to avoid confusion of multiple db definition. + + +### fdt-templ server + +This is cli version integration of `fdt-templ` server. + +Reason: +1. For Perform any changeset to any DB it should never be connected to any DB. +2. If Server connected to db file like `*-wal` and `*-shm` which will cause db curruption if done any operations. +![image](https://hackmd.io/_uploads/B1QZvgUO-g.png) + +3. Currently only ipm db need server -> b2 db upload which also means we can do db status, copy only for ipm db. + + +For this case we have 2 options: +1. Complete Server Shutdown while doing changeset. +2. Tell `./server` bin to disconnect the `ipm` db without stoping server. + +`fdt-templ` cli + +1. `./server --disconnect `: This will trigger db connection close function. +2. `./server --connect ` : This will initiates db connection. + + +Pro: + +1. Can reduce complete downtime of server. +2. Can use in-memory cache for serving `ipm` db. +3. Can add queue for ipm installation command insertion. + + +Cons: +1. Takes more time to implement. +2. Other than reducing downtime there is no much benefit. + +Please let me know your thoughts on this. + +### Changeset Script Template + +1. This Include teamplate version +2. This also include Common Functions +```py +# Template Version: v1 +# : _ +# is a short description of the change. + +## Predifned Imports and Functions +import sqlite3 +import urllib.parse +import os +import time + +## Import Common Functions +from changeset import db_status, db_download, db_upload # Still many more should be added. + + +def main(): + # Check status of db. + # If status is outdated_db, then download db from b2. + # If status is ready_to_upload, then upload db to b2. + # If status is up_to_date, then do nothing. + pass + +if __name__ == "__main__": + main() +``` + + +### `changeset.py` common functions + +1. Mainly all helper commands will be defined here. +2. This will reduce defining again and again. +3. It consist of `b2m` cli commands. Further I we can add more based on requirements. + +```py +# Common Functions +#!/usr/bin/env python3 +import subprocess + +def db_status(db_name): + print(f"Executing: {db_name}") + try: + subprocess.run(["../b2m", "--status", db_name], check=True) + except subprocess.CalledProcessError as e: + print(f"Error checking status for {db_name}: {e}") + +def db_upload(db_name): + print(f"Executing: {db_name}") + try: + subprocess.run(["../b2m", "--upload", db_name], check=True) + except subprocess.CalledProcessError as e: + print(f"Error uploading {db_name}: {e}") + +def db_download(db_name): + """ + Function description. + """ + + print(f"Executing: {db_name}") + try: + subprocess.run(["../b2m", "--download", db_name], check=True) + except subprocess.CalledProcessError as e: + print(f"Error downloading {db_name}: {e}") + + +``` + +## Complete Steps Involved in Changeset Creation and Execution + + +### Requirements + +This is a changeset example for Server -> B2. + +There are 2 types of changeset: +1. ipm-db-v3.db -> ipm-db-v3.db (6:00 PM Backup) +2. ipm-db-v3.db -> ipm-db-v4.db (Manual Triggered On Major DB Version Bump) + +#### Scenario 1: ipm-db-v3.db -> ipm-db-v3.db (6:00 PM Backup) + +This will be added for cron job based script which trigger at 6:00 PM for ipm db backup. + +Assume These Constraints : +1. Current ipm-db-v3.db Version States at 2PM. + +| DB Location | version | DB Bump Version | +| ----------- | ------- | --------------- | +| b2 | v1 | v3 | +| server | v1 | v3 | + + +This version is hash of `ipm-db-v3.db`. + +> Note: Hash is generated using `b3sum` command. +> This will be created when db is downloaded from b2 -> server. Hash will be uptodate with b2 version of db. + + +There will be 3 states: +1. Server DB has new data and ready to upload. (NO Version Conflict) +2. Server DB has new data but Outdated and Need Proper Migration (Version Conflict) +3. DB is Up to Date (NO Version Conflict) + +Case 1: Server DB has new data and ready to upload. (NO Version Conflict) +Initials State +| DB Location | version | DB Bump Version | +| ----------- | ------- | --------------- | +| b2 | v1 | v3 | +| server | v2* | v3 | + +Final State +| DB Location | version | DB Bump Version | +| ----------- | ------- | --------------- | +| b2 | v2 | v3 | +| server | v2 | v3 | + + +`*` : This is a marker to show that DB is updated by new data. + +Case 2: Server DB is Outdated and Need Proper miragtion (Version Conflict) + +Initial State +| DB Location | version | DB Bump Version | +| ----------- | ------- | --------------- | +| b2 | v2 | v3 | +| server | v2* | v3 | + +Final State +| DB Location | version | DB Bump Version | +| ----------- | ------- | --------------- | +| b2 | v3 | v3 | +| server | v3 | v3 | + +Case 3: DB is Up to Date (NO Version Conflict) + +Initial State +| DB Location | version | DB Bump Version | +| ----------- | ------- | --------------- | +| b2 | v1 | v3 | +| server | v1 | v3 | + +Final State +| DB Location | version | DB Bump Version | +| ----------- | ------- | --------------- | +| b2 | v1 | v3 | +| server | v1 | v3 | + + +**At 6PM:** +Changeset script is triggered at 6pm daily. + + +1. Disconnect Fdt Server from IPM DB. +2. Check Status of DB using `b2m`. + 1. Case 1:`b2m` will result **Ready to upload** as both DB. + 1. Step 1: Copy DB from changeset db location to server db location. + 2. Step 2: Reconnect Fdt Server to IPM DB. + 3. Step 3: Trigger `b2m` to upload DB to b2. + 4. Step 4: Reconnect Fdt Server to IPM DB. + 5. Step 5: Verify fdt Server is connected to IPM DB. + 2. Case 2:`b2m` will result **Outdated DB** as both DB. + 1. Step 1: Download DB from b2 to changeset db location. + 2. Step 2: Copy DB from changeset db location to server db location. + 3. Step 3: Migrate New Data from Server DB to Downloaded DB. + 4. Step 4: Trigger `b2m` to upload DB to b2. + 5. Step 5: Copy DB from server db location to changeset db location. + 6. Step 6: Reconnect Fdt Server to IPM DB. + 3. Case 3:`b2m` will result **Up to date** as both DB. + 1. Step 1: Reconnect Fdt Server to IPM DB. + 2. Step 2: Exit Changeset Script. + + +Here is example of how script looks like + +```py +# Template Version: v1 +# : _ +# + +## Predifned Imports and Functions +import sqlite3 +import urllib.parse +import os +import time + +DB_NAME = "ipm-db-v5.db" ## This will be Config defined but for example I have taken this +## Import Common Functions +from changeset import db_status, db_download, db_upload # Still many more should be added. + +def db_migration(db_name): + ## Donwload b2 db to changeset db location which will be predefined. + status = db_download(db_name) + if status == "downloaded": + ## Now we have the db in our changeset db location. + ## Now we need to migrate new data from the server db to the new db. + return True + else: + return False + +## This will be added to common functions +def copy_db(db_name): + try: + subprocess.run(["cp", db_name, "changeset/dbs/"], check=True) + return True + except subprocess.CalledProcessError as e: + print(f"Error copying {db_name}: {e}") + return False + +def handle_db_status(db_name): + status = db_status(db_name) + if status == "outdated_db": + if copy_db(db_name): + if db_migration(db_name): + if db_upload(db_name): + copy_db(db_name) + print("DB Migration successful") + else: + print("Error: db_upload failed") + else: + print("Error: db_migration failed") + else: + print("Error: copy_db failed") + elif status == "ready_to_upload": + db_upload(db_name) + elif status == "up_to_date": + pass + else: + print(f"Error: Unknown status {status}") + + +def main(db_name): + handle_db_status(db_name) + + +if __name__ == "__main__": + + main(DB_NAME) +``` +#### Scenario 2: ipm-db-v3.db -> ipm-db-v4.db (Manual Triggered On Major DB Version Bump) + + + +This is I am currently thinking. + +This is nothing but same as `outdated_db` case in scenario 1. But need to make this script more robust. in identifying the db version and bump version and automaticaly update the data. + + + +## Goal + +1. Writing Test Script for all the states which comes under b2m. +2. Write Clear Documentation on Explaing the Scenario and the script implementation. +3. Create a Proper Design on the b2m cli. +4. Final Implemention Logic. + + + +## Expectation + +1. Writing Final Implementation Docs. + + +# Implementation Of Changeset + + +For Example of db I will be taking `ipm-db-v1.db`, `ipm-db-v2.db` and `ipm-db-v3.db`. + +Any Update in the db should be bumped to new version as default which will be handled by `b2m` cli. + +## Structure + +This Consist of 4 main parts. +1. `changeset` directory +2. `b2m` cli +3. `db.toml` file +5. `changeset_script` template +6. `changeset.py` common function + +### Create changeset script + +Folder Structure: +``` +changeset/ +├── scripts/ +│ ├── _.py +│ └── ... +├── dbs/ +| ├── _/ +| | ├── _b2.db +| | ├── _server.db +| | └── ... +| └── ... +├── logs/ +| ├── _.log +| └── ... +├── changeset.py # common functions. +└── README.md +``` + + +1. `_.py` will be generated by `b2m` cli. +```shell +./b2m --create-changeset +``` +2. Create a changeset script in `changeset/scripts` directory. +3. It consist of 2 subdirectories. + 1. `scripts`: This will contain changeset script. + 2. `dbs`: (Details Explaination Below) + 1. Temporary dbs until changeset is executed. + 2. This is for safety purpose. + 3. Any Db download, upload should be done in this file. +3. `logs`: + 1. This will contain logs of changeset script. + 2. This is for logging purpose. +4. `changeset.py`: Common functions used in changeset script. + 1. Such as `b2m upload `, `b2m download `, `b2m status `, etc. + + +### B2M CLI + +1. `b2m` cli will be used to create, execute, and manage changeset script. +2. `b2m` cli will be placed in `frontend` directory. +3. `b2m` cli will be having following commands. There are 2 types of commands. + 1. User specific commands: These commands are used regurly by us. + 1. `b2m create-changeset `: Create a changeset script. + 1. This will create a changeset script in `changeset/scripts` directory. (Added Detailed Description Above) + 2. `b2m execute-changeset `: Execute a changeset script. + 1. This will execute the changeset script. It can also be done by `python ` just adding for making it easy to execute. + 2. Db specific commands: These commands are used and predifned in `changeset.py`. (Added Detailed Descript Above) + 1. `b2m status `: Check status of db. + 1. This will check the status of db. + 2. It will check the status of db. + 2. `b2m upload `: Upload db to b2. + 1. This will upload the db to b2 form `changeset/dbs//_b2.db`. (Added Detailed Description Above) + 3. `b2m download `: Download db from b2. + 1. This will download the db from b2 to `changeset/dbs//_b2.db`. (Added Detailed Description Above) + 4. `b2m fetch-db-toml`: Fetch db.toml from b2. + 1. This will fetch db.toml from b2 to `db/all_dbs/db.toml`. (Added Detailed Description Above) + +Reasons: + +1. We will be using only 2 commands `b2m create-changeset ` and `b2m execute-changeset `. +2. Other commands will defined under `changeset.py`. + + +### Db.toml + +This is added to remove any hardcoded values in `fdt-templ` server. + +```toml +[db] +ipmdb = "ipm-db-v2.db" +emojidb = "emoji-db-v2.db" +path = "/frontend/db/alldbs/" +``` + +This can also be defined in `fdt-dev.toml`, `fdt-prod.toml`, `fdt-staging.toml` but +for defining I have choosed `db.toml` to avoid confusion of multiple db definition. + +There are 2 situation for db version update happen: +1. Team +2. Server Dialy cron job (ipm db only) + + + +Changeset script will automatically bump the db version and update in `db.toml`. +For tracking this change we have 2 options: +1. git based. +2. b2 bucket based using b2m. + +I have choosed option 2. + +1. git based: + 1. This involves `git pull origin main` before checking any db status. + 2. Once updated it should be pushed to git by `git push origin main`. + + Pros: + 1. Easy to track changes. + 2. It will be git native + Cons: + 1. This will create 2 types of db versioning system. + 2. b2m uses b2 bucket for full db versioning system. If we use git for `db.toml` it will create confusion. + 3. If there are any conflicts in pushing to git it will be very hard to resolve. + 4. Need to implement seperate functions to handle it. +2. b2 based + 1. This uses existing b2m db versioning system. + 2. db.toml file will be present in b2 bucket. + Pros: + 1. Adding on top of existing b2m db versioning system. + 2. Easier to integrate with b2m as b2m already managing db `metadata`,`hash` and `lock` safely. + 3. This will create seperate versioning system for dbs independent with git. + 4. Any db ops done will be done using b2 bucket as source of truth. + 5. Anyone starting server will be depending on b2m for checking db.toml fetched from b2m. + Cons: + 1. Integrating b2m to `make start-prod` or `make run` command to identify any db changes. + + + + +### Changeset Script Template + +1. This Include teamplate version +2. This also include Common Functions +```py +# Template Version: v1 +# : _ +# is a short description of the change. + +## Predifned Imports and Functions +import sqlite3 +import urllib.parse +import os +import time + +## Import Common Functions +from changeset import db_status, db_download, db_upload # Still many more should be added. + + +def main(): + # Check status of db. + # If status is outdated_db, then download db from b2. + # If status is ready_to_upload, then upload db to b2. + # If status is up_to_date, then do nothing. + pass + +if __name__ == "__main__": + main() +``` + + +### `changeset.py` common functions + +1. Mainly all helper commands will be defined here. +2. This will reduce defining again and again. +3. It consist of `b2m` cli commands. Further I we can add more based on requirements. + +```py +# Common Functions +#!/usr/bin/env python3 +import subprocess + +def db_status(db_name): + print(f"Executing: {db_name}") + try: + subprocess.run(["../b2m", "--status", db_name], check=True) + except subprocess.CalledProcessError as e: + print(f"Error checking status for {db_name}: {e}") + +def db_upload(db_name): + print(f"Executing: {db_name}") + try: + subprocess.run(["../b2m", "--upload", db_name], check=True) + except subprocess.CalledProcessError as e: + print(f"Error uploading {db_name}: {e}") + +def db_download(db_name): + """ + Function description. + """ + + print(f"Executing: {db_name}") + try: + subprocess.run(["../b2m", "--download", db_name], check=True) + except subprocess.CalledProcessError as e: + print(f"Error downloading {db_name}: {e}") + + +``` + + + + +## Working Flow Of Changeset Script + +Assume these constraints: + +On Changeset Script Trigger From (Team or Cron Job): + +1. B2m status will return `ready_to_upload` or `outdated_db` or `up_to_date`. +2. This will be defined in `b2m status` command. + + +There will be 3 states: +1. `ready_to_upload`: `ipm-db-v1.db` has new data (NO Version Conflict) +2. `outdated_db`: `ipm-db-v1.db` has new data but Outdated and Need Proper Migration (Version Conflict) +3. `up_to_date`: `ipm-db-v1.db` is Up to Date (NO Version Conflict) + +Case 1: `ready_to_upload`: `ipm-db-v1.db` has new data (NO Version Conflict) +Initials State + +| DB Location | version | New Data | +| ----------- | ------------- | -------- | +| b2 | ipm-db-v1.db | No | +| server | ipm-db-v1.db | Yes | + +There is new data in `ipm-db-v1.db` in server side, so we will bump the version `ipm-db-v1.db` to `ipm-db-v2.db` and upload it to b2. + +Final State +| DB Location | version | New Data | +| ----------- | ------------- | -------- | +| b2 | ipm-db-v2.db | No | +| server | ipm-db-v2.db | No | + + +Case 2: `outdated_db`: Server DB is Outdated and Need Proper miragtion (Version Conflict) +Case 2.1: B2 has `ipm-db-v2.db` and server has `ipm-db-v1.db`. +Initial State +| DB Location | version | New Data | +| ----------- | ------- | -------- | +| b2 | ipm-db-v2.db | Yes | +| server | ipm-db-v1.db | Yes | + +There is new data in `ipm-db-v1.db` in server side and new version `ipm-db-v2.db` in b2 side. +1. Download to `ipm-db-v2.db` to `changeset/dbs/_/` from b2. +2. Copy `ipm-db-v1.db` to `changeset/dbs/_/` from `db/alldb.db`. +3. DB Migration + 1. We have Export/Select new from `ipm-db-v1.db` to `ipm-db-v2.db` in `changeset/dbs/_/` via sql query which have be defined `_.py`. +4. rename `ipm-db-v2.db` to `ipm-db-v3.db`. +5. Stop FDT Server with `make stop-prod`. +6. copy `ipm-db-v3.db` to `db/alldb`. +7. Update `db.toml` with `version = "v3"`. +8. Start FDT Server with `make start-prod`. +9. Upload `ipm-db-v3.db` from `changeset/dbs/_/` to b2. +10. Once Sucessful, Remove `changeset/dbs/_/`.(or keep it in `changeset/dbs/_/backup/` folder for safety) + +Final State +| DB Location | version | New Data | +| ----------- | ------- | -------- | +| b2 | ipm-db-v3.db | No | +| server | ipm-db-v3.db | No | + + + +Case 2.2: B2 has `ipm-db-v1.db` and server has `ipm-db-v1.db` but both have new data. + + +> Note: This case should never and will never happen. +> I have added how to this case is handled. + + +We can have discord notification that this type of case has failed due to this + + +Case 3: DB is Up to Date (NO Version Conflict) + +Initial State +| DB Location | version | New Data | +| ----------- | ------- | --------------- | +| b2 | v1 | No | +| server | v1 | No | + +Final State +| DB Location | version | New Data | +| ----------- | ------- | --------------- | +| b2 | v1 | No | +| server | v1 | No | + diff --git a/b2-manager/docs/fdtdb.md b/b2-manager/docs/fdtdb.md new file mode 100644 index 0000000000..a1d5834d0b --- /dev/null +++ b/b2-manager/docs/fdtdb.md @@ -0,0 +1,1079 @@ +## b2m + +There are lot of user confusions about using this script. +This include user knowing core features. + +## Problem + +1. Users cannot easily determine if the database is up-to-date due to confusing lock messages. +2. Redundant "DB" text in "Download DB" options. +3. Lack of upload speed/progress indication. + +## Goal + +I will be guiding you step by step to solve this and how to modify changes. + +### Phase 1.1 + +1. I will be going to define how to update existing code to new logic. +2. Solving Status check issues +3. User can't able understand current status check logic. +4. It is currently checking diff and if there is diff then it is asking download. +5. Remove this logic of checking. +6. Let's have `.metadata.json` file. + +#### Metadata File content + +```json +{ + "file_id": "banner-db", + "hash": "a1b2c3d4...", + "timestamp": 1738058400, // Unix Timestamp + "size_bytes": 1048576, + "uploader": "ganesh", + "hostname": "archlinux-laptop", + "platform": "linux", + "tool_version": "v1.4.0", + "upload_duration_sec": 2.5, + "datetime": "2026-01-28 10:00:00 UTC", + "events": [ + { + "sequence_id": 1, + "datetime": "2026-01-28 10:00:00 UTC", + "timestamp": 1738057400, + "hash": "a1b2c3d4e5...", + "size_bytes": 1048576, + "uploader": "ganesh", + "hostname": "archlinux-laptop", + "platform": "linux", + "tool_version": "v1.4.0", + "upload_duration_sec": 2.5 + } + ] +} +``` + +#### New Status Check Logic Status Check + +We store all necessary information in the **metadata file**. + +1. **Calculate Local Data** + +- Read Local File (Binary Mode) +- Generate BLAKE3 Hash (String) +- Get Local Modification Time (Unix Timestamp) + +2. **Construct Version Filename** + +- Format: `.metadata.json` +- _Example: banner-db.metadata.json_ + +3. **Read Metadata File** + +- Read the metadata file (JSON) +- Parse the JSON into a struct + +4. **Compare Data** + +- Compare the local data with the metadata file +- If the local data matches the metadata file, the database is up to date +- If the local data does not match the metadata file, the database is out of date + +#### The Status Check Algorithm + +**Step 1: Fetch & Parse** + +1. Run `rclone lsf b2-config:hexmos/freedevtools/content/db/version/` to get the list of metadata files. +2. Run `rclone lsf b2-config:hexmos/freedevtools/content/db/lock/` to get the list of lock files. +3. Calculate the **BLAKE3 Hash** and **ModTime** of your **Local** database. + +**Step 2: Comparison Conditions** +![image](https://hackmd.io/_uploads/BJQuIjP8We.png) + +![image](https://hackmd.io/_uploads/SkqWSoPI-e.png) + +``` +Check DB Lock Status + No DB not locked: Does Remote Hash exist? + Yes: Is Local Hash == Remote Hash? + Yes: Status : Upto Date + No: Is Local Time > Remote Time? + Yes: Status : Local Newer Ready to Upload 🔼 + No: Status : Outdated DB Download Now 🔽 DB Overwrite Warning + No: Status = Upload DB. + Yes DB is Locked: Status = Show User Uploading 🔼..... + + +``` + +#### Removal + +1. Remove Current status UI and Update based on new logic. +2. Remove UI mention of Download DB's db and replace with download db. +3. Remove Lock DB and Upload option (it should be done but don't show to user) don't change any logic + +#### UI + +1. Upload page should have single select DB. +2. New status check feature. +3. Just show Upload , back, main menu options. + +#### Hints and Tips + +1. If you have any questions, please ask me. + +## Phase 1.2 + +1. Cancel any opearations in the middle safely. +2. In Download DB's page, if user do ctr+c, it should cancel the operation safely. +3. In Status page, if user do ctr+c, it should cancel the operation safely. +4. In Upload page, if user do ctr+c, it should cancel the operation safely. + 1. That is if upload should first stop uploading and then update metadata saying upload failed in the event's and release lock. + +```json +{ + "file_id": "banner-db", + "hash": "a1b2c3d4...", + "timestamp": 1738058400, + "size_bytes": 1048576, + "uploader": "ganesh", + "hostname": "archlinux-laptop", + "platform": "linux", + "tool_version": "v1.4.0", + "upload_duration_sec": 2.5, + "datetime": "2026-01-28 10:00:00 UTC", + "status": "success", + "events": [ + { + "sequence_id": 2, + "datetime": "2026-01-29 10:00:00 UTC", + "timestamp": 1738058400, + "hash": "a1b2c3feqwe...", + "size_bytes": 1048576, + "uploader": "ganesh", + "hostname": "archlinux-laptop", + "platform": "linux", + "tool_version": "v1.4.1", + "upload_duration_sec": 2.5, + "status": "cancelled" + }, + { + "sequence_id": 1, + "datetime": "2026-01-28 10:00:00 UTC", + "timestamp": 1738057400, + "hash": "a1b2c3d4e5...", + "size_bytes": 1048576, + "uploader": "ganesh", + "hostname": "archlinux-laptop", + "platform": "linux", + "tool_version": "v1.4.0", + "upload_duration_sec": 2.5, + "status": "success" + } + ] +} +``` + +For Success Upload + +```json +{ + "file_id": "banner-db", + "hash": "a1b2c3feqwe...", + "timestamp": 1738058400, + "size_bytes": 1048576, + "uploader": "ganesh", + "hostname": "archlinux-laptop", + "platform": "linux", + "tool_version": "v1.4.1", + "upload_duration_sec": 2.5, + "datetime": "2026-01-29 10:00:00 UTC", + "status": "success", + "events": [ + { + "sequence_id": 2, + "datetime": "2026-01-29 10:00:00 UTC", + "timestamp": 1738058400, + "hash": "a1b2c3feqwe...", + "size_bytes": 1048576, + "uploader": "ganesh", + "hostname": "archlinux-laptop", + "platform": "linux", + "tool_version": "v1.4.1", + "upload_duration_sec": 2.5, + "status": "success" + }, + { + "sequence_id": 1, + "datetime": "2026-01-28 10:00:00 UTC", + "timestamp": 1738057400, + "hash": "a1b2c3d4e5...", + "size_bytes": 1048576, + "uploader": "ganesh", + "hostname": "archlinux-laptop", + "platform": "linux", + "tool_version": "v1.4.0", + "upload_duration_sec": 2.5, + "status": "success" + } + ] +} +``` + +### Phase 1.3 + +1. Explain current status check logic. + +``` +EXISTING DATABASES +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ DB NAME ┃ STATUS ┃ +┣━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +┃ banner-db.db ┃ Unknown Status ┃ +┃ cheatsheets-db-v4.db ┃ Unknown Status ┃ +┃ cheatsheets-db-v3.db ┃ Unknown Status ┃ +┃ emoji-db-v4.db ┃ Unknown Status ┃ +┃ emoji-db-v3.db ┃ Unknown Status ┃ +┃ ipm-db-v5.db ┃ Unknown Status ┃ +┃ ipm-db-v4.db ┃ Unknown Status ┃ +┃ ipm-db-v3.db ┃ Unknown Status ┃ +┃ man-pages-db-v4.db ┃ Unknown Status ┃ +┃ man-pages-db-v3.db ┃ Unknown Status ┃ +┃ mcp-db-v5.db ┃ Unknown Status ┃ +┃ mcp-db-v4.db ┃ Remote Only (Download Available 🔽) ┃ +┃ mcp-db-v3.db ┃ Unknown Status ┃ +┃ png-icons-db-v4.db ┃ Unknown Status ┃ +┃ png-icons-db-v3.db ┃ Unknown Status ┃ +┃ svg-icons-db-v4.db ┃ Unknown Status ┃ +┃ svg-icons-db-v3.db ┃ Unknown Status ┃ +┃ test-db.db ┃ Unknown Status ┃ +┃ tldr-db-v4.db ┃ Unknown Status ┃ +┃ tldr-db-v3.db ┃ Unknown Status ┃ +┗━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + + +``` + +### Phaze 2 + +1. Remove all the unwanted code / not used function from this project directory +2. Remove all the parallel processing code. +3. metadata's should be downloaded to local directory and then processed. +4. reading metadata should be done in a single thread. +5. update local metadata and then upload to remote. +6. if there is no metadata, of any db then create it and upload to remote. + +### Progress + +![alt text](image.png) + +### Issues solved + +1. Code Review. +2. Cancel any opearations in the middle safely. +3. Upload Safe cancelation. +4. Download bug fix. + +Here are the detailed meanings for each status message displayed in the CLI: + +### Locking Logic (Highest Priority) + +| Status | Message | Meaning | +| :---------------- | :------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **LockedByOther** | `%s is Uploading ⬆️` | Another user is currently uploading this DB. You cannot sync or upload until they finish. | +| **LockedByYou** | `Ready to Upload ⬆️` | **Currently confusing**. Indicates YOU have an active lock on this DB (perhaps from a previous failed/incomplete upload). Should likely be changed to "Locked (Retry Upload)". | + +### Existence Logic (When not locked) + +| Status | Message | Meaning | +| :------------- | :-------------------------- | :------------------------------------------------------------------------- | +| **NewLocal** | `New DB (Upload Ready ⬆️)` | You created this DB locally, and it hasn't been uploaded to the cloud yet. | +| **RemoteOnly** | `Remote Only (Download 🔽)` | This DB exists in the cloud but not on your machine. You can download it. | + +### Metadata Logic (When both Local and Remote exist) + +| Status | Message | Meaning | +| :-------------------- | :----------------------------- | :-------------------------------------------------------------------------------------------------- | +| **NoMetadata** | `No Meta (Upload ⬆️)` | Both exist, but no `metadata.json` found in B2. Treat as orphan/new. Upload to fix. | +| **UploadCancelled** | `❌ Upload Cancelled Retry ⬆️` | Last upload was explicitly cancelled by user. Metadata records this "cancelled" state. | +| **RecievedStaleMeta** | `Ready to Upload ⬆️` | Inconsistent state (metadata exists but remote file missing, or similar). Treat as ready to upload. | + +### Version Comparison (Standard State) + +| Status | Message | Meaning | +| :-------------- | :--------------------- | :---------------------------------------------------------------------------------------------- | +| **UpToDate** | `Up to Date ✅` | Local BLAKE3 matches Remote BLAKE3. | +| **LocalNewer** | `Local Newer ⬆️` | Hashes differ, and local modification time is **after** remote timestamp. You should upload. | +| **RemoteNewer** | `Remote Newer 🔽` | Hashes differ, and local modification time is **before** remote timestamp. You should download. | +| **Error** | `Error (Read/Stat ❌)` | File permission or IO errors during check. | + +## Status Definitions + +| Category | Statuses Included | UI Display | Meaning & Action | +| :------------------ | :-------------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **UpToDate** | `UpToDate` | `Up to Date ✅`
_(Green)_ | **Meaning**: The local database and remote database are identical (hashes match).
**Action**: No action required. You are in UptoDate. ✅ | +| **Download Needed** | `RemoteOnly`
`RemoteNewer` | `Remote Ahead (Download Now 🔽)`
_(Yellow)_ | **RemoteOnly**: Database exists in the cloud but not on your machine.
**RemoteNewer**: The cloud version has a newer timestamp than your local copy.
**Action**: Use the **Download** feature to get the latest version. | +| **Upload Needed** | `NewLocal`
`LocalNewer`
`NoMetadata`
`UploadCancelled` | `Local Ahead (Upload Now ⬆️)` | **NewLocal**: You created this DB locally; it's not in the cloud yet.
**LocalNewer**: You modified the DB locally; it's ahead of the cloud version.
**NoMetadata**: Remote file exists but is missing version info (orphan).
**UploadCancelled**: Your last upload attempt was stopped mid-way.
**Action**: Select the database and **Upload** to sync your changes/fixes. | +| **Locked** | `LockedByOther`
`LockedByYou`
`Uploading` (TODO) | `%s is Uploading ⬆️`
_(Yellow)_
`Ready to Upload ⬆️`
_(Green)_
`Ready to Upload ⬆️`
_(Green)
`You are Uploading ⬆️` (TODO)
_(Green) \_ | **LockedByOther**: Another user is currently uploading this database. A lock file prevents concurrent edits.
**LockedByYou**: You have an active lock (possibly from a previous incomplete upload).
**Uploading**: You are uploading this DB. (TODO)
**Action**: If locked by other, wait. If locked by you, retry upload. If You are Uploading wait. | +| **Errors** | `Error`
`Unknown` | `Error (Read ❌)`
_(Red)_ | **Meaning**: The tool could not read the local file or filesystem stats (permissions/IO error).
**Action**: Check local file permissions/path availability. | +| | | | | + +## Phase 3 + +### Goal + +The main goal of this phase is making the tool more user-friendly and stable. +This also means implementing TODOs and fixing any remaining issues. +Iterating with module implementation in mind. + +Readability of the code and ease of use are key priorities. + +### Phase 3.0 + +Implement common UI status display function based on the Status Definitions table above. + +Any questions? Please ask + +Make sure don't change any existing functionality. +Just Have above UI display. + +### Phase 3.1 + +Implement TODOs Adding check for `Uploading` status display. + +### Expectation + +1. This should be checked while the status found the db is locked and directly saying retry. +2. Check the metadata.json file of selected db name. +3. If the metadata.json file has the status `cancel` then the db is uploading. +4. Also better to have a condition like this whenver upload is selected created lock update latest db metadata.json file status to `uploading` and start upload. +5. So, Whenever status check is done it will say you are uploading. + +### Phase 3.2 + +Implementing single UI/UX page for all the operations. +Similar to lazygit UI/UX. + +Single page no need to switch between pages. + +### User Interface + +Using gocui https://github.com/jroimartin/gocui which lazygit uses for ui. + +Window 1 +This will be having only one window. +Full screen with no renderHeader + +This window will be having 3 columns. + +1. DB Name +2. Status +3. Progree Bar Which already implemented just need to display inside this specific row. + +### User Experience (Keyboard Shortcuts) + +u - upload +p - download +c - cancel +ctr + c - exit +ctr + r - refresh status check + +### Implementation Guidelines + +1. Implement using gocui +2. Use the existing progress bar implementation +3. Use Above UI/UX design +4. Corrently ctl + c was implemented to stop/ cancel any operation now it should be handled with c with row specific. +5. on ctr + c it should ask prompt to confirm to exit main ui. +6. No need to change any core functionality. +7. Action u/p/c should be handled with row specific. +8. Finaly make sure to remove all unused code of previous ui (ex renderHeader, clearScreen, etc). + +## Issues + +1. There is no table used which was used in previous ui. +2. Keyboard should be in button of the terminal +3. Can't able to ctr + c y/n. +4. can't able to use up and down arrow keys to navigate between rows. + 4 + +## Phase 4 + +This Phase is final improvement and validation phase. + +### Expectation + +1. Progress should only disaplay progree bar with percentage and speed and time remaining which can be extracted from the rclone output. +2. Progreess should also include 5% Locking 5% Updating metadata.json operations and 80% rclone progress bar and final 5% Updating metadata.json and 5% Unlocking operations. +3. The above progress bar should have locking,metadata.json update , unlocking and metadata.json update operations messge in the same progree column. +4. Finaly I want table to be static with no dynamic column seperation with `|` which is casuing ui issues in table and with 3 columns DB Name, Status, Progress. +5. So, for table i recomment have 25% DB Name, 25% Status, 50% Progress. +6. Even though the table updated with status it should not cause the issues with table column misplacement. +7. Majorly on this is done we can progress with validation phase +8. There should be no changes with current functionality. + +### Improvemnt + +now progress bar is going out of column + +do one thing inside the progress bar column have hiddle 2 columns 10% width from overlall screen should be these messges and remaing 40 % should be progress bar make it has ETA + +have this +bar := progressbar.NewOptions(1000, +progressbar.OptionSetWriter(ansi.NewAnsiStdout()), //you should install "github.com/k0kubun/go-ansi" +progressbar.OptionEnableColorCodes(true), +progressbar.OptionShowBytes(true), +progressbar.OptionSetWidth(15), +progressbar.OptionSetDescription("[cyan][1/3][reset] Writing moshable file..."), +progressbar.OptionSetTheme(progressbar.Theme{ +Saucer: "[green]=[reset]", +SaucerHead: "[green]>[reset]", +SaucerPadding: " ", +BarStart: "[", +BarEnd: "]", +})) +for i := 0; i < 1000; i++ { +bar.Add(1) +time.Sleep(5 \* time.Millisecond) +} + +ETA should be only present when there is rclone progress bar is running. + +Remaining it 5% progress it should not have ETA. +Also 5% progress should not proceed untill it is successfull completed. + +### Refactoring + +1. Move ui.go into ui folder. +2. Move Keybindings UI and config into ui/keybindings.go +3. Move main table to ui/table.go +4. Move keyboard operations to ui/operations.go +5. Move progress bar to ui/progress.go + +So, finaly we should have 5 files in ui folder. + +1. ui.go +2. keybindings.go +3. table.go +4. operations.go +5. progress.go + +Make sure nothing is broken after moving + +## Phase 5 + +Implement Edge cases + +1. Force Upload DB if lock is present. +2. This should be done with pop up with yes/no confirmation. +3. This for edge case to avoid deadlock conditions. +4. Update doc/workflow/upload.md file with this implementation. + +Implemeting Another Edge cases + +1. whenver user tries to do ctr+c safely cancel the operation and remove all the lock files and update the metadata.json file with status `cancel` and exit. + +## Final Validation + +1. checking race conditions in local code. +2. there are multiple files use parallelism used. +3. So, I want find all these files and check if they are using parallelism correctly. +4. Add a doc in parallelism.md file with all the files and their parallelism implementation. under docs/concurrent/parallelism.md +5. Make sure u mention where the parallelism is used and mention it in docs breilfy. + +6. If found any issues with parallelism implementation then fix it. + +## Fixing UI Issues + +1. Status fetch should show small +2. Adding status fetch animation in buttom whenever user presses ctr + r or there is refresh is going on. + +similar lazygit + +1. Configuration (The Frames) + The actual characters used for the spinner are defined in + pkg/config/user_config.go + . It uses a standard set of 4 characters and runs at 50ms per frame. + +go +// pkg/config/user_config.go +Spinner: SpinnerConfig{ +Frames: []string{"|", "/", "-", "\\"}, +Rate: 50, // 50 milliseconds per frame +}, 2. Logic (The Animation) +The logic to pick the correct frame based on the current time is in +pkg/gui/presentation/loader.go +. It uses the current Unix timestamp in milliseconds to "index" into the frames array. + +go +// pkg/gui/presentation/loader.go +// Loader dumps a string to be displayed as a loader +func Loader(now time.Time, config config.SpinnerConfig) string { +milliseconds := now.UnixMilli() +// Divide time by rate to get the "frame number", then modulo by frame count +index := milliseconds / int64(config.Rate) % int64(len(config.Frames)) +return config.Frames[index] +} + +The Layout Logic +File: +pkg/gui/controllers/helpers/window_arrangement_helper.go + +In the +GetWindowDimensions +function, the layout is defined as a tree of boxes. + +Bottom Line Detection: It checks if the bottom info section should be shown. This includes the app status. +go +showInfoSection := args.UserConfig.Gui.ShowBottomLine || +args.InSearchPrompt || +args.IsAnyModeActive || +args.AppStatus != "" +Vertical Stacking: It creates a root column with two children: +Top: The main content (side panels + main view). This has Weight: 1, meaning it takes up all available remaining space. +Bottom: The info section. This has Size: 1 (if shown), meaning it takes exactly 1 row at the bottom. +go +root := &boxlayout.Box{ +Direction: boxlayout.ROW, // Main layout row +Children: []\*boxlayout.Box{ +// ... (Top Section: Side panels + Main View) +{ +Direction: boxlayout.COLUMN, +Size: infoSectionSize, // 1 if visible, 0 if hidden +Children: infoSectionChildren(args), +}, +}, +} +Horizontal Stacking (The Bottom Bar): deeper in +infoSectionChildren +, it decides how to arrange the items in that bottom row. +If +AppStatus +exists (e.g. "Fetching..."), it adds a box for it. +It adds spacer boxes to push content to the left or right as needed. +So, essentially, it reserves the last row of the terminal for this status bar whenever there is a status message or if "show bottom line" is enabled in settings. + +3. Finaly there is a bug in showing DB status if the user updated DB it is showing db is outdated please download. +4. Even Though there is hash + date check some bug is there we need to fix it. +5. If someone is uploading to other it showing You are Uploading which is wrong. + +## Phase 6: Overall Logic Changes and New Implementation + +This phase introduces a "Local-Version" () tracking system. This acts as a synchronization anchor (snapshot) to distinguish between **"Local Changes"** (safe to upload) and **"Remote Changes"** (unsafe to upload without pulling). + +### Phase 6.1: Updating Status Check Logic + +The status check workflow is refactored to collect and compare three distinct states: + +1. \*\*\*\*: Remote Metadata (The current state on the server). +2. \*\*\*\*: Local DB File (The current state of the file on disk). +3. \*\*\*\*: Local-Version Metadata (The snapshot of the file when we last synced). + +#### 1. Updated Parallel Data Collection + +The `FetchDBStatusData` function (in `core/status.go`) is expanded to include a **5th parallel operation**. + +| Operation | Source | Action | +| ------------------------- | ------------- | ----------------------------------------------------------------------------------- | +| **A. List Local DBs** | `os.ReadDir` | Scans `db/all_dbs/*.db`. | +| **B. List Remote DBs** | `rclone lsf` | Lists files in B2 bucket. | +| **C. Fetch Locks** | `rclone lsf` | Lists `locks/` directory on B2. | +| **D. Download Metadata** | `rclone sync` | Updates `db/metadata/` from B2. | +| **E. Load Local-Version** | `os.ReadFile` | **(NEW)** Reads `db/all_dbs/.b2m/local-version/*.metadata.json` for every DB found. | + +#### 2. Updated Status Calculation Logic + +The `CalculateDBStatus` function is updated to use \*\*\*\* as the baseline. + +**Variables:** + +- \*\*\*\*: Hash from Remote Metadata. +- \*\*\*\*: Hash from `db/all_dbs/.b2m/local-version/.metadata.json`. +- \*\*\*\*: Hash calculated from the current local `.db` file. + +**Logic Flow (Priority Order):** + +| Priority | Condition | Status | UI Display | Color | +| ------------------ | ---------------------------------------- | ----------------- | ------------------ | ------ | +| **1. Lock** | Generic Lock exists | **LockedByOther** | `Locked by [User]` | Red | +| | Lock by Self + Meta="uploading" | **Uploading** | `Uploading...` | Yellow | +| | Lock by Self | **LockedByYou** | `Locked by You` | Yellow | +| **2. Existence** | Remote Missing, Local Exists | **NewLocal** | `New (Local)` | Green | +| | Remote Exists, Local Missing | **RemoteOnly** | `Download Needed` | Blue | +| **3. History ()** | **No file found** (First run or deleted) | | | | +| | If | **UpToDate** | `Synced` | Green | +| | If | **RemoteNewer** | `Remote Newer` | Blue | +| **4. Consistency** | ** file exists** | | | | +| | **Case A: Remote Changed** | | | | +| | If | **Outdated** | `Remote Newer` | Blue | +| | **Case B: Remote Unchanged** | | | | +| | If | **UpToDate** | `Synced` | Green | +| | If | **LocalNewer** | `Ready to Upload` | Cyan | + +> **Critical Conflict Rule:** If **Case A** is true (), the user is **blocked** from uploading, even if they have local changes. They _must_ pull the latest remote changes first. + +--- + +### Phase 6.2: Local-Version Persistence (New Implementation) + +This logic ensures the anchor is updated only upon **successful** synchronization events. + +**Location:** `db/all_dbs/.b2m/local-version/` +**File:** `.metadata.json` + +#### 1. Hook into "Download Workflow" + +Inside `DownloadDatabase` (in `core/rclone.go`), immediately after a **successful** `rclone copy`: + +1. **Retrieve**: Get the fresh metadata object that was just downloaded/synced from B2. +2. **Write**: Save this JSON object to `db/all_dbs/.b2m/local-version/.metadata.json`. +3. **Result**: is now equal to (Remote) and (Local). Status becomes `Synced`. + +#### 2. Hook into "Upload Workflow" + +Inside `PerformUpload` (in `core/upload.go`), immediately after **Phase 3: Finalization** (Metadata Upload): + +1. **Construct**: Use the _exact_ metadata object that was just generated and uploaded to B2 (containing the new Hash, Timestamp, and `status: success`). +2. **Write**: Overwrite `db/all_dbs/.b2m/local-version/.metadata.json` with this object. +3. **Result**: is updated to match the new and current . Status becomes `Synced`. + +#### 3. Standard JSON Structure + +The content of the local-version file must strictly follow this schema: + +```json +{ + "file_id": "tldr-db-v3", + "hash": "e9b08b8989454296be811cbdbf37ef3220c89a20842849dd23bcdf13bb0faaf2", + "timestamp": 1769187739, + "size_bytes": 28958720, + "uploader": "gk", + "hostname": "jarvis", + "platform": "linux", + "tool_version": "v1.0", + "upload_duration_sec": 26.36, + "datetime": "2026-01-23 17:02:19 UTC", + "status": "success" +} +``` + +### Phase 6.3: Helper Functions (Implementation Guide) + +You will need a helper to manage these files. + +```go +// core/helpers.go + +// UpdateLocalVersion writes the metadata to db/all_dbs/.b2m/local-version/.metadata.json +func UpdateLocalVersion(dbName string, meta model.Metadata) error { + // 1. Define Path: db/all_dbs/.b2m/local-version/ + dbName + .metadata.json + // 2. Marshal 'meta' struct to Indented JSON + // 3. os.WriteFile(path, data, 0644) + return nil +} + +// GetLocalVersion reads the metadata from the local-version directory +func GetLocalVersion(dbName string) (*model.Metadata, error) { + // 1. Read file + // 2. Unmarshal + // 3. Return *model.Metadata or nil if not found + return nil +} + +``` + +--- + +### Phase 6.4: Impact on "Overwrite Warning" + +With this logic, the CLI can now smartly warn the user: + +- **Old Logic:** "Remote exists. Overwrite?" (Vague) +- **New Logic ():** +- If `Status == LocalNewer`: **No warning needed.** (We know we started from the current remote state). +- If `Status == Outdated`: **HARD STOP / Warning.** "Remote has changed since you last downloaded. Overwriting will lose remote data. Please download first." + +This is excellent progress. You have now completed the backend logic (Phase 6.1) and the persistence layer (Phase 6.2). Your system can now mathematically prove whether a database is safe to upload or if it requires a pull. + +However, **logic is not enough**. You now need to enforce the rules in the UI layer. Currently, your CLI might calculate `RemoteNewer`, but if the user selects "Upload," does the code actually stop them? + +Let's move to **Phase 6.5: UI Safeguards & Conflict Resolution**. + +--- + +### Phase 6.5: Enforcing Rules in the UI + +We need to modify the interaction layer to respect the new statuses we calculate. + +#### Goal + +1. **Block Uploads** when status is `RemoteNewer` (Outdated). +2. **Allow Uploads** when status is `LocalNewer` (Safe). +3. **Prompt for Overwrite** when status is `RemoteNewer` but the user tries to "Download" (to warn them they will lose local changes). + +#### Step 1: Update `HandleAction` (CLI Interaction) + +You need to modify the function where you handle the user's keypress (e.g., in `ui/list.go` or wherever your main input loop lives). + +**Logic to Implement:** + +- **IF** User presses `u` (Upload): +- Check Status of selected item. +- **Case `Synced**`: Print "Already up to date." +- **Case `RemoteNewer` (Outdated)**: **REJECT ACTION.** +- Show Error Message: _"Conflict: Remote database has changed. You must DOWNLOAD/PULL first."_ +- _Do not allow the upload to proceed._ + +- **Case `LockedByOther**`: **REJECT ACTION.** +- Show Error Message: _"Locked by [User]. Cannot upload."_ + +- **Case `LocalNewer**`: **ALLOW.** Proceed to `PerformUpload`. + +- **IF** User presses `d` (Download): +- Check Status of selected item. +- **Case `LocalNewer**`: **WARNING REQUIRED.** +- The user has local changes that they haven't uploaded. If they download now, they destroy their work. +- Prompt: _"Warning: You have unsaved local changes. Overwrite with remote version? (y/n)"_ + +- **Case `RemoteNewer**`: **ALLOW.** Proceed to `DownloadDatabase`. + +#### Step 2: Refactor `main.go` / Action Handler + +I recommend creating a `ValidateAction(db model.DBInfo, action string) error` function to keep your UI code clean. + +**Proposed Helper Function:** + +```go +// core/validation.go + +func ValidateAction(db model.DBInfo, action string) error { + switch action { + case "upload": + if db.Status == model.StatusRemoteNewer { + return fmt.Errorf("CONFLICT: Remote is newer. Please download first.") + } + if db.Status == model.StatusLockedByOther { + return fmt.Errorf("LOCKED: Database is locked by %s.", db.LockOwner) + } + // Add other blocking conditions... + + case "download": + if db.Status == model.StatusLocalNewer { + // This isn't a hard error, but a signal that the UI needs to ask for confirmation + return fmt.Errorf("WARNING_LOCAL_CHANGES") + } + } + return nil +} + + +## Phase 7: New Feature + +Situation : +1. If user has 2 hours update he can just lock by using `l` key +2. This should show a message +2. This will crete lock file and update metadata with `status: "updating"` +3. +``` + +## TODO + +1. Custom Lock and unlock +2. new version of db will not have any previous data check. + +## Phase 8: + +This phase is for other to test the system + +in this directory the scipt is b2-manager/testing/test.sh + +but actual project are manily localted frontend + +so create make command to update db which is located in frontend + +```bash +make b2m-test +``` + +curerrntly db location is this frontend/db/all_dbs/test-db.db +in this db add +SELECT c.\* FROM category AS c + +Update this table with random words each time and append to next table make sure u only do this for this db frontend/db/all_dbs/test-db.db + +also make sure use sqlite cli to do this opration + +{ +"SELECT c.\* FROM category AS c where c.slug = \"aggregators\"": [ +{ +"slug" : "aggregators", +"name" : "asdfasd", +"description" : "Servers for accessing many apps and tools through a single MCP server.", +"count" : 19, +"updated_at" : "2025-11-26T16:31:40.312Z" +} +]} + +## Testing Phase + +1. If user has old db it should show Download DB now. +2. If user has updated an outdated db it should show Download DB now. with Warning of potential data loss +3. if other is user uploading it should user is uploading. +4. if user only uploading it should show you are uploading. +5. if user is updating db and he locked so other don't upload it should show user is updating. + +# **Testing Procedure: `b2m` Database Synchronization & Concurrency** + +**Date:** February 8, 2026 +**Testers:** @athreya7023, @lince_m +**Location:** `frontend` directory + +### **1. Prerequisites & Setup** + +Before beginning the test, ensure you are in the `frontend` directory and have disconnected from all active database connections. + +- **Build the executable:** + +```bash +make build-b2m + +``` + +### **2. Initialization & Status Verification** + +1. **Launch the tool:** + +```bash +./b2m + +``` + +2. **Verify DB Status:** + +- Allow the tool time to load the database list. +- **Success Criteria:** The status for `test-db.db` must display as **"Outdated DB"**. + +### **3. Local Update Simulation** + +While the first terminal is running, open a **second terminal** in the `frontend` directory to simulate a local change. + +1. **Run the test update command:** + +```bash +make b2m-test + +``` + +- _Note: This will update one row in `test-db.db`._ + +2. **Refresh the view:** + +- Return to the first terminal and press `Ctrl + R`. +- **Success Criteria:** The status should reflect the updated state. + +### **4. Concurrency & Locking Test (Collaborative)** + +**Scenario:** Two users attempting to upload changes simultaneously to test locking mechanisms. + +#### **Step A: Conflict Warning Check** + +1. **Both testers (@athreya7023 & @lince_m):** Attempt to upload by pressing `u`. +2. **Expected Result:** Both users should see a **warning prompt**. +3. **Action:** Press `n` (No) to cancel the upload. + +#### **Step B: Remote Download Check** + +1. **Action:** Press `p` to pull/download the remote database. +2. **Success Criteria:** The local database syncs successfully with the remote version. + +#### **Step C: Sequential Lock Verification** + +1. **Tester 1 (@lince_m):** Initiate an upload (`u`) and confirm. +2. **Tester 2 (@athreya7023):** Attempt to upload (`u`) _while_ Tester 1 is uploading. +3. **Success Criteria:** + +- **Tester 1:** Upload proceeds. +- **Tester 2:** Receives a warning stating that **Lince is currently uploading**. + +4. **Final Verification:** Once Tester 1 finishes, Tester 2 tries to upload again. + +- **Expected Result:** Tester 2 **cannot** upload (due to version mismatch/outdated DB requiring a pull first). + +## TODO + +1. Custom Lock and unlock +2. new version of db will not have any previous data check. + +### Problem + +For more than 2 hour update of db there is no status or notification to other user. + +> say i'm running a script for 2 hours which is also doing a db insertion or updation. +> According to this logic, if someone in between updated the db and pushed it, after pulling the latest db, i have to run the script again right? + +This issue can be solved by adding a new feature of custom lock. + +1. User can lock the db for update by pressing `l` key. +2. This should show {user} is updating {db_name}. +3. This `l` commad should only create lock and update metadata with `status: "updating"` +4. For other with current implementation it will show {user} is updating {db_name} and will not allow to upload. +5. Also should show small warning notification suggesting to downloader that db is updating +6. This keybinding should be added to keybinding list which is impelementated `ui/keybindings.go`. + +## Problem 2 + +1. If user trying to upload new version he should only update when there is no + +## Design and Structure + +1. @core/rclone.go have only command rclone function only. + + + +# Migration Integration +## Phase 1 + + +## Cases + + + +| Case No | Local | Remote | Action | +| ------- | ----- | ------ | ------------------- | +| 1 | New | Old | Upload | +| 2 | Old | New | Download | +| 3 | New | New | Download and Upload | + + +Case 3 + +Local New and Remote New +Assume New DB. + +1. Data Modification +2. Data insertion +3. Data Deletion + + +It sounds like you have built a clever workaround for sharing an SQLite database using B2, but you have hit the classic wall of **distributed database concurrency**. + +To be completely candid: sharing a single raw `.sqlite` file over cloud storage for multiple developers to actively write to is a known anti-pattern. Because SQLite is a local file, when Dev A and Dev B both make local changes and try to sync, you get a "split-brain" scenario. + +"Normal engineers" solve this by separating **Schema Migrations** (changes to tables/structure) from **Data Synchronization** (day-to-day row updates). + +Here are the most efficient, low-effort options to solve your problem, ranging from adapting your current B2 setup to adopting industry-standard practices. + +### Option 2: The `sqldiff` Auto-Merge Approach (Best for your current B2 setup) + +If you don't want developers to manually write `.sql` files for every data entry, you can automate the extraction of their local changes using SQLite's built-in `sqldiff` utility. + +`sqldiff` compares two SQLite databases and generates the exact `INSERT`, `UPDATE`, and `DELETE` statements needed to make the first database match the second. + +**The Workflow Script:** + +1. **Download:** The script downloads the latest `remote.db` from B2. + +2. **Diff:** The script runs `sqldiff remote.db local.db > my_changes.sql`. This captures _only_ the new work the developer did locally. + +3. **Merge:** The script applies `my_changes.sql` to the newly downloaded `remote.db`. + +4. **Upload:** The script uploads the newly merged `remote.db` back to B2 and replaces the developer's `local.db` with it. + + +**Why this fits your needs:** + +- **No Lost Updates:** Even if a developer has to download a new DB, their local updates are saved into a `.sql` file first and re-applied automatically. + +- **Small Work:** You just need to download the `sqldiff` binary (provided officially by SQLite) and wrap it in a 10-line bash or python script. + +- **Warning:** You will face **Primary Key collisions**. If Dev A inserts a row with `ID=10` and Dev B inserts a row with `ID=10`, `sqldiff` will conflict. _Fix this by using UUIDs for your primary keys instead of auto-incrementing integers._ + +I already have Lock + Safe Upload Workflow with hash check and other things + +Since you already enforce download-before-upload, extend it: + +## Safe workflow + +``` +1. Acquire lock +2. Download latest DB +3. Run migrations automatically +4. Apply local changes +5. Generate migration file +6. Commit migration +7. Upload DB +8. Release lock +``` + +Migration generation can be partially automated via scripts. + +--- + +# Recommended Setup for Your Team (Best Fit) + +For 5 engineers + SQLite + B2: + +## Use this: + +**Migration files + migration runner script** + +Minimal stack: + +``` +/db + migrations/ + migrate.py + schema.sql +``` + +Runner script: + +- checks migration table + +- runs missing files + +- records them + +- runs automatically before upload + + +--- + +## Dev workflow + +``` +git pull +python migrate.py +work +create new migration file +git commit +push +``` + +--- + +## Rebuild DB anytime + +``` +delete db.sqlite +python migrate.py +``` + +Fully reproducible. + +--- + +# What Engineers Normally Avoid + +Do NOT rely on: + +- manual DB edits + +- uploading full DB as truth + +- editing rows directly without migration logs + +- schema changes without versioning + + +Those always break collaboration. diff --git a/b2-manager/docs/workflow/migrations.md b/b2-manager/docs/workflow/migrations.md new file mode 100644 index 0000000000..c04a2f3f59 --- /dev/null +++ b/b2-manager/docs/workflow/migrations.md @@ -0,0 +1,103 @@ +# Database Migration Standard + +> This document defines the standard operating procedure for managing distributed SQLite databases within the `b2-manager` ecosystem. + +## Problem + +Sharing a single raw `.db` file over cloud storage (B2) for multiple developers to actively write to causes data loss and "split-brain" scenarios whenever concurrent edits occur. The last uploader overwrites the work of others. + +## Goal + +Enable safe, concurrent data evolution for multiple developers without a central database server, ensuring zero data loss and easy conflict resolution. + +## Current Reality + +- Developers make changes to their local SQLite file. +- To share changes, they must upload the entire binary file. +- If two developers upload around the same time, one version is lost. +- There is no history of _what_ changed, only the final state. + +## Expected Result + +- A workflow where developers can verify their local changes are safe to merge. +- A system that captures "what changed" rather than just the final binary. +- ability to "replay" changes on top of the latest remote version. + +## Solution + +We separate **Schema Migrations** (structure/data changes) from the **Database Artifact** (the binary file). + +### The Migration Artifact + +Instead of sharing the `.db` file directly, we share **Python Migration Scripts**. + +- **Format**: `YYYYMMDDHHMMSS_.py` +- **Creation**: Generated via `./b2m migrations create ` +- **Content**: A Python script that connects to the SQLite DB and executes SQL commands (or python logic) to apply changes. + +### The Safe Workflow (Lock-Last) + +#### Phase 1: Local Development (Unlocked) + +1. **Download**: `./b2m download ` (Get latest snapshot) +2. **Work**: Modify data or schema locally. +3. **Generate SQL (Optional)**: Use `sqldiff` to capture your changes. + - `sqldiff --primary-key remote_snapshot.db local.db > changes.sql` +4. **Create Migration Script**: `./b2m migrations create add_users_table` + - Creates: `scripts/20260215103000123456789_add_users_table.py` +5. **Implement**: detailed logic or paste the `sqldiff` output into the python script to apply your changes. +6. **Test**: Run the script locally against your DB. +7. **Commit**: `git add scripts/ && git commit` + +#### Phase 2: Synchronization (Locked) + +1. **Download Latest**: Ensure you have the absolute latest DB from B2. +2. **Apply New Migrations**: Run the new script(s) against this fresh DB. +3. **Verify**: Check data integrity. +4. **Lock**: Acquire global lock. +5. **Upload**: Upload the new DB binary. +6. **Unlock**: Release lock. + +## 6. Scenarios + +To accurately handle conflicts, we define the following states based on the "Base Version" the developer started with vs. the "Remote Version" currently on B2. + +### Version Scenarios + +| Case | Local Base | Remote State | Monitor / Check | Action | +| :---- | :--------- | :----------- | :---------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------ | +| **1** | **v1** | **v1** | **Automatic** | **Safe Upload**.
Run local migration -> Upload v2. | +| **2** | **v1** | **v2** | **Potential Conflict**
Check if v2 changes overlap with yours. | **Rebase Required**.
1. Download v2.
2. Re-apply your migration on top of v2.
3. Verify & Upload v3. | +| **3** | **v1** | **Locked** | **Blocked** | **Wait**. Another upload is in progress. Retry later. | + +### Data Source Types + +The risk of data loss depends on the _source_ of the update. + +1. **External Data Injection (Critical)** + - _Source_: LLM output, API streams, User input. + - _Risk_: **High**. If this data is inserted directly into the local DB and then wiped by a "Download" to resolve a conflict, the data is lost forever. + - _Mitigation_: Must be captured in a CSV/JSON or Migration Script _before_ insertion. + +2. **Internal Data Update (Recoverable)** + - _Source_: Deterministic logic, re-computable values. + - _Risk_: Low. Can be re-run if needed. But still, conflicts are possible. + +## 7. Actual Result + + + +## 7. Difference + + diff --git a/frontend/changeset/changeset.py b/frontend/changeset/changeset.py new file mode 100644 index 0000000000..a9ad52c61a --- /dev/null +++ b/frontend/changeset/changeset.py @@ -0,0 +1,34 @@ +# Common Functions +#!/usr/bin/env python3 +import subprocess + +def db_status(db_name): + print(f"Executing: {db_name}") + try: + subprocess.run(["../b2m", "--status", db_name], check=True) + except subprocess.CalledProcessError as e: + print(f"Error checking status for {db_name}: {e}") + +def db_upload(db_name): + print(f"Executing: {db_name}") + try: + subprocess.run(["../b2m", "--upload", db_name], check=True) + except subprocess.CalledProcessError as e: + print(f"Error uploading {db_name}: {e}") + +def db_download(db_name): + """ + This function will download the db from b2. + This db will be present in `/changeset/dbs/` directory. + This is mainly done to avoid any data loss. + + + + """ + + print(f"Executing: {db_name}") + try: + subprocess.run(["../b2m", "--download", db_name], check=True) + except subprocess.CalledProcessError as e: + print(f"Error downloading {db_name}: {e}") + diff --git a/frontend/changeset/changeset_scripts/20260214162434583026694_descroption.py b/frontend/changeset/changeset_scripts/20260214162434583026694_descroption.py new file mode 100644 index 0000000000..070d8424a1 --- /dev/null +++ b/frontend/changeset/changeset_scripts/20260214162434583026694_descroption.py @@ -0,0 +1,62 @@ +# Template Version: v1 +# : _ +# + +## Predifned Imports and Functions +import sqlite3 +import urllib.parse +import os +import time + +DB_NAME = "ipm-db-v5.db" +## Import Common Functions +from changeset import db_status, db_download, db_upload # Still many more should be added. + +def db_migration(db_name): + ## Donwload b2 db to changeset db location which will be predefined. + status = db_download(db_name) + if status == "downloaded": + ## Now we have the db in our changeset db location. + ## Now we need to migrate new data from the server db to the new db. + return True + else: + return False + + +def copy_db(db_name): + try: + subprocess.run(["cp", db_name, "changeset/dbs/"], check=True) + return True + except subprocess.CalledProcessError as e: + print(f"Error copying {db_name}: {e}") + return False + +def handle_db_status(db_name): + status = db_status(db_name) + if status == "outdated_db": + if copy_db(db_name): + if db_migration(db_name): + if db_upload(db_name): + copy_db(db_name) + print("DB Migration successful") + else: + print("Error: db_upload failed") + else: + print("Error: db_migration failed") + else: + print("Error: copy_db failed") + elif status == "ready_to_upload": + db_upload(db_name) + elif status == "up_to_date": + pass + else: + print(f"Error: Unknown status {status}") + + +def main(db_name): + handle_db_status(db_name) + + +if __name__ == "__main__": + + main(DB_NAME) \ No newline at end of file diff --git a/frontend/package-lock.json b/frontend/package-lock.json index bc0b36be25..338375478a 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -131,7 +131,6 @@ "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.5", @@ -504,7 +503,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=18" }, @@ -528,7 +526,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=18" } @@ -3274,7 +3271,6 @@ "integrity": "sha512-MWtvHrGZLFttgeEj28VXHxpmwYbor/ATPYbBfSFZEIRK0ecCFLl2Qo55z52Hss+UV9CRN7trSeq1zbgx7YDWWg==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "csstype": "^3.2.2" } @@ -3285,7 +3281,6 @@ "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", "devOptional": true, "license": "MIT", - "peer": true, "peerDependencies": { "@types/react": "^19.2.0" } @@ -3877,7 +3872,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -5979,7 +5973,6 @@ "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", "license": "MIT", - "peer": true, "bin": { "jiti": "bin/jiti.js" } @@ -7087,7 +7080,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -7543,7 +7535,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.3.tgz", "integrity": "sha512-Ku/hhYbVjOQnXDZFv2+RibmLFGwFdeeKHFcOTlrt7xplBnya5OGn/hIRDsqDiSUcfORsDC7MPxwork8jBwsIWA==", "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -7574,7 +7565,6 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.3.tgz", "integrity": "sha512-yELu4WmLPw5Mr/lmeEpox5rw3RETacE++JgHqQzd2dg+YbJuat3jH4ingc+WPZhxaoFzdv9y33G+F7Nl5O0GBg==", "license": "MIT", - "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -8406,7 +8396,6 @@ "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.19.tgz", "integrity": "sha512-3ofp+LL8E+pK/JuPLPggVAIaEuhvIz4qNcf3nA1Xn2o/7fb7s/TYpHhwGDv1ZU3PkBluUVaF8PyCHcm48cKLWQ==", "license": "MIT", - "peer": true, "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", @@ -8614,7 +8603,6 @@ "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -8731,7 +8719,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -8950,7 +8937,6 @@ "integrity": "sha512-dZwN5L1VlUBewiP6H9s2+B3e3Jg96D0vzN+Ry73sOefebhYr9f94wwkMNN/9ouoU8pV1BqA1d1zGk8928cx0rg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", @@ -9064,7 +9050,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -9229,7 +9214,6 @@ "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz", "integrity": "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==", "license": "ISC", - "peer": true, "bin": { "yaml": "bin.mjs" }, From 9b9607170aba592a5cec3f7e3572d7e0fcd50043 Mon Sep 17 00:00:00 2001 From: Ganesh Kumar Date: Wed, 25 Feb 2026 20:06:11 +0530 Subject: [PATCH 2/5] Implement Database Changeset System Phase 1.1 --- b2-manager/Changeset.md | 1114 ++++-------------------- frontend/changeset/changeset.py | 24 +- frontend/md/changset-implementation.md | 475 ++++++++++ 3 files changed, 674 insertions(+), 939 deletions(-) create mode 100644 frontend/md/changset-implementation.md diff --git a/b2-manager/Changeset.md b/b2-manager/Changeset.md index 12663b8168..a46432246d 100644 --- a/b2-manager/Changeset.md +++ b/b2-manager/Changeset.md @@ -1,907 +1,4 @@ -# 17th Feb 2026 - -# DB Changeset Script Policy - -To Upload server db / local db to b2 without any data loss. - -## States of DB - -Db will be present in these 3 locations mainly: - -1. b2: -- It is the source of truth for the database version. -- Any db updated and inserted, should be done via changeset script integrated with `b2m`. - -2. Server: -- This is the production db. -- Data will be inserted regularly from API and inserted/updated data should reflect in live server. -- Inserted/Updated Data should be exportable during changeset phase which I will be further defining on how it is done in changeset script. -3. Local -- These are the db which are present in the local machine of the developer. -- These db are used for the development purposes. -- Any new feature or changes in the db can be done with only changeset script. - - -## Changeset Script - -I have divided into two types of changeset scripts based on where it is triggered from: - -1. Server -> B2: No version confict (happy flow) -2. Server -> B2: Version conflict (Changeset required) -4. Team Member -> B2: No version confict (happy flow) -5. Team Member -> B2: Version conflict (Changeset required) - -Both of these should be well defined by the developer in the changeset script. -### Automated Changeset Script -In this changeset script, It should automatically applies changeset of data from server to b2. Both b2 and New data should be present before uploading to b2. - - -#### Scenario 1: Server -> B2: No version confict (happy flow) - -Assume These Constraints : -1. Current IPM DB Version States at 2PM. - -| DB Location | version | -| ----------- | ------- | -| b2 | v1 | -| server | v1 | - -**At 3PM:** -1. DB version in same state. -2. User runs ipm and generates IPM json for a new repo which didn't exist earlier. -3. The db gets updated in server db immediately. - -| DB Location | Initial Version | -| ----------- | --------------- | -| b2 | v1 | -| server | v2* | - -Using `*` to say db is updated and not present in b2. - -**At 6PM:** -Changeset script is triggered at 6pm daily. -Step 1: Check Status of DB using `b2m`. -Result:`b2m` will result **Ready to upload** as both DB. -Step 2: Trigger `b2m` to upload DB to b2. -DB Version After Upload to b2 - -| DB Location | Initial Version | -| ----------- | --------------- | -| b2 db | v2 | -| server db | v2 | - -#### Scenario 2: Server -> B2: Version conflict (Changeset required) -1. Current IPM DB Version States at 2PM. - -| DB Location | DB Version | -| ----------- | ---------- | -| b2 | v1 | -| server | v1 | - -**At 3PM:** -1. DB version in same state. -2. User runs ipm and generates IPM json for a new repo which didn't exist earlier. -3. The db gets updated in server db immediately. - -| DB Location | Initial Version | -| ----------- | --------------- | -| b2 | v1 | -| server | v2* | - -Using `*` to say db is updated and not present in b2. -**At 4PM:** - 1. IPM db was updated with bulk json insertion. - 2. This db is uploaded to b2 - -| DB Location | Initial Version | -| ----------- | --------------- | -| b2 | v2 | -| server | v2* | - -Using `*` to say db is updated and not present in b2. - -**At 6PM:** -Changeset script is triggered at 6pm. -Step 1: Check Status of DB using `b2m`. -Result:`b2m` will result **Outdated DB** as both DB. -Step 2: Changeset should export/backup db version `v2*` -Step 3: Download `v2` from `b2` via`b2m` -Step 4: Trigger builk insertion of json from `v2*` to `v2` hence creating `v3` -Step 4: Trigger `b2m` to upload `v3` db to b2. -DB Version After Upload to b2 - -| DB Location | Initial Version | -| ----------- | --------------- | -| b2 db | v3 | -| server db | v3 | - - - - - -#### Scenario 3: Team Member -> B2: No version confict (happy flow) - - -Assume These Constraints : -1. Current emoji DB Version States at 2PM. - -| DB Location | version | -| ----------- | ------- | -| b2 | v1 | -| Athreya | v1 | - -**At 2:10PM:** -1. DB version in same state. -2. Athreya uses Changeset Script to update emoji db - - -| DB Location | Initial Version | -| ----------- | --------------- | -| b2 | v1 | -| Athreya | v2* | - -Using `*` to say db is updated and not present in b2. - -3. Changeset script will be triggered by Athreya and it will continue with these steps. -Step 1: Check Status of DB using `b2m`. -Result:`b2m` will result **Ready to upload** as both DB. -Step 2: Trigger `b2m` to upload DB to b2. -DB Version After Upload to b2 - -| DB Location | Initial Version | -| ----------- | --------------- | -| b2 db | v2 | -| Athreya db | v2 | - - -#### Scenario 4: Team Member -> B2: Version conflict (Changeset required) - - -Assume These Constraints : -1. Current emoji DB Version States at 2PM. - -| DB Location | version | -| ----------- | ------- | -| b2 | v1 | -| Athreya | v1 | -| Lince | v1 | - -**At 2:10PM:** -1. DB version in same state. -2. Athreya uses Changeset Script to update emoji db - - -| DB Location | Initial Version | -| ----------- | --------------- | -| b2 | v1 | -| Athreya | v2* | -| Lince | v1 | - -Using `*` to say db is updated and not present in b2. - -Also Currently Athreya's db is updating DB - - -**At 2:11PM:** - -1. Lince uses Changeset Script to update emoji db - - -| DB Location | Initial Version | -| ----------- | --------------- | -| b2 | v1 | -| Athreya | v2* | -| Lince | v2* | - -Using `*` to say db is updated and not present in b2. - -3. Changeset script will be triggered by Lince and it will continue with these steps. -Step 1: Check Status of DB using `b2m`. -Result:`b2m` will result **Ready to upload** as both DB. -Step 2: Trigger `b2m` to upload DB to b2. -DB Version After Upload to b2 - -| DB Location | Initial Version | -| ----------- | --------------- | -| b2 db | v2 | -| Athreya db | v2* | -| Lince db | v2 | - - -**At 2:12PM:** - -4. Changeset script will be triggered by Athreya and it will continue with these steps. -Step 1: Check Status of DB using `b2m`. -Result:`b2m` will result **Outdated DB** as both DB. -Step 2: Changeset should export/backup db version `v2*` -Step 3: Download `v2` from `b2` via`b2m` -Step 4: Trigger builk insertion of json from `v2*` to `v2` hence creating `v3` -Step 5: Trigger `b2m` to upload `v3` db to b2. -DB Version After Upload to b2 - -| DB Location | Initial Version | -| ----------- | --------------- | -| b2 | v3 | -| Athreya | v3 | -| Lince db | v2 | - - - - -### Default Action Performed by Changeset Script. - -These rules should be followed on start of changeset script. - -This is my assumption, you can correct me if I am wrong. -1. `*-wal` file and the `*-shm` file should not be present. There should be no active connection to the db. -2. `fdt-templ` should have cli to disconnect to slected changeset db's. -3. Make sure Server should still be serving other pages even the db which is being migrated via recent cache. -4. Trigger `b2m` to check status of db. -5. If `b2m` returns **Ready to upload** then trigger `b2m` to upload db to b2. -6. If `b2m` returns **Outdated DB** then - 1. trigger `b2m` to export new data from db version `v2*` or create cp of `v2*` to predifned directory. - 2. download `v2` from `b2` via `b2m` - 3. Insertion of new data from `v2*` to `v2` hence creating `v3` - 4. trigger `b2m` to upload `v3` db to b2. -7. `fdt-templ` should have cli to connect to slected changeset db's. - - - -### How To Use `b2m` cli in Changeset Script. - -> This is proposal of `b2m` cli integration. - -`b2m` cli should have - - -1. `--status` flag to check status of db. - Return `ready_to_upload`, `outdated_db` and `up_to_date`. -Multiple DBs can be checked at once. -```shell -./b2m --status -``` -This Will Check DB Version Status. - 1. `ready_to_upload` - Local DB is up to date and ready to upload to `b2`. - 2. `outdated_db` - Local DB is outdated and needs to download from `b2` and proceed with changeset and then upload to `b2`. - 3. `up_to_date` - Local DB is up to date with `b2`. - -Single DB Check -```shell -./b2m --status -``` -This will be for single DB amd Result will be same. - -2. `--upload` flag to upload db to b2. - return `success` or `failed`. - - -```shell -./b2m --upload -``` -This will upload selected DBs to b2. - - - -3. `--download` flag to download db from b2. - return `success` or `failed`. - -```shell -./b2m --download -``` -This will download selected DBs from b2. -> Note: This will override local DBs. Have a backup of local DBs before downloading. - - - - -# 18th Feb 2026 - -## Goal - -Implementation of Changeset Script Where Team can use it for db version changes, Db Upload to b2 and Db Download from b2 without any data loss/Server Downtime. - - -## Current State - -Identified 4 Scenarios where db version changes can happen. -Many Comments on Proposal - -I have added ✅ and ❌ to the comments to indicate that the comment has been addressed or not. - - -1. How To execute changeset script? ✅ -2. Is there any command for it? ✅ -3. What is the naming convention for that script? ✅ -4. How does it look like? ✅ -5. Download : Isnt this destructive? Some safeguards can be put right? ❌ -7. even in scenario 2,3and 4 changeset should be used to create the v2* db ❌ -1. when git pull happens? cuz changesets *.py are commited to git right for every conflict case ❌ -2. after changing versions how are you handing in code? we have paths hardcoded in code man-pages-db-v2.db ❌ - - -## Problem - -Team Coudn't Understand the Proposal especially in implementation of the changeset script. -1. No Clear Explaination of the the changeset script, How it works. -2. How ipm-db-v2.db -> ipm-db-v3.db is handled? -3. How is changescript will handle hardcoded values in `fdt-templ` server? -4. Does git pull play major role in this? - -## Expectations - -In this iteration on defineing changeset script and it's deps. - -1. Defining Changeset Script Template and how to create changeset script. -2. Explaination of how to execute changeset script. -3. How Change Set script handles ipm-db-v2.db -> ipm-db-v3.db? -4. Changeset script should have these main criteria. - 1. Script will be generated using `b2m` cli. - 2. File will be placed in `changeset` with structure mentioned below by `b2m`. - 3. `changeset_cron`: cron job script (ex: 6:00PM db changeset). - `changeset`: Will be common changeset script - ```shell - changeset/ - ├── changeset_scripts/ - │ ├── _.py or .py - │ └── ... - │ changeset.py # common functions. - └── README.md - ``` - 3. Script should have same template with version tag inside it. (#Template Version: v1) - 4. All the migration (sql scripts), Data extraction (sql scripts), Data insertion (sql scripts) should be in one file. - 5. Script should be executable using `python ` -5. Final Expectation Full template to write code for changeset script. - - -## Solution -### 1. Changeset Script Generation - -1. This will be done by `b2m` cli. -```shell -./b2m --changeset -``` -2. This will create a changeset script in `changeset` directory. -3. It will be generated based on predifned template. - - -### 2. Defining Changeset Script Template - - -Template Proposal: - -Template should look like this. - -1. Predifned Imports and Functions -2. These can be done by creating custom common library where changeset script can import it. -```py -# Template Version: v1 -# : _ -# is a short description of the change. - -## Predifned Imports and Functions -import sqlite3 -import urllib.parse -import os -import time - -def upload(db_name): - # Define cli command to upload db to b2. - # This will be added once cli defined - # Example: - # os.system(f"./b2m --upload {db_name}") - pass - -def download(db_name): - # Define cli command to download db from b2. - # This will be added once cli defined - # Example: - # os.system(f"./b2m --download {db_name}") - pass - -def status(db_name): - # Define cli command to export db to json. - # This will be added once cli defined - # Example: - # os.system(f"./b2m --status {db_name}") - pass - -def update(db_name): - # Use migration code. - # This consist of sql script to update db. - # - pass -### There will be still more need to define those gradualy. - -def main(): - # Check status of db. - # If status is outdated_db, then download db from b2. - # If status is ready_to_upload, then upload db to b2. - # If status is up_to_date, then do nothing. - pass - -if __name__ == "__main__": - main() -``` - - -### 3. Exectuing Changeset Script - -1. This will be done by `b2m` cli. -```shell -./b2m --execute -``` -2. This will execute the changeset script. -3. It will update the db and upload it to b2. - - -## Result - -1. Defined script template, execution methods. - - -## Difference - -1. Main comments not addressed. - - - -# 19th Feb 2026 - - -## Goal 2 - -I have added ✅ and ❌ to the comments to indicate that the comment has been addressed or not. - -1. Address lot of ambiguity in the proposal.(Did 2 iteration before posting for review)✅ -2. How ipm-db-v2.db -> ipm-db-v3.db is handled? (Trying to automate this also) so, not yet solved ❌ -3. How changescript will handle hardcoded values in `fdt-templ` server?✅ -4. Does git pull play major role in this?❌ As, of now I don't think we need push from server side. - -## Expectation -Defineing these points. -1. How to create, execute changeset script? -2. Template of changeset script. -3. Common functions used. -4. Give Example of how changeset script will be handling ipm db changeset. -5. Steps involved in changeset script. with proper description. -6. `db.toml` file will be used to define the db version. - - -## Proposal - - -In this Proposal first I will define all the structure's which include any cli and steps involved in defining changeset script. - - -**Structure** - -This Consist of 4 main parts. -1. `changeset` directory -2. `b2m` cli -3. `db.toml` file -4. `fdt-templ` server -5. `changeset_script` template -6. `changeset.py` common function - -### Create changeset script - -Folder Structure: -``` -changeset/ -├── scripts/ -│ ├── _.py -│ └── ... -├── dbs/ -| ├── _/ -| | ├── _b2.db -| | ├── _server.db -| | └── ... -| └── ... -├── logs/ -| ├── _.log -| └── ... -├── changeset.py # common functions. -└── README.md -``` - - -1. `_.py` will be generated by `b2m` cli. -```shell -./b2m --create-changeset -``` -2. Create a changeset script in `changeset/scripts` directory. -3. It consist of 2 subdirectories. - 1. `scripts`: This will contain changeset script. - 2. `dbs`: (Details Explaination Below) - 1. Temporary dbs until changeset is executed. - 2. This is for safety purpose. - 3. Any Db download, upload should be done in this file. -3. `logs`: - 1. This will contain logs of changeset script. - 2. This is for logging purpose. -4. `changeset.py`: Common functions used in changeset script. - 1. Such as `b2m --upload `, `b2m --download `, `b2m --status `, etc. - - -Reason For dbs Directory: - - -Goal: -1. Move new data added to ipm db in master to b2. - -Understanding Situation with IPM Db: -1. B2 has new data and Master has new data. -2. To Keep Both data we need to download latest db from b2 and insert new data to b2 db. - - -Options to do this: - -1. Use same `db/all_dbs` directory for changeset operations. (Can be done but need to be careful) -2. Use changeset directory for operation. (safe) - - -Option 1: - -1. Export New data to prediffned json file to `db/all_dbs` directory. -2. Download latest db from b2 to `db/all_dbs` directory. -3. Insert new data to b2 db. -4. Upload b2 db to b2. - -Option 2: - -1. Create a copy of original ipm db to `changeset/dbs/_` directory name `ipm-v3.master.db`. -2. Download latest db from b2 to `changeset/dbs/_` directory name `ipm-v3.b2.db` (it can be `ipm-v4.b2.db` depends on version present in b2). -3. Insert new data from `ipm-v3.master.db` to `ipm-v3.b2.db`. -4. Upload `ipm-v3.b2.db` to b2. -5. copy `ipm-v3.b2.db` to `db/all_dbs` directory. -6. Remove `changeset/dbs/_` directory if all the operations are successfully done. - -I choosed option 2 to define any db changeset operations should be done in changeset directory. - - - - -### B2M CLI - -1. `b2m` cli will be used to create, execute, and manage changeset script. -2. `b2m` cli will be placed in `frontend` directory. -3. `b2m` cli will be having following commands. There are 2 types of commands. - 1. User specific commands: These commands are used regurly by us. - 1. `b2m --create-changeset `: Create a changeset script. - 1. This will create a changeset script in `changeset/scripts` directory. (Added Detailed Description Above) - 2. `b2m --execute `: Execute a changeset script. - 1. This will execute the changeset script. It can also be done by `python ` just adding for making it easy to execute. - 2. Db specific commands: These commands are used and predifned in `changeset.py`. (Added Detailed Descript Above) - 1. `b2m --status `: Check status of db. - 1. This will check the status of db. - 2. It will check the status of db. - 2. `b2m --upload `: Upload db to b2. - 1. This will upload the db to b2 form `changeset/dbs//_b2.db`. (Added Detailed Description Above) - 3. `b2m --download `: Download db from b2. - 1. This will download the db from b2 to `changeset/dbs//_b2.db`. (Added Detailed Description Above) - -Reasons: - -1. Potentially use only 2 commands `b2m --create-changeset ` and `b2m --execute `. -2. Other commands will defined under `changeset.py`. - - -### Db.toml - -This is added to remove any hardcoded values in `fdt-templ` server. - -```toml -[db] -ipmdb = "ipm-db-v2.db" -emojidb = "emoji-db-v2.db" -path = "/frontend/db/alldbs/" -``` - -This can also be defined in `fdt-dev.toml`, `fdt-prod.toml`, `fdt-staging.toml` but -for defining I have choosed `db.toml` to avoid confusion of multiple db definition. - - -### fdt-templ server - -This is cli version integration of `fdt-templ` server. - -Reason: -1. For Perform any changeset to any DB it should never be connected to any DB. -2. If Server connected to db file like `*-wal` and `*-shm` which will cause db curruption if done any operations. -![image](https://hackmd.io/_uploads/B1QZvgUO-g.png) - -3. Currently only ipm db need server -> b2 db upload which also means we can do db status, copy only for ipm db. - - -For this case we have 2 options: -1. Complete Server Shutdown while doing changeset. -2. Tell `./server` bin to disconnect the `ipm` db without stoping server. - -`fdt-templ` cli - -1. `./server --disconnect `: This will trigger db connection close function. -2. `./server --connect ` : This will initiates db connection. - - -Pro: - -1. Can reduce complete downtime of server. -2. Can use in-memory cache for serving `ipm` db. -3. Can add queue for ipm installation command insertion. - - -Cons: -1. Takes more time to implement. -2. Other than reducing downtime there is no much benefit. - -Please let me know your thoughts on this. - -### Changeset Script Template - -1. This Include teamplate version -2. This also include Common Functions -```py -# Template Version: v1 -# : _ -# is a short description of the change. - -## Predifned Imports and Functions -import sqlite3 -import urllib.parse -import os -import time - -## Import Common Functions -from changeset import db_status, db_download, db_upload # Still many more should be added. - - -def main(): - # Check status of db. - # If status is outdated_db, then download db from b2. - # If status is ready_to_upload, then upload db to b2. - # If status is up_to_date, then do nothing. - pass - -if __name__ == "__main__": - main() -``` - - -### `changeset.py` common functions - -1. Mainly all helper commands will be defined here. -2. This will reduce defining again and again. -3. It consist of `b2m` cli commands. Further I we can add more based on requirements. - -```py -# Common Functions -#!/usr/bin/env python3 -import subprocess - -def db_status(db_name): - print(f"Executing: {db_name}") - try: - subprocess.run(["../b2m", "--status", db_name], check=True) - except subprocess.CalledProcessError as e: - print(f"Error checking status for {db_name}: {e}") - -def db_upload(db_name): - print(f"Executing: {db_name}") - try: - subprocess.run(["../b2m", "--upload", db_name], check=True) - except subprocess.CalledProcessError as e: - print(f"Error uploading {db_name}: {e}") - -def db_download(db_name): - """ - Function description. - """ - - print(f"Executing: {db_name}") - try: - subprocess.run(["../b2m", "--download", db_name], check=True) - except subprocess.CalledProcessError as e: - print(f"Error downloading {db_name}: {e}") - - -``` - -## Complete Steps Involved in Changeset Creation and Execution - - -### Requirements - -This is a changeset example for Server -> B2. - -There are 2 types of changeset: -1. ipm-db-v3.db -> ipm-db-v3.db (6:00 PM Backup) -2. ipm-db-v3.db -> ipm-db-v4.db (Manual Triggered On Major DB Version Bump) - -#### Scenario 1: ipm-db-v3.db -> ipm-db-v3.db (6:00 PM Backup) - -This will be added for cron job based script which trigger at 6:00 PM for ipm db backup. - -Assume These Constraints : -1. Current ipm-db-v3.db Version States at 2PM. - -| DB Location | version | DB Bump Version | -| ----------- | ------- | --------------- | -| b2 | v1 | v3 | -| server | v1 | v3 | - - -This version is hash of `ipm-db-v3.db`. - -> Note: Hash is generated using `b3sum` command. -> This will be created when db is downloaded from b2 -> server. Hash will be uptodate with b2 version of db. - - -There will be 3 states: -1. Server DB has new data and ready to upload. (NO Version Conflict) -2. Server DB has new data but Outdated and Need Proper Migration (Version Conflict) -3. DB is Up to Date (NO Version Conflict) - -Case 1: Server DB has new data and ready to upload. (NO Version Conflict) -Initials State -| DB Location | version | DB Bump Version | -| ----------- | ------- | --------------- | -| b2 | v1 | v3 | -| server | v2* | v3 | - -Final State -| DB Location | version | DB Bump Version | -| ----------- | ------- | --------------- | -| b2 | v2 | v3 | -| server | v2 | v3 | - - -`*` : This is a marker to show that DB is updated by new data. - -Case 2: Server DB is Outdated and Need Proper miragtion (Version Conflict) - -Initial State -| DB Location | version | DB Bump Version | -| ----------- | ------- | --------------- | -| b2 | v2 | v3 | -| server | v2* | v3 | - -Final State -| DB Location | version | DB Bump Version | -| ----------- | ------- | --------------- | -| b2 | v3 | v3 | -| server | v3 | v3 | - -Case 3: DB is Up to Date (NO Version Conflict) - -Initial State -| DB Location | version | DB Bump Version | -| ----------- | ------- | --------------- | -| b2 | v1 | v3 | -| server | v1 | v3 | - -Final State -| DB Location | version | DB Bump Version | -| ----------- | ------- | --------------- | -| b2 | v1 | v3 | -| server | v1 | v3 | - - -**At 6PM:** -Changeset script is triggered at 6pm daily. - - -1. Disconnect Fdt Server from IPM DB. -2. Check Status of DB using `b2m`. - 1. Case 1:`b2m` will result **Ready to upload** as both DB. - 1. Step 1: Copy DB from changeset db location to server db location. - 2. Step 2: Reconnect Fdt Server to IPM DB. - 3. Step 3: Trigger `b2m` to upload DB to b2. - 4. Step 4: Reconnect Fdt Server to IPM DB. - 5. Step 5: Verify fdt Server is connected to IPM DB. - 2. Case 2:`b2m` will result **Outdated DB** as both DB. - 1. Step 1: Download DB from b2 to changeset db location. - 2. Step 2: Copy DB from changeset db location to server db location. - 3. Step 3: Migrate New Data from Server DB to Downloaded DB. - 4. Step 4: Trigger `b2m` to upload DB to b2. - 5. Step 5: Copy DB from server db location to changeset db location. - 6. Step 6: Reconnect Fdt Server to IPM DB. - 3. Case 3:`b2m` will result **Up to date** as both DB. - 1. Step 1: Reconnect Fdt Server to IPM DB. - 2. Step 2: Exit Changeset Script. - - -Here is example of how script looks like - -```py -# Template Version: v1 -# : _ -# - -## Predifned Imports and Functions -import sqlite3 -import urllib.parse -import os -import time - -DB_NAME = "ipm-db-v5.db" ## This will be Config defined but for example I have taken this -## Import Common Functions -from changeset import db_status, db_download, db_upload # Still many more should be added. - -def db_migration(db_name): - ## Donwload b2 db to changeset db location which will be predefined. - status = db_download(db_name) - if status == "downloaded": - ## Now we have the db in our changeset db location. - ## Now we need to migrate new data from the server db to the new db. - return True - else: - return False - -## This will be added to common functions -def copy_db(db_name): - try: - subprocess.run(["cp", db_name, "changeset/dbs/"], check=True) - return True - except subprocess.CalledProcessError as e: - print(f"Error copying {db_name}: {e}") - return False - -def handle_db_status(db_name): - status = db_status(db_name) - if status == "outdated_db": - if copy_db(db_name): - if db_migration(db_name): - if db_upload(db_name): - copy_db(db_name) - print("DB Migration successful") - else: - print("Error: db_upload failed") - else: - print("Error: db_migration failed") - else: - print("Error: copy_db failed") - elif status == "ready_to_upload": - db_upload(db_name) - elif status == "up_to_date": - pass - else: - print(f"Error: Unknown status {status}") - - -def main(db_name): - handle_db_status(db_name) - - -if __name__ == "__main__": - - main(DB_NAME) -``` -#### Scenario 2: ipm-db-v3.db -> ipm-db-v4.db (Manual Triggered On Major DB Version Bump) - - - -This is I am currently thinking. - -This is nothing but same as `outdated_db` case in scenario 1. But need to make this script more robust. in identifying the db version and bump version and automaticaly update the data. - - - -## Goal - -1. Writing Test Script for all the states which comes under b2m. -2. Write Clear Documentation on Explaing the Scenario and the script implementation. -3. Create a Proper Design on the b2m cli. -4. Final Implemention Logic. - - - -## Expectation - -1. Writing Final Implementation Docs. - - -# Implementation Of Changeset - +# Changeset Implementation Details For Example of db I will be taking `ipm-db-v1.db`, `ipm-db-v2.db` and `ipm-db-v3.db`. @@ -909,12 +6,12 @@ Any Update in the db should be bumped to new version as default which will be ha ## Structure -This Consist of 4 main parts. +This Consist of 5 main parts. 1. `changeset` directory 2. `b2m` cli 3. `db.toml` file -5. `changeset_script` template -6. `changeset.py` common function +4. `changeset_script` template +5. `changeset.py` common function ### Create changeset script @@ -948,7 +45,7 @@ changeset/ 2. `dbs`: (Details Explaination Below) 1. Temporary dbs until changeset is executed. 2. This is for safety purpose. - 3. Any Db download, upload should be done in this file. + 3. Any Db download, upload should be done in this folder. 3. `logs`: 1. This will contain logs of changeset script. 2. This is for logging purpose. @@ -975,7 +72,7 @@ changeset/ 3. `b2m download `: Download db from b2. 1. This will download the db from b2 to `changeset/dbs//_b2.db`. (Added Detailed Description Above) 4. `b2m fetch-db-toml`: Fetch db.toml from b2. - 1. This will fetch db.toml from b2 to `db/all_dbs/db.toml`. (Added Detailed Description Above) + 1. This will fetch db.toml from b2 to `db/all_dbs/db.toml`. Reasons: @@ -991,7 +88,7 @@ This is added to remove any hardcoded values in `fdt-templ` server. [db] ipmdb = "ipm-db-v2.db" emojidb = "emoji-db-v2.db" -path = "/frontend/db/alldbs/" +path = "/frontend/db/all_dbs/" ``` This can also be defined in `fdt-dev.toml`, `fdt-prod.toml`, `fdt-staging.toml` but @@ -1109,7 +206,6 @@ def db_download(db_name): - ## Working Flow Of Changeset Script Assume these constraints: @@ -1118,14 +214,14 @@ On Changeset Script Trigger From (Team or Cron Job): 1. B2m status will return `ready_to_upload` or `outdated_db` or `up_to_date`. 2. This will be defined in `b2m status` command. - +3. New data queries will be present in `ipm-db-v1.sql` in `db/all_dbs` this will be defined rijul/ any other team member. There will be 3 states: 1. `ready_to_upload`: `ipm-db-v1.db` has new data (NO Version Conflict) 2. `outdated_db`: `ipm-db-v1.db` has new data but Outdated and Need Proper Migration (Version Conflict) 3. `up_to_date`: `ipm-db-v1.db` is Up to Date (NO Version Conflict) -Case 1: `ready_to_upload`: `ipm-db-v1.db` has new data (NO Version Conflict) +### Case 1: `ready_to_upload`: `ipm-db-v1.db` has new data (NO Version Conflict) Initials State | DB Location | version | New Data | @@ -1142,7 +238,7 @@ Final State | server | ipm-db-v2.db | No | -Case 2: `outdated_db`: Server DB is Outdated and Need Proper miragtion (Version Conflict) +### Case 2: `outdated_db`: Server DB is Outdated and Need Proper miragtion (Version Conflict) Case 2.1: B2 has `ipm-db-v2.db` and server has `ipm-db-v1.db`. Initial State | DB Location | version | New Data | @@ -1150,18 +246,54 @@ Initial State | b2 | ipm-db-v2.db | Yes | | server | ipm-db-v1.db | Yes | +Ex: + +At 10:00 AM, + +- User installed `ipm i msi-ec` and selected option. +- This triggered api and via llm + github readme json genereated. +- Once json generated queries generated and executed in `ipm-db-v1.db`. +- On successfull execution, these queries will be stored in `ipm-db-v1.sql` in `/db/all_dbs`. (name of the `ipm-db-v1.sql` should be always fetched from db.toml `ipm-db-v1.db` but in the place of `.db` it should be stored as `.sql`) +```sql +insert into table_name (column1, column2, column3, ...) +``` + + +At 2:00 PM, + +- There was major bump of `ipm-db-v1.db` to `ipm-db-v2.db`. +- Need to deploy `ipm-db-v2.db` to server. +- b2m should not allow direct download via b2m cli-ui or via b2m cli commands. +- This operation must be done via changeset script. +- Trigger changeset script manualy. + +``` +./b2m execute-changeset +or +python3 /.py +``` + There is new data in `ipm-db-v1.db` in server side and new version `ipm-db-v2.db` in b2 side. -1. Download to `ipm-db-v2.db` to `changeset/dbs/_/` from b2. -2. Copy `ipm-db-v1.db` to `changeset/dbs/_/` from `db/alldb.db`. -3. DB Migration - 1. We have Export/Select new from `ipm-db-v1.db` to `ipm-db-v2.db` in `changeset/dbs/_/` via sql query which have be defined `_.py`. -4. rename `ipm-db-v2.db` to `ipm-db-v3.db`. -5. Stop FDT Server with `make stop-prod`. -6. copy `ipm-db-v3.db` to `db/alldb`. -7. Update `db.toml` with `version = "v3"`. -8. Start FDT Server with `make start-prod`. -9. Upload `ipm-db-v3.db` from `changeset/dbs/_/` to b2. -10. Once Sucessful, Remove `changeset/dbs/_/`.(or keep it in `changeset/dbs/_/backup/` folder for safety) + +1. Status Check by b2m and it will return `outdated_db`. +2. Download `ipm-db-v2.db` from b2 to `changeset/dbs/_/`. +3. Copy `ipm-db-v1.sql` from `/db/all_dbs` to `changeset/dbs/_/`. +4. All the new data will be present in `ipm-db-v1.sql` in will be defined by backend on new data insert/update/delete. +5. `ipm-db-v1.sql` will be having `insert`, `update`, `replace` and `delete` queries. + +6. Both `insert` and `replace` queries will be executed in `ipm-db-v2.db`. But `delete` query will be done soft-delete in `ipm-db-v2.db`. +This should be defined in function + +7. rename `ipm-db-v2.db` to `ipm-db-v3.db`. +8. Stop FDT Server with `make stop-prod`. +9. copy `ipm-db-v3.db` to `db/all_dbs`. +10. Update `db.toml` with `ipm-db-v3.db`. +11. Start FDT Server with `make start-prod`. +12. Delete `ipm-db-v1.sql` (as we we will be having 2 backups `ipm-db-v1.sql` in `changeset/dbs/_/` and `ipm-db-v1.db` in `db/all_dbs`). +13. Upload `ipm-db-v3.db` from `changeset/dbs/_/` to b2. +13. Once Sucessful, Remove `changeset/dbs/_/`.(or keep it in `changeset/dbs/_/backup/` folder for safety) + + Final State | DB Location | version | New Data | @@ -1170,18 +302,16 @@ Final State | server | ipm-db-v3.db | No | - Case 2.2: B2 has `ipm-db-v1.db` and server has `ipm-db-v1.db` but both have new data. > Note: This case should never and will never happen. > I have added how to this case is handled. - -We can have discord notification that this type of case has failed due to this +We can have discord notification that this type of case has failed due to this. -Case 3: DB is Up to Date (NO Version Conflict) +### Case 3: DB is Up to Date (NO Version Conflict) Initial State | DB Location | version | New Data | @@ -1195,3 +325,129 @@ Final State | b2 | v1 | No | | server | v1 | No | + + + + + +### Defining the Changeset Script + +1. Create a changeset script using b2m cmd. +``` +./b2m create-changeset +``` + +This will create a changeset script in `changeset/scripts` folder with name `_.py`. +For example: `phrase` is `ipm-new-data-backup` +``` +./b2m create-changeset ipm-new-data-backup +``` +A new changeset script will be created in `changeset/scripts` folder with name `1735288266296000000_ipm-new-data-backup.py`. +2. Template will be consist of this. + + +This is simple example of how the changeset script should look like. + +This Will have some rules on writing the changeset script. + +1. Logging should be done in the changeset script and it should be present in `changeset/logs` folder. +2. Use the predifned functions in the changeset script. +3. Use the predifned variables in the changeset script. + +```py +# Template Version: v1 +# script_name : 1735288266296000000_ipm-new-data-backup +# phrase : ipm-new-data-backup + +## Predifned Imports and Functions +import sqlite3 +import urllib.parse +import os +import time + +DB_NAME = "ipm-db" + +## Import Common Functions +from changeset import db_status, db_download, db_upload # Still many more should be added. + +def inserted_queries(db_name): + """ + This function should insert the new data in the db. + These will be in ipm-db-v1.sql file. + """ + # execute queries from `changeset/dbs/1735288266296000000_ipm-new-data-backup/ipm-db-v1.sql` file to `changeset/dbs/1735288266296000000_ipm-new-data-backup/ipm-db-v2.db` file. + + return None + + +def main(): + # Check status of db. + status = db_status(DB_NAME) + # If status is outdated_db, then download db from b2. + if status == "outdated_db": + db_download(DB_NAME) + cp_queries(DB_NAME) + inserted_queries(DB_NAME) + rename_db(DB_NAME) + stop_server() + copy_db(DB_NAME) + update_db(DB_NAME) + start_server() + upload_db(DB_NAME) + # If status is ready_to_upload, then upload db to b2. + if status == "ready_to_upload": + stop_server() + copy_db(DB_NAME) + start_server() + db_upload(DB_NAME) + # If status is up_to_date, then do nothing. + elif status == "up_to_date": + pass + +if __name__ == "__main__": + main() +``` +4. To execute the changeset script, use the following command. + +in frontend directory +``` +./b2m execute-changeset 1735288266296000000_ipm-new-data-backup +or +python3 changeset/scripts/1735288266296000000_ipm-new-data-backup.py +``` + + +### Phase 1 + +Implementing structure of changeset directory and b2m cli. +This Consist of 5 main parts. +1. `changeset` directory +2. `b2m` cli +3. `db.toml` file +4. `changeset_script` template +5. `changeset.py` common function + + +In Phase 1.1 let's implement the structure of changeset directory + +### Phase 1.1: Implementing structure of changeset directory + +``` +changeset/ +├── dbs/ +│ ├── _/ +│ │ ├── ipm-db-v1.db +│ │ └── ipm-db-v2.db +│ └── backup/ +│ ├── _/ +│ │ ├── ipm-db-v1.db +│ │ └── ipm-db-v2.db +│ └── ... +├── logs/ +│ ├── _.log +│ └── ... +├── scripts/ +│ ├── _.py +│ └── ... +└── changeset.py +``` \ No newline at end of file diff --git a/frontend/changeset/changeset.py b/frontend/changeset/changeset.py index a9ad52c61a..99851e118c 100644 --- a/frontend/changeset/changeset.py +++ b/frontend/changeset/changeset.py @@ -1,13 +1,24 @@ -# Common Functions #!/usr/bin/env python3 +# Common Functions import subprocess def db_status(db_name): print(f"Executing: {db_name}") try: - subprocess.run(["../b2m", "--status", db_name], check=True) + result = subprocess.run(["../b2m", "--status", db_name], capture_output=True, text=True, check=True) + # Assuming b2m outputs the status to stdout. We might want to return it. + # But based on the template, we'll return the stdout stripped. + # Wait, the spec doesn't explicitly return it in the provided code snippet, but the template code expects return: + # status = db_status(DB_NAME) + # So I will return the output + output_lines = result.stdout.strip().split('\n') + # If the b2m command outputs extra info, the status might be the last line. + # For safety I will just return the whole stripped output, + # or maybe the script just expects exactly 'outdated_db', 'ready_to_upload', 'up_to_date' + return result.stdout.strip() except subprocess.CalledProcessError as e: print(f"Error checking status for {db_name}: {e}") + return None def db_upload(db_name): print(f"Executing: {db_name}") @@ -18,17 +29,10 @@ def db_upload(db_name): def db_download(db_name): """ - This function will download the db from b2. - This db will be present in `/changeset/dbs/` directory. - This is mainly done to avoid any data loss. - - - + Function description. """ - print(f"Executing: {db_name}") try: subprocess.run(["../b2m", "--download", db_name], check=True) except subprocess.CalledProcessError as e: print(f"Error downloading {db_name}: {e}") - diff --git a/frontend/md/changset-implementation.md b/frontend/md/changset-implementation.md new file mode 100644 index 0000000000..312e1ef01a --- /dev/null +++ b/frontend/md/changset-implementation.md @@ -0,0 +1,475 @@ +# Changeset Implementation Details + +For Example of db I will be taking `ipm-db-v1.db`, `ipm-db-v2.db` and `ipm-db-v3.db`. + +Any Update in the db should be bumped to new version as default which will be handled by `b2m` cli. + +## Structure + +This Consist of 5 main parts. +1. `changeset` directory +2. `b2m` cli +3. `db.toml` file +4. `changeset_script` template +5. `changeset.py` common function + +### Create changeset script + +Folder Structure: +``` +changeset/ +├── scripts/ +│ ├── _.py +│ └── ... +├── dbs/ +| ├── _/ +| | ├── _b2.db +| | ├── _server.db +| | └── ... +| └── ... +├── logs/ +| ├── _.log +| └── ... +├── changeset.py # common functions. +└── README.md +``` + + +1. `_.py` will be generated by `b2m` cli. +```shell +./b2m --create-changeset +``` +2. Create a changeset script in `changeset/scripts` directory. +3. It consist of 2 subdirectories. + 1. `scripts`: This will contain changeset script. + 2. `dbs`: (Details Explaination Below) + 1. Temporary dbs until changeset is executed. + 2. This is for safety purpose. + 3. Any Db download, upload should be done in this folder. +3. `logs`: + 1. This will contain logs of changeset script. + 2. This is for logging purpose. +4. `changeset.py`: Common functions used in changeset script. + 1. Such as `b2m upload `, `b2m download `, `b2m status `, etc. + + +### B2M CLI + +1. `b2m` cli will be used to create, execute, and manage changeset script. +2. `b2m` cli will be placed in `frontend` directory. +3. `b2m` cli will be having following commands. There are 2 types of commands. + 1. User specific commands: These commands are used regurly by us. + 1. `b2m create-changeset `: Create a changeset script. + 1. This will create a changeset script in `changeset/scripts` directory. (Added Detailed Description Above) + 2. `b2m execute-changeset `: Execute a changeset script. + 1. This will execute the changeset script. It can also be done by `python ` just adding for making it easy to execute. + 2. Db specific commands: These commands are used and predifned in `changeset.py`. (Added Detailed Descript Above) + 1. `b2m status `: Check status of db. + 1. This will check the status of db. + 2. It will check the status of db. + 2. `b2m upload `: Upload db to b2. + 1. This will upload the db to b2 form `changeset/dbs//_b2.db`. (Added Detailed Description Above) + 3. `b2m download `: Download db from b2. + 1. This will download the db from b2 to `changeset/dbs//_b2.db`. (Added Detailed Description Above) + 4. `b2m fetch-db-toml`: Fetch db.toml from b2. + 1. This will fetch db.toml from b2 to `db/all_dbs/db.toml`. + +Reasons: + +1. We will be using only 2 commands `b2m create-changeset ` and `b2m execute-changeset `. +2. Other commands will defined under `changeset.py`. + + +### Db.toml + +This is added to remove any hardcoded values in `fdt-templ` server. + +```toml +[db] +ipmdb = "ipm-db-v2.db" +emojidb = "emoji-db-v2.db" +path = "/frontend/db/all_dbs/" +``` + +This can also be defined in `fdt-dev.toml`, `fdt-prod.toml`, `fdt-staging.toml` but +for defining I have choosed `db.toml` to avoid confusion of multiple db definition. + +There are 2 situation for db version update happen: +1. Team +2. Server Dialy cron job (ipm db only) + + + +Changeset script will automatically bump the db version and update in `db.toml`. +For tracking this change we have 2 options: +1. git based. +2. b2 bucket based using b2m. + +I have choosed option 2. + +1. git based: + 1. This involves `git pull origin main` before checking any db status. + 2. Once updated it should be pushed to git by `git push origin main`. + + Pros: + 1. Easy to track changes. + 2. It will be git native + Cons: + 1. This will create 2 types of db versioning system. + 2. b2m uses b2 bucket for full db versioning system. If we use git for `db.toml` it will create confusion. + 3. If there are any conflicts in pushing to git it will be very hard to resolve. + 4. Need to implement seperate functions to handle it. +2. b2 based + 1. This uses existing b2m db versioning system. + 2. db.toml file will be present in b2 bucket. + Pros: + 1. Adding on top of existing b2m db versioning system. + 2. Easier to integrate with b2m as b2m already managing db `metadata`,`hash` and `lock` safely. + 3. This will create seperate versioning system for dbs independent with git. + 4. Any db ops done will be done using b2 bucket as source of truth. + 5. Anyone starting server will be depending on b2m for checking db.toml fetched from b2m. + Cons: + 1. Integrating b2m to `make start-prod` or `make run` command to identify any db changes. + + + + +### Changeset Script Template + +1. This Include teamplate version +2. This also include Common Functions +```py +# Template Version: v1 +# : _ +# is a short description of the change. + +## Predifned Imports and Functions +import sqlite3 +import urllib.parse +import os +import time + +## Import Common Functions +from changeset import db_status, db_download, db_upload # Still many more should be added. + + +def main(): + # Check status of db. + # If status is outdated_db, then download db from b2. + # If status is ready_to_upload, then upload db to b2. + # If status is up_to_date, then do nothing. + pass + +if __name__ == "__main__": + main() +``` + + +### `changeset.py` common functions + +1. Mainly all helper commands will be defined here. +2. This will reduce defining again and again. +3. It consist of `b2m` cli commands. Further I we can add more based on requirements. + +```py +# Common Functions +#!/usr/bin/env python3 +import subprocess + +def db_status(db_name): + print(f"Executing: {db_name}") + try: + subprocess.run(["../b2m", "--status", db_name], check=True) + except subprocess.CalledProcessError as e: + print(f"Error checking status for {db_name}: {e}") + +def db_upload(db_name): + print(f"Executing: {db_name}") + try: + subprocess.run(["../b2m", "--upload", db_name], check=True) + except subprocess.CalledProcessError as e: + print(f"Error uploading {db_name}: {e}") + +def db_download(db_name): + """ + Function description. + """ + + print(f"Executing: {db_name}") + try: + subprocess.run(["../b2m", "--download", db_name], check=True) + except subprocess.CalledProcessError as e: + print(f"Error downloading {db_name}: {e}") + + +``` + + + +## Working Flow Of Changeset Script + +Assume these constraints: + +On Changeset Script Trigger From (Team or Cron Job): + +1. B2m status will return `ready_to_upload` or `outdated_db` or `up_to_date`. +2. This will be defined in `b2m status` command. +3. New data queries will be present in `ipm-db-v1.sql` in `db/all_dbs` this will be defined rijul/ any other team member. + +There will be 3 states: +1. `ready_to_upload`: `ipm-db-v1.db` has new data (NO Version Conflict) +2. `outdated_db`: `ipm-db-v1.db` has new data but Outdated and Need Proper Migration (Version Conflict) +3. `up_to_date`: `ipm-db-v1.db` is Up to Date (NO Version Conflict) + +### Case 1: `ready_to_upload`: `ipm-db-v1.db` has new data (NO Version Conflict) +Initials State + +| DB Location | version | New Data | +| ----------- | ------------- | -------- | +| b2 | ipm-db-v1.db | No | +| server | ipm-db-v1.db | Yes | + +There is new data in `ipm-db-v1.db` in server side, so we will bump the version `ipm-db-v1.db` to `ipm-db-v2.db` and upload it to b2. + +Final State +| DB Location | version | New Data | +| ----------- | ------------- | -------- | +| b2 | ipm-db-v2.db | No | +| server | ipm-db-v2.db | No | + + +### Case 2: `outdated_db`: Server DB is Outdated and Need Proper miragtion (Version Conflict) +Case 2.1: B2 has `ipm-db-v2.db` and server has `ipm-db-v1.db`. +Initial State +| DB Location | version | New Data | +| ----------- | ------- | -------- | +| b2 | ipm-db-v2.db | Yes | +| server | ipm-db-v1.db | Yes | + +Ex: + +At 10:00 AM, + +- User installed `ipm i msi-ec` and selected option. +- This triggered api and via llm + github readme json genereated. +- Once json generated queries generated and executed in `ipm-db-v1.db`. +- On successfull execution, these queries will be stored in `ipm-db-v1.sql` in `/db/all_dbs`. (name of the `ipm-db-v1.sql` should be always fetched from db.toml `ipm-db-v1.db` but in the place of `.db` it should be stored as `.sql`) +```sql +insert into table_name (column1, column2, column3, ...) +``` + + +At 2:00 PM, + +- There was major bump of `ipm-db-v1.db` to `ipm-db-v2.db`. +- Need to deploy `ipm-db-v2.db` to server. +- b2m should not allow direct download via b2m cli-ui or via b2m cli commands. +- This operation must be done via changeset script. +- Trigger changeset script manualy. + +``` +./b2m execute-changeset +or +python3 /.py +``` + +There is new data in `ipm-db-v1.db` in server side and new version `ipm-db-v2.db` in b2 side. + +1. Status Check by b2m and it will return `outdated_db`. +2. Download `ipm-db-v2.db` from b2 to `changeset/dbs/_/`. +3. Copy `ipm-db-v1.sql` from `/db/all_dbs` to `changeset/dbs/_/`. +4. All the new data will be present in `ipm-db-v1.sql` in will be defined by backend on new data insert/update/delete. +5. `ipm-db-v1.sql` will be having `insert`, `update`, `replace` and `delete` queries. + +6. Both `insert` and `replace` queries will be executed in `ipm-db-v2.db`. But `delete` query will be done soft-delete in `ipm-db-v2.db`. +This should be defined in function + +7. rename `ipm-db-v2.db` to `ipm-db-v3.db`. +8. Stop FDT Server with `make stop-prod`. +9. copy `ipm-db-v3.db` to `db/all_dbs`. +10. Update `db.toml` with `ipm-db-v3.db`. +11. Start FDT Server with `make start-prod`. +12. Delete `ipm-db-v1.sql` (as we we will be having 2 backups `ipm-db-v1.sql` in `changeset/dbs/_/` and `ipm-db-v1.db` in `db/all_dbs`). +13. Upload `ipm-db-v3.db` from `changeset/dbs/_/` to b2. +13. Once Sucessful, Remove `changeset/dbs/_/`.(or keep it in `changeset/dbs/_/backup/` folder for safety) + + + +Final State +| DB Location | version | New Data | +| ----------- | ------- | -------- | +| b2 | ipm-db-v3.db | No | +| server | ipm-db-v3.db | No | + + +Case 2.2: B2 has `ipm-db-v1.db` and server has `ipm-db-v1.db` but both have new data. + + +> Note: This case should never and will never happen. +> I have added how to this case is handled. + +We can have discord notification that this type of case has failed due to this. + + +### Case 3: DB is Up to Date (NO Version Conflict) + +Initial State +| DB Location | version | New Data | +| ----------- | ------- | --------------- | +| b2 | v1 | No | +| server | v1 | No | + +Final State +| DB Location | version | New Data | +| ----------- | ------- | --------------- | +| b2 | v1 | No | +| server | v1 | No | + + + + + + +### Defining the Changeset Script + +1. Create a changeset script using b2m cmd. +``` +./b2m create-changeset +``` + +This will create a changeset script in `changeset/scripts` folder with name `_.py`. +For example: `phrase` is `ipm-new-data-backup` +``` +./b2m create-changeset ipm-new-data-backup +``` +A new changeset script will be created in `changeset/scripts` folder with name `1735288266296000000_ipm-new-data-backup.py`. +2. Template will be consist of this. + + +This is simple example of how the changeset script should look like. + +This Will have some rules on writing the changeset script. + +1. Logging should be done in the changeset script and it should be present in `changeset/logs` folder. +2. Use the predifned functions in the changeset script. +3. Use the predifned variables in the changeset script. + +```py +# Template Version: v1 +# script_name : 1735288266296000000_ipm-new-data-backup +# phrase : ipm-new-data-backup + +## Predifned Imports and Functions +import sqlite3 +import urllib.parse +import os +import time + +DB_NAME = "ipm-db" + +## Import Common Functions +from changeset import db_status, db_download, db_upload # Still many more should be added. + +def inserted_queries(db_name): + """ + This function should insert the new data in the db. + These will be in ipm-db-v1.sql file. + """ + # execute queries from `changeset/dbs/1735288266296000000_ipm-new-data-backup/ipm-db-v1.sql` file to `changeset/dbs/1735288266296000000_ipm-new-data-backup/ipm-db-v2.db` file. + + return None + + +def main(): + # Check status of db. + status = db_status(DB_NAME) + # If status is outdated_db, then download db from b2. + if status == "outdated_db": + db_download(DB_NAME) + cp_queries(DB_NAME) + inserted_queries(DB_NAME) + rename_db(DB_NAME) + stop_server() + copy_db(DB_NAME) + update_db(DB_NAME) + start_server() + upload_db(DB_NAME) + # If status is ready_to_upload, then upload db to b2. + if status == "ready_to_upload": + stop_server() + copy_db(DB_NAME) + start_server() + db_upload(DB_NAME) + # If status is up_to_date, then do nothing. + elif status == "up_to_date": + pass + +if __name__ == "__main__": + main() +``` +4. To execute the changeset script, use the following command. + +in frontend directory +``` +./b2m execute-changeset 1735288266296000000_ipm-new-data-backup +or +python3 changeset/scripts/1735288266296000000_ipm-new-data-backup.py +``` + + +### Phase 1 + +Implementing structure of changeset directory and b2m cli. +This Consist of 5 main parts. +1. `changeset` directory +2. `b2m` cli +3. `db.toml` file +4. `changeset_script` template +5. `changeset.py` common function + + +In Phase 1.1 let's implement the structure of changeset directory + +### Phase 1.1: Implementing structure of changeset directory + +``` +frontend/ +├── changeset/ +│ ├── dbs/ +│ │ ├── _/ +│ │ │ ├── ipm-db-v1.db +│ │ │ └── ipm-db-v2.db +│ │ └── backup/ +│ │ ├── _/ +│ │ │ ├── ipm-db-v1.db +│ │ │ └── ipm-db-v2.db +│ │ └── ... +│ ├── logs/ +│ │ ├── _.log +│ │ └── ... +│ ├── scripts/ +│ │ ├── _.py +│ │ └── ... +│ └── changeset.py +``` + + + + +1. `_.py` will be generated by `b2m` cli. +```shell +./b2m --create-changeset +``` +2. Create a changeset script in `changeset/scripts` directory. +3. It consist of 2 subdirectories. + 1. `scripts`: This will contain changeset script. + 2. `dbs`: (Details Explaination Below) + 1. Temporary dbs until changeset is executed. + 2. This is for safety purpose. + 3. Any Db download, upload should be done in this folder. +3. `logs`: + 1. This will contain logs of changeset script. + 2. This is for logging purpose. +4. `changeset.py`: Common functions used in changeset script. + 1. Such as `b2m upload `, `b2m download `, `b2m status `, etc. + From c0bc11b677195c61c81fcdaadfcdd0e2718ce03f Mon Sep 17 00:00:00 2001 From: Ganesh Kumar Date: Wed, 25 Feb 2026 21:04:30 +0530 Subject: [PATCH 3/5] Implement Database Changeset Management CLI Phase 1.2 --- b2-manager/config/config.go | 38 +- b2-manager/core/changeset.go | 160 ++++++++ b2-manager/go.mod | 16 +- b2-manager/go.sum | 32 +- b2-manager/model/types.go | 5 + b2-manager/templates/changeset_template.py | 50 +++ b2-manager/ui/cli.go | 352 ++++++++++-------- frontend/changeset/changeset.py | 6 +- .../20260214162434583026694_descroption.py | 62 --- .../1772031633645610550_sample-phrase.py | 50 +++ frontend/md/changset-implementation.md | 41 ++ 11 files changed, 574 insertions(+), 238 deletions(-) create mode 100644 b2-manager/core/changeset.go create mode 100644 b2-manager/templates/changeset_template.py delete mode 100644 frontend/changeset/changeset_scripts/20260214162434583026694_descroption.py create mode 100644 frontend/changeset/scripts/1772031633645610550_sample-phrase.py diff --git a/b2-manager/config/config.go b/b2-manager/config/config.go index 2bc022fa19..1d9712086f 100644 --- a/b2-manager/config/config.go +++ b/b2-manager/config/config.go @@ -8,12 +8,16 @@ import ( "path/filepath" "strings" - "github.com/BurntSushi/toml" + "github.com/knadh/koanf/parsers/toml/v2" + "github.com/knadh/koanf/providers/file" + "github.com/knadh/koanf/v2" "b2m/core" "b2m/model" ) +var k = koanf.New(".") + // InitializeConfig sets up global configuration variables func InitializeConfig() error { var err error @@ -45,6 +49,12 @@ func InitializeConfig() error { model.AppConfig.LocalAnchorDir = filepath.Join(model.AppConfig.LocalB2MDir, "local-version") model.AppConfig.MigrationsDir = filepath.Join(model.AppConfig.ProjectRoot, "b2m-migration") + // Changeset Paths + model.AppConfig.ChangesetScriptsDir = filepath.Join(model.AppConfig.ProjectRoot, "changeset", "scripts") + model.AppConfig.ChangesetLogsDir = filepath.Join(model.AppConfig.ProjectRoot, "changeset", "logs") + model.AppConfig.ChangesetDBsDir = filepath.Join(model.AppConfig.ProjectRoot, "changeset", "dbs") + model.AppConfig.FrontendTomlPath = filepath.Join(model.AppConfig.ProjectRoot, "db", "all_dbs", "db.toml") + return nil } @@ -71,24 +81,20 @@ func loadTOMLConfig() error { return fmt.Errorf("couldn't find fdt-dev.toml file at %s: %w", tomlPath, err) } - var tomlConf struct { - B2M struct { - Discord string `toml:"b2m_discord_webhook"` - RootBucket string `toml:"b2m_remote_root_bucket"` - LocalDBDir string `toml:"b2m_db_dir"` - } `toml:"b2m"` - } - if _, err := toml.DecodeFile(tomlPath, &tomlConf); err != nil { - return fmt.Errorf("failed to decode fdt-dev.toml: %w", err) + // Load TOML file + if err := k.Load(file.Provider(tomlPath), toml.Parser()); err != nil { + return fmt.Errorf("failed to load fdt-dev.toml: %w", err) } - model.AppConfig.RootBucket = tomlConf.B2M.RootBucket - model.AppConfig.DiscordWebhookURL = tomlConf.B2M.Discord - if tomlConf.B2M.LocalDBDir != "" { - if filepath.IsAbs(tomlConf.B2M.LocalDBDir) { - model.AppConfig.LocalDBDir = tomlConf.B2M.LocalDBDir + model.AppConfig.RootBucket = k.String("b2m.b2m_remote_root_bucket") + model.AppConfig.DiscordWebhookURL = k.String("b2m.b2m_discord_webhook") + + localDBDir := k.String("b2m.b2m_db_dir") + if localDBDir != "" { + if filepath.IsAbs(localDBDir) { + model.AppConfig.LocalDBDir = localDBDir } else { - model.AppConfig.LocalDBDir = filepath.Join(model.AppConfig.ProjectRoot, tomlConf.B2M.LocalDBDir) + model.AppConfig.LocalDBDir = filepath.Join(model.AppConfig.ProjectRoot, localDBDir) } } diff --git a/b2-manager/core/changeset.go b/b2-manager/core/changeset.go new file mode 100644 index 0000000000..38aeb370c1 --- /dev/null +++ b/b2-manager/core/changeset.go @@ -0,0 +1,160 @@ +package core + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "text/template" + "time" + + "b2m/model" +) + +// CreateChangeset generates a new changeset python script from a template +func CreateChangeset(phrase string) error { + timestamp := time.Now().UnixNano() + filename := fmt.Sprintf("%d_%s.py", timestamp, phrase) + + scriptDir := model.AppConfig.ChangesetScriptsDir + if err := os.MkdirAll(scriptDir, 0755); err != nil { + return fmt.Errorf("failed to create scripts directory: %w", err) + } + + scriptPath := filepath.Join(scriptDir, filename) + + // Get template path + // Assuming b2m runs from frontend, the templates dir would be in ../b2-manager/templates/ + // We should probably rely on ProjectRoot + templatePath := filepath.Join(model.AppConfig.ProjectRoot, "..", "b2-manager", "templates", "changeset_template.py") + + tmpl, err := template.ParseFiles(templatePath) + if err != nil { + return fmt.Errorf("failed to parse template file %s: %w", templatePath, err) + } + + f, err := os.Create(scriptPath) + if err != nil { + return fmt.Errorf("failed to create script file: %w", err) + } + defer f.Close() + + data := struct { + Timestamp int64 + Phrase string + }{ + Timestamp: timestamp, + Phrase: phrase, + } + + if err := tmpl.Execute(f, data); err != nil { + return fmt.Errorf("failed to execute template: %w", err) + } + + fmt.Printf("Changeset script created at: %s\n", scriptPath) + return nil +} + +// ExecuteChangeset runs the specified python script securely +func ExecuteChangeset(scriptName string) error { + scriptDir := model.AppConfig.ChangesetScriptsDir + + // Ensure ".py" extension is present if not provided + if filepath.Ext(scriptName) != ".py" { + scriptName += ".py" + } + // Sanitize to prevent directory traversal + scriptName = filepath.Base(scriptName) + + scriptPath := filepath.Join(scriptDir, scriptName) + + if _, err := os.Stat(scriptPath); os.IsNotExist(err) { + return fmt.Errorf("script %s not found in %s", scriptName, scriptDir) + } + + fmt.Printf("Executing Changeset Script: %s\n", scriptPath) + + cmd := exec.Command("python3", scriptPath) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("script execution failed: %w", err) + } + + return nil +} + +// RunCLIStatus runs a status check for a specific database and prints the status natively for python +func RunCLIStatus(dbName string) error { + ctx := context.Background() + + // In order to get status, we fetch local DBs, Remote Metas, and Locks... + // FetchDBStatusData logic does exactly this. + statusData, err := FetchDBStatusData(ctx, nil) + if err != nil { + return fmt.Errorf("failed to fetch status data: %w", err) + } + + found := false + for _, info := range statusData { + // handle both exact match and extension-less + baseName := strings.TrimSuffix(info.DB.Name, filepath.Ext(info.DB.Name)) + reqBaseName := strings.TrimSuffix(dbName, filepath.Ext(dbName)) + + if baseName == reqBaseName || info.DB.Name == dbName { + found = true + // Translate core statuses to the 3 Python required statuses + switch info.StatusCode { + case model.StatusCodeLocalNewer, model.StatusCodeNewLocal, model.StatusCodeLockedByYou: + fmt.Println("ready_to_upload") + case model.StatusCodeUpToDate: + fmt.Println("up_to_date") + default: + fmt.Println("outdated_db") // fallback for safety + } + return nil + } + } + + if !found { + fmt.Println("outdated_db") + } + return nil +} + +// RunCLIUpload runs a database upload without UI components +func RunCLIUpload(dbName string) error { + ctx := context.Background() + // Using empty functions to keep it quiet + return PerformUpload(ctx, dbName, false, nil, nil) +} + +// RunCLIDownload runs a database download without UI components +func RunCLIDownload(dbName string) error { + ctx := context.Background() + return DownloadDatabase(ctx, dbName, true, nil) +} + +// RunCLIFetchDBToml downloads db.toml from backblaze +func RunCLIFetchDBToml() error { + ctx := context.Background() + remotePath := model.AppConfig.RootBucket + "db.toml" // Assuming it sits at root or specify correctly + localPath := model.AppConfig.FrontendTomlPath + + if err := os.MkdirAll(filepath.Dir(localPath), 0755); err != nil { + return fmt.Errorf("failed to create directory for db.toml: %w", err) + } + + // Use RcloneCopy to pull specific file to localPath + description := "Fetching db.toml" + err := RcloneCopy(ctx, "copyto", remotePath, localPath, description, true, nil) + if err != nil { + return fmt.Errorf("failed to fetch db.toml: %w", err) + } + + fmt.Printf("db.toml downloaded to %s\n", localPath) + return nil +} diff --git a/b2-manager/go.mod b/b2-manager/go.mod index a7602e2367..66c0029483 100644 --- a/b2-manager/go.mod +++ b/b2-manager/go.mod @@ -3,16 +3,28 @@ module b2m go 1.24.4 require ( - github.com/BurntSushi/toml v1.6.0 github.com/jedib0t/go-pretty/v6 v6.7.8 github.com/jroimartin/gocui v0.5.0 + github.com/knadh/koanf/parsers/toml/v2 v2.2.0 + github.com/knadh/koanf/providers/file v1.2.1 + github.com/knadh/koanf/v2 v2.3.2 + github.com/urfave/cli/v2 v2.27.7 ) require ( + github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/knadh/koanf/maps v0.1.2 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/nsf/termbox-go v1.1.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/rivo/uniseg v0.4.7 // indirect - golang.org/x/sys v0.30.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect + golang.org/x/sys v0.32.0 // indirect golang.org/x/term v0.29.0 // indirect golang.org/x/text v0.22.0 // indirect ) diff --git a/b2-manager/go.sum b/b2-manager/go.sum index 2d4e7d89f8..80184bcee3 100644 --- a/b2-manager/go.sum +++ b/b2-manager/go.sum @@ -1,25 +1,49 @@ -github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= -github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= +github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/jedib0t/go-pretty/v6 v6.7.8 h1:BVYrDy5DPBA3Qn9ICT+PokP9cvCv1KaHv2i+Hc8sr5o= github.com/jedib0t/go-pretty/v6 v6.7.8/go.mod h1:YwC5CE4fJ1HFUDeivSV1r//AmANFHyqczZk+U6BDALU= github.com/jroimartin/gocui v0.5.0 h1:DCZc97zY9dMnHXJSJLLmx9VqiEnAj0yh0eTNpuEtG/4= github.com/jroimartin/gocui v0.5.0/go.mod h1:l7Hz8DoYoL6NoYnlnaX6XCNR62G7J5FfSW5jEogzaxE= +github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo= +github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/parsers/toml/v2 v2.2.0 h1:2nV7tHYJ5OZy2BynQ4mOJ6k5bDqbbCzRERLUKBytz3A= +github.com/knadh/koanf/parsers/toml/v2 v2.2.0/go.mod h1:JpjTeK1Ge1hVX0wbof5DMCuDBriR8bWgeQP98eeOZpI= +github.com/knadh/koanf/providers/file v1.2.1 h1:bEWbtQwYrA+W2DtdBrQWyXqJaJSG3KrP3AESOJYp9wM= +github.com/knadh/koanf/providers/file v1.2.1/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA= +github.com/knadh/koanf/v2 v2.3.2 h1:Ee6tuzQYFwcZXQpc2MiVeC6qHMandf5SMUJJNoFp/c4= +github.com/knadh/koanf/v2 v2.3.2/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/nsf/termbox-go v1.1.1 h1:nksUPLCb73Q++DwbYUBEglYBRPZyoXJdrj5L+TkjyZY= github.com/nsf/termbox-go v1.1.1/go.mod h1:T0cTdVuOwf7pHQNtfhnEbzHbcNyCEcVU4YPpouCbVxo= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU= +github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4= +github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= +github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= diff --git a/b2-manager/model/types.go b/b2-manager/model/types.go index a12d4a5014..2191568a9b 100644 --- a/b2-manager/model/types.go +++ b/b2-manager/model/types.go @@ -172,6 +172,11 @@ type Config struct { LocalDBDir string MigrationsDir string + ChangesetScriptsDir string + ChangesetLogsDir string + ChangesetDBsDir string + FrontendTomlPath string + // Environment DiscordWebhookURL string diff --git a/b2-manager/templates/changeset_template.py b/b2-manager/templates/changeset_template.py new file mode 100644 index 0000000000..d08d2dbfc4 --- /dev/null +++ b/b2-manager/templates/changeset_template.py @@ -0,0 +1,50 @@ +# Template Version: v1 +# script_name : {{.Timestamp}}_{{.Phrase}} +# phrase : {{.Phrase}} + +## Predifned Imports and Functions +import sqlite3 +import urllib.parse +import os +import time + +DB_NAME = "ipm-db" + +## Import Common Functions +from changeset import db_status, db_download, db_upload # Still many more should be added. + +def inserted_queries(db_name): + """ + This function should insert the new data in the db. + These will be in ipm-db-v1.sql file. + """ + # execute queries from `changeset/dbs/{{.Timestamp}}_{{.Phrase}}/ipm-db-v1.sql` file to `changeset/dbs/{{.Timestamp}}_{{.Phrase}}/ipm-db-v2.db` file. + + return None + +def main(): + # Check status of db. + status = db_status(DB_NAME) + # If status is outdated_db, then download db from b2. + if status == "outdated_db": + db_download(DB_NAME) + cp_queries(DB_NAME) + inserted_queries(DB_NAME) + rename_db(DB_NAME) + stop_server() + copy_db(DB_NAME) + update_db(DB_NAME) + start_server() + upload_db(DB_NAME) + # If status is ready_to_upload, then upload db to b2. + if status == "ready_to_upload": + stop_server() + copy_db(DB_NAME) + start_server() + db_upload(DB_NAME) + # If status is up_to_date, then do nothing. + elif status == "up_to_date": + pass + +if __name__ == "__main__": + main() diff --git a/b2-manager/ui/cli.go b/b2-manager/ui/cli.go index dacb5ccb62..bf0455bff4 100644 --- a/b2-manager/ui/cli.go +++ b/b2-manager/ui/cli.go @@ -5,166 +5,216 @@ import ( "fmt" "os" + "github.com/urfave/cli/v2" + "b2m/config" "b2m/core" "b2m/model" ) -// HandleCLI processes command line arguments. +// HandleCLI processes command line arguments using urfave/cli. // If a command is handled, it may exit the program. func HandleCLI() { - if len(os.Args) > 1 { - command := os.Args[1] - - switch command { - case "--help": - printUsage() - os.Exit(0) - - case "--version": - fmt.Printf("b2m version %s\n", model.AppConfig.ToolVersion) - os.Exit(0) - - case "--generate-hash": - // Common Dependencies check - if err := config.CheckDependencies(); err != nil { - fmt.Printf("Error: %v\n", err) - os.Exit(1) - } - - // Warning and Confirmation - fmt.Println("\nWARNING: This operation regenerates all metadata from local files.") - fmt.Println("Ensure your local databases are synced with remote to avoid data loss.") - fmt.Println("This should ONLY be done when changing hashing algorithms or recovering from corruption.") - fmt.Print("\nAre you sure you want to proceed? (y/N): ") - - var confirmation string - fmt.Scanln(&confirmation) - if confirmation != "y" && confirmation != "Y" { - fmt.Println("Operation cancelled.") - os.Exit(0) - } - - // Clean up .b2m before generating metadata - if err := core.CleanupLocalMetadata(); err != nil { - fmt.Printf("Error: failed to cleanup metadata: %v\n", err) - core.LogError("Generate-Hash: Failed to cleanup metadata: %v", err) - os.Exit(1) - } - - // Explicitly clear hash cache - core.ClearHashCache() - - // Bootstrap system minimal - // Use background context for CLI tool mode - cliCtx := context.Background() - if err := core.BootstrapSystem(cliCtx); err != nil { - core.LogError("Startup Warning: %v", err) - } - core.HandleBatchMetadataGeneration() - config.Cleanup() - os.Exit(0) - - case "--reset": - fmt.Println("Resetting system state...") - // Clean up .b2m before starting normal execution - if err := core.CleanupLocalMetadata(); err != nil { - fmt.Printf("Error: failed to cleanup metadata: %v\n", err) - core.LogError("Reset: Failed to cleanup metadata: %v", err) - os.Exit(1) - } - - // Explicitly clear hash cache - core.ClearHashCache() - - config.Cleanup() - fmt.Println("Reset complete. Please restart the application.") - os.Exit(0) - - case "migrations": - if len(os.Args) < 3 { - fmt.Println("Usage: b2m migrations [args]") - fmt.Println("Available commands: create") - os.Exit(1) - } - - subCmd := os.Args[2] - switch subCmd { - case "create": - if len(os.Args) < 4 { - fmt.Println("Usage: b2m migrations create ") - fmt.Println("Error: missing required argument ") - os.Exit(1) - } - phrase := os.Args[3] - if phrase == "" { - fmt.Println("Error: phrase cannot be empty") - os.Exit(1) - } - // Config is already initialized by InitSystem - if err := core.CreateMigration(phrase); err != nil { - fmt.Fprintf(os.Stderr, "Error creating migration: %v\n", err) - os.Exit(1) - } - os.Exit(0) - default: - fmt.Printf("Unknown migration command: %s\n", subCmd) - fmt.Println("Usage: b2m migrations create ") - os.Exit(1) - } - - case "unlock": - if len(os.Args) < 3 { - fmt.Println("Usage: b2m unlock ") - os.Exit(1) - } - dbName := os.Args[2] - if err := core.SanitizeDBName(dbName); err != nil { - fmt.Printf("Error: Invalid database name: %v\n", err) - os.Exit(1) - } - - // Force unlock warning - fmt.Printf("WARNING: You are about to FORCE UNLOCK database '%s'.\n", dbName) - fmt.Println("This should ONLY be done if the lock is stale due to a crash or network issue.") - fmt.Println("If another user is actively writing to this database, forcing an unlock may cause DATA CORRUPTION.") - fmt.Print("Are you sure you want to proceed? (y/N): ") - - var confirmation string - fmt.Scanln(&confirmation) - if confirmation != "y" && confirmation != "Y" { - fmt.Println("Operation cancelled.") - os.Exit(0) - } + if len(os.Args) <= 1 { + return // Let the main func proceed to TUI + } - // Perform unlock (using force=true as CLI unlock implies admin override) - // Context needed - ctx := context.Background() - if err := core.UnlockDatabase(ctx, dbName, "CLI-User", true); err != nil { - fmt.Printf("Error unlocking database: %v\n", err) - os.Exit(1) - } - fmt.Println("Database unlocked successfully.") - os.Exit(0) + app := &cli.App{ + Name: "b2m", + Usage: "Backblaze B2 Database Manager", + Version: model.AppConfig.ToolVersion, + Commands: []*cli.Command{ + { + Name: "generate-hash", + Usage: "Generate new hash and create metadata in remote", + Action: func(cCtx *cli.Context) error { + if err := config.CheckDependencies(); err != nil { + return cli.Exit(fmt.Sprintf("Error: %v", err), 1) + } + + fmt.Println("\nWARNING: This operation regenerates all metadata from local files.") + fmt.Println("Ensure your local databases are synced with remote to avoid data loss.") + fmt.Println("This should ONLY be done when changing hashing algorithms or recovering from corruption.") + fmt.Print("\nAre you sure you want to proceed? (y/N): ") + + var confirmation string + fmt.Scanln(&confirmation) + if confirmation != "y" && confirmation != "Y" { + fmt.Println("Operation cancelled.") + return nil + } + + if err := core.CleanupLocalMetadata(); err != nil { + core.LogError("Generate-Hash: Failed to cleanup metadata: %v", err) + return cli.Exit(fmt.Sprintf("Error: failed to cleanup metadata: %v", err), 1) + } + core.ClearHashCache() + + cliCtx := context.Background() + if err := core.BootstrapSystem(cliCtx); err != nil { + core.LogError("Startup Warning: %v", err) + } + core.HandleBatchMetadataGeneration() + config.Cleanup() + return nil + }, + }, + { + Name: "reset", + Usage: "Remove local metadata caches and start fresh UI session", + Action: func(cCtx *cli.Context) error { + fmt.Println("Resetting system state...") + if err := core.CleanupLocalMetadata(); err != nil { + core.LogError("Reset: Failed to cleanup metadata: %v", err) + return cli.Exit(fmt.Sprintf("Error: failed to cleanup metadata: %v", err), 1) + } + core.ClearHashCache() + config.Cleanup() + fmt.Println("Reset complete. Please restart the application.") + return nil + }, + }, + { + Name: "migrations", + Usage: "Manage migration scripts", + Subcommands: []*cli.Command{ + { + Name: "create", + Usage: "Create a new migration script", + Action: func(cCtx *cli.Context) error { + if cCtx.NArg() == 0 { + return cli.Exit("Error: missing required argument ", 1) + } + phrase := cCtx.Args().First() + if err := core.CreateMigration(phrase); err != nil { + return cli.Exit(fmt.Sprintf("Error creating migration: %v", err), 1) + } + return nil + }, + }, + }, + }, + { + Name: "unlock", + Usage: "Force unlock a database", + Action: func(cCtx *cli.Context) error { + if cCtx.NArg() == 0 { + return cli.Exit("Usage: b2m unlock ", 1) + } + dbName := cCtx.Args().First() + if err := core.SanitizeDBName(dbName); err != nil { + return cli.Exit(fmt.Sprintf("Error: Invalid database name: %v", err), 1) + } + + fmt.Printf("WARNING: You are about to FORCE UNLOCK database '%s'.\n", dbName) + fmt.Println("This should ONLY be done if the lock is stale due to a crash or network issue.") + fmt.Println("If another user is actively writing to this database, forcing an unlock may cause DATA CORRUPTION.") + fmt.Print("Are you sure you want to proceed? (y/N): ") + + var confirmation string + fmt.Scanln(&confirmation) + if confirmation != "y" && confirmation != "Y" { + fmt.Println("Operation cancelled.") + return nil + } + + ctx := context.Background() + if err := core.UnlockDatabase(ctx, dbName, "CLI-User", true); err != nil { + return cli.Exit(fmt.Sprintf("Error unlocking database: %v", err), 1) + } + fmt.Println("Database unlocked successfully.") + return nil + }, + }, + { + Name: "create-changeset", + Usage: "Create a new changeset python script", + Action: func(cCtx *cli.Context) error { + if cCtx.NArg() == 0 { + return cli.Exit("Usage: b2m create-changeset ", 1) + } + phrase := cCtx.Args().First() + if err := core.CreateChangeset(phrase); err != nil { + return cli.Exit(fmt.Sprintf("Error creating changeset: %v", err), 1) + } + return nil + }, + }, + { + Name: "execute-changeset", + Usage: "Execute a given changeset script", + Action: func(cCtx *cli.Context) error { + if cCtx.NArg() == 0 { + return cli.Exit("Usage: b2m execute-changeset ", 1) + } + scriptName := cCtx.Args().First() + if err := core.ExecuteChangeset(scriptName); err != nil { + return cli.Exit(fmt.Sprintf("Error executing changeset: %v", err), 1) + } + return nil + }, + }, + { + Name: "status", + Usage: "Check status of a database (for scripting)", + Action: func(cCtx *cli.Context) error { + if cCtx.NArg() == 0 { + return cli.Exit("Usage: b2m status ", 1) + } + dbName := cCtx.Args().First() + if err := core.RunCLIStatus(dbName); err != nil { + return cli.Exit("", 1) // don't log generic error to Python script output + } + return nil + }, + }, + { + Name: "upload", + Usage: "Upload database directly (for scripting)", + Action: func(cCtx *cli.Context) error { + if cCtx.NArg() == 0 { + return cli.Exit("Usage: b2m upload ", 1) + } + dbName := cCtx.Args().First() + if err := core.RunCLIUpload(dbName); err != nil { + return cli.Exit(fmt.Sprintf("Error uploading database: %v", err), 1) + } + return nil + }, + }, + { + Name: "download", + Usage: "Download database directly (for scripting)", + Action: func(cCtx *cli.Context) error { + if cCtx.NArg() == 0 { + return cli.Exit("Usage: b2m download ", 1) + } + dbName := cCtx.Args().First() + if err := core.RunCLIDownload(dbName); err != nil { + return cli.Exit(fmt.Sprintf("Error downloading database: %v", err), 1) + } + return nil + }, + }, + { + Name: "fetch-db-toml", + Usage: "Fetch db.toml from B2 (for scripting)", + Action: func(cCtx *cli.Context) error { + if err := core.RunCLIFetchDBToml(); err != nil { + return cli.Exit(fmt.Sprintf("Error fetching db.toml: %v", err), 1) + } + return nil + }, + }, + }, + } - default: - fmt.Printf("Unknown command: %s\n", command) - printUsage() - os.Exit(1) - } + if err := app.Run(os.Args); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) } -} -func printUsage() { - fmt.Println("b2-manager - Backblaze B2 Database Manager") - fmt.Println("\nUsage:") - fmt.Println(" b2-manager [command]") - fmt.Println("\nCommands:") - fmt.Println(" --help Show this help message") - fmt.Println(" --version Show version information") - fmt.Println(" --generate-hash Generate new hash and create metadata in remote") - fmt.Println(" --reset Remove local metadata caches and start fresh UI session") - fmt.Println(" migrations create Create a new migration script") - fmt.Println(" unlock Force unlock a database") - fmt.Println("\nIf no command is provided, the TUI application starts normally.") + // Because app.Run succeeded (or printed help/version), and it's a CLI command, + // we want to exit so we don't drop down into the TUI. + os.Exit(0) } diff --git a/frontend/changeset/changeset.py b/frontend/changeset/changeset.py index 99851e118c..260ad2f690 100644 --- a/frontend/changeset/changeset.py +++ b/frontend/changeset/changeset.py @@ -5,7 +5,7 @@ def db_status(db_name): print(f"Executing: {db_name}") try: - result = subprocess.run(["../b2m", "--status", db_name], capture_output=True, text=True, check=True) + result = subprocess.run(["../b2m", "status", db_name], capture_output=True, text=True, check=True) # Assuming b2m outputs the status to stdout. We might want to return it. # But based on the template, we'll return the stdout stripped. # Wait, the spec doesn't explicitly return it in the provided code snippet, but the template code expects return: @@ -23,7 +23,7 @@ def db_status(db_name): def db_upload(db_name): print(f"Executing: {db_name}") try: - subprocess.run(["../b2m", "--upload", db_name], check=True) + subprocess.run(["../b2m", "upload", db_name], check=True) except subprocess.CalledProcessError as e: print(f"Error uploading {db_name}: {e}") @@ -33,6 +33,6 @@ def db_download(db_name): """ print(f"Executing: {db_name}") try: - subprocess.run(["../b2m", "--download", db_name], check=True) + subprocess.run(["../b2m", "download", db_name], check=True) except subprocess.CalledProcessError as e: print(f"Error downloading {db_name}: {e}") diff --git a/frontend/changeset/changeset_scripts/20260214162434583026694_descroption.py b/frontend/changeset/changeset_scripts/20260214162434583026694_descroption.py deleted file mode 100644 index 070d8424a1..0000000000 --- a/frontend/changeset/changeset_scripts/20260214162434583026694_descroption.py +++ /dev/null @@ -1,62 +0,0 @@ -# Template Version: v1 -# : _ -# - -## Predifned Imports and Functions -import sqlite3 -import urllib.parse -import os -import time - -DB_NAME = "ipm-db-v5.db" -## Import Common Functions -from changeset import db_status, db_download, db_upload # Still many more should be added. - -def db_migration(db_name): - ## Donwload b2 db to changeset db location which will be predefined. - status = db_download(db_name) - if status == "downloaded": - ## Now we have the db in our changeset db location. - ## Now we need to migrate new data from the server db to the new db. - return True - else: - return False - - -def copy_db(db_name): - try: - subprocess.run(["cp", db_name, "changeset/dbs/"], check=True) - return True - except subprocess.CalledProcessError as e: - print(f"Error copying {db_name}: {e}") - return False - -def handle_db_status(db_name): - status = db_status(db_name) - if status == "outdated_db": - if copy_db(db_name): - if db_migration(db_name): - if db_upload(db_name): - copy_db(db_name) - print("DB Migration successful") - else: - print("Error: db_upload failed") - else: - print("Error: db_migration failed") - else: - print("Error: copy_db failed") - elif status == "ready_to_upload": - db_upload(db_name) - elif status == "up_to_date": - pass - else: - print(f"Error: Unknown status {status}") - - -def main(db_name): - handle_db_status(db_name) - - -if __name__ == "__main__": - - main(DB_NAME) \ No newline at end of file diff --git a/frontend/changeset/scripts/1772031633645610550_sample-phrase.py b/frontend/changeset/scripts/1772031633645610550_sample-phrase.py new file mode 100644 index 0000000000..f2ae688e44 --- /dev/null +++ b/frontend/changeset/scripts/1772031633645610550_sample-phrase.py @@ -0,0 +1,50 @@ +# Template Version: v1 +# script_name : 1772031633645610550_sample-phrase +# phrase : sample-phrase + +## Predifned Imports and Functions +import sqlite3 +import urllib.parse +import os +import time + +DB_NAME = "ipm-db" + +## Import Common Functions +from changeset import db_status, db_download, db_upload # Still many more should be added. + +def inserted_queries(db_name): + """ + This function should insert the new data in the db. + These will be in ipm-db-v1.sql file. + """ + # execute queries from `changeset/dbs/1772031633645610550_sample-phrase/ipm-db-v1.sql` file to `changeset/dbs/1772031633645610550_sample-phrase/ipm-db-v2.db` file. + + return None + +def main(): + # Check status of db. + status = db_status(DB_NAME) + # If status is outdated_db, then download db from b2. + if status == "outdated_db": + db_download(DB_NAME) + cp_queries(DB_NAME) + inserted_queries(DB_NAME) + rename_db(DB_NAME) + stop_server() + copy_db(DB_NAME) + update_db(DB_NAME) + start_server() + upload_db(DB_NAME) + # If status is ready_to_upload, then upload db to b2. + if status == "ready_to_upload": + stop_server() + copy_db(DB_NAME) + start_server() + db_upload(DB_NAME) + # If status is up_to_date, then do nothing. + elif status == "up_to_date": + pass + +if __name__ == "__main__": + main() diff --git a/frontend/md/changset-implementation.md b/frontend/md/changset-implementation.md index 312e1ef01a..1fe87e9b79 100644 --- a/frontend/md/changset-implementation.md +++ b/frontend/md/changset-implementation.md @@ -473,3 +473,44 @@ frontend/ 4. `changeset.py`: Common functions used in changeset script. 1. Such as `b2m upload `, `b2m download `, `b2m status `, etc. +### Phase 1.2: Implementing b2m cli + +Currently b2m interative cli is done here + +@b2-manager/ + +refer Docs @b2-manager/docs + +for any integration + +All cli functions should setup and called via @b2-manager/ui/cli.go + +New deps like template and others should be added in @b2-manager/go.mod +and make sure template stayes under this @b2-manager/templates folder + +Still any doubts as in me before implementing this. + +**Integration with b2m cli** +1. `b2m` cli will be used to create, execute, and manage changeset script. +2. `b2m` cli will be placed in `frontend` directory. +3. `b2m` cli will be having following commands. There are 2 types of commands. + 1. User specific commands: These commands are used regurly by us. + 1. `b2m create-changeset `: Create a changeset script. + 1. This will create a changeset script in `changeset/scripts` directory. (Added Detailed Description Above) + 2. `b2m execute-changeset `: Execute a changeset script. + 1. This will execute the changeset script. It can also be done by `python ` just adding for making it easy to execute. + 2. Db specific commands: These commands are used and predifned in `changeset.py`. (Added Detailed Descript Above) + 1. `b2m status `: Check status of db. + 1. This will check the status of db. + 2. It will check the status of db. + 2. `b2m upload `: Upload db to b2. + 1. This will upload the db to b2 form `changeset/dbs//_b2.db`. (Added Detailed Description Above) + 3. `b2m download `: Download db from b2. + 1. This will download the db from b2 to `changeset/dbs//_b2.db`. (Added Detailed Description Above) + 4. `b2m fetch-db-toml`: Fetch db.toml from b2. + 1. This will fetch db.toml from b2 to `db/all_dbs/db.toml`. + + +Any New function implemented should be having test cases and should be tested properly merging also make sure take clarification from me to do anything before creating tests + + From e2d5c58fe30f09aeb43f23bc30119a08456e4234 Mon Sep 17 00:00:00 2001 From: Ganesh Kumar Date: Thu, 26 Feb 2026 19:22:37 +0530 Subject: [PATCH 4/5] Refactor Changeset Configuration and CLI Structure --- b2-manager/config/config.go | 8 ++-- b2-manager/core/changeset.go | 2 +- b2-manager/model/types.go | 2 +- b2-manager/ui/cli.go | 65 +++++++++++--------------- frontend/md/changset-implementation.md | 27 +++++++++-- 5 files changed, 57 insertions(+), 47 deletions(-) diff --git a/b2-manager/config/config.go b/b2-manager/config/config.go index 1d9712086f..fdc12b0b3a 100644 --- a/b2-manager/config/config.go +++ b/b2-manager/config/config.go @@ -50,9 +50,11 @@ func InitializeConfig() error { model.AppConfig.MigrationsDir = filepath.Join(model.AppConfig.ProjectRoot, "b2m-migration") // Changeset Paths - model.AppConfig.ChangesetScriptsDir = filepath.Join(model.AppConfig.ProjectRoot, "changeset", "scripts") - model.AppConfig.ChangesetLogsDir = filepath.Join(model.AppConfig.ProjectRoot, "changeset", "logs") - model.AppConfig.ChangesetDBsDir = filepath.Join(model.AppConfig.ProjectRoot, "changeset", "dbs") + model.AppConfig.ChangesetDir = filepath.Join(model.AppConfig.ProjectRoot, "changeset") + model.AppConfig.ChangesetScriptsDir = filepath.Join(model.AppConfig.ChangesetDir, "scripts") + model.AppConfig.ChangesetLogsDir = filepath.Join(model.AppConfig.ChangesetDir, "logs") + model.AppConfig.ChangesetDBsDir = filepath.Join(model.AppConfig.ChangesetDir, "dbs") + model.AppConfig.FrontendTomlPath = filepath.Join(model.AppConfig.ProjectRoot, "db", "all_dbs", "db.toml") return nil diff --git a/b2-manager/core/changeset.go b/b2-manager/core/changeset.go index 38aeb370c1..c293e8de96 100644 --- a/b2-manager/core/changeset.go +++ b/b2-manager/core/changeset.go @@ -141,8 +141,8 @@ func RunCLIDownload(dbName string) error { // RunCLIFetchDBToml downloads db.toml from backblaze func RunCLIFetchDBToml() error { ctx := context.Background() - remotePath := model.AppConfig.RootBucket + "db.toml" // Assuming it sits at root or specify correctly localPath := model.AppConfig.FrontendTomlPath + remotePath := model.AppConfig.RootBucket + filepath.Base(localPath) if err := os.MkdirAll(filepath.Dir(localPath), 0755); err != nil { return fmt.Errorf("failed to create directory for db.toml: %w", err) diff --git a/b2-manager/model/types.go b/b2-manager/model/types.go index 2191568a9b..f17268f058 100644 --- a/b2-manager/model/types.go +++ b/b2-manager/model/types.go @@ -171,7 +171,7 @@ type Config struct { LocalB2MDir string LocalDBDir string MigrationsDir string - + ChangesetDir string ChangesetScriptsDir string ChangesetLogsDir string ChangesetDBsDir string diff --git a/b2-manager/ui/cli.go b/b2-manager/ui/cli.go index bf0455bff4..0dde931cbb 100644 --- a/b2-manager/ui/cli.go +++ b/b2-manager/ui/cli.go @@ -25,8 +25,9 @@ func HandleCLI() { Version: model.AppConfig.ToolVersion, Commands: []*cli.Command{ { - Name: "generate-hash", - Usage: "Generate new hash and create metadata in remote", + Name: "generate-hash", + Category: "User Commands", + Usage: "Generate new hash and create metadata in remote", Action: func(cCtx *cli.Context) error { if err := config.CheckDependencies(); err != nil { return cli.Exit(fmt.Sprintf("Error: %v", err), 1) @@ -60,8 +61,9 @@ func HandleCLI() { }, }, { - Name: "reset", - Usage: "Remove local metadata caches and start fresh UI session", + Name: "reset", + Category: "User Commands", + Usage: "Remove local metadata caches and start fresh UI session", Action: func(cCtx *cli.Context) error { fmt.Println("Resetting system state...") if err := core.CleanupLocalMetadata(); err != nil { @@ -75,28 +77,9 @@ func HandleCLI() { }, }, { - Name: "migrations", - Usage: "Manage migration scripts", - Subcommands: []*cli.Command{ - { - Name: "create", - Usage: "Create a new migration script", - Action: func(cCtx *cli.Context) error { - if cCtx.NArg() == 0 { - return cli.Exit("Error: missing required argument ", 1) - } - phrase := cCtx.Args().First() - if err := core.CreateMigration(phrase); err != nil { - return cli.Exit(fmt.Sprintf("Error creating migration: %v", err), 1) - } - return nil - }, - }, - }, - }, - { - Name: "unlock", - Usage: "Force unlock a database", + Name: "unlock", + Category: "User Commands", + Usage: "Force unlock a database", Action: func(cCtx *cli.Context) error { if cCtx.NArg() == 0 { return cli.Exit("Usage: b2m unlock ", 1) @@ -127,8 +110,9 @@ func HandleCLI() { }, }, { - Name: "create-changeset", - Usage: "Create a new changeset python script", + Name: "create-changeset", + Category: "User Commands", + Usage: "Create a new changeset python script", Action: func(cCtx *cli.Context) error { if cCtx.NArg() == 0 { return cli.Exit("Usage: b2m create-changeset ", 1) @@ -141,8 +125,9 @@ func HandleCLI() { }, }, { - Name: "execute-changeset", - Usage: "Execute a given changeset script", + Name: "execute-changeset", + Category: "User Commands", + Usage: "Execute a given changeset script", Action: func(cCtx *cli.Context) error { if cCtx.NArg() == 0 { return cli.Exit("Usage: b2m execute-changeset ", 1) @@ -155,8 +140,9 @@ func HandleCLI() { }, }, { - Name: "status", - Usage: "Check status of a database (for scripting)", + Name: "status", + Category: "Changeset Commands", + Usage: "Check status of a database (for scripting)", Action: func(cCtx *cli.Context) error { if cCtx.NArg() == 0 { return cli.Exit("Usage: b2m status ", 1) @@ -169,8 +155,9 @@ func HandleCLI() { }, }, { - Name: "upload", - Usage: "Upload database directly (for scripting)", + Name: "upload", + Category: "Changeset Commands", + Usage: "Upload database directly (for scripting)", Action: func(cCtx *cli.Context) error { if cCtx.NArg() == 0 { return cli.Exit("Usage: b2m upload ", 1) @@ -183,8 +170,9 @@ func HandleCLI() { }, }, { - Name: "download", - Usage: "Download database directly (for scripting)", + Name: "download", + Category: "Changeset Commands", + Usage: "Download database directly (for scripting)", Action: func(cCtx *cli.Context) error { if cCtx.NArg() == 0 { return cli.Exit("Usage: b2m download ", 1) @@ -197,8 +185,9 @@ func HandleCLI() { }, }, { - Name: "fetch-db-toml", - Usage: "Fetch db.toml from B2 (for scripting)", + Name: "fetch-db-toml", + Category: "Changeset Commands", + Usage: "Fetch db.toml from B2 (for scripting)", Action: func(cCtx *cli.Context) error { if err := core.RunCLIFetchDBToml(); err != nil { return cli.Exit(fmt.Sprintf("Error fetching db.toml: %v", err), 1) diff --git a/frontend/md/changset-implementation.md b/frontend/md/changset-implementation.md index 1fe87e9b79..4d5fbd8d26 100644 --- a/frontend/md/changset-implementation.md +++ b/frontend/md/changset-implementation.md @@ -8,10 +8,10 @@ Any Update in the db should be bumped to new version as default which will be ha This Consist of 5 main parts. 1. `changeset` directory -2. `b2m` cli -3. `db.toml` file -4. `changeset_script` template -5. `changeset.py` common function +2. `b2m` cli (Need to do testing) +3. `db.toml` file (Need to use in all the scripts.) +4. `changeset_script` template Iterations needed +5. `changeset.py` common function Iterations needed ### Create changeset script @@ -514,3 +514,22 @@ Still any doubts as in me before implementing this. Any New function implemented should be having test cases and should be tested properly merging also make sure take clarification from me to do anything before creating tests + + +> Note: Not tested yet + +### Phase 1.3: Implementing db.toml + +`db.toml` file will present in `db/all_dbs/db.toml`. + +This file will be used to define the db versions and paths. + +```toml +[db] +ipmdb = "ipm-db-v2.db" +emojidb = "emoji-db-v2.db" +path = "/frontend/db/all_dbs/" +``` +we use frontend/internal/config/config.go for defining all the other configs. Now i want to define db.toml in frontend/internal/config/config.go and and use the same config data for all db related path location. + +I have added in context where all the templ server has deps of db \ No newline at end of file From 70b9710195b8d981f483d363158f0a9dc17a603f Mon Sep 17 00:00:00 2001 From: Ganesh Kumar Date: Thu, 26 Feb 2026 21:48:42 +0530 Subject: [PATCH 5/5] Standardize SQLite Database Path Management --- frontend/db.toml | 11 ++ frontend/internal/config/config.go | 121 ++++++++++++++++-- frontend/internal/db/banner/utils.go | 27 ++-- frontend/internal/db/cheatsheets/queries.go | 19 ++- frontend/internal/db/cheatsheets/utils.go | 15 ++- frontend/internal/db/emojis/queries.go | 20 ++- frontend/internal/db/emojis/utils.go | 16 ++- .../internal/db/installerpedia/queries.go | 110 +++++++++------- frontend/internal/db/man_pages/queries.go | 21 ++- frontend/internal/db/man_pages/utils.go | 16 ++- frontend/internal/db/mcp/mcp_db.go | 19 ++- frontend/internal/db/mcp/utils.go | 15 ++- frontend/internal/db/png_icons/png_queries.go | 21 ++- frontend/internal/db/png_icons/utils.go | 16 ++- frontend/internal/db/svg_icons/queries.go | 21 ++- frontend/internal/db/svg_icons/utils.go | 15 ++- frontend/internal/db/tldr/queries.go | 21 ++- frontend/md/changset-implementation.md | 4 +- frontend/scripts/analyze_svg_icon_404.go | 14 +- frontend/scripts/find_correct_manpage_urls.go | 6 +- .../scripts/man-pages/create_html_content.go | 12 +- .../scripts/man-pages/drop_content_column.go | 6 +- 22 files changed, 416 insertions(+), 130 deletions(-) create mode 100644 frontend/db.toml diff --git a/frontend/db.toml b/frontend/db.toml new file mode 100644 index 0000000000..41aefedadd --- /dev/null +++ b/frontend/db.toml @@ -0,0 +1,11 @@ +[db] +path = "db/all_dbs/" +bannerdb = "banner-db.db" +cheatsheetsdb = "cheatsheets-db-v5.db" +emojidb = "emoji-db-v5.db" +ipmdb = "ipm-db-v6.db" +manpagesdb = "man-pages-db-v5.db" +mcpdb = "mcp-db-v6.db" +pngiconsdb = "png-icons-db-v5.db" +svgiconsdb = "svg-icons-db-v5.db" +tldrdb = "tldr-db-v5.db" diff --git a/frontend/internal/config/config.go b/frontend/internal/config/config.go index 2fb90074a5..921dd16027 100644 --- a/frontend/internal/config/config.go +++ b/frontend/internal/config/config.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "github.com/pelletier/go-toml/v2" ) @@ -21,10 +22,24 @@ type Config struct { B2AccountID string `toml:"b2_account_id"` B2ApplicationKey string `toml:"b2_application_key"` MeiliWriteKey string `toml:"meili_write_key"` - GeminiKeys string `toml:"gemini_keys"` + GeminiKeys string `toml:"gemini_keys"` EnableAds bool `toml:"enable_ads"` Ads map[string][]string `toml:"ads"` - FdtPgDB FdtPgDBConfig `toml:"fdt_pg_db"` + FdtPgDB FdtPgDBConfig `toml:"fdt_pg_db"` +} + +// DBTomlConfig holds the dynamic database paths and filenames from db.toml +type DBTomlConfig struct { + Path string `toml:"path"` + BannerDB string `toml:"bannerdb"` + CheatsheetsDB string `toml:"cheatsheetsdb"` + EmojiDB string `toml:"emojidb"` + IpmDB string `toml:"ipmdb"` + ManPagesDB string `toml:"manpagesdb"` + McpDB string `toml:"mcpdb"` + PngIconsDB string `toml:"pngiconsdb"` + SvgIconsDB string `toml:"svgiconsdb"` + TldrDB string `toml:"tldrdb"` } // FdtPgDBConfig holds PostgreSQL database configuration for Free DevTools @@ -38,6 +53,12 @@ type FdtPgDBConfig struct { var appConfig *Config +var ( + DBConfig *DBTomlConfig + dbTomlOnce sync.Once + dbTomlErr error +) + // loadNodeEnvFromDotEnv reads NODE_ENV from .env file // Returns the value if found, otherwise returns empty string func loadNodeEnvFromDotEnv() string { @@ -82,6 +103,10 @@ func LoadConfig() (*Config, error) { return appConfig, nil } + if err := LoadDBToml(); err != nil { + log.Printf("Warning: Failed to load db.toml: %v", err) + } + // First try to read from .env file env := loadNodeEnvFromDotEnv() // If not found in .env, try environment variable @@ -105,17 +130,17 @@ func LoadConfig() (*Config, error) { NodeEnv: "dev", B2AccountID: "", B2ApplicationKey: "", - MeiliWriteKey: "", - GeminiKeys: "", + MeiliWriteKey: "", + GeminiKeys: "", EnableAds: false, Ads: make(map[string][]string), - FdtPgDB: FdtPgDBConfig{ - Host: "", - Port: "5432", - User: "freedevtools_user", - Password: "", - DBName: "freedevtools", - }, + FdtPgDB: FdtPgDBConfig{ + Host: "", + Port: "5432", + User: "freedevtools_user", + Password: "", + DBName: "freedevtools", + }, } return appConfig, nil } @@ -163,8 +188,8 @@ func GetConfig() *Config { NodeEnv: "dev", B2AccountID: "", B2ApplicationKey: "", - MeiliWriteKey: "", - GeminiKeys: "", + MeiliWriteKey: "", + GeminiKeys: "", EnableAds: false, } } else { @@ -364,3 +389,73 @@ func LoadConfigFromPath(path string) (*Config, error) { return &config, nil } + +// LoadDBToml loads database versions and paths from db.toml in a thread-safe manner +func LoadDBToml() error { + dbTomlOnce.Do(func() { + dbTomlErr = loadDBTomlInternal() + }) + return dbTomlErr +} + +func loadDBTomlInternal() error { + var tomlPath string + fallbackPaths := []string{ + "db.toml", + "../db.toml", + "../../db.toml", + } + + for _, p := range fallbackPaths { + if _, err := os.Stat(p); !os.IsNotExist(err) { + tomlPath = p + break + } + } + + if tomlPath == "" { + return fmt.Errorf("could not find db.toml file") + } + + data, err := os.ReadFile(tomlPath) + if err != nil { + return fmt.Errorf("failed to read db.toml from %s: %w", tomlPath, err) + } + + // Wrap struct to match the [db] section in TOML + var wrapper struct { + DB DBTomlConfig `toml:"db"` + } + + if err := toml.Unmarshal(data, &wrapper); err != nil { + return fmt.Errorf("failed to parse db.toml: %w", err) + } + + basePathStr := wrapper.DB.Path + if basePathStr == "" { + basePathStr = "db/all_dbs/" + } + + // Prepend paths safely + safeJoin := func(filename string) string { + if filename == "" { + return "" + } + return filepath.Join(basePathStr, filename) + } + + DBConfig = &DBTomlConfig{ + Path: basePathStr, + BannerDB: safeJoin(wrapper.DB.BannerDB), + CheatsheetsDB: safeJoin(wrapper.DB.CheatsheetsDB), + EmojiDB: safeJoin(wrapper.DB.EmojiDB), + IpmDB: safeJoin(wrapper.DB.IpmDB), + ManPagesDB: safeJoin(wrapper.DB.ManPagesDB), + McpDB: safeJoin(wrapper.DB.McpDB), + PngIconsDB: safeJoin(wrapper.DB.PngIconsDB), + SvgIconsDB: safeJoin(wrapper.DB.SvgIconsDB), + TldrDB: safeJoin(wrapper.DB.TldrDB), + } + + return nil +} diff --git a/frontend/internal/db/banner/utils.go b/frontend/internal/db/banner/utils.go index f8e02932ac..b2223b4c76 100644 --- a/frontend/internal/db/banner/utils.go +++ b/frontend/internal/db/banner/utils.go @@ -2,8 +2,8 @@ package banner import ( "database/sql" - "log" - "path/filepath" + "fdt-templ/internal/config" + "fmt" "sync" "time" @@ -17,12 +17,22 @@ var ( // GetDB returns a singleton database connection func GetDB() (*sql.DB, error) { - var err error + var initErr error dbOnce.Do(func() { - dbPath := filepath.Join("db", "all_dbs", "banner-db.db") - dbInstance, err = sql.Open("sqlite3", dbPath+"?mode=ro") - if err != nil { - log.Printf("Failed to open banner database: %v", err) + // Ensure config is loaded + if e := config.LoadDBToml(); e != nil { + initErr = fmt.Errorf("failed to load db.toml for Banner DB: %w", e) + return + } + dbPath := config.DBConfig.BannerDB + if dbPath == "" { + initErr = fmt.Errorf("Banner DB path is empty in db.toml") + return + } + + dbInstance, initErr = sql.Open("sqlite3", dbPath+"?mode=ro") + if initErr != nil { + initErr = fmt.Errorf("failed to open banner database: %w", initErr) return } // Set connection pool settings @@ -30,7 +40,7 @@ func GetDB() (*sql.DB, error) { dbInstance.SetMaxIdleConns(1) dbInstance.SetConnMaxLifetime(time.Hour) }) - return dbInstance, err + return dbInstance, initErr } // CloseDB closes the database connection @@ -39,4 +49,3 @@ func CloseDB() { dbInstance.Close() } } - diff --git a/frontend/internal/db/cheatsheets/queries.go b/frontend/internal/db/cheatsheets/queries.go index 5b5a4f8b7c..f5a8399194 100644 --- a/frontend/internal/db/cheatsheets/queries.go +++ b/frontend/internal/db/cheatsheets/queries.go @@ -8,6 +8,7 @@ import ( "time" db_config "fdt-templ/db/config" + "fdt-templ/internal/config" _ "github.com/mattn/go-sqlite3" ) @@ -43,12 +44,24 @@ func (db *DB) Close() error { // GetDB returns a database instance func GetDB() (*DB, error) { - dbPath := GetDBPath() + if err := config.LoadDBToml(); err != nil { + return nil, fmt.Errorf("failed to load db.toml for Cheatsheets DB: %w", err) + } + dbPath := config.DBConfig.CheatsheetsDB + if dbPath == "" { + return nil, fmt.Errorf("Cheatsheets DB path is empty in db.toml") + } + absPath, err := filepath.Abs(dbPath) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to resolve absolute path for Cheatsheets DB: %w", err) + } + + db, err := NewDB(absPath) + if err != nil { + return nil, fmt.Errorf("failed to open Cheatsheets DB: %w", err) } - return NewDB(absPath) + return db, nil } // GetTotalCheatsheets returns the total number of cheatsheets diff --git a/frontend/internal/db/cheatsheets/utils.go b/frontend/internal/db/cheatsheets/utils.go index 41cfceedf9..950b28a533 100644 --- a/frontend/internal/db/cheatsheets/utils.go +++ b/frontend/internal/db/cheatsheets/utils.go @@ -3,12 +3,21 @@ package cheatsheets import ( "crypto/sha256" "encoding/binary" - "path/filepath" + "fdt-templ/internal/config" + "fmt" ) // GetDBPath returns the path to the cheatsheets database -func GetDBPath() string { - return filepath.Join("db", "all_dbs", "cheatsheets-db-v5.db") +func GetDBPath() (string, error) { + if config.DBConfig == nil { + if err := config.LoadDBToml(); err != nil { + return "", err + } + } + if config.DBConfig.CheatsheetsDB == "" { + return "", fmt.Errorf("Cheatsheets DB path is empty in db.toml") + } + return config.DBConfig.CheatsheetsDB, nil } // HashURLToKeyInt generates a hash ID from category and slug. diff --git a/frontend/internal/db/emojis/queries.go b/frontend/internal/db/emojis/queries.go index 103da5b6c1..4c7df42f73 100644 --- a/frontend/internal/db/emojis/queries.go +++ b/frontend/internal/db/emojis/queries.go @@ -12,6 +12,7 @@ import ( "time" db_config "fdt-templ/db/config" + "fdt-templ/internal/config" _ "github.com/mattn/go-sqlite3" ) @@ -468,13 +469,24 @@ func (db *DB) GetSitemapEmojis() ([]SitemapEmoji, error) { // GetDB returns a database instance using the default path func GetDB() (*DB, error) { - dbPath := GetDBPath() - // Resolve to absolute path + if err := config.LoadDBToml(); err != nil { + return nil, fmt.Errorf("failed to load db.toml for Emoji DB: %w", err) + } + dbPath := config.DBConfig.EmojiDB + if dbPath == "" { + return nil, fmt.Errorf("Emoji DB path is empty in db.toml") + } + absPath, err := filepath.Abs(dbPath) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to resolve absolute path for Emoji DB: %w", err) + } + + db, err := NewDB(absPath) + if err != nil { + return nil, fmt.Errorf("failed to open Emoji DB: %w", err) } - return NewDB(absPath) + return db, nil } // GetEmojiCategoryUpdatedAt returns the updated_at timestamp for a category diff --git a/frontend/internal/db/emojis/utils.go b/frontend/internal/db/emojis/utils.go index b0e6241f0f..b461c5c564 100644 --- a/frontend/internal/db/emojis/utils.go +++ b/frontend/internal/db/emojis/utils.go @@ -1,11 +1,19 @@ package emojis import ( - "path/filepath" + "fdt-templ/internal/config" + "fmt" ) // GetDBPath returns the path to the emoji database -func GetDBPath() string { - // Assuming we're running from project root - return filepath.Join("db", "all_dbs", "emoji-db-v6.db") +func GetDBPath() (string, error) { + if config.DBConfig == nil { + if err := config.LoadDBToml(); err != nil { + return "", err + } + } + if config.DBConfig.EmojiDB == "" { + return "", fmt.Errorf("Emoji DB path is empty in db.toml") + } + return config.DBConfig.EmojiDB, nil } diff --git a/frontend/internal/db/installerpedia/queries.go b/frontend/internal/db/installerpedia/queries.go index 5d8706e2bf..b940b0f770 100644 --- a/frontend/internal/db/installerpedia/queries.go +++ b/frontend/internal/db/installerpedia/queries.go @@ -2,11 +2,15 @@ package installerpedia import ( "database/sql" + "fmt" "path/filepath" + "fdt-templ/internal/config" + _ "github.com/mattn/go-sqlite3" ) -var IPM_DB_FILE = "ipm-db-v6.db" + +// Configuration resolution happens in GetDB() and GetWriteDB() to avoid startup panics type DB struct { conn *sql.DB @@ -43,10 +47,20 @@ func ParseRepoListRow(row RawRepoListRow) RepoData { } // ------------------------- -// DB init +// DB init // ------------------------- func GetDB() (*DB, error) { - dbPath := filepath.Join(".", "db", "all_dbs", IPM_DB_FILE) + if config.DBConfig == nil { + if err := config.LoadDBToml(); err != nil { + return nil, fmt.Errorf("failed to load db.toml for Installerpedia DB: %w", err) + } + } + dbPathConfig := config.DBConfig.IpmDB + if dbPathConfig == "" { + return nil, fmt.Errorf("IPM DB path is empty in db.toml") + } + // IPM_DB_FILE already contains the full path including db/all_dbs/ due to safeJoin + dbPath := filepath.Join(".", dbPathConfig) // Match man_pages read-only + immutable configuration connStr := "file:" + dbPath + "?mode=ro&_immutable=1" @@ -70,20 +84,26 @@ func GetDB() (*DB, error) { return &DB{conn: conn}, nil } -// GetWriteDB opens the database with write permissions for the API. func GetWriteDB() (*DB, error) { - dbPath := filepath.Join(".", "db", "all_dbs", IPM_DB_FILE) + if err := config.LoadDBToml(); err != nil { + return nil, fmt.Errorf("failed to load db.toml for Installerpedia DB: %w", err) + } + dbPathConfig := config.DBConfig.IpmDB + if dbPathConfig == "" { + return nil, fmt.Errorf("IPM DB path is empty in db.toml") + } + dbPath := filepath.Join(".", dbPathConfig) - // Remove mode=ro and add WAL for concurrent write/read - connStr := "file:" + dbPath + "?_journal=WAL&_sync=NORMAL" - conn, err := sql.Open("sqlite3", connStr) + // Remove mode=ro and add WAL for concurrent write/read + connStr := "file:" + dbPath + "?_journal=WAL&_sync=NORMAL" + conn, err := sql.Open("sqlite3", connStr) if err != nil { return nil, err } - // Standard write-safe pool settings + // Standard write-safe pool settings conn.SetMaxOpenConns(1) // SQLite handles writes best with a single connection - conn.SetMaxIdleConns(1) // Fix: replaced the undefined method + conn.SetMaxIdleConns(1) // Fix: replaced the undefined method if err := conn.Ping(); err != nil { return nil, err } @@ -93,24 +113,24 @@ func GetWriteDB() (*DB, error) { // GetConn exported helper to let the API use the internal connection func (db *DB) GetConn() *sql.DB { - return db.conn + return db.conn } // ------------------------- // Categories // ------------------------- func (db *DB) GetRepoCategories() ([]RepoCategory, error) { - // Define the list of allowed categories - fixedCategories := []string{ - "tool", "library", "cli", "server", "framework", - "plugin", "mobile", "desktop", "sdk", "sample", - "api", "container", "graphics", - } - - // Use the IN clause to filter at the source - // Note: If this list grows huge, consider a separate table or a join, - // but for 13 strings, this is perfectly fine. - query := ` + // Define the list of allowed categories + fixedCategories := []string{ + "tool", "library", "cli", "server", "framework", + "plugin", "mobile", "desktop", "sdk", "sample", + "api", "container", "graphics", + } + + // Use the IN clause to filter at the source + // Note: If this list grows huge, consider a separate table or a join, + // but for 13 strings, this is perfectly fine. + query := ` SELECT repo_type, COUNT(*) FROM ipm_data WHERE repo_type IN (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) @@ -118,29 +138,30 @@ func (db *DB) GetRepoCategories() ([]RepoCategory, error) { ORDER BY COUNT(*) DESC ` - // Convert slice to interface slice for the Query method - args := make([]any, len(fixedCategories)) - for i, v := range fixedCategories { - args[i] = v - } - - rows, err := db.conn.Query(query, args...) - if err != nil { - return nil, err - } - defer rows.Close() - - var result []RepoCategory - for rows.Next() { - var c RepoCategory - if err := rows.Scan(&c.Name, &c.Count); err != nil { - return nil, err - } - result = append(result, c) - } - - return result, nil + // Convert slice to interface slice for the Query method + args := make([]any, len(fixedCategories)) + for i, v := range fixedCategories { + args[i] = v + } + + rows, err := db.conn.Query(query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + var result []RepoCategory + for rows.Next() { + var c RepoCategory + if err := rows.Scan(&c.Name, &c.Count); err != nil { + return nil, err + } + result = append(result, c) + } + + return result, nil } + // ------------------------- // Overview // ------------------------- @@ -269,7 +290,6 @@ func (db *DB) GetRepo(hashID int64) (*RepoData, error) { return &parsed, nil } - // ------------------------- // Row parsing // ------------------------- diff --git a/frontend/internal/db/man_pages/queries.go b/frontend/internal/db/man_pages/queries.go index d36c169bfc..77add6b4e3 100644 --- a/frontend/internal/db/man_pages/queries.go +++ b/frontend/internal/db/man_pages/queries.go @@ -8,6 +8,7 @@ import ( "time" db_config "fdt-templ/db/config" + "fdt-templ/internal/config" _ "github.com/mattn/go-sqlite3" ) @@ -607,13 +608,25 @@ func (db *DB) GetManPageUpdatedAt(hashID int64) (string, error) { return updatedAt, nil } -// GetDB returns a database instance using the default path +// GetDB returns a database instance func GetDB() (*DB, error) { - dbPath := GetDBPath() + if err := config.LoadDBToml(); err != nil { + return nil, fmt.Errorf("failed to load db.toml for Man Pages DB: %w", err) + } + dbPath := config.DBConfig.ManPagesDB + if dbPath == "" { + return nil, fmt.Errorf("Man Pages DB path is empty in db.toml") + } + // Resolve to absolute path absPath, err := filepath.Abs(dbPath) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to resolve absolute path for Man Pages DB: %w", err) + } + + db, err := NewDB(absPath) + if err != nil { + return nil, fmt.Errorf("failed to open Man Pages DB: %w", err) } - return NewDB(absPath) + return db, nil } diff --git a/frontend/internal/db/man_pages/utils.go b/frontend/internal/db/man_pages/utils.go index ac70ef270a..f5532137b0 100644 --- a/frontend/internal/db/man_pages/utils.go +++ b/frontend/internal/db/man_pages/utils.go @@ -3,13 +3,21 @@ package man_pages import ( "crypto/sha256" "encoding/binary" - "path/filepath" + "fdt-templ/internal/config" + "fmt" ) // GetDBPath returns the path to the man pages database -func GetDBPath() string { - // Assuming we're running from project root - return filepath.Join("db", "all_dbs", "man-pages-db-v6.db") +func GetDBPath() (string, error) { + if config.DBConfig == nil { + if err := config.LoadDBToml(); err != nil { + return "", err + } + } + if config.DBConfig.ManPagesDB == "" { + return "", fmt.Errorf("Man Pages DB path is empty in db.toml") + } + return config.DBConfig.ManPagesDB, nil } // HashURLToKey generates a hash ID from mainCategory, subCategory, and slug diff --git a/frontend/internal/db/mcp/mcp_db.go b/frontend/internal/db/mcp/mcp_db.go index ba1d242d6a..ed28b9b30c 100644 --- a/frontend/internal/db/mcp/mcp_db.go +++ b/frontend/internal/db/mcp/mcp_db.go @@ -6,6 +6,7 @@ import ( "path/filepath" db_config "fdt-templ/db/config" + "fdt-templ/internal/config" _ "github.com/mattn/go-sqlite3" ) @@ -41,10 +42,22 @@ func (db *DB) Close() error { // GetDB returns a database instance using the default path func GetDB() (*DB, error) { - dbPath := GetDBPath() + if err := config.LoadDBToml(); err != nil { + return nil, fmt.Errorf("failed to load db.toml for MCP DB: %w", err) + } + dbPath := config.DBConfig.McpDB + if dbPath == "" { + return nil, fmt.Errorf("MCP DB path is empty in db.toml") + } + absPath, err := filepath.Abs(dbPath) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to resolve absolute path for MCP DB: %w", err) + } + + db, err := NewDB(absPath) + if err != nil { + return nil, fmt.Errorf("failed to open MCP DB: %w", err) } - return NewDB(absPath) + return db, nil } diff --git a/frontend/internal/db/mcp/utils.go b/frontend/internal/db/mcp/utils.go index d46bb20601..b245444e3e 100644 --- a/frontend/internal/db/mcp/utils.go +++ b/frontend/internal/db/mcp/utils.go @@ -3,12 +3,21 @@ package mcp import ( "crypto/sha256" "encoding/binary" - "path/filepath" + "fdt-templ/internal/config" + "fmt" ) // GetDBPath returns the path to the mcp database -func GetDBPath() string { - return filepath.Join("db", "all_dbs", "mcp-db-v6.db") +func GetDBPath() (string, error) { + if config.DBConfig == nil { + if err := config.LoadDBToml(); err != nil { + return "", err + } + } + if config.DBConfig.McpDB == "" { + return "", fmt.Errorf("MCP DB path is empty in db.toml") + } + return config.DBConfig.McpDB, nil } // HashToID generates a hash ID from a string key diff --git a/frontend/internal/db/png_icons/png_queries.go b/frontend/internal/db/png_icons/png_queries.go index cb4fa69b67..51f8841a66 100644 --- a/frontend/internal/db/png_icons/png_queries.go +++ b/frontend/internal/db/png_icons/png_queries.go @@ -6,6 +6,7 @@ import ( "path/filepath" db_config "fdt-templ/db/config" + "fdt-templ/internal/config" _ "github.com/mattn/go-sqlite3" ) @@ -472,13 +473,25 @@ func (db *DB) GetIconUpdatedAt(sourceFolder string, iconName string) (string, er return updatedAt, nil } -// GetDB returns a database instance using the default path +// GetDB returns a database instance func GetDB() (*DB, error) { - dbPath := GetDBPath() + if err := config.LoadDBToml(); err != nil { + return nil, fmt.Errorf("failed to load db.toml for PNG Icons DB: %w", err) + } + dbPath := config.DBConfig.PngIconsDB + if dbPath == "" { + return nil, fmt.Errorf("PNG Icons DB path is empty in db.toml") + } + // Resolve to absolute path absPath, err := filepath.Abs(dbPath) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to resolve absolute path for PNG Icons DB: %w", err) + } + + db, err := NewDB(absPath) + if err != nil { + return nil, fmt.Errorf("failed to open PNG Icons DB: %w", err) } - return NewDB(absPath) + return db, nil } diff --git a/frontend/internal/db/png_icons/utils.go b/frontend/internal/db/png_icons/utils.go index 54e99eea37..297f3f22b2 100644 --- a/frontend/internal/db/png_icons/utils.go +++ b/frontend/internal/db/png_icons/utils.go @@ -3,8 +3,9 @@ package png_icons import ( "crypto/sha256" "encoding/binary" + "fdt-templ/internal/config" + "fmt" "net/url" - "path/filepath" "strings" ) @@ -40,7 +41,14 @@ func HashClusterToKey(cluster string) int64 { } // GetDBPath returns the path to the PNG icons database -func GetDBPath() string { - // Assuming we're running from project root - return filepath.Join("db", "all_dbs", "png-icons-db-v6.db") +func GetDBPath() (string, error) { + if config.DBConfig == nil { + if err := config.LoadDBToml(); err != nil { + return "", err + } + } + if config.DBConfig.PngIconsDB == "" { + return "", fmt.Errorf("PNG Icons DB path is empty in db.toml") + } + return config.DBConfig.PngIconsDB, nil } diff --git a/frontend/internal/db/svg_icons/queries.go b/frontend/internal/db/svg_icons/queries.go index 40d7d7fa9c..e7b012ce99 100644 --- a/frontend/internal/db/svg_icons/queries.go +++ b/frontend/internal/db/svg_icons/queries.go @@ -6,6 +6,7 @@ import ( "path/filepath" db_config "fdt-templ/db/config" + "fdt-templ/internal/config" _ "github.com/mattn/go-sqlite3" ) @@ -437,15 +438,27 @@ func (db *DB) GetIconByCategoryAndName(category, iconName string) (*Icon, error) return &icon, nil } -// GetDB returns a database instance using the default path +// GetDB returns a database instance func GetDB() (*DB, error) { - dbPath := GetDBPath() + if err := config.LoadDBToml(); err != nil { + return nil, fmt.Errorf("failed to load db.toml for SVG Icons DB: %w", err) + } + dbPath := config.DBConfig.SvgIconsDB + if dbPath == "" { + return nil, fmt.Errorf("SVG Icons DB path is empty in db.toml") + } + // Resolve to absolute path absPath, err := filepath.Abs(dbPath) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to resolve absolute path for SVG Icons DB: %w", err) + } + + db, err := NewDB(absPath) + if err != nil { + return nil, fmt.Errorf("failed to open SVG Icons DB: %w", err) } - return NewDB(absPath) + return db, nil } func (db *DB) GetClusterUpdatedAt(hashName string) (string, error) { diff --git a/frontend/internal/db/svg_icons/utils.go b/frontend/internal/db/svg_icons/utils.go index c9b433f6b4..1ede7af4a6 100644 --- a/frontend/internal/db/svg_icons/utils.go +++ b/frontend/internal/db/svg_icons/utils.go @@ -3,9 +3,9 @@ package svg_icons import ( "crypto/sha256" "encoding/binary" + "fdt-templ/internal/config" "fmt" "net/url" - "path/filepath" "strings" ) @@ -50,7 +50,14 @@ func HashClusterToKeyInt(cluster string) int64 { } // GetDBPath returns the path to the SVG icons database -func GetDBPath() string { - // Assuming we're running from project root - return filepath.Join("db", "all_dbs", "svg-icons-db-v5.db") +func GetDBPath() (string, error) { + if config.DBConfig == nil { + if err := config.LoadDBToml(); err != nil { + return "", err + } + } + if config.DBConfig.SvgIconsDB == "" { + return "", fmt.Errorf("SVG Icons DB path is empty in db.toml") + } + return config.DBConfig.SvgIconsDB, nil } diff --git a/frontend/internal/db/tldr/queries.go b/frontend/internal/db/tldr/queries.go index c06dc2f633..1214f0fb25 100644 --- a/frontend/internal/db/tldr/queries.go +++ b/frontend/internal/db/tldr/queries.go @@ -6,6 +6,7 @@ import ( "path/filepath" db_config "fdt-templ/db/config" + "fdt-templ/internal/config" _ "github.com/mattn/go-sqlite3" ) @@ -47,12 +48,24 @@ func (db *DB) Close() error { // GetDB returns a database instance func GetDB() (*DB, error) { - // Standard path for tldr db - dbPath, err := filepath.Abs("db/all_dbs/tldr-db-v6.db") + if err := config.LoadDBToml(); err != nil { + return nil, fmt.Errorf("failed to load db.toml for TLDR DB: %w", err) + } + dbPath := config.DBConfig.TldrDB + if dbPath == "" { + return nil, fmt.Errorf("TLDR DB path is empty in db.toml") + } + + absPath, err := filepath.Abs(dbPath) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to resolve absolute path for TLDR DB: %w", err) + } + + db, err := NewDB(absPath) + if err != nil { + return nil, fmt.Errorf("failed to open TLDR DB: %w", err) } - return NewDB(dbPath) + return db, nil } // GetAllClusters retrieves all clusters (platforms) diff --git a/frontend/md/changset-implementation.md b/frontend/md/changset-implementation.md index 4d5fbd8d26..29b9092db5 100644 --- a/frontend/md/changset-implementation.md +++ b/frontend/md/changset-implementation.md @@ -532,4 +532,6 @@ path = "/frontend/db/all_dbs/" ``` we use frontend/internal/config/config.go for defining all the other configs. Now i want to define db.toml in frontend/internal/config/config.go and and use the same config data for all db related path location. -I have added in context where all the templ server has deps of db \ No newline at end of file +I have added in context where all the templ server has deps of db + +all db version should defined same as it is present in thec ode \ No newline at end of file diff --git a/frontend/scripts/analyze_svg_icon_404.go b/frontend/scripts/analyze_svg_icon_404.go index 99d0c3dda4..4ec0694b3a 100644 --- a/frontend/scripts/analyze_svg_icon_404.go +++ b/frontend/scripts/analyze_svg_icon_404.go @@ -6,15 +6,20 @@ import ( "os" "strings" - _ "github.com/mattn/go-sqlite3" "fdt-templ/internal/db/svg_icons" + + _ "github.com/mattn/go-sqlite3" ) func main() { log.SetOutput(os.Stdout) log.SetFlags(0) - db, err := sql.Open("sqlite3", svg_icons.GetDBPath()) + dbPath, err := svg_icons.GetDBPath() + if err != nil { + log.Fatalf("Error getting DB path: %v", err) + } + db, err := sql.Open("sqlite3", dbPath) if err != nil { log.Fatalf("Error opening database: %v", err) } @@ -30,7 +35,7 @@ func main() { // Check if icon exists with exact name log.Printf("1. Checking for icon with name: '%s' in category: '%s'", iconName, category) - + // Get cluster first var clusterID int64 var clusterSourceFolder string @@ -73,7 +78,7 @@ func main() { } found = true log.Printf(" Found: ID=%d, name='%s', cluster='%s'", id, name, cluster) - + // Check if name contains problematic characters if strings.Contains(name, "/") || strings.Contains(name, "linear-gradient") { log.Printf(" ⚠️ WARNING: Icon name contains '/' or 'linear-gradient' - this is malformed!") @@ -131,4 +136,3 @@ func main() { log.Println("2. The icon.URL field contains malformed data") log.Println("3. A link was generated incorrectly from SVG/CSS content") } - diff --git a/frontend/scripts/find_correct_manpage_urls.go b/frontend/scripts/find_correct_manpage_urls.go index e067595ce3..4370a906d9 100644 --- a/frontend/scripts/find_correct_manpage_urls.go +++ b/frontend/scripts/find_correct_manpage_urls.go @@ -18,7 +18,10 @@ func main() { log.SetFlags(0) // Open database - dbPath := man_pages.GetDBPath() + dbPath, err := man_pages.GetDBPath() + if err != nil { + log.Fatalf("Error getting DB path: %v", err) + } db, err := sql.Open("sqlite3", dbPath) if err != nil { log.Fatalf("Error opening database: %v", err) @@ -151,4 +154,3 @@ func findManPageBySlug(db *sql.DB, slug string) string { url.PathEscape(foundSubCategory), url.PathEscape(foundSlug)) } - diff --git a/frontend/scripts/man-pages/create_html_content.go b/frontend/scripts/man-pages/create_html_content.go index 9dd5176427..21a1552db8 100644 --- a/frontend/scripts/man-pages/create_html_content.go +++ b/frontend/scripts/man-pages/create_html_content.go @@ -67,7 +67,7 @@ func renderContentToHTML(content man_pages_db.ManPageContent) string { } htmlBuilder.WriteString(``) - + // Minify HTML by removing unnecessary whitespace between tags return minifyHTML(htmlBuilder.String()) } @@ -77,16 +77,19 @@ func minifyHTML(html string) string { // Remove whitespace between tags (but preserve whitespace inside text nodes) // This regex matches > followed by whitespace followed by < html = regexp.MustCompile(`>\s+<`).ReplaceAllString(html, "><") - + // Remove leading/trailing whitespace from the entire string html = strings.TrimSpace(html) - + return html } func main() { // Get database path - dbPath := man_pages_db.GetDBPath() + dbPath, err := man_pages_db.GetDBPath() + if err != nil { + log.Fatalf("Error getting DB path: %v", err) + } absPath, err := filepath.Abs(dbPath) if err != nil { log.Fatalf("Failed to resolve database path: %v", err) @@ -244,4 +247,3 @@ func updateBatch(conn *sql.DB, batch []struct { return nil } - diff --git a/frontend/scripts/man-pages/drop_content_column.go b/frontend/scripts/man-pages/drop_content_column.go index 2f32be0f39..6b6c4e7cbe 100644 --- a/frontend/scripts/man-pages/drop_content_column.go +++ b/frontend/scripts/man-pages/drop_content_column.go @@ -13,7 +13,10 @@ import ( func main() { // Get database path - dbPath := man_pages_db.GetDBPath() + dbPath, err := man_pages_db.GetDBPath() + if err != nil { + log.Fatalf("Error getting DB path: %v", err) + } absPath, err := filepath.Abs(dbPath) if err != nil { log.Fatalf("Failed to resolve database path: %v", err) @@ -61,4 +64,3 @@ func main() { log.Println("✓ Successfully dropped 'content' column") } -