From b9f100ae5144f71ddd8709ded4265ef897327c24 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 23 Aug 2025 07:07:05 +0000 Subject: [PATCH 1/4] Initial plan From 566c375eabcb0183c5f190b8e63a8e940cb59418 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 23 Aug 2025 07:15:23 +0000 Subject: [PATCH 2/4] Update RST documentation files - intro, config, examples, design, build, and api Co-authored-by: clauspruefer <17313789+clauspruefer@users.noreply.github.com> --- doc/conf.py/source/api.rst | 71 +++++++++-- doc/conf.py/source/build.rst | 148 +++++++++++++++++++--- doc/conf.py/source/config.rst | 143 +++++++++++++++++---- doc/conf.py/source/design.rst | 214 +++++++++++++++++++++++++------- doc/conf.py/source/examples.rst | 207 +++++++++++++++++++++++++----- doc/conf.py/source/intro.rst | 22 ++-- 6 files changed, 671 insertions(+), 134 deletions(-) diff --git a/doc/conf.py/source/api.rst b/doc/conf.py/source/api.rst index 62dbfd5..73428be 100644 --- a/doc/conf.py/source/api.rst +++ b/doc/conf.py/source/api.rst @@ -1,15 +1,70 @@ .. api -============================= -Internal Class Representation -============================= +============= +API Reference +============= -pgdbpool -======== +Complete API documentation for pgdbpool classes and functions. + +pgdbpool Module +=============== + +Core Classes +------------ + +.. automodule:: pgdbpool.pool + :members: Connection, Handler, Query + :special-members: __init__, __enter__, __exit__ + :exclude-members: __weakref__ + :show-inheritance: + +Utility Functions +----------------- + +.. automodule:: pgdbpool.pool + :members: conn_iter + :exclude-members: __weakref__ + +Exception Classes +----------------- .. automodule:: pgdbpool.pool - :members: conn_iter, conn_iter_locked, Connection, Query, Handler - :special-members: - :private-members: + :members: DBConnectionError, DBQueryError, DBOfflineError, UnconfiguredGroupError :exclude-members: __weakref__ :show-inheritance: + +Class Method Reference +====================== + +Connection Class Methods +------------------------- + +The Connection class provides static methods for pool management: + +- **init(config)**: Initialize connection pool with configuration +- **get_threading_model()**: Return current threading model ('threaded' or 'non-threaded') +- **get_max_pool_size(group)**: Get maximum connections for a group +- **get_next_connection(group)**: Get next available connection from group +- **connect(connection)**: Establish database connection +- **reconnect(connection)**: Reconnect failed database connection +- **check_db(connection)**: Verify database connection health + +Handler Context Manager +----------------------- + +The Handler class provides a context manager for database operations: + +- **__init__(group)**: Initialize handler for connection group +- **__enter__()**: Enter context, acquire connection +- **__exit__()**: Exit context, release connection +- **query(statement, params=None)**: Execute SQL query +- **query_prepared(params)**: Execute prepared statement +- **commit()**: Commit transaction (for autocommit=False groups) + +Query Static Methods +-------------------- + +The Query class provides static methods for SQL execution: + +- **execute(connection, sql_statement, sql_params=None)**: Execute SQL statement +- **execute_prepared(connection, sql_params)**: Execute prepared statement diff --git a/doc/conf.py/source/build.rst b/doc/conf.py/source/build.rst index 4b60817..84222af 100644 --- a/doc/conf.py/source/build.rst +++ b/doc/conf.py/source/build.rst @@ -1,47 +1,159 @@ .. build -============ -Build Module -============ +================= +Building pgdbpool +================= -1. Consider Using Environment -============================= +Guide for building pgdbpool from source code. + +1. Development Environment Setup +================================ -Python PEP 405 proposes Virtual Environments. Think about using. +**Virtual Environment (Recommended)** -2. Clone Repository -=================== +Python PEP 405 virtual environments provide isolated development environments: + +.. code-block:: bash + + # Create virtual environment + python3 -m venv pgdbpool-dev + source pgdbpool-dev/bin/activate # Linux/macOS + # pgdbpool-dev\Scripts\activate # Windows + + # Upgrade pip + pip install --upgrade pip + +2. Source Code Acquisition +=========================== + +**Clone from GitHub:** .. code-block:: bash git clone https://github.com/clauspruefer/python-dbpool.git cd python-dbpool -3. Build As Non Root User -========================= +**Download Release Archive:** + +.. code-block:: bash + + # Download latest release + wget https://github.com/clauspruefer/python-dbpool/archive/refs/tags/v1.0rc1.tar.gz + tar -xzf v1.0rc1.tar.gz + cd python-dbpool-1.0rc1 + +3. Install Dependencies +======================= + +**Runtime Dependencies:** + +.. code-block:: bash + + # Install PostgreSQL adapter + pip install psycopg2-binary + + # Or compile from source (requires libpq-dev) + # apt-get install libpq-dev # Debian/Ubuntu + # pip install psycopg2 + +**Development Dependencies:** + +.. code-block:: bash + + # Install testing framework + pip install pytest pytest-cov + + # Install documentation tools (optional) + pip install sphinx sphinx-rtd-theme + +4. Build Distribution Package +============================= + +**Create Source Distribution:** .. code-block:: bash - python3 setup.py sdist + # Modern way using build + pip install build + python -m build --sdist -4. Install As Root User + # Legacy way using setuptools + python setup.py sdist + +**Create Wheel Distribution:** + +.. code-block:: bash + + python -m build --wheel + +5. Installation Methods ======================= +**Development Installation (Editable):** + .. code-block:: bash - sudo pip3 install ./dist/package-0.1.tar.gz + # Install in development mode + pip install -e . -or global on restrictive PIP package manager systems ... +**Local Package Installation:** .. code-block:: bash - sudo pip3 install ./dist/package-0.1.tar.gz --break-system-packages + # Install from built package + pip install dist/pgdbpool-1.0rc1.tar.gz -5. Run Tests -============ +**System-wide Installation:** -To ensure everything works correctly, run tests (after module installation). +.. code-block:: bash + + # Install system-wide (requires sudo) + sudo pip install dist/pgdbpool-1.0rc1.tar.gz + + # For restrictive systems + sudo pip install dist/pgdbpool-1.0rc1.tar.gz --break-system-packages + +6. Testing +========== + +**Run Test Suite:** .. code-block:: bash + # Run all tests pytest + + # Run with coverage report + pytest --cov=pgdbpool --cov-report=html + + # Run specific test file + pytest test/unit/test_pool.py + +**Test Configuration:** + +Tests require a PostgreSQL database. Set environment variables: + +.. code-block:: bash + + export PSQL_PWD="your_test_db_password" + export TEST_DB_HOST="localhost" + export TEST_DB_NAME="test_database" + export TEST_DB_USER="test_user" + +7. Documentation Building +========================= + +**Build HTML Documentation:** + +.. code-block:: bash + + cd doc/conf.py + sphinx-build -b html source build/html + +**View Documentation:** + +.. code-block:: bash + + # Open in browser + open build/html/index.html # macOS + xdg-open build/html/index.html # Linux diff --git a/doc/conf.py/source/config.rst b/doc/conf.py/source/config.rst index 88cce0e..be9c7df 100644 --- a/doc/conf.py/source/config.rst +++ b/doc/conf.py/source/config.rst @@ -4,29 +4,40 @@ Configuration ============= -Basic configuration explanation. See :ref:`examples-label` subsection for more detailed information -(working Apache WSGI Script). +Configuration guide for pgdbpool. See :ref:`examples-label` for detailed working examples (including Apache WSGI scripts). -1. Root Dict -============ +1. Root Configuration Structure +=============================== -The root dictionary must contain keys `db` for **Database Connection Config** (see ) and `groups` -for **Group Config** (see ) . +The root configuration dictionary must contain: + +- ``db``: Database connection configuration (dict or list of dicts for multi-DB support) +- ``groups``: Connection group configuration +- ``type`` (optional): Threading model specification (``threaded`` or ``non-threaded``) .. code-block:: python config = { 'db': { + # Single database configuration }, + # OR for multiple databases: + # 'db': [ + # { # Database 1 config }, + # { # Database 2 config } + # ], 'groups': { - } + # Connection groups + }, + 'type': 'threaded' # optional, defaults to 'threaded' } -2. Database Connection -====================== +2. Database Connection Configuration +==================================== + +The database connection configuration supports both single and multiple database endpoints. -The Database Connection Configuration JSON schema. It consists of base configuration properties -and the "groups" dictionary to specify (multiple) group data. +**Single Database Configuration:** .. code-block:: python @@ -42,9 +53,59 @@ and the "groups" dictionary to specify (multiple) group data. 'query_timeout': 120, 'session_tmp_buffer': 128 } + } + +**Multiple Database Configuration (Load Balancing):** + +.. code-block:: python + + config = { + 'db': [ + { + 'host': 'db1.example.com', + 'name': 'dbname', + 'user': 'username', + 'pass': 'userpass', + 'ssl': False, + 'connect_timeout': 30, + 'connection_retry_sleep': 1, + 'query_timeout': 120, + 'session_tmp_buffer': 128 + }, + { + 'host': 'db2.example.com', + 'name': 'dbname', + 'user': 'username', + 'pass': 'userpass', + 'ssl': False, + 'connect_timeout': 30, + 'connection_retry_sleep': 1, + 'query_timeout': 120, + 'session_tmp_buffer': 128 + } + ] + } + +3. Threading Model Configuration +================================== -3. Database Connection Properties +The threading model can be configured to optimize for different deployment scenarios: + +.. code-block:: python + + config = { + 'type': 'threaded', # or 'non-threaded' + 'db': { ... }, + 'groups': { ... } + } + +**Threading Models:** + +- ``threaded`` (default): Uses thread-safe connection handling with locks. Optimal for threaded web servers (Apache, Gunicorn with threads). +- ``non-threaded``: Removes locking overhead. Optimal for process-based servers (Gunicorn with workers) or single-threaded applications. + +4. Database Connection Properties ================================= .. list-table:: Database Connection Properties @@ -74,13 +135,13 @@ and the "groups" dictionary to specify (multiple) group data. - - - - - Database Auth Usern + - Database Auth Username * - pass - string - - - - - Database Auth Pass + - Database Auth Password * - ssl - bool - @@ -112,7 +173,7 @@ and the "groups" dictionary to specify (multiple) group data. - 128 - Session Buffer Memory -4. Group Configuration +5. Group Configuration ====================== .. code-block:: python @@ -129,7 +190,7 @@ and the "groups" dictionary to specify (multiple) group data. } -5. Group Configuration Properties +6. Group Configuration Properties ================================= .. list-table:: Group Properties @@ -155,11 +216,10 @@ and the "groups" dictionary to specify (multiple) group data. - True - Autocommit on / off -6. Internal Default (Values) +7. Internal Default Values ============================ -The following schema represensts the internal Python structures. Some values (e.g. groups.id.connections) -are used to internally store values and should not be overwritten. +The following schema represents the internal Python structures. Some values (e.g., ``groups.id.connections``) are used internally and should not be modified manually. .. code-block:: python @@ -187,10 +247,10 @@ are used to internally store values and should not be overwritten. } } -7. Multi Group Example +8. Multi-Group Configuration Example ====================== -To specify a) autocommit and b) non autocommit database connection to the same endpoint. +Example configuration with separate groups for autocommit and non-autocommit connections to the same database endpoint: .. code-block:: python @@ -212,3 +272,44 @@ To specify a) autocommit and b) non autocommit database connection to the same e } } } + +9. Multi-Database Load Balancing Example +========================================= + +Configure multiple database endpoints for automatic load balancing and failover: + +.. code-block:: python + + config = { + 'type': 'threaded', + 'db': [ + { + 'host': 'primary-db.example.com', + 'name': 'myapp', + 'user': 'appuser', + 'pass': 'securepassword', + 'ssl': 'require', + 'connect_timeout': 30, + 'query_timeout': 120 + }, + { + 'host': 'secondary-db.example.com', + 'name': 'myapp', + 'user': 'appuser', + 'pass': 'securepassword', + 'ssl': 'require', + 'connect_timeout': 30, + 'query_timeout': 120 + } + ], + 'groups': { + 'read_write': { + 'connection_count': 20, + 'autocommit': True + }, + 'transactions': { + 'connection_count': 10, + 'autocommit': False + } + } + } diff --git a/doc/conf.py/source/design.rst b/doc/conf.py/source/design.rst index df7dcd6..d4a17b8 100644 --- a/doc/conf.py/source/design.rst +++ b/doc/conf.py/source/design.rst @@ -1,71 +1,191 @@ .. design -====== -Design -====== +================== +Architecture Design +================== -1. Current Model -================ +Overview of pgdbpool's architecture patterns and deployment models. -Common Web-Application-Server use a threaded or event-based (also threaded) model. -The following diagrams illustrate how the DB-Connection-Pool works in detail. +1. Threading Models +=================== -.. code-block:: bash +pgdbpool supports both threaded and non-threaded deployment models to optimize for different server architectures. +**Threaded Model (Default)** + +Designed for threaded web servers like Apache with mod_wsgi, Gunicorn with threads, or Flask development servers: + +.. code-block:: text + + +-----------------------------------------------------------------------------+ + | Web Server (Apache/Gunicorn+threads) | + |-----------------------------------------------------------------------------| + | pgdbpool Module (Thread-Safe) | + | - Connection locking enabled | + | - Thread-safe connection iteration | +-----------------------------------------------------------------------------+ - | Web-Server / ESB Middleware | - |-----------------------------------------------------------------------------+ - | Python-App / DB-Pool Module | - | | + | Thread 1 | Thread 2 | Thread 3 | Thread 4 | Thread N | + | Handler | Handler | Handler | Handler | Handler | +-----------------------------------------------------------------------------+ - | Script1.py (T1) | Script1.py (T2) | ScriptX.py (T3) | ScriptX.py (T4) | - | PCon1 | PCon2 | PCon3 | PCon4 | + | | | | | + +--------+---------------+---------------+---------------+--------------+----+ + | Connection Pool (Shared across threads) | + | - Automatic load balancing across DB endpoints | + | - Thread-safe connection allocation | +-----------------------------------------------------------------------------+ - | | | | - +-------------------+------------------+------------------+-------------------+ - | Node 1 / Con 1 | Node 1 / Con 2 | Node 1 / Con 3 | Node 1 / Con 4 | - |-------------------+------------------+------------------+-------------------| - | Database Cluster | + | | | | | + +--------+---------------+---------------+---------------+--------------+----+ + | DB Node 1 | DB Node 1 | DB Node 2 | DB Node 2 | DB Node N | + | Connection | Connection | Connection | Connection | Connection | +-----------------------------------------------------------------------------+ -- Currently you have to care about load-sharing to multiple hosts by yourself. -- Also Python Global Interpreter Lock reduces Performance / Scalability. +**Non-Threaded Model** + +Optimized for process-based servers like Gunicorn with workers, uWSGI, or single-threaded applications: + +.. code-block:: text + + +-----------------------------------------------------------------------------+ + | Web Server (Gunicorn+workers/uWSGI) | + |-----------------------------------------------------------------------------| + | pgdbpool Module (Non-Threaded) | + | - No connection locking overhead | + | - Optimized for single-threaded access | + +-----------------------------------------------------------------------------+ + | Process 1 | Process 2 | Process 3 | Process 4 | Process N | + | Handler | Handler | Handler | Handler | Handler | + | (Isolated) | (Isolated) | (Isolated) | (Isolated) | (Isolated) | + +-----------------------------------------------------------------------------+ + | | | | | + +--------+---------------+---------------+---------------+--------------+----+ + | Per-Process Connection Pool | + | - Load balancing across DB endpoints | + | - No inter-process locking required | + +-----------------------------------------------------------------------------+ -2. Process Based / FalconAS -=========================== +2. Multi-Database Load Balancing +================================= -FalconAS is a high speed Python Application Server / Data-Aggregator Middleware. +pgdbpool now supports automatic load balancing across multiple database endpoints: -.. code-block:: bash +.. code-block:: text +-----------------------------------------------------------------------------+ - | Web-Server / FalconAS / ESB Middleware | - |-----------------------------------------------------------------------------+ - | Python-App / DB-Pool Module | - | | + | Application Layer | +-----------------------------------------------------------------------------+ - | Script1.py (P1) | Script1.py (P2) | Script2.py (P3) | Script2.py (P4) | - | 1 Interpreter | 1 Interpreter | 1 Interpreter | 1 Interpreter | - | PCon1 | PCon2 | PCon3 | PCon4 | + | pgdbpool Connection Manager | + | - Database endpoint rotation | + | - Automatic failover handling | + | - Connection health monitoring | +-----------------------------------------------------------------------------+ - | | | | - +-------------------+------------------+------------------+-------------------+ - | Node 1 / Con 1 | Node 1 / Con 2 | Node 2 / Con 3 | Node 2 / Con 4 | - |-------------------+------------------+------------------+-------------------| - | Database Cluster | + | + +-----------------------------------+-----------------------------------+ + | | | + +-------------------+ +-------------------+ + + | Primary Database | | Secondary Database| | + | - Read/Write | | - Read/Write | | + | - High Priority | | - Backup/Replica | | + +-------------------+ +-------------------+ + + | Connection Pool | | Connection Pool | | + | Group 1: 15 conns | | Group 1: 15 conns | | + | Group 2: 10 conns | | Group 2: 10 conns | | + +-------------------+ +-------------------+ + + +**Key Features:** + +- **Transparent Load Balancing**: Connections are automatically distributed across available database endpoints +- **High Availability**: Automatic failover when database endpoints become unavailable +- **Scalability**: Add database replicas without application code changes +- **Performance**: Reduced load on individual database servers + +3. Connection Group Architecture +================================= + +Connection groups allow fine-grained control over connection behavior: + +**Autocommit Groups** (``autocommit: true``) + - Immediate transaction commits + - Optimal for read operations and simple writes + - Lower overhead, higher throughput + +**Transaction Groups** (``autocommit: false``) + - Manual transaction control with ``commit()`` + - Essential for complex multi-statement transactions + - ACID compliance for critical operations + +.. code-block:: text + + +-----------------------------------------------------------------------------+ + | Application Handlers | +-----------------------------------------------------------------------------+ + | | | | | + | READ_POOL | WRITE_POOL | TRANSACTION_POOL | BATCH_POOL | + | autocommit: | autocommit: | autocommit: | autocommit: | + | true | true | false | false | + | | | | | + | Simple | Quick writes | Complex | Bulk operations | + | queries | Single statements | transactions | Data imports | + +-----------------------------------------------------------------------------+ + +4. Deployment Recommendations +============================= + +**Threaded Model Use Cases:** + +- Apache with mod_wsgi (default) +- Gunicorn with ``--workers=1 --threads=N`` +- Flask/Django development servers +- Applications with moderate concurrency + +**Non-Threaded Model Use Cases:** + +- Gunicorn with ``--workers=N --threads=1`` +- uWSGI in worker mode +- High-performance applications avoiding GIL contention +- Microservices with dedicated processes + +**Configuration Guidelines:** + +.. list-table:: Threading Model Selection + :widths: 20 40 40 + :header-rows: 1 + + * - Server Type + - Recommended Model + - Configuration + * - Apache + mod_wsgi + - ``threaded`` + - Standard threading with connection locking + * - Gunicorn (threads) + - ``threaded`` + - Thread-safe connection handling + * - Gunicorn (workers) + - ``non-threaded`` + - Process isolation, no locking overhead + * - uWSGI (workers) + - ``non-threaded`` + - Optimized for process-based architecture + * - Development + - ``threaded`` + - General compatibility, easier debugging + +5. Performance Considerations +============================= + +**Connection Pool Sizing:** -- 1 Client Request will be handled by a *static preloaded* Python Interpreter in 1 OS Process. -- 1 Process does not need DB Pooling because Client Requests are serial. -- 1 Process holds 1 DB Connection to 1 Backend Node (non-loadbalanced). -- Security is increased because of Process Segmentation / Separation. -- Loadbalancing can easily be implemented into this model. +- **Threaded**: Pool size should accommodate peak concurrent threads +- **Non-threaded**: Smaller pools per process (typically 3-10 connections) +- **Multi-DB**: Connections distributed automatically across endpoints -.. warning:: +**Memory Usage:** - This model currently is not supported by **pgdbpool** module but will be implemented in - the next major release. +- Each connection consumes ~8-16MB depending on ``session_tmp_buffer`` +- Non-threaded model typically uses less memory per connection +- Monitor total memory usage: ``(connections × buffer_size × processes)`` -.. note:: +**Latency Optimization:** - Loadbalancing will be implemented *transparently* for both models in a next major release. +- Place database replicas geographically close to application servers +- Use appropriate ``connect_timeout`` and ``query_timeout`` values +- Consider connection keep-alive settings for high-frequency access diff --git a/doc/conf.py/source/examples.rst b/doc/conf.py/source/examples.rst index e60a1df..5208140 100644 --- a/doc/conf.py/source/examples.rst +++ b/doc/conf.py/source/examples.rst @@ -6,66 +6,209 @@ Examples ======== -The following example shows a working example for 1) Database Config 2) WSGI Python Script running on -Apache Web-Server. +Working examples for database configuration and web server integration with pgdbpool. .. note:: - This assumes a working Apache Python WSGI Config. + These examples assume a properly configured web server environment. -1. Database / Group Config -========================== +1. Single Database Configuration +================================= + +Basic configuration for a single PostgreSQL database: .. code-block:: python - # File DBConfig.py + # File: DBConfig.py import os - DBName = 'db1' - DBUser = 'db1' - DBHost = 'mypostgres' - DBPass = os.environ['PSQL_PWD'] + # Database configuration + DB_NAME = 'myapp_db' + DB_USER = 'myapp_user' + DB_HOST = 'localhost' + DB_PASS = os.environ['PSQL_PASSWORD'] - config1 = { + config = { + 'type': 'threaded', # Threading model 'db': { - 'host': DBHost, - 'name': DBName, - 'user': DBUser, - 'pass': DBPass, - 'ssl': 'disable', + 'host': DB_HOST, + 'name': DB_NAME, + 'user': DB_USER, + 'pass': DB_PASS, + 'ssl': 'prefer', # Use SSL when possible 'connect_timeout': 30, 'connection_retry_sleep': 1, 'query_timeout': 30000, 'session_tmp_buffer': 128 }, 'groups': { - 'db1': { - 'connection_count': 3, - 'autocommit': True, + 'default': { + 'connection_count': 10, + 'autocommit': True + }, + 'transactions': { + 'connection_count': 5, + 'autocommit': False } } } -2. Apache WSGI Script -===================== +2. Apache WSGI Application +=========================== + +Example WSGI application using pgdbpool: .. code-block:: python + # File: app.py import DBConfig - from dbpool import pool as dbpool + from pgdbpool import pool as dbpool - dbpool.Connection.init(DBConfig.config1) + # Initialize connection pool + dbpool.Connection.init(DBConfig.config) def application(environ, start_response): + """WSGI application entry point.""" + + if environ['REQUEST_METHOD'].upper() == 'GET': + try: + start_response('200 OK', [ + ('Content-Type', 'application/json; charset=UTF-8') + ]) + + # Simple query with autocommit + sql = "SELECT id, name, email FROM users WHERE active = true" + + with dbpool.Handler('default') as db: + results = [] + for row in db.query(sql): + results.append({ + 'id': row[0], + 'name': row[1], + 'email': row[2] + }) + + import json + yield json.dumps(results).encode('utf-8') + + except Exception as e: + start_response('500 Internal Server Error', [ + ('Content-Type', 'text/plain') + ]) + yield b"Internal Server Error" - if environ['REQUEST_METHOD'].upper() == 'POST': +3. Multi-Database Load Balancing Configuration +=============================================== - try: - start_response('200 OK', [('Content-Type', 'application/json; charset=UTF-8')]) +Configure multiple database endpoints for high availability: - sql = """SELECT col1, col2 FROM testtable1"""; +.. code-block:: python - with dbpool.Handler('db1') as db: - for rec in db.query(sql): - yield rec - except Exception as e: - yield "500 Internal Server Error" + # File: MultiDBConfig.py + import os + + config = { + 'type': 'threaded', + 'db': [ + { + 'host': 'primary-db.example.com', + 'name': 'myapp', + 'user': 'appuser', + 'pass': os.environ['DB_PASSWORD'], + 'ssl': 'require', + 'connect_timeout': 30, + 'query_timeout': 120, + 'session_tmp_buffer': 256 + }, + { + 'host': 'replica-db.example.com', + 'name': 'myapp', + 'user': 'appuser', + 'pass': os.environ['DB_PASSWORD'], + 'ssl': 'require', + 'connect_timeout': 30, + 'query_timeout': 120, + 'session_tmp_buffer': 256 + } + ], + 'groups': { + 'read_pool': { + 'connection_count': 15, + 'autocommit': True + }, + 'write_pool': { + 'connection_count': 10, + 'autocommit': False + } + } + } + +4. Transaction Management Example +================================= + +Example showing manual transaction control with commit(): + +.. code-block:: python + + from pgdbpool import pool as dbpool + + # Initialize with multi-DB config + dbpool.Connection.init(MultiDBConfig.config) + + def transfer_funds(from_account, to_account, amount): + """Example transaction with manual commit.""" + + try: + with dbpool.Handler('write_pool') as db: + # Start transaction (autocommit=False for write_pool) + + # Debit from source account + db.query( + "UPDATE accounts SET balance = balance - %s WHERE id = %s", + (amount, from_account) + ) + + # Credit to destination account + db.query( + "UPDATE accounts SET balance = balance + %s WHERE id = %s", + (amount, to_account) + ) + + # Log the transaction + db.query( + "INSERT INTO transaction_log (from_id, to_id, amount) VALUES (%s, %s, %s)", + (from_account, to_account, amount) + ) + + # Commit transaction + db.commit() + return True + + except Exception as e: + # Transaction will be automatically rolled back + print(f"Transaction failed: {e}") + return False + +5. Non-Threaded Configuration +============================= + +Configuration for process-based web servers (e.g., Gunicorn with workers): + +.. code-block:: python + + # File: ProcessConfig.py + config = { + 'type': 'non-threaded', # Removes locking overhead + 'db': { + 'host': 'localhost', + 'name': 'myapp', + 'user': 'appuser', + 'pass': os.environ['DB_PASSWORD'], + 'ssl': 'prefer' + }, + 'groups': { + 'worker_pool': { + 'connection_count': 5, # Fewer connections per process + 'autocommit': True + } + } + } diff --git a/doc/conf.py/source/intro.rst b/doc/conf.py/source/intro.rst index c7fe951..a2f0647 100644 --- a/doc/conf.py/source/intro.rst +++ b/doc/conf.py/source/intro.rst @@ -4,7 +4,7 @@ Intro / Module Description ========================== -The **pgdbpool** Python Module is a tiny PostgreSQL Database Connection De-Multiplexer primarily scoped for Web- / Application Server. +The **pgdbpool** Python Module is a lightweight PostgreSQL Database Connection Pool and Load Balancer primarily designed for Web and Application Servers. 1. Basic Install ================ @@ -26,7 +26,7 @@ The **pgdbpool** Python Module is a tiny PostgreSQL Database Connection De-Multi 3. Build Dependencies ===================== -On current Debian 12 / Ubuntu 22.04.3, 24.04.1 install the following additional packages (Documentation Rendering & Testing). +On current Debian 12 / Ubuntu 22.04+ / Ubuntu 24.04+ install the following additional packages for documentation rendering and testing: .. code-block:: bash @@ -49,11 +49,17 @@ To run all tests (unit and integration) after pip package installation. 5. Current Features =================== -- Connection Pooling in (threaded) Web-Server Environment (Single Destination DB-Node) -- Automatic DB Reconnection -- PostgreSQL Prepared Queries Module +- **Multi-Database Support**: Connection pooling with multiple database endpoints for load balancing +- **Threading Models**: Support for both threaded and non-threaded environments +- **Connection Pooling**: Efficient connection pooling in Web-Server environments +- **Automatic Reconnection**: Automatic database reconnection on connection failures +- **PostgreSQL Prepared Queries**: Support for prepared SQL statements +- **Transaction Control**: Manual commit() procedure for non-autocommit connections -6. Planned Features -=================== +6. Architecture Benefits +======================== -- Connection Load Balancing to multiple (auto-scaled) Database Nodes +- **Scalability**: Distributes connections across multiple database nodes +- **Reliability**: Automatic failover and reconnection capabilities +- **Performance**: Optimized connection reuse and reduced overhead +- **Flexibility**: Configurable for different deployment scenarios From b5d0c7f7ceaed21a7a0f18effabcbaaf9687358e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 23 Aug 2025 07:17:13 +0000 Subject: [PATCH 3/4] Final documentation improvements - fix table formatting, update CHANGELOG, and enhance index Co-authored-by: clauspruefer <17313789+clauspruefer@users.noreply.github.com> --- CHANGELOG.md | 14 +++++++-- doc/conf.py/source/config.rst | 2 +- doc/conf.py/source/index.rst | 54 ++++++++++++++++++++++++++++++----- 3 files changed, 59 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b129f67..91ca86c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,19 @@ # Changelog -## Version 0.98rc1 +## Version 1.0rc1 -- Finish Documentation including Docstrings -- Fix Metadata +- **Multi-Database Support**: Added support for multiple database endpoints with automatic load balancing +- **Threading Models**: Support for both threaded and non-threaded deployment scenarios +- **Improved Architecture**: Enhanced connection handling and pool management +- **Documentation**: Complete rewrite of documentation to reflect new features +- **API Improvements**: Better error handling and connection management ## Version 0.99 - Add Handler.commit() procedure used for autocommit=False connections +## Version 0.98rc1 + +- Finish Documentation including Docstrings +- Fix Metadata + diff --git a/doc/conf.py/source/config.rst b/doc/conf.py/source/config.rst index be9c7df..60a311b 100644 --- a/doc/conf.py/source/config.rst +++ b/doc/conf.py/source/config.rst @@ -207,7 +207,7 @@ The threading model can be configured to optimize for different deployment scena - int - Quantity - - - + - Required - Connection Count * - autocommit - bool diff --git a/doc/conf.py/source/index.rst b/doc/conf.py/source/index.rst index a6eff59..1e801ff 100644 --- a/doc/conf.py/source/index.rst +++ b/doc/conf.py/source/index.rst @@ -1,23 +1,63 @@ -.. python-xml-microparser documentation master file, created by +.. pgdbpool documentation master file, created by sphinx-quickstart on Tue Feb 6 08:37:14 2018. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Python Module "pgdbpool" Documentation -====================================== +pgdbpool Documentation +====================== -Contents: +PostgreSQL Database Connection Pool and Load Balancer + +pgdbpool is a lightweight Python module that provides efficient connection pooling and load balancing for PostgreSQL databases, designed for web and application servers. + +**Key Features:** + +- Multi-database endpoint support with automatic load balancing +- Both threaded and non-threaded deployment models +- Automatic reconnection and failover capabilities +- Transaction control with manual commit support +- High-performance connection pooling + +**Quick Start:** + +.. code-block:: python + + from pgdbpool import pool as dbpool + + config = { + 'db': {'host': 'localhost', 'name': 'mydb', 'user': 'user', 'pass': 'pass'}, + 'groups': {'default': {'connection_count': 10, 'autocommit': True}} + } + + dbpool.Connection.init(config) + + with dbpool.Handler('default') as db: + results = db.query("SELECT * FROM users") + +Documentation Contents +====================== .. toctree:: - :maxdepth: 3 + :maxdepth: 2 + :caption: Getting Started intro - design build + +.. toctree:: + :maxdepth: 2 + :caption: Configuration & Usage + config - api examples +.. toctree:: + :maxdepth: 2 + :caption: Advanced Topics + + design + api + Indices and tables ================== From 9363d7a7836e86fa8b467ed879cfc7f21fdacfb1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 23 Aug 2025 07:43:21 +0000 Subject: [PATCH 4/4] Fix code block comments to use lowercase formatting Co-authored-by: clauspruefer <17313789+clauspruefer@users.noreply.github.com> --- doc/conf.py/source/build.rst | 34 +++++++++++++++---------------- doc/conf.py/source/config.rst | 10 ++++----- doc/conf.py/source/examples.rst | 36 ++++++++++++++++----------------- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/doc/conf.py/source/build.rst b/doc/conf.py/source/build.rst index 84222af..8251cbf 100644 --- a/doc/conf.py/source/build.rst +++ b/doc/conf.py/source/build.rst @@ -15,12 +15,12 @@ Python PEP 405 virtual environments provide isolated development environments: .. code-block:: bash - # Create virtual environment + # create virtual environment python3 -m venv pgdbpool-dev source pgdbpool-dev/bin/activate # Linux/macOS # pgdbpool-dev\Scripts\activate # Windows - # Upgrade pip + # upgrade pip pip install --upgrade pip 2. Source Code Acquisition @@ -37,7 +37,7 @@ Python PEP 405 virtual environments provide isolated development environments: .. code-block:: bash - # Download latest release + # download latest release wget https://github.com/clauspruefer/python-dbpool/archive/refs/tags/v1.0rc1.tar.gz tar -xzf v1.0rc1.tar.gz cd python-dbpool-1.0rc1 @@ -49,10 +49,10 @@ Python PEP 405 virtual environments provide isolated development environments: .. code-block:: bash - # Install PostgreSQL adapter + # install postgresql adapter pip install psycopg2-binary - # Or compile from source (requires libpq-dev) + # or compile from source (requires libpq-dev) # apt-get install libpq-dev # Debian/Ubuntu # pip install psycopg2 @@ -60,10 +60,10 @@ Python PEP 405 virtual environments provide isolated development environments: .. code-block:: bash - # Install testing framework + # install testing framework pip install pytest pytest-cov - # Install documentation tools (optional) + # install documentation tools (optional) pip install sphinx sphinx-rtd-theme 4. Build Distribution Package @@ -73,11 +73,11 @@ Python PEP 405 virtual environments provide isolated development environments: .. code-block:: bash - # Modern way using build + # modern way using build pip install build python -m build --sdist - # Legacy way using setuptools + # legacy way using setuptools python setup.py sdist **Create Wheel Distribution:** @@ -93,24 +93,24 @@ Python PEP 405 virtual environments provide isolated development environments: .. code-block:: bash - # Install in development mode + # install in development mode pip install -e . **Local Package Installation:** .. code-block:: bash - # Install from built package + # install from built package pip install dist/pgdbpool-1.0rc1.tar.gz **System-wide Installation:** .. code-block:: bash - # Install system-wide (requires sudo) + # install system-wide (requires sudo) sudo pip install dist/pgdbpool-1.0rc1.tar.gz - # For restrictive systems + # for restrictive systems sudo pip install dist/pgdbpool-1.0rc1.tar.gz --break-system-packages 6. Testing @@ -120,13 +120,13 @@ Python PEP 405 virtual environments provide isolated development environments: .. code-block:: bash - # Run all tests + # run all tests pytest - # Run with coverage report + # run with coverage report pytest --cov=pgdbpool --cov-report=html - # Run specific test file + # run specific test file pytest test/unit/test_pool.py **Test Configuration:** @@ -154,6 +154,6 @@ Tests require a PostgreSQL database. Set environment variables: .. code-block:: bash - # Open in browser + # open in browser open build/html/index.html # macOS xdg-open build/html/index.html # Linux diff --git a/doc/conf.py/source/config.rst b/doc/conf.py/source/config.rst index 60a311b..18aab62 100644 --- a/doc/conf.py/source/config.rst +++ b/doc/conf.py/source/config.rst @@ -19,15 +19,15 @@ The root configuration dictionary must contain: config = { 'db': { - # Single database configuration + # single database configuration }, - # OR for multiple databases: + # or for multiple databases: # 'db': [ - # { # Database 1 config }, - # { # Database 2 config } + # { # database 1 config }, + # { # database 2 config } # ], 'groups': { - # Connection groups + # connection groups }, 'type': 'threaded' # optional, defaults to 'threaded' } diff --git a/doc/conf.py/source/examples.rst b/doc/conf.py/source/examples.rst index 5208140..ee5571b 100644 --- a/doc/conf.py/source/examples.rst +++ b/doc/conf.py/source/examples.rst @@ -18,23 +18,23 @@ Basic configuration for a single PostgreSQL database: .. code-block:: python - # File: DBConfig.py + # file: DBConfig.py import os - # Database configuration + # database configuration DB_NAME = 'myapp_db' DB_USER = 'myapp_user' DB_HOST = 'localhost' DB_PASS = os.environ['PSQL_PASSWORD'] config = { - 'type': 'threaded', # Threading model + 'type': 'threaded', # threading model 'db': { 'host': DB_HOST, 'name': DB_NAME, 'user': DB_USER, 'pass': DB_PASS, - 'ssl': 'prefer', # Use SSL when possible + 'ssl': 'prefer', # use ssl when possible 'connect_timeout': 30, 'connection_retry_sleep': 1, 'query_timeout': 30000, @@ -59,11 +59,11 @@ Example WSGI application using pgdbpool: .. code-block:: python - # File: app.py + # file: app.py import DBConfig from pgdbpool import pool as dbpool - # Initialize connection pool + # initialize connection pool dbpool.Connection.init(DBConfig.config) def application(environ, start_response): @@ -75,7 +75,7 @@ Example WSGI application using pgdbpool: ('Content-Type', 'application/json; charset=UTF-8') ]) - # Simple query with autocommit + # simple query with autocommit sql = "SELECT id, name, email FROM users WHERE active = true" with dbpool.Handler('default') as db: @@ -103,7 +103,7 @@ Configure multiple database endpoints for high availability: .. code-block:: python - # File: MultiDBConfig.py + # file: MultiDBConfig.py import os config = { @@ -151,7 +151,7 @@ Example showing manual transaction control with commit(): from pgdbpool import pool as dbpool - # Initialize with multi-DB config + # initialize with multi-DB config dbpool.Connection.init(MultiDBConfig.config) def transfer_funds(from_account, to_account, amount): @@ -159,32 +159,32 @@ Example showing manual transaction control with commit(): try: with dbpool.Handler('write_pool') as db: - # Start transaction (autocommit=False for write_pool) + # start transaction (autocommit=False for write_pool) - # Debit from source account + # debit from source account db.query( "UPDATE accounts SET balance = balance - %s WHERE id = %s", (amount, from_account) ) - # Credit to destination account + # credit to destination account db.query( "UPDATE accounts SET balance = balance + %s WHERE id = %s", (amount, to_account) ) - # Log the transaction + # log the transaction db.query( "INSERT INTO transaction_log (from_id, to_id, amount) VALUES (%s, %s, %s)", (from_account, to_account, amount) ) - # Commit transaction + # commit transaction db.commit() return True except Exception as e: - # Transaction will be automatically rolled back + # transaction will be automatically rolled back print(f"Transaction failed: {e}") return False @@ -195,9 +195,9 @@ Configuration for process-based web servers (e.g., Gunicorn with workers): .. code-block:: python - # File: ProcessConfig.py + # file: ProcessConfig.py config = { - 'type': 'non-threaded', # Removes locking overhead + 'type': 'non-threaded', # removes locking overhead 'db': { 'host': 'localhost', 'name': 'myapp', @@ -207,7 +207,7 @@ Configuration for process-based web servers (e.g., Gunicorn with workers): }, 'groups': { 'worker_pool': { - 'connection_count': 5, # Fewer connections per process + 'connection_count': 5, # fewer connections per process 'autocommit': True } }