diff --git a/examples/binary-mode/binary_loopback_example.py b/examples/binary-mode/loopback/binary_loopback_example.py similarity index 100% rename from examples/binary-mode/binary_loopback_example.py rename to examples/binary-mode/loopback/binary_loopback_example.py diff --git a/examples/binary-mode/upload/README.md b/examples/binary-mode/upload/README.md new file mode 100644 index 0000000..d4cd230 --- /dev/null +++ b/examples/binary-mode/upload/README.md @@ -0,0 +1,85 @@ +# Binary Upload Example + +Upload a binary file from a Notecard to a remote server via Notehub, using the note-python SDK's chunked upload mechanism. + +This example includes two scripts: + +- **`binary_upload_example.py`** — Runs on the host (e.g. Raspberry Pi) connected to a Notecard. Reads `blues_logo.png`, chunks it through the Notecard's binary buffer, and sends it to Notehub via `web.post`. +- **`receive_binary.py`** — A minimal HTTP server that receives the binary data routed from Notehub and saves it to disk. + +## Prerequisites + +- A [Blues Notecard](https://blues.com/products/notecard/) connected via USB serial +- A [Notehub](https://notehub.io) account and project +- Python 3.7+ +- `pyserial` and `note-python` installed (`pip install pyserial note-python`) +- [ngrok](https://ngrok.com/) (or another tunnel) to expose the receive server publicly + +## Setup + +### 1. Start the receive server + +On the machine where you want to receive files: + +```bash +python3 receive_binary.py +``` + +This starts an HTTP server on port 8080 (pass a different port as an argument if needed). Files are saved to the current directory. + +### 2. Expose the server with ngrok + +In a separate terminal: + +```bash +ngrok http 8080 +``` + +Copy the HTTPS forwarding URL (e.g. `https://abc123.ngrok.io`). + +### 3. Create a Notehub proxy route + +In [Notehub](https://notehub.io), go to your project's **Routes** and create a new **General HTTP/HTTPS** route: + +- **Route alias**: `upload` +- **URL**: your ngrok HTTPS URL + +### 4. Configure and run the upload script + +Edit `binary_upload_example.py` and set: + +- **`PRODUCT_UID`** — your Notehub product UID (e.g. `com.your-company:your-project`) +- **`ROUTE_ALIAS`** — the route alias from step 3 (default: `upload`) +- **Serial port** — update the `serial.Serial(...)` path to match your Notecard's port + +Then run: + +```bash +python3 binary_upload_example.py +``` + +The script will: + +1. Connect the Notecard to Notehub and wait for a connection +2. Read `blues_logo.png` (~222 KB) +3. Upload it in 64 KB chunks, printing progress after each chunk +4. Print a summary with total bytes, duration, and throughput + +### 5. Check the output + +The receive server prints a line for each file received: + +``` +Listening on port 8080. Saving files to: /your/current/directory +Press Ctrl+C to stop. + +[14:23:01] Received 222,511 bytes -> blues_logo.png +``` + +## Chunk size tuning + +The `MAX_CHUNK_SIZE` constant in `binary_upload_example.py` controls how large each chunk is. The Notecard's binary buffer can hold ~250 KB, but large single-chunk uploads over cellular can time out. The default of 64 KB is a good balance between throughput and reliability. Lower it to 32 KB if you experience timeouts on slow connections. + +## File naming + +Files are named using the Notecard's `label` field (sent as the `X-Notecard-Label` HTTP header by Notehub). If no label is present, the server generates a timestamped filename with an extension inferred from the file's magic bytes (`.png`, `.jpg`, `.pdf`, `.bin`, etc.). diff --git a/examples/binary-mode/upload/binary_upload_example.py b/examples/binary-mode/upload/binary_upload_example.py new file mode 100644 index 0000000..d684e70 --- /dev/null +++ b/examples/binary-mode/upload/binary_upload_example.py @@ -0,0 +1,90 @@ +"""note-python binary upload example. + +This example uploads binary data to a Notehub proxy route using the +high-speed chunked upload mechanism. The data is staged through the +Notecard's binary buffer and sent to Notehub via web.post. + +Before running this example: +1. Create a Proxy Route in your Notehub project (e.g. pointing to + https://httpbin.org/post or your own endpoint). +2. Set PRODUCT_UID below to your Notehub product UID. +3. Set ROUTE_ALIAS to the alias of your proxy route. + +Targets Raspberry Pi and other Linux systems. +""" +import os +import sys +import time + +sys.path.insert(0, os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..'))) + +import serial # noqa: E402 + +import notecard # noqa: E402 +from notecard import hub # noqa: E402 +from notecard.upload import upload # noqa: E402 + + +PRODUCT_UID = 'com.your-company:your-product' +ROUTE_ALIAS = 'upload' +# Keep chunks small enough to reliably transfer over cellular. +# The Notecard buffer can hold ~250KB, but pushing that much data +# in a single web.post over cellular often times out. +MAX_CHUNK_SIZE = 65536 # 64 KB + + +def on_progress(info): + """Print upload progress after each chunk.""" + print(f' Chunk {info["chunk"]}/{info["total_chunks"]} ' + f'- {info["percent_complete"]:.1f}% ' + f'- {info["avg_bytes_per_sec"]:.0f} B/s ' + f'- ETA {info["eta_secs"]:.1f}s') + + +def run_example(): + """Connect to Notecard and upload binary data to Notehub.""" + port = serial.Serial('/dev/ttyUSB0', 115200) + card = notecard.OpenSerial(port, debug=True) + + # Connect the Notecard to Notehub. + hub.set(card, product=PRODUCT_UID, mode='continuous') + + # Wait for the Notecard to connect to Notehub. + print('Waiting for Notehub connection...') + while True: + rsp = hub.status(card) + connected = rsp.get('connected', False) + status_msg = rsp.get('status', '') + if connected: + print('Connected to Notehub.') + break + print(f' Not yet connected: {status_msg}') + time.sleep(2) + + # Read the image file to upload. + image_path = os.path.join(os.path.dirname(__file__), 'blues_logo.png') + with open(image_path, 'rb') as f: + data = f.read() + + print(f'Uploading {image_path} ({len(data)} bytes) ' + f'to route "{ROUTE_ALIAS}"...') + + result = upload( + card, + data, + route=ROUTE_ALIAS, + label='blues_logo.png', + content_type='image/png', + max_chunk_size=MAX_CHUNK_SIZE, + progress_cb=on_progress, + ) + + print(f'Upload complete: {result["bytes_uploaded"]} bytes ' + f'in {result["chunks"]} chunks, ' + f'{result["duration_secs"]:.1f}s ' + f'({result["bytes_per_sec"]:.0f} B/s)') + + +if __name__ == '__main__': + run_example() diff --git a/examples/binary-mode/upload/blues_logo.png b/examples/binary-mode/upload/blues_logo.png new file mode 100644 index 0000000..ae29c97 Binary files /dev/null and b/examples/binary-mode/upload/blues_logo.png differ diff --git a/examples/binary-mode/upload/receive_binary.py b/examples/binary-mode/upload/receive_binary.py new file mode 100644 index 0000000..90e4144 --- /dev/null +++ b/examples/binary-mode/upload/receive_binary.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Minimal HTTP server that receives binary files routed from Notehub. + +Receives binary files via a General HTTP/HTTPS route and saves them to +the current directory. + +Usage: + python3 receive_binary.py [port] + + Default port: 8080 + +Setup: + 1. Run this script (optionally with ngrok to expose it publicly): + python3 receive_binary.py + ngrok http 8080 + + 2. In Notehub, create a General HTTP/HTTPS route pointing to this + server's URL (or your ngrok URL). + + 3. Send a binary file from your Notecard: + {"req": "web.post", "route": "", "binary": true, ...} + + Files are saved to the current directory with a name derived from the + Notecard's "label" field, or falling back to a timestamped filename + with an extension inferred from the file's magic bytes. +""" + +import os +import sys +import time +from http.server import BaseHTTPRequestHandler, HTTPServer + +# Magic bytes used to infer file extensions +MAGIC_SIGNATURES = [ + (b"\x89PNG", "png"), + (b"\xff\xd8\xff", "jpg"), + (b"%PDF", "pdf"), + (b"GIF8", "gif"), + (b"PK\x03\x04", "zip"), + (b"\x1f\x8b", "gz"), +] + +DEFAULT_PORT = 8080 + + +def decode_chunked(data: bytes) -> bytes: + """Decode HTTP chunked transfer encoding.""" + output = b"" + pos = 0 + while pos < len(data): + end = data.find(b"\r\n", pos) + if end == -1: + break + size_str = data[pos:end].decode(errors="ignore").strip() + if not size_str: + break + try: + size = int(size_str, 16) + except ValueError: + break + if size == 0: + break + pos = end + 2 + output += data[pos:pos + size] + pos += size + 2 + return output + + +def infer_extension(data: bytes) -> str: + """Infer file extension from magic bytes.""" + for magic, ext in MAGIC_SIGNATURES: + if data[: len(magic)] == magic: + return ext + return "bin" + + +def make_filename(label: str, data: bytes) -> str: + """Return the label if provided, or a timestamped filename.""" + if label: + return label + ext = infer_extension(data) + timestamp = time.strftime("%Y%m%d_%H%M%S") + return f"received_{timestamp}.{ext}" + + +class BinaryReceiveHandler(BaseHTTPRequestHandler): + """Handle incoming binary POST requests from Notehub.""" + + def do_POST(self): + """Receive a binary file and save it to disk.""" + # Read body, handling both Content-Length and chunked encoding + content_length = self.headers.get("Content-Length") + transfer_encoding = self.headers.get("Transfer-Encoding", "") + + if content_length: + raw = self.rfile.read(int(content_length)) + else: + raw = self.rfile.read() + + if "chunked" in transfer_encoding.lower(): + body = decode_chunked(raw) + else: + body = raw + + if not body: + self._respond(400, "Empty body") + return + + # Notehub sets X-Notecard-Label to the note's label field + label = self.headers.get("X-Notecard-Label", "").strip() + filename = make_filename(label, body) + + filepath = os.path.join(os.getcwd(), filename) + with open(filepath, "wb") as f: + f.write(body) + + print(f"[{time.strftime('%H:%M:%S')}] Received {len(body):,} bytes -> {filename}") + self._respond(200, "OK") + + def _respond(self, code: int, message: str): + self.send_response(code) + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(message.encode()) + + def log_message(self, format, *args): + """Suppress default request logging.""" + pass + + +def main(): + """Start the binary receive server.""" + port = int(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_PORT + server = HTTPServer(("", port), BinaryReceiveHandler) + print(f"Listening on port {port}. Saving files to: {os.getcwd()}") + print("Press Ctrl+C to stop.\n") + try: + server.serve_forever() + except KeyboardInterrupt: + print("\nStopped.") + + +if __name__ == "__main__": + main() diff --git a/notecard/upload.py b/notecard/upload.py new file mode 100644 index 0000000..e1203fd --- /dev/null +++ b/notecard/upload.py @@ -0,0 +1,220 @@ +"""High-speed binary file upload to Notehub via the Notecard.""" + +import sys +import time + +from notecard.cobs import cobs_encode +from notecard.notecard import Notecard + +if sys.implementation.name == 'cpython': + import hashlib + + def _md5_hash(data): + """Create an MD5 digest of the given data.""" + return hashlib.md5(data).hexdigest() +else: + from .md5 import digest as _md5_hash + +BINARY_STAGE_RETRIES = 50 +WEB_POST_RETRIES = 20 +WEB_POST_RETRY_DELAY_SECS = 15 + +try: + _monotonic = time.monotonic +except AttributeError: + _monotonic = time.time + + +def _stage_binary_chunk(card, chunk_data): + """Stage a binary chunk into the Notecard's binary buffer. + + Performs card.binary.put + raw byte transmit + verification, with + retries on failure. Mirrors the Go implementation's inner binary + transfer retry loop. + + Args: + card (Notecard): The Notecard object. + chunk_data (bytearray): The raw chunk data to stage. + + Raises: + Exception: If staging fails after all retries. + """ + encoded = cobs_encode(bytearray(chunk_data), ord('\n')) + req = { + 'req': 'card.binary.put', + 'cobs': len(encoded), + } + encoded.append(ord('\n')) + expected_len = len(chunk_data) + + tries_left = BINARY_STAGE_RETRIES + while tries_left > 0: + try: + card.lock() + rsp = card.Transaction(req, lock=False) + if 'err' in rsp: + raise Exception(rsp['err']) + card.transmit(encoded, delay=False) + except Exception: + tries_left -= 1 + if tries_left == 0: + raise + continue + finally: + card.unlock() + + try: + rsp = card.Transaction({'req': 'card.binary'}) + except Exception: + tries_left -= 1 + if tries_left == 0: + raise + continue + + if 'err' in rsp: + err_msg = rsp['err'] + if '{bad-bin}' in err_msg or '{io}' in err_msg: + tries_left -= 1 + if tries_left == 0: + raise Exception( + f'Failed to stage binary data: {err_msg}') + continue + raise Exception(f'Failed to stage binary data: {err_msg}') + + actual_len = rsp.get('length', 0) + if actual_len != expected_len: + tries_left -= 1 + if tries_left == 0: + raise Exception( + f'Binary length mismatch: expected {expected_len}, ' + f'got {actual_len}.') + continue + + return + + raise Exception('Failed to stage binary data after retries.') + + +def upload(card, data, route, target=None, label=None, + content_type='application/octet-stream', max_chunk_size=0, + progress_cb=None): + """Upload binary data to a Notehub proxy route via the Notecard. + + The data is chunked to fit in the Notecard's binary buffer, staged + via card.binary.put, and sent to Notehub via web.post with + binary:true. + + Args: + card (Notecard): The Notecard object. + data (bytes or bytearray): The binary data to upload. + route (str): The Notehub proxy route alias. + target (str, optional): URL path appended to the route (sent as + ``name`` in the web.post request). + label (str, optional): Filename label for the upload. + content_type (str): MIME type. Default ``application/octet-stream``. + max_chunk_size (int): Maximum chunk size in bytes. 0 means use the + Notecard's maximum buffer capacity. + progress_cb (callable, optional): Called after each chunk with a dict + containing progress information. + + Returns: + dict: Upload statistics with keys ``bytes_uploaded``, ``chunks``, + ``duration_secs``, and ``bytes_per_sec``. + + Raises: + ValueError: If ``route`` is empty or ``data`` is empty. + Exception: If the upload fails. + """ + if not route: + raise ValueError('route must not be empty.') + if not data: + raise ValueError('data must not be empty.') + + rsp = card.Transaction({'req': 'card.binary', 'reset': True}) + if 'err' in rsp and '{bad-bin}' not in rsp['err']: + raise Exception( + f'Error querying card.binary: {rsp["err"]}') + + buf_capacity = rsp.get('max', 0) + if buf_capacity == 0: + raise Exception( + 'Notecard binary buffer capacity is zero or not reported.') + + if max_chunk_size > 0: + chunk_size = min(max_chunk_size, buf_capacity) + else: + chunk_size = buf_capacity + + total_len = len(data) + total_chunks = (total_len + chunk_size - 1) // chunk_size + upload_start = _monotonic() + bytes_sent = 0 + + for chunk_idx in range(total_chunks): + offset = chunk_idx * chunk_size + end = min(offset + chunk_size, total_len) + chunk_data = data[offset:end] + chunk_len = len(chunk_data) + chunk_md5 = _md5_hash(chunk_data) + + _stage_binary_chunk(card, chunk_data) + + web_req = { + 'req': 'web.post', + 'route': route, + 'binary': True, + 'content': content_type, + 'offset': offset, + 'status': chunk_md5, + } + if target: + web_req['name'] = target + if label: + web_req['label'] = label + # Only set total for multi-chunk (segmented) uploads, matching + # the Go implementation. This tells Notehub to expect multiple + # segments and reassemble them. + if total_chunks > 1: + web_req['total'] = total_len + + web_tries = WEB_POST_RETRIES + while web_tries > 0: + rsp = card.Transaction(web_req) + result_code = rsp.get('result', 0) + if result_code >= 300 or 'err' in rsp: + web_tries -= 1 + if web_tries == 0: + err_detail = rsp.get('err', f'HTTP {result_code}') + raise Exception( + f'web.post failed after retries: {err_detail}') + time.sleep(WEB_POST_RETRY_DELAY_SECS) + _stage_binary_chunk(card, chunk_data) + continue + break + + bytes_sent += chunk_len + elapsed = _monotonic() - upload_start + current_bps = chunk_len / elapsed if elapsed > 0 else 0 + avg_bps = bytes_sent / elapsed if elapsed > 0 else 0 + remaining = total_len - bytes_sent + eta = remaining / avg_bps if avg_bps > 0 else 0 + + if progress_cb: + progress_cb({ + 'chunk': chunk_idx + 1, + 'total_chunks': total_chunks, + 'bytes_sent': bytes_sent, + 'total_bytes': total_len, + 'percent_complete': (bytes_sent / total_len) * 100, + 'bytes_per_sec': current_bps, + 'avg_bytes_per_sec': avg_bps, + 'eta_secs': eta, + }) + + duration = _monotonic() - upload_start + return { + 'bytes_uploaded': bytes_sent, + 'chunks': total_chunks, + 'duration_secs': duration, + 'bytes_per_sec': bytes_sent / duration if duration > 0 else 0, + } diff --git a/notecard/web.py b/notecard/web.py index 461c61d..22a02f9 100644 --- a/notecard/web.py +++ b/notecard/web.py @@ -95,7 +95,7 @@ def get(card, async_=None, binary=None, body=None, content=None, file=None, max= @validate_card_object -def post(card, async_=None, binary=None, body=None, content=None, file=None, max=None, name=None, note=None, offset=None, payload=None, route=None, seconds=None, status=None, total=None, verify=None): +def post(card, async_=None, binary=None, body=None, content=None, file=None, label=None, max=None, name=None, note=None, offset=None, payload=None, route=None, seconds=None, status=None, total=None, verify=None): """Perform a simple HTTP or HTTPS `POST` request against an external endpoint, and returns the response to the Notecard. Args: @@ -105,6 +105,7 @@ def post(card, async_=None, binary=None, body=None, content=None, file=None, max body (dict): The JSON body to send with the request. content (str): The MIME type of the body or payload of the response. Default is `application/json`. file (str): The name of the local-only Database Notefile (`.dbx`) to be used if the web request is issued asynchronously and you wish to store the response. + label (str): A label for the upload, typically a filename. max (int): The maximum size of the response from the remote server, in bytes. Useful if a memory-constrained host wants to limit the response size. name (str): A web URL endpoint relative to the host configured in the Proxy Route. URL parameters may be added to this argument as well (e.g. `/addReading?id=1`). note (str): The unique Note ID for the local-only Database Notefile (`.dbx`). Only used with asynchronous web requests (see `file` argument above). @@ -130,6 +131,8 @@ def post(card, async_=None, binary=None, body=None, content=None, file=None, max req["content"] = content if file: req["file"] = file + if label: + req["label"] = label if max is not None: req["max"] = max if name: @@ -154,7 +157,7 @@ def post(card, async_=None, binary=None, body=None, content=None, file=None, max @validate_card_object -def put(card, async_=None, binary=None, body=None, content=None, file=None, max=None, name=None, note=None, offset=None, payload=None, route=None, seconds=None, status=None, total=None, verify=None): +def put(card, async_=None, binary=None, body=None, content=None, file=None, label=None, max=None, name=None, note=None, offset=None, payload=None, route=None, seconds=None, status=None, total=None, verify=None): """Perform a simple HTTP or HTTPS `PUT` request against an external endpoint, and returns the response to the Notecard. Args: @@ -164,6 +167,7 @@ def put(card, async_=None, binary=None, body=None, content=None, file=None, max= body (dict): The JSON body to send with the request. content (str): The MIME type of the body or payload of the response. Default is `application/json`. file (str): The name of the local-only Database Notefile (`.dbx`) to be used if the web request is issued asynchronously and you wish to store the response. + label (str): A label for the upload, typically a filename. max (int): The maximum size of the response from the remote server, in bytes. Useful if a memory-constrained host wants to limit the response size. Default (and maximum value) is 8192. name (str): A web URL endpoint relative to the host configured in the Proxy Route. URL parameters may be added to this argument as well (e.g. `/updateReading?id=1`). note (str): The unique Note ID for the local-only Database Notefile (`.dbx`). Only used with asynchronous web requests (see `file` argument above). @@ -189,6 +193,8 @@ def put(card, async_=None, binary=None, body=None, content=None, file=None, max= req["content"] = content if file: req["file"] = file + if label: + req["label"] = label if max is not None: req["max"] = max if name: diff --git a/test/test_upload.py b/test/test_upload.py new file mode 100644 index 0000000..d065419 --- /dev/null +++ b/test/test_upload.py @@ -0,0 +1,521 @@ +import os +import sys +import pytest +from unittest.mock import MagicMock, patch, call + +sys.path.insert(0, + os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +import notecard # noqa: E402 +from notecard.upload import ( # noqa: E402 + upload, + _stage_binary_chunk, + _md5_hash, + BINARY_STAGE_RETRIES, + WEB_POST_RETRIES, +) + + +@pytest.fixture +def card(): + """Create a mock Notecard with the methods used by upload.""" + c = notecard.Notecard() + c.Transaction = MagicMock() + c.lock = MagicMock() + c.unlock = MagicMock() + c.transmit = MagicMock() + return c + + +@pytest.fixture +def sample_data(): + """A small sample payload for testing.""" + return bytearray(range(64)) + + +class TestUploadValidation: + def test_raises_on_empty_route(self, card, sample_data): + with pytest.raises(ValueError, match='route must not be empty'): + upload(card, sample_data, route='') + + def test_raises_on_none_route(self, card, sample_data): + with pytest.raises(ValueError, match='route must not be empty'): + upload(card, sample_data, route=None) + + def test_raises_on_empty_data(self, card): + with pytest.raises(ValueError, match='data must not be empty'): + upload(card, b'', route='my-route') + + def test_raises_on_none_data(self, card): + with pytest.raises(ValueError, match='data must not be empty'): + upload(card, None, route='my-route') + + def test_raises_on_zero_buffer_capacity(self, card, sample_data): + card.Transaction.return_value = {'max': 0} + with pytest.raises(Exception, match='capacity is zero'): + upload(card, sample_data, route='my-route') + + +class TestSingleChunkUpload: + def test_single_chunk_upload(self, card, sample_data): + """Data fits in one chunk -- one stage + one web.post.""" + buf_max = len(sample_data) + 100 + + call_count = [0] + + def transaction_side_effect(req, **kwargs): + call_count[0] += 1 + r = req.get('req', req.get('cmd', '')) + + if r == 'card.binary' and req.get('reset'): + return {'max': buf_max} + if r == 'card.binary.put': + return {} + if r == 'card.binary': + return {'length': len(sample_data)} + if r == 'web.post': + return {'result': 200} + return {} + + card.Transaction.side_effect = transaction_side_effect + + result = upload(card, sample_data, route='my-route') + + assert result['bytes_uploaded'] == len(sample_data) + assert result['chunks'] == 1 + assert result['duration_secs'] >= 0 + assert result['bytes_per_sec'] >= 0 + + def test_single_chunk_sets_web_post_fields(self, card, sample_data): + """Verify the web.post request has the correct fields.""" + buf_max = len(sample_data) + 100 + captured_web_req = {} + + def transaction_side_effect(req, **kwargs): + r = req.get('req', '') + if r == 'card.binary' and req.get('reset'): + return {'max': buf_max} + if r == 'card.binary.put': + return {} + if r == 'card.binary': + return {'length': len(sample_data)} + if r == 'web.post': + captured_web_req.update(req) + return {'result': 200} + return {} + + card.Transaction.side_effect = transaction_side_effect + + upload(card, sample_data, route='my-route', target='/path', + label='file.bin', content_type='image/png') + + assert captured_web_req['route'] == 'my-route' + assert captured_web_req['binary'] is True + assert captured_web_req['name'] == '/path' + assert captured_web_req['label'] == 'file.bin' + assert captured_web_req['content'] == 'image/png' + assert captured_web_req['offset'] == 0 + # total should NOT be set for single-chunk uploads + assert 'total' not in captured_web_req + assert captured_web_req['status'] == _md5_hash(sample_data) + + def test_omits_name_and_label_when_not_provided(self, card, sample_data): + """target and label should be omitted from web.post when None.""" + buf_max = len(sample_data) + 100 + captured_web_req = {} + + def transaction_side_effect(req, **kwargs): + r = req.get('req', '') + if r == 'card.binary' and req.get('reset'): + return {'max': buf_max} + if r == 'card.binary.put': + return {} + if r == 'card.binary': + return {'length': len(sample_data)} + if r == 'web.post': + captured_web_req.update(req) + return {'result': 200} + return {} + + card.Transaction.side_effect = transaction_side_effect + + upload(card, sample_data, route='my-route') + + assert 'name' not in captured_web_req + assert 'label' not in captured_web_req + + +class TestMultiChunkUpload: + def test_multi_chunk_upload(self, card): + """Data requires multiple chunks.""" + data = bytearray(range(256)) * 4 # 1024 bytes + buf_max = 300 # forces ~4 chunks + chunk_idx = [0] + expected_chunks = (len(data) + buf_max - 1) // buf_max + + def transaction_side_effect(req, **kwargs): + r = req.get('req', '') + if r == 'card.binary' and req.get('reset'): + return {'max': buf_max} + if r == 'card.binary.put': + return {} + if r == 'card.binary': + # Return length matching the current chunk size + offset = chunk_idx[0] * buf_max + end = min(offset + buf_max, len(data)) + return {'length': end - offset} + if r == 'web.post': + chunk_idx[0] += 1 + return {'result': 200} + return {} + + card.Transaction.side_effect = transaction_side_effect + + result = upload(card, data, route='my-route') + + assert result['bytes_uploaded'] == len(data) + assert result['chunks'] == expected_chunks + + def test_sets_total_for_multi_chunk(self, card): + """total field should be set in web.post for multi-chunk uploads.""" + data = bytearray(range(256)) * 4 # 1024 bytes + buf_max = 300 + captured_web_reqs = [] + + chunk_idx = [0] + + def transaction_side_effect(req, **kwargs): + r = req.get('req', '') + if r == 'card.binary' and req.get('reset'): + return {'max': buf_max} + if r == 'card.binary.put': + return {} + if r == 'card.binary': + offset = chunk_idx[0] * buf_max + end = min(offset + buf_max, len(data)) + return {'length': end - offset} + if r == 'web.post': + captured_web_reqs.append(dict(req)) + chunk_idx[0] += 1 + return {'result': 200} + return {} + + card.Transaction.side_effect = transaction_side_effect + + upload(card, data, route='my-route') + + assert len(captured_web_reqs) > 1 + for web_req in captured_web_reqs: + assert web_req['total'] == len(data) + + def test_respects_max_chunk_size(self, card): + """max_chunk_size limits chunk size even when buffer is larger.""" + data = bytearray(100) + buf_max = 1000 + max_chunk = 30 + expected_chunks = (len(data) + max_chunk - 1) // max_chunk + web_post_offsets = [] + + def transaction_side_effect(req, **kwargs): + r = req.get('req', '') + if r == 'card.binary' and req.get('reset'): + return {'max': buf_max} + if r == 'card.binary.put': + # Track the chunk size from the COBS-encoded length hint + # is not directly the raw size, so we track via web.post + # offset math instead. Just return success. + return {} + if r == 'card.binary': + # The verification call: we need to return the length of + # the chunk that was just staged. Compute from the last + # web.post offset we haven't sent yet. + chunk_idx = len(web_post_offsets) + offset = chunk_idx * max_chunk + end = min(offset + max_chunk, len(data)) + return {'length': end - offset} + if r == 'web.post': + web_post_offsets.append(req['offset']) + return {'result': 200} + return {} + + card.Transaction.side_effect = transaction_side_effect + + result = upload(card, data, route='r', max_chunk_size=max_chunk) + + assert result['chunks'] == expected_chunks + assert web_post_offsets == [i * max_chunk + for i in range(expected_chunks)] + + +class TestStageBinaryRetry: + def test_retries_on_transmit_exception(self, card): + """_stage_binary_chunk retries when transmit raises.""" + chunk = bytearray(b'\x01\x02\x03') + + card.Transaction.side_effect = [ + {}, # first card.binary.put + {}, # second card.binary.put (retry) + {'length': len(chunk)}, # verification after retry + ] + card.transmit.side_effect = [ + Exception('transmit error'), + None, + ] + + _stage_binary_chunk(card, chunk) + + assert card.transmit.call_count == 2 + + def test_retries_on_verification_error(self, card): + """_stage_binary_chunk retries when verification has an error.""" + chunk = bytearray(b'\x01\x02\x03') + + card.Transaction.side_effect = [ + {}, # first card.binary.put + {'err': '{bad-bin}'}, # first verification fails + {}, # second card.binary.put (retry) + {'length': len(chunk)}, # second verification ok + ] + + _stage_binary_chunk(card, chunk) + + assert card.transmit.call_count == 2 + + def test_retries_on_length_mismatch(self, card): + """_stage_binary_chunk retries when verified length doesn't match.""" + chunk = bytearray(b'\x01\x02\x03') + + card.Transaction.side_effect = [ + {}, # first card.binary.put + {'length': 999}, # first verification: wrong length + {}, # second card.binary.put (retry) + {'length': len(chunk)}, # second verification ok + ] + + _stage_binary_chunk(card, chunk) + + assert card.transmit.call_count == 2 + + def test_fails_after_all_retries_exhausted(self, card): + """_stage_binary_chunk raises after BINARY_STAGE_RETRIES failures.""" + chunk = bytearray(b'\x01\x02\x03') + + # Every verification fails + side_effects = [] + for _ in range(BINARY_STAGE_RETRIES): + side_effects.append({}) # card.binary.put + side_effects.append({'err': '{bad-bin}'}) # verification + card.Transaction.side_effect = side_effects + + with pytest.raises(Exception, match='Failed to stage binary data'): + _stage_binary_chunk(card, chunk) + + def test_locks_and_unlocks_on_each_attempt(self, card): + """lock/unlock called for each staging attempt.""" + chunk = bytearray(b'\x01\x02\x03') + + card.Transaction.side_effect = [ + {}, # card.binary.put + {'length': len(chunk)}, # verification + ] + + _stage_binary_chunk(card, chunk) + + card.lock.assert_called_once() + card.unlock.assert_called_once() + + def test_unlocks_even_on_exception(self, card): + """unlock is called even when transmit raises.""" + chunk = bytearray(b'\x01\x02\x03') + + card.Transaction.return_value = {} + card.transmit.side_effect = Exception('fail') + + with pytest.raises(Exception): + # Set retries to 1 via patching to avoid long test + with patch('notecard.upload.BINARY_STAGE_RETRIES', 1): + _stage_binary_chunk(card, chunk) + + assert card.unlock.call_count >= 1 + + +class TestWebPostRetry: + @patch('notecard.upload.WEB_POST_RETRY_DELAY_SECS', 0) + def test_retries_on_http_error(self, card, sample_data): + """web.post is retried when result >= 300.""" + buf_max = len(sample_data) + 100 + web_post_count = [0] + + def transaction_side_effect(req, **kwargs): + r = req.get('req', '') + if r == 'card.binary' and req.get('reset'): + return {'max': buf_max} + if r == 'card.binary.put': + return {} + if r == 'card.binary': + return {'length': len(sample_data)} + if r == 'web.post': + web_post_count[0] += 1 + if web_post_count[0] == 1: + return {'result': 500} + return {'result': 200} + return {} + + card.Transaction.side_effect = transaction_side_effect + + result = upload(card, sample_data, route='my-route') + + assert web_post_count[0] == 2 + assert result['bytes_uploaded'] == len(sample_data) + + @patch('notecard.upload.WEB_POST_RETRY_DELAY_SECS', 0) + def test_retries_on_web_post_err(self, card, sample_data): + """web.post is retried when response has 'err' field.""" + buf_max = len(sample_data) + 100 + web_post_count = [0] + + def transaction_side_effect(req, **kwargs): + r = req.get('req', '') + if r == 'card.binary' and req.get('reset'): + return {'max': buf_max} + if r == 'card.binary.put': + return {} + if r == 'card.binary': + return {'length': len(sample_data)} + if r == 'web.post': + web_post_count[0] += 1 + if web_post_count[0] == 1: + return {'err': 'some network error'} + return {'result': 200} + return {} + + card.Transaction.side_effect = transaction_side_effect + + upload(card, sample_data, route='my-route') + + assert web_post_count[0] == 2 + + @patch('notecard.upload.WEB_POST_RETRY_DELAY_SECS', 0) + @patch('notecard.upload.WEB_POST_RETRIES', 2) + def test_fails_after_web_post_retries_exhausted(self, card, sample_data): + """upload raises after WEB_POST_RETRIES web.post failures.""" + buf_max = len(sample_data) + 100 + + def transaction_side_effect(req, **kwargs): + r = req.get('req', '') + if r == 'card.binary' and req.get('reset'): + return {'max': buf_max} + if r == 'card.binary.put': + return {} + if r == 'card.binary': + return {'length': len(sample_data)} + if r == 'web.post': + return {'result': 500} + return {} + + card.Transaction.side_effect = transaction_side_effect + + with pytest.raises(Exception, match='web.post failed after retries'): + upload(card, sample_data, route='my-route') + + @patch('notecard.upload.WEB_POST_RETRY_DELAY_SECS', 0) + def test_re_stages_binary_on_web_post_retry(self, card, sample_data): + """Binary data is re-staged before each web.post retry.""" + buf_max = len(sample_data) + 100 + web_post_count = [0] + binary_put_count = [0] + + def transaction_side_effect(req, **kwargs): + r = req.get('req', '') + if r == 'card.binary' and req.get('reset'): + return {'max': buf_max} + if r == 'card.binary.put': + binary_put_count[0] += 1 + return {} + if r == 'card.binary': + return {'length': len(sample_data)} + if r == 'web.post': + web_post_count[0] += 1 + if web_post_count[0] == 1: + return {'result': 500} + return {'result': 200} + return {} + + card.Transaction.side_effect = transaction_side_effect + + upload(card, sample_data, route='my-route') + + # Initial stage + re-stage for the retry + assert binary_put_count[0] == 2 + + +class TestProgressCallback: + def test_progress_callback_called_per_chunk(self, card): + """progress_cb is called once per chunk with correct fields.""" + data = bytearray(100) + buf_max = 40 + expected_chunks = (len(data) + buf_max - 1) // buf_max + progress_calls = [] + chunk_idx = [0] + + def transaction_side_effect(req, **kwargs): + r = req.get('req', '') + if r == 'card.binary' and req.get('reset'): + return {'max': buf_max} + if r == 'card.binary.put': + return {} + if r == 'card.binary': + offset = chunk_idx[0] * buf_max + end = min(offset + buf_max, len(data)) + return {'length': end - offset} + if r == 'web.post': + chunk_idx[0] += 1 + return {'result': 200} + return {} + + card.Transaction.side_effect = transaction_side_effect + + def progress_cb(info): + progress_calls.append(info) + + upload(card, data, route='my-route', progress_cb=progress_cb) + + assert len(progress_calls) == expected_chunks + + # Verify first callback + first = progress_calls[0] + assert first['chunk'] == 1 + assert first['total_chunks'] == expected_chunks + assert first['total_bytes'] == len(data) + assert 'bytes_sent' in first + assert 'percent_complete' in first + assert 'bytes_per_sec' in first + assert 'avg_bytes_per_sec' in first + assert 'eta_secs' in first + + # Verify last callback + last = progress_calls[-1] + assert last['chunk'] == expected_chunks + assert last['bytes_sent'] == len(data) + assert last['percent_complete'] == 100.0 + + def test_no_error_when_progress_callback_is_none(self, card, sample_data): + """Upload works fine with no progress callback.""" + buf_max = len(sample_data) + 100 + + def transaction_side_effect(req, **kwargs): + r = req.get('req', '') + if r == 'card.binary' and req.get('reset'): + return {'max': buf_max} + if r == 'card.binary.put': + return {} + if r == 'card.binary': + return {'length': len(sample_data)} + if r == 'web.post': + return {'result': 200} + return {} + + card.Transaction.side_effect = transaction_side_effect + + upload(card, sample_data, route='my-route', + progress_cb=None)