basyn

#!/usr/bin/env python3

# Copyright (c) 2021-2026 Stan Orlov <stvor768@gmail.com>
#
# Inspired by the concept of "Bscp" (https://github.com/vog/bscp) by Volker Diels-Grabsch <v@njh.eu>
#
# SPDX-License-Identifier: MIT
# This software is distributed under the MIT License.
# You are free to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies, provided the copyright notice is retained.
# See the LICENSE file or https://opensource.org/licenses/MIT for full terms.

import sys
import zlib
import struct
import os.path
import hashlib

# Command code constants
CMD_INIT = 100
CMD_QUIT = 255
CMD_HASH = 10
CMD_SUBHASH = 11
CMD_READ = 12
CMD_SUBREAD = 13
CMD_WRITE = 14


# SLAVE MODE SECTION
# This section contains code for slave mode. It will be uploaded to remote PC,
# and started there in Python interpreter via SSH to serve master PC's requests.
# In case of "local" connection, pipeline with separate Python instance will be used.

def slaveMode():
    # Slave mode handler.
    # This simple function loops to:
    #   1) listen to stdin, waiting for a 1-byte command;
    #   2) read command-related data;
    #   3) perform simple command-related operations;
    #   4) send 0x00 byte response followed by corresponding data to stdout.
    cin, cout = sys.stdin.buffer, sys.stdout.buffer  # Binary I/O streams for communication

    sendOk = lambda: cout.write(struct.pack('<B', 0))

    # Reporting OK status, so master PC can ensure that we are listening
    sendOk()
    cout.flush()

    # Local 'globals'
    hashName = ''
    bufferSize = 0
    chunkSize = 0
    compLevel = 6
    localFile = None

    lastHashBuffer = ''
    lastPosition = 0

    try:
        # Waiting for command and trying to handle it
        while True:
            (cmdCode,) = struct.unpack('<B', cin.read(1))

            # QUIT command - just exit
            if cmdCode == CMD_QUIT:
                break

            # INIT command. It gets device file name, hash name, opens the file and returns size
            elif cmdCode == CMD_INIT:
                # Reading all necessary integers
                (fileNameLength, hashNameLength, bufferSize, chunkSize, compLevel) = struct.unpack('<QQQQB',
                                                                                                   cin.read(8 * 4 + 1))
                # ...and strings
                fileName = cin.read(fileNameLength).decode('utf-8')
                hashName = cin.read(hashNameLength).decode('utf-8')


                # Checking device file existence
                if not os.path.exists(fileName):
                    cout.write(('Device %s is not found' % (fileName,)).encode('utf-8'))
                    exit(0)

                # Opening file and determining device size, available for writing
                localFile = open(fileName, 'r+b')
                localFile.seek(0, 2)
                localSize = localFile.tell()
                localFile.seek(0)

                # Sending 'OK' and device size to the master
                sendOk()
                cout.write(struct.pack('<Q', localSize))
                cout.flush()

            # CMD_HASH command computes hash of device data starting from given position with given length
            elif cmdCode == CMD_HASH:
                (position, bytesLeft) = struct.unpack('<QQ', cin.read(8 * 2))

                # Reading requested length (bytesLeft) by bufferSize chunks, and updating them to hash
                hashFunc = hashlib.new(hashName)
                localFile.seek(position)
                while bytesLeft > 0:
                    bytesToRead = min(bytesLeft, bufferSize)
                    lastHashBuffer = localFile.read(bytesToRead)
                    lastPosition = position
                    hashFunc.update(lastHashBuffer)
                    bytesLeft -= bytesToRead

                # Sending 'OK' and digest
                sendOk()
                cout.write(hashFunc.digest())
                cout.flush()

            # CMD_SUBHASH command computes an array of hashes, one for every chunkSize in lastHashBuffer (read by
            # last use of CMD_HASH
            elif cmdCode == CMD_SUBHASH:
                length = len(lastHashBuffer)
                chunkCount = int(length / chunkSize)
                if length % chunkSize > 0:
                    chunkCount += 1

                # Getting chunk hashes
                hashes = [
                    hashlib.new(hashName, lastHashBuffer[chunkSize * i:min(length, chunkSize * (i + 1))]).digest()
                    for i in range(chunkCount)]

                # Sending them
                sendOk()
                for i in range(chunkCount):
                    cout.write(hashes[i])
                cout.flush()

            # CMD_READ reads data from given position and length, compresses and returns it to the master
            elif cmdCode == CMD_READ:
                (position, length) = struct.unpack('<QQ', cin.read(8 * 2))

                # May we use cached data?
                if position == lastPosition and len(lastHashBuffer) == length and lastHashBuffer != '':
                    buf = lastHashBuffer
                else:
                    localFile.seek(position)
                    buf = localFile.read(length)

                compBuffer = zlib.compress(buf, compLevel)

                sendOk()
                cout.write(struct.pack('<Q', len(compBuffer)))  # Compressed data length
                cout.write(compBuffer)  # Compressed data
                cout.flush()

            # CMD_SUBREAD reads i-th chunk from lastHashBuffer, read by CMD_HASH
            elif cmdCode == CMD_SUBREAD:
                length = len(lastHashBuffer)
                (i,) = struct.unpack('<Q', cin.read(8))

                buf = lastHashBuffer[chunkSize * i:min(length, chunkSize * (i + 1))]
                compBuffer = zlib.compress(buf, compLevel)
                sendOk()
                cout.write(struct.pack('<Q', len(compBuffer)))  # Compressed data length
                cout.write(compBuffer)  # Compressed data
                cout.flush()

            # CMD_WRITE writes data from master to device into given position.
            elif cmdCode == CMD_WRITE:
                (position, compLength) = struct.unpack('<QQ', cin.read(8 * 2))
                compBuffer = cin.read(compLength)
                localFile.seek(position)
                localFile.write(zlib.decompress(compBuffer))
                os.fsync(localFile.fileno())  # Ensure data is written to disk before confirming

                sendOk()
                cout.flush()

    finally:
        if localFile is not None:
            localFile.close()


# END SLAVE MODE -- DO NOT REMOVE. This magic string is delimiter for uploadable slave part of this script.

# MASTER MODE SECTION

# Additional imports for master mode
import time
import datetime
import getopt
import subprocess

# Hash function name to ID mapping
HASH_FUNCTIONS = {
    "sha1": 0,
    "sha224": 1,
    "sha256": 2,
    "sha384": 3,
    "sha512": 4,
    "blake2b": 5,
    "blake2s": 6,
    "md5": 7
}

# Lambda function to get system time in milliseconds
getTime = lambda: int(round(time.time() * 1000))


class IOCounter:
    """I/O wrapper and counter class for tracking bytes transferred."""
    
    def __init__(self):
        """Initialize I/O counter with zero totals."""
        self.inStream = None
        self.outStream = None
        self.inTotal = 0
        self.outTotal = 0

    def read(self, size=None):
        """Read data from input stream and track bytes received.
        
        Args:
            size: Number of bytes to read. If None, reads all available data.
            
        Returns:
            Bytes read from stream.
            
        Raises:
            IOError: If read was interrupted (received less than requested).
        """
        if size is None:
            s = self.inStream.read()
        else:
            s = self.inStream.read(size)
            if len(s) < size:
                raise IOError("Interrupted read from remote side")
        self.inTotal += len(s)
        return s

    def write(self, s):
        """Write data to output stream and track bytes sent.
        
        Args:
            s: Bytes to write to stream.
        """
        self.outStream.write(s)
        self.outTotal += len(s)
        self.outStream.flush()


class SlaveError(Exception):
    """Exception raised for errors on the slave side."""
    pass


class SettingsParseError(Exception):
    """Exception raised for command-line argument parsing errors."""
    pass


class ActionError(Exception):
    """Exception raised for synchronization action errors."""
    pass


class Settings:
    """Settings container and command-line argument parser."""
    
    def __init__(self):
        """Initialize settings with default values."""
        self.localPath = ''
        self.remotePath = ''
        self.bitmapPath = ''
        self.remoteHost = ''
        self.port = '22'
        self.action = ''
        self.mode = ''
        self.recheck = False
        self.stat = False
        self.verbose = False
        self.hashName = 'sha1'
        self.digestSize = hashlib.new(self.hashName).digest_size
        self.bufferSize = 2 * 1024 * 1024 * 1
        self.chunkSize = 0
        self.zipLevel = 9
        self.user = ''
        self.debug = False
        self.showProgress = False
        self.resetBitmap = False
        self.updateBitmap = False

    def displayHelp(self):
        """Display usage information and available command-line options."""
        print(
            """
basyn <options>
Options (* for mandatory, = for value):
 * -l, --local=     - Local device (e.g., /dev/sda1)
 * -r, --remote=    - Remote device (e.g., /dev/vg_disk_backups/copy-sda1)
   -h, --host=      - Remote SSH hostname or IP address (e.g., 10.0.0.1 or bkp.corp.site).
                      If omitted, localhost is assumed. No SSH will be used.
   -p  --port=      - Remote SSH port (default: 22).
   -u  --user=      - Remote SSH user (default: same as current local user).
   -a  --action=    - One of the actions:
                      PUSH - use local device as source, remote - as destination;
                      PULL  - use remote device as source, local - as destination.
                      If omitted, no data transfer is performed - only device existence is verified.
 * -m  --mode=      - One of the synchronization modes (mandatory if any --action selected):
                      SYNC - copy only changed blocks, detected by parallel hash computation and comparison;
                      COPY - copy whole data (recommended for the initial run).
       --recheck    - Re-check data after sync (or immediately, if no --action and --mode is specified).
       --stat       - Show transfer statistics upon completion.
       --verbose    - Be verbose in stdout about what is script doing (usable for logging).
       --progress   - Show progress indicator to stderr.
       --debug      - Display debug info to stderr (may be extensive for large devices).
       --hash=      - Hash function (SHA1 (by default), SHA224, SHA256, SHA384, SHA512, BLAKE2B,
                      BLAKE2S, MD5).
       --buffer=    - Buffer size in KB for sequential comparison (default: 2048 KB, i.e., 2 MB)
       --chunk=     - Chunk size in kilobytes for detailed comparison.
                      Allows trading lesser traffic (by transferring only actually changed small chunks, instead
                      of whole --buffer) for CPU time (used for secondary data hashing bypass)
                      Not used by default.
       --zlevel=    - ZLIB compression level (0 - no compression, saves CPU, 9 - best compression, saves
                      bandwidth. Default: 9)
   -b  --bitmap=    - Bitmap file. Use a bitmap file to compare data from the source device instead of reading the 
                      destination device. If the file does not exist, it will be created.
                      This option significantly alters the behavior of several other parameters:
                      - The destination device is never read. Only hashes from the source device are compared to
                      those stored in the bitmap file. 
                      - When this option is used, you may omit the destination device (--remote for PUSH, 
                      and --local for PULL). In this case, the bitmap file will simply be updated by source device 
                      hashes, without writing any data to destination device.
                      - Can only be used with --mode=SYNC.
       --reset-bitmap - Force reset (recreate) the bitmap file before sync. The existing bitmap will be
                      started from scratch (all blocks marked as changed). Use this when you want to rebuild the
                      bitmap completely, when switching to a different destination device, or when you suspect the
                      destination or bitmap were heavily corrupted by independent write, and you don't want destination 
                      device to be re-read, but don't mind all data to be transferred again.
                      You may also just delete bitmap file manually.
                      Requires --bitmap option. Cannot be combined with --update-bitmap.
       --update-bitmap - Perform a regular SYNC (reads both source and destination, transfers only changed blocks),
                      and simultaneously update the bitmap file with current source hashes. Use this when the
                      destination device may have been partially modified, the bitmap is out of sync with it, but you want to save
                      traffic by not transferring unchanged data, and full re-read of destination device isn't a problem.
                      Requires --bitmap option. Cannot be combined with --reset-bitmap.
Exit codes:
  0 - A-OK. Data is identical.
  1 - Command line parsing error (displayed). No data was changed.
  2 - Copy / sync error (displayed). Data may have partially changed on receiving device, no match guarantees.
""")

    def parseArguments(self):
        """Parse command-line arguments and set extracted values to instance fields.
        
        Raises:
            SettingsParseError: If arguments are invalid or required options are missing.
        """
        try:
            opts, args = getopt.getopt(sys.argv[1:], 'l:r:h:p:m:u:a:b:',
                                       ['local=', 'remote=', 'host=', 'port=', 'action=', 'mode=', 'recheck', 'stat',
                                        'verbose', 'hash=', 'buffer=', 'chunk=', 'zlevel=', 'user=', 'debug',
                                        'progress', 'bitmap=', 'reset-bitmap', 'update-bitmap'])

            for opt, arg in opts:
                if opt in ("-l", "--local"):
                    self.localPath = arg
                elif opt in ("-r", "--remote"):
                    self.remotePath = arg
                elif opt in ("-h", "--host"):
                    self.remoteHost = arg
                elif opt in ("-p", "--port"):
                    self.port = str(int(arg))
                elif opt in ("-u", "--user"):
                    self.user = arg
                elif opt in ("-b", "--bitmap"):
                    self.bitmapPath = arg
                elif opt in ("-a", "--action"):
                    self.action = str(arg).upper()
                elif opt in ("-m", "--mode"):
                    self.mode = str(arg).upper()
                elif opt == "--recheck":
                    self.recheck = True
                elif opt == "--stat":
                    self.stat = True
                elif opt == "--verbose":
                    self.verbose = True
                elif opt == "--hash":
                    self.hashName = arg.lower()
                elif opt == "--buffer":
                    self.bufferSize = 1024 * int(arg)
                elif opt == "--chunk":
                    self.chunkSize = 1024 * int(arg)
                elif opt == "--zlevel":
                    self.zipLevel = int(arg)
                elif opt == "--debug":
                    self.debug = True
                elif opt == "--progress":
                    self.showProgress = True
                elif opt == "--reset-bitmap":
                    self.resetBitmap = True
                elif opt == "--update-bitmap":
                    self.updateBitmap = True

        except getopt.GetoptError as e:
            raise SettingsParseError(str(e))

        except ValueError as e:
            raise SettingsParseError("Argument conversion error: " + str(e))

        self.digestSize = hashlib.new(self.hashName).digest_size

        if self.action not in ("", "PUSH", "PULL"):
            raise SettingsParseError("Unknown action - '%s'" % (self.action,))
        if self.action and self.mode not in ("COPY", "SYNC"):
            raise SettingsParseError("Unknown mode - '%s' for action '%s'" % (self.mode, self.action))
        if not self.action and self.mode:
            raise SettingsParseError("Option -m/--mode can be only used with -a/--action")
        if not self.localPath and (not self.bitmapPath or self.action == "PUSH"):
            raise SettingsParseError("Option must be provided: -l/--local")
        if not self.remotePath and (not self.bitmapPath or self.action == "PULL"):
            raise SettingsParseError("Option must be provided: -r/--remote")
        if not self.remoteHost and self.user:
            raise SettingsParseError("Option -u/--user can be only used with -h/--host")
        if self.chunkSize > self.bufferSize:
            raise SettingsParseError("--chunk can't be greater than --buffer")
        if self.mode != "SYNC" and self.bitmapPath:
            raise SettingsParseError("--bitmap can be used with --mode==SYNC only")
        if self.resetBitmap and not self.bitmapPath:
            raise SettingsParseError("--reset-bitmap requires --bitmap option")
        if self.updateBitmap and not self.bitmapPath:
            raise SettingsParseError("--update-bitmap requires --bitmap option")
        if self.updateBitmap and self.resetBitmap:
            raise SettingsParseError("--update-bitmap and --reset-bitmap cannot be used together")
        if self.hashName:
            if self.hashName not in HASH_FUNCTIONS:
                raise ValueError(f"Unsupported hash function: {self.hashName}")
            self.digestSize = hashlib.new(self.hashName).digest_size


def displayAbout():
    """Display information about this script."""
    print('''
Block device Advanced SYNcronization utility
Synchronizes data from block device / file to another local or network (via SSH) block device / file\n''')


def byteToHex(byteStr):
    """Convert byte string to hexadecimal string representation.
    
    Args:
        byteStr: Bytes to convert.
        
    Returns:
        Hexadecimal string representation.
    """
    return ''.join(["%02x" % x for x in byteStr]).strip()


def logDebug(desc):
    """Write string to debug console (stderr).
    
    Args:
        desc: Message to write.
    """
    sys.stderr.write(desc + '\n')


def logVerbose(desc):
    """Write timestamped message to stdout.
    
    Args:
        desc: Message to write.
    """
    print("%s: %s" % (datetime.datetime.today().strftime("%x %X"), desc))


def logProgress(desc):
    """Write string to stderr with carriage return to update the same line.
    
    Args:
        desc: Progress message to display.
    """
    sys.stderr.write(desc + '\r')
    sys.stderr.flush()


def prepareSlave(settings):
    """Set up SSH connection (or local pipe) and upload slave script.
    
    Reads the slave portion of this script (up to END SLAVE MODE marker),
    uploads it to the remote host via SSH (or starts local Python process),
    and returns an I/O wrapper for communication.
    
    Args:
        settings: Settings object containing connection parameters.
        
    Returns:
        IOCounter object wrapping the process streams.
        
    Raises:
        SlaveError: If slave initialization fails.
    """
    # Reading this script from the beginning to the "#END SLAVE MODE" line
    scriptPath = sys.argv[0]

    lines = []
    with open(scriptPath, "r") as textFile:
        for line in textFile:
            if "# END SLAVE MODE" in line:
                break
            if line[0][0] == '#':
                # TODO: more complicated comment and white-line skip
                continue
            lines.append(line)

    # Appending it with slaveMode() call
    lines.append("\nslaveMode()\n")

    # Command lines
    script = "".join(lines)

    if not settings.remoteHost:
        localCommand = ('python3', '-c', script)
        if settings.verbose:
            logVerbose('Using pipeline connection')
    else:
        # Escape double quotes and backslashes in the script for SSH command
        escapedScript = script.replace('\\', '\\\\').replace('"', '\\"')
        remoteCommand = 'python3 -c "%s"' % (escapedScript,)
        if settings.user:
            host = '%s@%s' % (settings.user, settings.remoteHost)
        else:
            host = settings.remoteHost

        localCommand = ('ssh', host, '-p' + settings.port, remoteCommand)
        if settings.verbose:
            logVerbose('Using SSH connection to host %s port %s' % (settings.remoteHost, settings.port))

    # Starting SSH with slave script and returning process instance
    slave = subprocess.Popen(localCommand, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    io = IOCounter()
    io.inStream, io.outStream = slave.stdout, slave.stdin

    # Testing if script is working
    checkCommand(io)

    if settings.verbose:
        logVerbose('Connection established, slave instance is on-line')

    return io


def initRemoteDevice(io, settings):
    """Initialize remote device on slave process and return its size.
    
    Sends INIT command with settings to the slave and receives device size.
    
    Args:
        io: IOCounter object for communication.
        settings: Settings object containing device parameters.
        
    Returns:
        Size of the remote device in bytes.
        
    Raises:
        SlaveError: If device not found or initialization fails.
    """
    # Sending the part of the settings
    sendCommand(io, CMD_INIT)

    remotePathBuf = settings.remotePath.encode("utf-8")
    hashNameBuf = settings.hashName.encode("utf-8")

    io.write(struct.pack("<QQQQB", len(remotePathBuf), len(hashNameBuf), settings.bufferSize,
                         settings.chunkSize, settings.zipLevel))
    io.write(remotePathBuf)
    io.write(hashNameBuf)

    # Checking for OK, meaning that device exists, and getting remote device size
    checkCommand(io)
    (remoteSize,) = struct.unpack("<Q", io.read(8))

    return remoteSize


def sendCommand(io, commandCode):
    """Send command code to remote slave script.
    
    Args:
        io: IOCounter object for communication.
        commandCode: Command code constant (CMD_*).
    """
    io.write(struct.pack('<B', commandCode))


def checkCommand(io):
    """Check if command was successfully executed on the slave.
    
    Reads response byte from slave. If it's 0x00, the command succeeded.
    Otherwise, reads error message and raises SlaveError.
    
    Args:
        io: IOCounter object for communication.
        
    Raises:
        SlaveError: If slave returned non-zero status.
    """
    resp = io.read(1)

    (status,) = struct.unpack("<B", resp)
    if status == 0:
        return

    raise SlaveError((resp + io.read()).decode("utf-8"))


class Syncer:
    """Encapsulation of sync and auxiliary methods with required data fields"""

    def __init__(self, settings):
        """Initialize Syncer with settings.
        
        Opens local and remote devices, determines sync size, and validates configuration.
        
        Args:
            settings: Settings object with synchronization parameters.
            
        Raises:
            ActionError: If device validation fails.
        """
        self.settings = settings
        self.startTime = 0

        # Preparing slave, if needed
        self.remoteIO = None
        self.remoteSize = 0
        if self.settings.remotePath:
            self.remoteIO = prepareSlave(self.settings)
            self.remoteSize = initRemoteDevice(self.remoteIO, settings)

        # Preparing local file, if needed
        self.localFile = None
        self.localSize = 0
        if self.settings.localPath:
            self.localFile = open(settings.localPath, 'r+b')
            self.localFile.seek(0,2)
            self.localSize = self.localFile.tell()
            self.localFile.seek(0)

        # Determining size to sync, checking size of devices
        self.syncSize = 0
        if self.localFile and self.remoteIO:
            self.syncSize = min(self.localSize, self.remoteSize)
            self.doSizeCheck()
        elif self.localFile:
            self.syncSize = self.localSize
        elif self.remoteIO:
            self.syncSize = self.remoteSize

        self.bitmapFile = None
        self.bitmapSize = 0
        self.bitmapOffset = 0

    def perform(self):
        """Perform the requested synchronization action.
        
        Executes PUSH/PULL with SYNC/COPY mode, displays statistics if requested,
        and performs recheck if enabled.
        """
        if self.settings.action != "":
            if self.settings.verbose:
                logVerbose('Starting to %s / %s data with buffer size %iB' % (self.settings.action, self.settings.mode,
                                                                              self.settings.bufferSize,))
            self.startTime = getTime()

            if self.settings.bitmapPath:
                self.initBitmap()

            if self.settings.mode == "SYNC":
                if self.bitmapFile and not self.settings.updateBitmap:
                    self.doBSync()
                else:
                    self.doSync()

            elif self.settings.mode == "COPY":
                self.doCopy()

            if self.settings.verbose:
                logVerbose('%s / %s finished' % (self.settings.action, self.settings.mode))

        if self.settings.stat:
            self.displayStat()

        if self.settings.recheck:
            self.doRecheck()

    def __del__(self):
        """Clean up resources: close local file and bitmap file."""
        # Closing local file
        if hasattr(self, 'localFile') and self.localFile:
            self.localFile.close()

        # Closing bitmap
        if hasattr(self, 'bitmapFile') and self.bitmapFile:
            self.bitmapFile.close()

    def initBitmap(self):
        """Initialize bitmap file for hash-based synchronization.
        
        Creates new bitmap file or validates existing one. The bitmap stores
        hashes of device blocks for comparison without reading destination.
        
        Raises:
            ActionError: If bitmap validation fails.
            ValueError: If bitmap parameters don't match current settings.
        """
        MAGIC_NUMBER = 239840023593485091

        hash_id = HASH_FUNCTIONS[self.settings.hashName]
        need_hashes = (self.syncSize + self.settings.bufferSize - 1) // self.settings.bufferSize
        header_format = "<Q Q Q B"  # 8+8+8+1 = 25 bytes

        reset_bitmap = False

        if self.settings.resetBitmap:
            reset_bitmap = True
            if self.settings.verbose:
                logVerbose('Bitmap reset requested - will recreate bitmap')            

        try:
            # Open bitmap file
            self.bitmapFile = open(self.settings.bitmapPath, "r+b")
        except FileNotFoundError:
            # Create file with current settings
            self.bitmapFile = open(self.settings.bitmapPath, "w+b")
            reset_bitmap = True            

        if reset_bitmap:
            hash_block_size = need_hashes * self.settings.digestSize

            header = struct.pack(
                header_format,
                MAGIC_NUMBER,
                self.syncSize,
                self.settings.bufferSize,
                hash_id
            )
            self.bitmapFile.seek(0)
            self.bitmapFile.truncate()
            self.bitmapFile.write(header)
            self.bitmapFile.write(bytes(hash_block_size))
        # At this point bitmap file already exists, correct or not
        self.bitmapFile.seek(0,2)
        bitmapSize = self.bitmapFile.tell()
        self.bitmapFile.seek(0)

        header_size = struct.calcsize(header_format)
        header_data = self.bitmapFile.read(header_size)

        if len(header_data) < header_size:
            raise ActionError("Bitmap file is too small to contain header")

        # Unpack header fields
        magic, file_device_size, file_buffer_size, file_hash_id = struct.unpack(header_format, header_data)

        # Validate magic number
        if magic != MAGIC_NUMBER:
            raise ActionError("Invalid bitmap file")

        # Validate device size
        if file_device_size != self.syncSize:
            raise ActionError(f"Device size mismatch: expected - {self.syncSize}, got in bitmap - {file_device_size}")

        # Validate buffer size
        if file_buffer_size != self.settings.bufferSize:
            raise ValueError(f"Buffer size mismatch: expected {self.settings.bufferSize}, got {file_buffer_size}")

        # Validate hash function
        if file_hash_id != hash_id:
            # TODO change to names
            raise ValueError(f"Hash function mismatch: expected {hash_id}, got {file_hash_id}")

        # Calculate expected hash block size
        expected_hash_size = self.settings.digestSize
        expected_num_hashes = (file_device_size + file_buffer_size - 1) // file_buffer_size
        expected_hash_block_size = expected_num_hashes * expected_hash_size

        # Validate actual file size
        actual_hash_block_size = bitmapSize - header_size
        if actual_hash_block_size != expected_hash_block_size:
            raise ValueError(f"Hash block size mismatch. Bitmap file is damaged")

        # Store offset to first hash for later use
        self.bitmapOffset = header_size

        return True

    def displayStat(self):
        """Displays statistics"""
        usedTime = 0.001 * (getTime() - self.startTime)
        if self.remoteIO:
            bytesRx, bytesTx, bytesTotal = (self.remoteIO.inTotal, self.remoteIO.outTotal, self.remoteIO.inTotal +
                                            self.remoteIO.outTotal)
            ratio = 100.0 * bytesTotal / self.syncSize if self.syncSize != 0 else 100
            realBandwidth = 0.0
            effectiveBandwidth = 0.0
            
            if usedTime > 0:
                realBandwidth = 1.0 * bytesTotal / (1024 * usedTime)
                effectiveBandwidth = 1.0 * self.syncSize / (1024 * usedTime)

            print('STAT. Device size:%iB. Traffic: Rx:%iB, Tx:%iB, Both:%iB Ratio: %.4f%%. Time: %.2fs. \
        Bandwidth: Real:%.2fkB/s, Effective:%.2fkB/s' % (
                    self.syncSize, bytesRx, bytesTx, bytesTotal, ratio, usedTime, realBandwidth, effectiveBandwidth))
        else:
            # No remote IO, show only basic stats
            print('STAT. Device size:%iB. Time: %.2fs.' % (self.syncSize, usedTime))

    def doRecheck(self):
        """Re-check both sides by computing and comparing hashes block-by-block.
        
        Compares hashes of local and remote blocks sequentially, stopping at first mismatch.
        This approach is faster and allows for proper interruption handling.
        
        Raises:
            ActionError: If hashes don't match after synchronization.
        """
        if self.settings.verbose:
            logVerbose('Starting block-by-block re-check')

        self.localFile.seek(0)
        currentPosition = 0
        
        while currentPosition < self.syncSize:
            length = min(self.settings.bufferSize, self.syncSize - currentPosition)

            # Request remote hash for this block
            sendCommand(self.remoteIO, CMD_HASH)
            self.remoteIO.write(struct.pack("<QQ", currentPosition, length))

            # Compute local hash for this block
            localBuf = self.localFile.read(length)
            localDigest = hashlib.new(self.settings.hashName, localBuf).digest()

            # Get remote hash
            checkCommand(self.remoteIO)
            remoteDigest = self.remoteIO.read(self.settings.digestSize)

            # Compare hashes
            if localDigest != remoteDigest:
                if self.settings.verbose:
                    logVerbose('Mismatch found at position %d' % currentPosition)
                    logVerbose('Local hash %s does NOT MATCH remote hash %s' % (
                        byteToHex(localDigest), byteToHex(remoteDigest)))
                elif self.settings.showProgress:
                    print("")
                raise ActionError('Local and remote hashes does NOT MATCH at position %d' % currentPosition)

            currentPosition += length

            # Progress indication
            if self.settings.showProgress:
                logProgress('RECHECK: %.2f%%' % (100.0 * currentPosition / self.syncSize,))

        # All blocks matched
        if self.settings.verbose:
            logVerbose('All blocks matched - devices are identical')
        elif self.settings.showProgress:
            print("")

    def doSizeCheck(self):
        """Check if there is enough space on the receiving device.
        
        Validates that destination device is large enough for the operation.
        
        Raises:
            ActionError: If device sizes are incompatible for the requested action.
        """
        if self.settings.verbose:
            logVerbose('Local device %s size is %iB' % (self.settings.localPath, self.localSize))
            logVerbose('Remote device %s size is %iB' % (self.settings.remotePath, self.remoteSize))

        if min(self.localSize, self.remoteSize) == 0:
            raise ActionError('One of the devices is size 0. There is nothing to sync.')

        if self.settings.action == "PUSH" and self.remoteSize < self.localSize:
            raise ActionError(
                'Cannot PUSH because remote device size (%i) is smaller than local (%i)' % (self.remoteSize,
                                                                                            self.localSize))

        if self.settings.action == "PULL" and self.remoteSize > self.localSize:
            raise ActionError(
                'Cannot PULL because remote device size (%i) is greater than local (%i)' % (self.remoteSize,
                                                                                            self.localSize))

    def doSync(self):
        """Perform SYNC operation: transfer only changed blocks.

        Compares hashes of local and remote blocks, transferring only those that differ.
        If --update-bitmap is set, writes current source hashes into the bitmap file for
        every block processed.
        Supports optional chunk-level comparison for fine-grained transfers.
        """
        FSYNC_TIME_INTERVAL = 5.0  # seconds

        getChunk = lambda n: buf[self.settings.chunkSize * n:min(length, self.settings.chunkSize * (n + 1))]

        self.localFile.seek(0)
        blockIndex = 0
        last_fsync_time = time.time()

        try:
            while self.localFile.tell() < self.syncSize:
                match, buf, currentPosition, length, sourceDigest = self._syncCheckStep()

                # Update bitmap with current source hash (--update-bitmap mode)
                if self.bitmapFile and self.settings.updateBitmap:
                    self.bitmapFile.seek(self.bitmapOffset + blockIndex * self.settings.digestSize)
                    self.bitmapFile.write(sourceDigest)

                # Checking if local and remote hashes match each other, and, if not, transferring the data buffer
                if not match:
                    # Straight syncing with big buffer (no chunks)
                    if self.settings.chunkSize == 0:
                        if self.settings.action == "PUSH":
                            self._syncPushTransfer(buf, currentPosition, length)

                        if self.settings.action == "PULL":
                            self._syncPullTransfer(currentPosition, length)

                    # Syncing with chunks
                    else:
                        chunkCount, localDigestList, remoteDigestList = self._syncAnalyzeChunks(currentPosition, length,
                                                                                                getChunk)
                        # Comparing hashes and writing only necessary small chunks
                        for i in range(0, chunkCount):
                            if localDigestList[i] != remoteDigestList[i]:
                                if self.settings.action == "PUSH":
                                    self.__syncPushTransferChunked(currentPosition, i, getChunk)

                                if self.settings.action == "PULL":
                                    self.__syncPullTransferChunked(currentPosition, i)

                        # In case of PULL we used to write chunks in "random" positions, so we should return to expected pos
                        self.localFile.seek(currentPosition + length)

                blockIndex += 1

                # Periodic fsync of bitmap
                if self.bitmapFile and self.settings.updateBitmap:
                    current_time = time.time()
                    if current_time - last_fsync_time >= FSYNC_TIME_INTERVAL:
                        if self.settings.debug:
                            logDebug(f'Periodic bitmap fsync after {current_time - last_fsync_time:.1f}s')
                        self.bitmapFile.flush()
                        os.fsync(self.bitmapFile.fileno())
                        last_fsync_time = current_time

                # Progress indication
                if self.settings.showProgress:
                    logProgress('SYNC progress: %.2f%%' % (100.0 * currentPosition / self.syncSize,))

        finally:
            if self.bitmapFile and self.settings.updateBitmap:
                if self.settings.debug:
                    logDebug('Final bitmap fsync')
                self.bitmapFile.flush()
                os.fsync(self.bitmapFile.fileno())

    def doBSync(self):
        """Performs synchronization using bitmap file for comparison."""
        getChunk = lambda n: buf[self.settings.chunkSize * n:min(length, self.settings.chunkSize * (n + 1))]

        FSYNC_TIME_INTERVAL = 5.0  # seconds

        # Reset position
        if self.localFile:
            self.localFile.seek(0)

        blockIndex = 0
        currentPosition = 0
        last_fsync_time = time.time()

        try:
            while currentPosition < self.syncSize:
                match, buf, currentPosition, length = self._bsyncCheckStep(blockIndex)

                # Checking if source and bitmap hashes match each other, and, if not, transferring the data buffer
                if not match:
                    # Transfer data if destination device is present
                    if (self.settings.action == "PUSH" and self.settings.remotePath) or \
                       (self.settings.action == "PULL" and self.settings.localPath):

                        # Straight syncing with big buffer (no chunks)
                        if self.settings.chunkSize == 0:
                            if self.settings.action == "PUSH":
                                self._syncPushTransfer(buf, currentPosition, length)

                            if self.settings.action == "PULL":
                                self._syncPullTransfer(currentPosition, length)

                        # Syncing with chunks
                        else:
                            chunkCount, localDigestList, remoteDigestList = self._syncAnalyzeChunks(currentPosition, length,
                                                                                                    getChunk)
                            # Comparing hashes and writing only necessary small chunks
                            for i in range(0, chunkCount):
                                if localDigestList[i] != remoteDigestList[i]:
                                    if self.settings.action == "PUSH":
                                        self.__syncPushTransferChunked(currentPosition, i, getChunk)

                                    if self.settings.action == "PULL":
                                        self.__syncPullTransferChunked(currentPosition, i)

                            # In case of PULL we used to write chunks in "random" positions, so we should return to expected pos
                            if self.settings.action == "PULL" and self.localFile:
                                self.localFile.seek(currentPosition + length)

                # Move to next block
                blockIndex += 1
                currentPosition += length

                # Periodic fsync based on time
                current_time = time.time()
                if current_time - last_fsync_time >= FSYNC_TIME_INTERVAL:
                    if self.settings.debug:
                        logDebug(f'Periodic fsync after {current_time - last_fsync_time:.1f}s')
                    os.fsync(self.bitmapFile.fileno())
                    last_fsync_time = current_time

                # Progress indication
                if self.settings.showProgress:
                    logProgress('BSYNC progress: %.2f%%' % (100.0 * currentPosition / self.syncSize,))

        finally:
            # Final fsync - always, regardless of how we exited (normal, Ctrl+C, exception)
            if self.settings.debug:
                logDebug('Final bitmap fsync')
            self.bitmapFile.flush()
            os.fsync(self.bitmapFile.fileno())

    def doCopy(self):
        """Perform COPY operation: transfer all data block-by-block.

        Copies entire device contents from source to destination with compression.
        """
        # Copying data block-by-block with compression
        self.localFile.seek(0)
        while self.localFile.tell() < self.syncSize:
            currentPosition = self.localFile.tell()
            length = min(self.settings.bufferSize, self.syncSize - currentPosition)

            # To remote device if PUSH
            if self.settings.action == "PUSH":
                buf = self.localFile.read(length)
                self._syncPushTransfer(buf, currentPosition, length)

            # Or from remote device if PULL
            if self.settings.action == "PULL":
                self._syncPullTransfer(currentPosition, length)

            # Progress indication
            if self.settings.showProgress:
                logProgress('COPY progress: %.2f%%' % (100.0 * currentPosition / self.syncSize,))

    def _syncCheckStep(self):
        """Compare hashes of one buffer-sized block on local and remote devices.

        Also returns the source digest (local for PUSH, remote for PULL)
        for optional bitmap update.

        Returns:
            Tuple of (match, buffer, position, length, sourceDigest) where:
                match: True if hashes match
                buffer: local data buffer
                position: current position in device
                length: length of data read
                sourceDigest: hash of source side (local for PUSH, remote for PULL)
        """
        currentPosition = self.localFile.tell()
        length = min(self.settings.bufferSize, self.syncSize - currentPosition)

        # An order for slave to read remote data and compute hash
        sendCommand(self.remoteIO, CMD_HASH)
        self.remoteIO.write(struct.pack("<QQ", currentPosition, length))

        # Reading local data and computing hash
        buf = self.localFile.read(length)
        localDigest = hashlib.new(self.settings.hashName, buf).digest()

        # Waiting for slave to respond and getting remote hash
        checkCommand(self.remoteIO)
        remoteDigest = self.remoteIO.read(self.settings.digestSize)

        # Determine which side is source for bitmap purposes
        if self.settings.action == "PUSH":
            sourceDigest = localDigest
        else:  # PULL
            sourceDigest = remoteDigest

        if self.settings.debug:
            logDebug('Checking pos %d. Local hash is %s, remote is %s' % (currentPosition,
                                                                          byteToHex(localDigest),
                                                                          byteToHex(remoteDigest)))
        return remoteDigest == localDigest, buf, currentPosition, length, sourceDigest

    def _bsyncCheckStep(self, blockIndex):
        """Check step for bitmap sync: compare source hash with bitmap hash.
        
        Reads stored hash from bitmap and compares it with current source hash.
        Updates bitmap if hashes differ.
        
        Args:
            blockIndex: Index of the block to check.
            
        Returns:
            Tuple of (match, buffer, position, length) where:
                match: True if hashes match
                buffer: source data buffer (or None for PULL)
                position: current position in device
                length: length of data
        """

        # For bitmap sync, always calculate position from blockIndex
        # (destination file is not read, so tell() would be unreliable)
        currentPosition = blockIndex * self.settings.bufferSize
        length = min(self.settings.bufferSize, self.syncSize - currentPosition)

        # Calculate hash position in bitmap file
        bitmapHashPosition = self.bitmapOffset + blockIndex * self.settings.digestSize

        # Read existing hash from bitmap
        self.bitmapFile.seek(bitmapHashPosition)
        storedDigest = self.bitmapFile.read(self.settings.digestSize)

        # Get source data and compute its hash
        if self.settings.action == "PUSH":
            # Read from local device
            buf = self.localFile.read(length)
            sourceDigest = hashlib.new(self.settings.hashName, buf).digest()

        elif self.settings.action == "PULL":
            # Request hash from remote device
            sendCommand(self.remoteIO, CMD_HASH)
            self.remoteIO.write(struct.pack("<QQ", currentPosition, length))
            checkCommand(self.remoteIO)
            sourceDigest = self.remoteIO.read(self.settings.digestSize)
            buf = None  # For PULL, buf will be fetched later if needed

        # Compare source hash with stored hash
        match = (sourceDigest == storedDigest)

        if self.settings.debug:
            logDebug('BSync block %d at pos %d: stored hash %s, source hash %s, match=%s' % (
                blockIndex, currentPosition, byteToHex(storedDigest), 
                byteToHex(sourceDigest), match))

        # Update bitmap with new hash if they don't match
        if not match:
            self.bitmapFile.seek(bitmapHashPosition)
            self.bitmapFile.write(sourceDigest)
            self.bitmapFile.flush()

        return match, buf, currentPosition, length

    def _syncPushTransfer(self, buf, currentPosition, length):
        """Transfer buffer from local to remote device (PUSH).
        
        Args:
            buf: Data buffer to transfer.
            currentPosition: Position in device to write to.
            length: Length of data.
        """
        # Compressing buffer
        compBuffer = zlib.compress(buf, self.settings.zipLevel)
        compLength = len(compBuffer)

        # Writing it to remote
        sendCommand(self.remoteIO, CMD_WRITE)
        self.remoteIO.write(struct.pack("<QQ", currentPosition, compLength))
        self.remoteIO.write(compBuffer)
        checkCommand(self.remoteIO)

        # Debug logging
        if self.settings.debug:
            logDebug('Transferred local->remote raw:%iB, comp:%iB, pos:%i' % (
                length, compLength, currentPosition))

    def _syncPullTransfer(self, currentPosition, length):
        """Transfer buffer from remote to local device (PULL).
        
        Args:
            currentPosition: Position in device to read from.
            length: Length of data to read.
        """
        # Reading compressed data from remote
        sendCommand(self.remoteIO, CMD_READ)
        self.remoteIO.write(struct.pack("<QQ", currentPosition, length))
        checkCommand(self.remoteIO)
        (compLength,) = struct.unpack('<Q', self.remoteIO.read(8))
        compBuffer = self.remoteIO.read(compLength)

        # Writing it to local
        self.localFile.seek(currentPosition)
        self.localFile.write(zlib.decompress(compBuffer))
        self.localFile.seek(currentPosition + length)

        # Debug logging
        if self.settings.debug:
            logDebug('Transferred remote->local raw:%iB, comp:%iB, pos:%i' % (
                length, compLength, currentPosition))

    def _syncAnalyzeChunks(self, currentPosition, length, getChunkLambda):
        """Analyze chunks: compute and compare hashes for each chunk in buffer.
        
        Args:
            currentPosition: Starting position of buffer.
            length: Total length of buffer.
            getChunkLambda: Lambda function to extract chunk by index.
            
        Returns:
            Tuple of (chunkCount, localDigestList, remoteDigestList).
        """
        #getChunk = lambda n: buf[self.settings.chunkSize * n:min(length, self.settings.chunkSize * (n + 1))]

        chunkCount = int(length / self.settings.chunkSize)
        if length % self.settings.chunkSize > 0:
            chunkCount += 1

        if self.settings.debug:
            logDebug('Splitting buffer at %i with length %i for %i chunks for detailed check' % (
                currentPosition, length, chunkCount))

        # An order for slave to start computing its hashes for every chunk. It takes time.
        sendCommand(self.remoteIO, CMD_SUBHASH)

        # Meanwhile, we can calculate our own
        localDigestList = [hashlib.new(self.settings.hashName, getChunkLambda(i)).digest()
                           for i in range(chunkCount)]

        # Reading remote hashes, they supposed to be ready by now
        checkCommand(self.remoteIO)
        remoteDigestList = [self.remoteIO.read(self.settings.digestSize) for i in range(chunkCount)]

        return chunkCount, localDigestList, remoteDigestList

    def __syncPushTransferChunked(self, currentPosition, chunkIndex, getChunkLambda):
        """Transfer single chunk from local to remote device (PUSH).
        
        Args:
            currentPosition: Starting position of buffer.
            chunkIndex: Index of chunk to transfer.
            getChunkLambda: Lambda function to extract chunk by index.
        """
        # Getting compressed chunk local data
        chunkPosition = currentPosition + self.settings.chunkSize * chunkIndex
        chunk = getChunkLambda(chunkIndex)
        compChunk = zlib.compress(chunk, self.settings.zipLevel)
        compChunkLength = len(compChunk)

        # Writing chunk to the remote device
        sendCommand(self.remoteIO, CMD_WRITE)
        self.remoteIO.write(struct.pack("<QQ", chunkPosition, compChunkLength))
        self.remoteIO.write(compChunk)
        checkCommand(self.remoteIO)

        # Debug logging
        if self.settings.debug:
            logDebug('Transferred local->remote chunk #%i (raw:%iB, comp:%iB, pos:%i)' % (
                chunkIndex, len(chunk), compChunkLength, chunkPosition))

    def __syncPullTransferChunked(self, currentPosition, chunkIndex):
        """Transfer single chunk from remote to local device (PULL).
        
        Args:
            currentPosition: Starting position of buffer.
            chunkIndex: Index of chunk to transfer.
        """
        chunkPosition = currentPosition + self.settings.chunkSize * chunkIndex

        # Reading chunk from remote
        sendCommand(self.remoteIO, CMD_SUBREAD)
        self.remoteIO.write(struct.pack("<Q", chunkIndex))
        checkCommand(self.remoteIO)
        (compChunkLength,) = struct.unpack('<Q', self.remoteIO.read(8))
        compBuffer = self.remoteIO.read(compChunkLength)
        chunk = zlib.decompress(compBuffer)

        # Writing it locally
        self.localFile.seek(chunkPosition)
        self.localFile.write(chunk)
        self.localFile.seek(chunkPosition + len(chunk))

        # Debug logging
        if self.settings.debug:
            logDebug('Transferred remote->local chunk #%i (raw:%iB, comp:%iB, pos:%i)' % (
                chunkIndex, len(chunk), compChunkLength, chunkPosition))


def masterMode():
    """Main entry point for master mode.
    
    Parses settings, creates Syncer instance, and performs requested operation.
    Handles exceptions and sets appropriate exit codes.
    """
    settings = Settings()
    try:
        # Parsing config
        settings.parseArguments()

        syncer = Syncer(settings)
        try:
            syncer.perform()
        finally:
            del syncer

    except SettingsParseError as e:
        displayAbout()
        print(str(e))
        settings.displayHelp()
        exit(1)

    except IOError as e:
        print("I/O error: " + str(e))
        exit(2)

    except ActionError as e:
        print(str(e))
        exit(2)

    except SlaveError as e:
        print("Error at remote side: " + str(e))
        exit(2)

    except KeyboardInterrupt:
        print("\nInterrupted by user")
        exit(2)

    exit(0)


if __name__ == "__main__":
    masterMode()