main.py

from src.support_ticket.utils.utils import load_config
from src.support_ticket.utils.utils import get_logger
from src.support_ticket.utils.utils import load_json
from stream import stream
from single_stream import single_stream

def main():
    config_path = "config.yaml"
    config = load_config(config_path)
    logger = get_logger(config)    
    
    if config["Process"]["Stream"]:
        logger.info("Streamning Data ... ")

        input_query_path = config["Data"]["ticket_test"]
        ticket_data = load_json(input_query_path)

        batch_size = config["Process"]["Stream_batch"]
        num_batches = len(ticket_data) // batch_size

        for i in range(num_batches + 1):
            start_index = i * batch_size
            end_index = start_index + batch_size
            batch = ticket_data[start_index:end_index]
            print("batch data size: ", len(batch))
            logger.info(f"Streamning Batch: {i}")
            # Only process the ticket if batch is not empty
            if batch:
                stream(batch)
            break
    else:
        single_stream()

if __name__ == "__main__":
    main()