1+ import logging
12import os
23import time
34import requests
45from concurrent .futures import ThreadPoolExecutor
56
7+ logger = logging .getLogger (__name__ )
8+ logging .basicConfig (level = logging .INFO )
9+
610def process_file (client , file_path , output_folder , workflow_id , delete = True , failed_file = None ):
711 # Upload local file
812 file_url = client .upload_file (file_path = file_path )
@@ -13,15 +17,17 @@ def process_file(client, file_path, output_folder, workflow_id, delete=True, fai
1317 }
1418 create_job_info = client .create_job (job_name = os .path .basename (file_path ), workflow_id = workflow_id , params = workflow_params )
1519 job_id = create_job_info ['id' ]
16- print ('Job Created:' , job_id )
20+ file_name = os .path .basename (file_path )
21+ logger .info (f"Job Created for { file_name } with id { job_id } " )
22+
1723
1824 # Polling for job results
1925 while True :
2026 job_info = client .get_job (job_id = job_id )
2127 if job_info ['status' ] == 'SUCCEEDED' :
2228 break
2329 if job_info ['status' ] == 'FAILED' :
24- print ( ' Job failed' )
30+ logger . error ( f" Job for { file_name } failed" )
2531 # create failed.json file if it does not exist
2632 if not os .path .exists (failed_file ):
2733 with open (failed_file , 'w' ) as f :
@@ -30,10 +36,10 @@ def process_file(client, file_path, output_folder, workflow_id, delete=True, fai
3036 with open (failed_file , 'a' ) as f :
3137 f .write (file_path + '\n ' )
3238 break
33- print ( 'Job not ready yet. Waiting for 5 seconds...' )
39+ logger . info ( f"Waiting for job completion for { file_name } " )
3440 time .sleep (5 )
3541
36- print ( ' Job Status:' , job_info ['status' ])
42+ logger . info ( f" Job completed with status { job_info ['status' ]} for { file_name } " )
3743
3844 # Download the result
3945 result = job_info ['result' ]
@@ -52,10 +58,10 @@ def process_file(client, file_path, output_folder, workflow_id, delete=True, fai
5258 with open (output_path , 'wb' ) as f :
5359 f .write (response .content )
5460
55- print ( ' Job Result downloaded.' )
61+ logger . info ( f" Job result downloaded for { file_name } " )
5662 if delete :
5763 client .delete_job (job_id = job_id )
58- print ( 'File deleted.' )
64+ logger . info ( f"Job for { file_name } deleted from server" )
5965
6066def process_folder (input_folder , output_folder , workflow_id , parallelism = 5 , delete = True , client = None ):
6167 # add failed.json file to output_folder to control all failed files
0 commit comments