@@ -18,6 +18,7 @@ const folders = require('../library/folder-setup');
1818const escape = require ( 'escape-html' ) ;
1919
2020const Logger = require ( '../library/logger' ) ;
21+ const { describeCron} = require ( "../library/cron-utilities" ) ;
2122const xigLog = Logger . getInstance ( ) . child ( { module : 'xig' } ) ;
2223
2324const router = express . Router ( ) ;
@@ -1500,6 +1501,9 @@ function getMetadata(key) {
15001501function downloadFile ( url , destination , maxRedirects = 5 ) {
15011502 return new Promise ( ( resolve , reject ) => {
15021503 xigLog . info ( `Starting download from ${ url } ` ) ;
1504+ if ( globalStats ) {
1505+ globalStats . task ( 'XIG Download' , `Downloading from ${ url } ` )
1506+ }
15031507 const downloadMeta = {
15041508 url : url ,
15051509 finalUrl : url ,
@@ -1545,6 +1549,9 @@ function downloadFile(url, destination, maxRedirects = 5) {
15451549 }
15461550
15471551 if ( response . statusCode !== 200 ) {
1552+ if ( globalStats ) {
1553+ globalStats . task ( 'XIG Download' , `Download failed: ${ response . statusCode } ` )
1554+ }
15481555 reject ( Object . assign (
15491556 new Error ( `Download failed with HTTP ${ response . statusCode } ` ) ,
15501557 { downloadMeta }
@@ -1555,6 +1562,9 @@ function downloadFile(url, destination, maxRedirects = 5) {
15551562 // Check content length
15561563 const maxSize = 10 * 1024 * 1024 * 1024 ; // 10GB limit
15571564 if ( downloadMeta . contentLength && downloadMeta . contentLength > maxSize ) {
1565+ if ( globalStats ) {
1566+ globalStats . task ( 'XIG Download' , `Download failed: too large` )
1567+ }
15581568 reject ( Object . assign ( new Error ( 'File too large' ) , { downloadMeta } ) ) ;
15591569 return ;
15601570 }
@@ -1565,6 +1575,9 @@ function downloadFile(url, destination, maxRedirects = 5) {
15651575 downloadMeta . downloadedBytes += chunk . length ;
15661576 if ( downloadMeta . downloadedBytes > maxSize ) {
15671577 request . destroy ( ) ;
1578+ if ( globalStats ) {
1579+ globalStats . task ( 'XIG Download' , `Download failed: file too large` ) ;
1580+ }
15681581 fs . unlink ( destination , ( ) => { } ) ; // Clean up
15691582 reject ( Object . assign ( new Error ( 'File too large' ) , { downloadMeta } ) ) ;
15701583 return ;
@@ -1577,21 +1590,33 @@ function downloadFile(url, destination, maxRedirects = 5) {
15771590 fileStream . close ( ) ;
15781591 downloadMeta . durationMs = Date . now ( ) - downloadMeta . startTime ;
15791592 xigLog . info ( `Download completed successfully. Downloaded ${ downloadMeta . downloadedBytes } bytes to ${ destination } ` ) ;
1593+ if ( globalStats ) {
1594+ globalStats . task ( 'XIG Download' , `Downloaded ${ downloadMeta . downloadedBytes } bytes to ${ destination } ` ) ;
1595+ }
15801596 resolve ( downloadMeta ) ;
15811597 } ) ;
15821598
15831599 fileStream . on ( 'error' , ( err ) => {
1600+ if ( globalStats ) {
1601+ globalStats . task ( 'XIG Download' , `Download failed` ) ;
1602+ }
15841603 fs . unlink ( destination , ( ) => { } ) ; // Delete partial file
15851604 reject ( Object . assign ( err , { downloadMeta } ) ) ;
15861605 } ) ;
15871606 } ) ;
15881607
15891608 request . on ( 'error' , ( err ) => {
1609+ if ( globalStats ) {
1610+ globalStats . task ( 'XIG Download' , `Download Error` ) ;
1611+ }
15901612 reject ( Object . assign ( err , { downloadMeta } ) ) ;
15911613 } ) ;
15921614
15931615 request . setTimeout ( 300000 , ( ) => { // 5 minutes timeout
15941616 request . destroy ( ) ;
1617+ if ( globalStats ) {
1618+ globalStats . task ( 'XIG Download' , `Download Timeout` ) ;
1619+ }
15951620 reject ( Object . assign ( new Error ( 'Download timeout after 5 minutes' ) , { downloadMeta } ) ) ;
15961621 } ) ;
15971622
@@ -2918,6 +2943,9 @@ async function initializeXigModule(stats) {
29182943 } , 5000 ) ;
29192944 }
29202945
2946+ if ( globalStats ) {
2947+ globalStats . addTask ( 'XIG Download' , describeCron ( this . config . crawler . schedule ) ) ;
2948+ }
29212949 // Check if auto-update is enabled
29222950 // Note: This assumes we're called only when XIG is enabled
29232951 cron . schedule ( '0 2 * * *' , ( ) => {
0 commit comments