-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathcontent.js
More file actions
2157 lines (1908 loc) · 105 KB
/
content.js
File metadata and controls
2157 lines (1908 loc) · 105 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*
===========================================================
TABLE OF CONTENTS — LOGICAL CHUNKS
===========================================================
Chunk 1 – Top-Level Setup
Chunk 2 – Constructor
Chunk 3 – Logging Helpers
Chunk 4 – Scrolling Logic
Chunk 5 – Period Key + Telegram
Chunk 6 – Scraping
Chunk 7 – Reward Helper Functions
Chunk 8 – Locking
Chunk 9 – ReloadGuard
Chunk 10 – Daily Pre-Send Checks
Chunk 11 – computeRewardsSinceBaseline
Chunk 12 – getDailySummary
Chunk 13 – Daily Scheduler
Chunk 14 – checkAndNotify
Chunk 15 – previousValues persistence
Chunk 16 – Lifecycle
Chunk 17 – Interim Summary
Chunk 18 – Full Daily Summary
===========================================================
NOTES FOR LLM ASSISTANTS
-----------------------------------------------------------
Use the table of contents to understand the overall layout.
Each chunk is marked with "START OF CHUNK X" and
"END OF CHUNK X" comments in the file.
You may request a chunk by number for clarity.
===========================================================
*/
// ===========================================================
// START OF CHUNK 1 — Top-Level Setup
// ===========================================================
const ITERATION = 'Iteration 21.4';
console.log(`Initializing monitor — ${ITERATION}`);
const sleep = ms => new Promise(r => setTimeout(r, ms));
// Improved auto-scroller: requires the page height to remain stable for a
// number of consecutive checks before resolving. This helps with slow or
// incremental lazy-loading where elements append after short delays.
class ValueMonitor {
// ===========================================================
// END OF CHUNK 1 — Top-Level Setup
// ===========================================================
// ===========================================================
// START OF CHUNK 2 — Constructor
// ===========================================================
constructor() {
// config/state
this.telegramToken = '';
this.chatId = '';
this.previousValues = null;
this.checkInterval = null;
this._dailyTimerId = null;
this.isChecking = false;
// identity/keys/timeouts
this._instanceId = Math.random().toString(36).slice(2);
this._dailyLockKey = 'dailyLock';
this._dailyStatsKey = 'dailyStats';
this._dailyBaselineKey = 'dailyBaseline'; // CRITICAL: Dedicated daily baseline (single source of truth for "today")
this._lastSuccessfulKey = 'lastSuccessfulDailyReport';
this._dailyLockTimeoutMs = 2 * 60 * 1000;
this._lastDailySentCooldownKey = 'lastDailySentCooldown';
// new keys/guards
this._dailyPlannedKey = 'dailyPlanned';
this._lastDailySentKey = 'lastDailySentAt';
this._dailyLockBaseKey = 'dailyLock';
this._dailyLockHoldMs = 3 * 60 * 1000; // hold daily lock for 3 minutes
// processing lock to avoid race between periodic checks and daily summary
this._processingLockKey = 'processingLock';
this._processingLockTimeoutMs = 2 * 60 * 1000; // 2 minutes
this._dailyMaxPreSendRetries = 5;
this._dailyPreSendBaseBackoffMs = 300;
this._dailyScheduleJitterMs = 10 * 1000; // reduced jitter ±10s
this._defaultFallbackHours = 48;
this.notifySummaryMode = false;
this._telegramMaxMessageChars = 4000;
this._suspiciousDeltaLimit = 200;
this._tempBaselineKey = 'tempDailyBaseline';
this._cumulativePeriodicKey = 'cumulativePeriodicRewards';
this._lastDailyResetKey = 'lastDailyResetKey';
// New for accumulation/correction (primary: accumulation, fallback: re-calc)
this._accumulatedRewardsKey = 'accumulatedRewardsToday';
this._mismatchTolerance = 2; // Points for flagging validation mismatches (accumulate vs. re-calc)
this._maxRecoveryRetries = 2; // Cap correction attempts
this._lastAccumulationResetKey = 'lastAccumulationResetDay'; // For double-send protection
}
// ===========================================================
// END OF CHUNK 2 — Constructor
// ===========================================================
// ===========================================================
// START OF CHUNK 3 — Logging Helpers
// ===========================================================
// logging shorthands (preserve outputs)
log(...a){ console.log(...a); }
warn(...a){ console.warn(...a); }
error(...a){ console.error(...a); }
// DEBUG: Print current baseline state to console
async debugDailyBaseline() {
const baseline = await new Promise(res => chrome.storage.local.get([this._dailyBaselineKey], r => res(r?.[this._dailyBaselineKey] || null)));
const currentDay = await this.getReportBasedDayKey();
const currentValues = await this.getCurrentValues();
console.log('=== DAILY BASELINE DIAGNOSTIC ===');
console.log('Current Report Day:', currentDay);
console.log('Baseline exists:', !!baseline);
if (baseline) {
console.log('Baseline dayKey:', baseline.dayKey);
console.log('Baseline matches current day:', baseline.dayKey === currentDay);
console.log('Baseline models count:', Object.keys(baseline.models || {}).length);
console.log('Baseline points:', baseline.points);
console.log('Baseline timestamp:', new Date(baseline.timestamp).toLocaleString());
}
console.log('Current models count:', Object.keys(currentValues?.models || {}).length);
console.log('Current points:', currentValues?.points);
console.log('===================================');
}
// DEBUG: Force a fresh daily summary computation and log results
async debugDailySummaryNow() {
console.log('>>> MANUAL DAILY SUMMARY COMPUTATION <<<');
const summary = await this.computeRewardsSinceBaseline();
console.log('Summary Result:', {
dailyDownloads: summary.dailyDownloads,
dailyPrints: summary.dailyPrints,
dailyBoosts: summary.dailyBoosts,
rewardPointsTotal: summary.rewardPointsTotal,
pointsGained: summary.pointsGained,
modelsWithChanges: Object.keys(summary.modelChanges).length,
rewardsEarned: summary.rewardsEarned.length
});
if (summary.modelChanges && Object.keys(summary.modelChanges).length > 0) {
console.log('Models with activity today:', Object.values(summary.modelChanges).slice(0, 5).map(m => ({
name: m.name,
downloads: m.downloadsGained,
prints: m.printsGained,
boosts: m.boostsGained
})));
}
console.log('<<< END MANUAL COMPUTATION >>>');
return summary;
}
// DEBUG: Reset daily baseline to force re-creation (for testing purposes)
async debugResetDailyBaseline() {
console.log('⚠️ RESETTING DAILY BASELINE FOR TESTING');
const currentValues = await this.getCurrentValues();
if (currentValues) {
const currentDay = await this.getReportBasedDayKey();
const dailyBaseline = {
models: currentValues.models || {},
points: currentValues.points || 0,
timestamp: Date.now(),
dayKey: currentDay
};
await new Promise(res => chrome.storage.local.set({ [this._dailyBaselineKey]: dailyBaseline }, res));
console.log('✓ Baseline reset to current state for day:', currentDay);
console.log(' Models:', Object.keys(dailyBaseline.models).length);
console.log(' Points:', dailyBaseline.points);
console.log(' Now any activity on your models will be captured on the next dailySummary!');
}
}
// ===========================================================
// END OF CHUNK 3 — Logging Helpers
// ===========================================================
// ===========================================================
// START OF CHUNK 4 — Scrolling Logic
// ===========================================================
async autoScrollToFullBottom() {
const BASE_DELAY_MS = 600;
const MAX_LOOPS = 10;
const REQUIRED_STABLE = 3;
const MAX_RETRIES = 2;
let attempt = 0;
while (attempt <= MAX_RETRIES) {
let lastHeight = 0;
let stableCount = 0;
let loopCount = 0;
while (loopCount < MAX_LOOPS) {
// Lazy-load nudge
window.scrollTo(0, document.body.scrollHeight - 300);
await sleep(200);
// Full bottom
window.scrollTo(0, document.body.scrollHeight);
await sleep(BASE_DELAY_MS);
const newHeight = document.body.scrollHeight;
if (newHeight === lastHeight) {
stableCount++;
} else {
stableCount = 0;
}
if (stableCount >= REQUIRED_STABLE) {
break;
}
lastHeight = newHeight;
loopCount++;
}
if (loopCount >= MAX_LOOPS) {
console.warn('autoScroll: reached MAX_LOOPS without stabilizing scroll height.');
}
// Settling delay
await sleep(800);
// Validate model count
const currentModelEls = document.querySelectorAll('[data-trackid]');
const currentModelCount = currentModelEls.length;
const previousModelCount = Object.keys(this.previousValues?.models || {}).length;
const drop = previousModelCount - currentModelCount;
if (drop >= 2 && attempt < MAX_RETRIES) {
console.warn(`autoScroll: model count dropped by ${drop}; retrying (${attempt + 1}/2).`);
attempt++;
continue;
} else if (drop < 2 && attempt > 0) {
console.log('autoScroll: model count recovered after retry.');
} else if (drop >= 2 && attempt === MAX_RETRIES) {
console.warn('autoScroll: model count still low after retries; proceeding with scrape.');
}
break;
}
}
// ===========================================================
// END OF CHUNK 4 — Scrolling Logic
// ===========================================================
// ===========================================================
// START OF CHUNK 5 — Period Key + Telegram
// ===========================================================
// compute the MakerWorld day key based on daily report time
// if current time is BEFORE today's report time, we are in yesterday's day
// if current time is AFTER today's report time, we are in today's new day
async getReportBasedDayKey() {
const cfg = await new Promise(res => chrome.storage.sync.get(['dailyNotificationTime'], r =>
res(r && r.dailyNotificationTime ? r.dailyNotificationTime : '12:00')));
const [hourStr, minuteStr] = String(cfg).split(':');
const hour = Number.isFinite(Number(hourStr)) ? Number(hourStr) : 12;
const minute = Number.isFinite(Number(minuteStr)) ? Number(minuteStr) : 0;
const now = new Date();
const candidate = new Date(now.getFullYear(), now.getMonth(), now.getDate(), hour, minute, 0, 0);
if (candidate > now) candidate.setDate(candidate.getDate() - 1);
const pad = n => String(n).padStart(2,'0');
return `${candidate.getFullYear()}-${pad(candidate.getMonth()+1)}-${pad(candidate.getDate())}`;
}
// reset derived daily state if the MakerWorld day has changed
async resetDailyStateIfNeeded() {
const currentDayKey = await this.getReportBasedDayKey();
const lastResetDayKey = await new Promise(res => chrome.storage.local.get([this._lastDailyResetKey], r => res(r?.[this._lastDailyResetKey] || null)));
this.log(`DIAGNOSTIC [resetDailyStateIfNeeded]: currentDayKey: ${currentDayKey}, lastResetDayKey: ${lastResetDayKey}, match: ${lastResetDayKey === currentDayKey}`);
if (lastResetDayKey === currentDayKey) {
this.log(`DIAGNOSTIC [resetDailyStateIfNeeded]: Already reset for this day, skipping`);
return; // no-op: already reset for this day
}
// Day has changed: clear periodic accumulators and previousValues, establish new daily baseline
this.log(`Daily state rollover: previous day was ${lastResetDayKey}, now ${currentDayKey}`);
this.log(`DIAGNOSTIC [resetDailyStateIfNeeded]: ⚠️ DAY BOUNDARY DETECTED - Establishing fresh baseline!`);
// CRITICAL: Capture current values as the new daily baseline at day boundary
const currentValues = await this.getCurrentValues();
if (currentValues) {
const dailyBaseline = {
models: currentValues.models || {},
points: currentValues.points || 0,
timestamp: Date.now(),
dayKey: currentDayKey // Store day marker in baseline itself
};
await new Promise(res => chrome.storage.local.set({ [this._dailyBaselineKey]: dailyBaseline }, res));
this.log(`Daily baseline established for ${currentDayKey}: ${Object.keys(dailyBaseline.models || {}).length} models, ${dailyBaseline.points} points`);
this.log(`DIAGNOSTIC [resetDailyStateIfNeeded]: ✓ Baseline stored to ${this._dailyBaselineKey}`);
} else {
this.log(`DIAGNOSTIC [resetDailyStateIfNeeded]: ⚠️ Could not get currentValues to establish baseline!`);
}
const keysToRemove = [
'previousValues', // Clear to start fresh periodic baseline each day
this._cumulativePeriodicKey // Clear cumulative periodic rewards
// Do NOT remove _tempBaselineKey, _dailyStatsKey, or locks - owned by daily summary
];
await new Promise(res => chrome.storage.local.remove(keysToRemove, res));
// Reset accumulated rewards to 0 for new day
await new Promise(res => chrome.storage.local.set({ [this._accumulatedRewardsKey]: 0 }, res));
this.log('Daily reset: cleared previousValues and accumulators for new day.');
// Store the new day key
await new Promise(res => chrome.storage.local.set({ [this._lastDailyResetKey]: currentDayKey }, res));
}
// period key uses user's dailyNotificationTime or 12:00 default
async getCurrentPeriodKey() {
const cfg = await new Promise(res => chrome.storage.sync.get(['dailyNotificationTime'], r =>
res(r && r.dailyNotificationTime ? r.dailyNotificationTime : '12:00')));
const [hourStr, minuteStr] = String(cfg).split(':');
const hour = Number.isFinite(Number(hourStr)) ? Number(hourStr) : 12;
const minute = Number.isFinite(Number(minuteStr)) ? Number(minuteStr) : 0;
const now = new Date();
const candidate = new Date(now.getFullYear(), now.getMonth(), now.getDate(), hour, minute, 0, 0);
if (candidate > now) candidate.setDate(candidate.getDate() - 1);
const pad = n => String(n).padStart(2,'0');
const offset = -candidate.getTimezoneOffset();
const sign = offset >= 0 ? '+' : '-';
const offsetHours = pad(Math.floor(Math.abs(offset)/60));
const offsetMins = pad(Math.abs(offset)%60);
return `${candidate.getFullYear()}-${pad(candidate.getMonth()+1)}-${pad(candidate.getDate())}T${pad(candidate.getHours())}:${pad(candidate.getMinutes())}:00${sign}${offsetHours}:${offsetMins}`;
}
// split large telegram messages into parts keeping paragraphs
_splitMessageIntoParts(message='', maxLen=this._telegramMaxMessageChars) {
if (!message) return [];
if (message.length <= maxLen) return [message];
const parts=[]; const paragraphs = message.split('\n\n'); let current='';
for (const p of paragraphs) {
const chunk = (current ? '\n\n' : '') + p;
if ((current + chunk).length > maxLen) {
if (current) { parts.push(current); current = p; if (current.length > maxLen) { let s=0; while (s < current.length){ parts.push(current.slice(s, s+maxLen)); s+=maxLen;} current=''; } }
else { let s=0; while (s < p.length){ parts.push(p.slice(s, s+maxLen)); s+=maxLen; } current=''; }
} else current += chunk;
}
if (current) parts.push(current);
return parts;
}
// Telegram send helpers with one retry
async sendTelegramMessage(message, attempt=1) {
if (!this.telegramToken || !this.chatId) { this.error('Missing Token or Chat ID'); return false; }
let parts = this._splitMessageIntoParts(message, this._telegramMaxMessageChars);
if (parts.length > 1) {
parts = parts.map((part, i) => `Part ${i + 1} of ${parts.length}\n\n${part}`);
}
for (const part of parts) {
const payload = { chat_id: this.chatId, text: part, parse_mode: 'HTML' };
this.log('→ Telegram payload (part):', { len: part.length });
try {
const res = await fetch(`https://api.telegram.org/bot${this.telegramToken}/sendMessage`, { method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify(payload) });
const body = await res.json();
if (!res.ok) {
this.error('← Telegram API error:', body);
if (attempt < 2) { this.log('Retrying Telegram send...'); await new Promise(r=>setTimeout(r,1000)); return this.sendTelegramMessage(message, attempt+1); }
return false;
}
this.log('← Telegram API ok:', body);
} catch (err) {
this.error('Error sending message:', err);
if (attempt < 2) { this.log('Retrying Telegram send...'); await new Promise(r=>setTimeout(r,1000)); return this.sendTelegramMessage(message, attempt+1); }
return false;
}
await new Promise(r=>setTimeout(r,200));
}
return true;
}
async sendTelegramMessageWithPhoto(message, photoUrl) {
if (!this.telegramToken || !this.chatId || !photoUrl) { this.log('Falling back to text message (missing token/chat/photo).'); return this.sendTelegramMessage(message); }
try {
this.log('Attempting to send photo:', { photoUrl, chatId: this.chatId });
const imgRes = await fetch(photoUrl);
if (!imgRes.ok) throw new Error(`Image download failed: ${imgRes.status}`);
const blob = await imgRes.blob();
const form = new FormData(); form.append('chat_id', this.chatId); form.append('caption', message); form.append('photo', blob, 'model_image.jpg');
const res = await fetch(`https://api.telegram.org/bot${this.telegramToken}/sendPhoto`, { method:'POST', body: form });
const result = await res.json();
this.log('Telegram response:', result);
if (!res.ok) throw new Error(`Telegram Error: ${res.status}`);
return true;
} catch (err) {
this.error('Error sending photo:', err);
return this.sendTelegramMessage(message);
}
}
// ===========================================================
// END OF CHUNK 5 — Period Key + Telegram
// ===========================================================
// ===========================================================
// START OF CHUNK 6 — Scraping
// ===========================================================
// scraping/parsing
parseNumber(text){ if (!text) return 0; text = String(text).trim().toLowerCase(); if (text.includes('k')){ const base = parseFloat(text.replace('k','')); if (Number.isFinite(base)) return Math.round(base*1000); } const n = parseInt(text.replace(/[^\d]/g,''),10); return Number.isFinite(n)? n:0; }
async getCurrentValues() {
try {
const currentValues = { models: {}, points: 0, timestamp: Date.now() };
try {
const pointsContainer = document.querySelector('.mw-css-1541sxf');
this.log('Found points container:', !!pointsContainer);
if (pointsContainer) {
const pts = pointsContainer.textContent.trim().match(/[\d,]+(\.\d+)?/);
if (pts && pts[0]) { currentValues.points = parseFloat(pts[0].replace(/,/g,'')); this.log('Points found:', currentValues.points); }
}
} catch (e){ this.error('Error extracting points:', e); }
const downloadElements = document.querySelectorAll('[data-trackid]');
this.log(`[DIAGNOSTIC] Found ${downloadElements.length} elements with [data-trackid]`);
if (downloadElements.length === 0) {
this.warn('[DIAGNOSTIC] No elements found with [data-trackid] - page may have changed structure');
this.log('[DIAGNOSTIC] Page HTML (first 2000 chars):', document.body.innerHTML.substring(0, 2000));
}
downloadElements.forEach((element, index) => {
const modelId = element.getAttribute('data-trackid');
const modelTitle = element.querySelector('h3.translated-text');
const name = modelTitle?.textContent.trim() || 'Model';
const imageUrl = element.querySelector('img')?.getAttribute('src') || '';
// Detect exclusive badge via SVG color
const isExclusive = !!element.querySelector(
'.design-icons-box svg path[fill="#B1FF42"]'
);
let permalink = null;
const anchor = element.querySelector('a[href*="/models/"], a[href*="/model/"], a[href*="/models/"]');
if (anchor?.href) permalink = anchor.href;
// DIAGNOSTIC: Check what selectors actually find
const allMetrics = element.querySelectorAll('.mw-css-xlgty3 span');
this.log(`[DIAGNOSTIC] Model ${index} (${name}): found ${allMetrics.length} metrics with .mw-css-xlgty3 span`);
if (allMetrics.length === 0) {
// Try alternative selectors to find the metrics
this.log(`[DIAGNOSTIC] No metrics found for "${name}", trying alternatives...`);
const allSpans = element.querySelectorAll('span');
this.log(`[DIAGNOSTIC] Total spans in element: ${allSpans.length}`);
allSpans.forEach((span, i) => {
if (i < 20) { // Log first 20 spans
this.log(`[DIAGNOSTIC] Span ${i}: class="${span.className}" text="${span.textContent.substring(0, 50)}"`);
}
});
// Log the entire element structure for first model only
if (index === 0) {
this.log('[DIAGNOSTIC] Full element HTML:', element.outerHTML.substring(0, 1000));
}
}
if (allMetrics.length >= 3) {
const lastThree = Array.from(allMetrics).slice(-3);
const boosts = this.parseNumber(lastThree[0]?.textContent || '0');
const downloads = this.parseNumber(lastThree[1]?.textContent || '0');
const prints = this.parseNumber(lastThree[2]?.textContent || '0');
currentValues.models[modelId] = { id: modelId, permalink, name, boosts, downloads, prints, imageUrl, isExclusive };
// DIAGNOSTIC: Log first 3 scraped models for comparison
if (index < 3) {
this.log(`[DIAGNOSTIC] SCRAPED Model ${index}: "${name}" -> dl=${downloads}, pr=${prints}, bo=${boosts}`);
}
this.log(`Model "${name}":`, { id: modelId, boosts, downloads, prints, permalink });
} else this.log(`Not enough metrics for ${name} (found ${allMetrics.length})`);
});
const modelCount = Object.keys(currentValues.models).length;
this.log(`[DIAGNOSTIC] Final model count: ${modelCount}`);
if (modelCount === 0) {
this.warn('[DIAGNOSTIC] *** ZERO models scraped - returning empty object ***');
}
// Enhance for jitter detection/retry (Change 1) — MUST AWAIT async method
return await this._detectAndRetryOnJitter(currentValues, 0);
} catch (err) { this.error('Error extracting values:', err); return null; }
}
// Helper: Detect jitter via equiv sum variance and retry
async _detectAndRetryOnJitter(currentValues, retryCount = 0) {
try {
const currentEquivSum = Object.values(currentValues.models).reduce((sum, m) => sum + this.calculateDownloadsEquivalent(m.downloads || 0, m.prints || 0), 0);
const prevData = await new Promise(res => chrome.storage.local.get(['previousValues'], res));
const prevEquivSum = prevData?.previousValues?.models ? Object.values(prevData.previousValues.models).reduce((sum, m) => sum + this.calculateDownloadsEquivalent(m.downloads || 0, m.prints || 0), 0) : 0;
this.log(`[DIAGNOSTIC] _detectAndRetryOnJitter: currentEquivSum=${currentEquivSum}, prevEquivSum=${prevEquivSum}, currentModels=${Object.keys(currentValues.models).length}, retryCount=${retryCount}`);
// Check if previousValues is from TODAY (same period key) before comparing
let isFromToday = false;
if (prevData?.previousValues && prevData.previousValues.periodKey) {
const currentPeriodKey = await this.getCurrentPeriodKey();
isFromToday = prevData.previousValues.periodKey === currentPeriodKey;
}
// Threshold for flagged variance (15 pts)
if (isFromToday && prevEquivSum > 0 && Math.abs(currentEquivSum - prevEquivSum) > 15 && retryCount < 2) {
this.log(`Scraping jitter detected: Equiv sum ${currentEquivSum} vs prev ${prevEquivSum}. Retrying after DOM settle (attempt ${retryCount + 1}).`);
await new Promise(resolve => setTimeout(resolve, 2500)); // Allow 2.5s for DOM to stabilize
await this.autoScrollToFullBottom(); // Ensure fresh scroll
const retryValues = await this.getCurrentValues(); // Recursive retry
this.log(`[DIAGNOSTIC] After retry scrape: got ${Object.keys(retryValues.models).length} models`);
return this._detectAndRetryOnJitter(retryValues, retryCount + 1);
}
return currentValues;
} catch (err) {
this.error('_detectAndRetryOnJitter failed:', err);
return currentValues; // Fallback: return original values on error
}
}
// ===========================================================
// END OF CHUNK 6 — Scraping
// ===========================================================
// ===========================================================
// START OF CHUNK 7 — Reward Helper Functions
// ===========================================================
// reward math
getRewardInterval(totalDownloads){ if (totalDownloads <= 50) return 10; if (totalDownloads <= 500) return 25; if (totalDownloads <= 1000) return 50; return 100; }
nextRewardDownloads(totalDownloads){ const interval = this.getRewardInterval(totalDownloads); const mod = totalDownloads % interval; return (totalDownloads === 0 || mod === 0) ? totalDownloads + interval : totalDownloads + (interval - mod); }
getRewardPointsForDownloads(thresholdDownloads){ if (thresholdDownloads <= 50) return 15; if (thresholdDownloads <= 500) return 12; if (thresholdDownloads <= 1000) return 20; return 30; }
calculateDownloadsEquivalent(downloads, prints){ return Number(downloads||0) + (Number(prints||0) * 2); }
getRewardCategory(downloads, prints) {
const total = this.calculateDownloadsEquivalent(downloads, prints);
if (total <= 49) return 1;
if (total <= 499) return 2;
if (total <= 999) return 3;
return 4;
}
// ===========================================================
// END OF CHUNK 7 — Reward Helper Functions
// ===========================================================
// ===========================================================
// ACCUMULATION & CORRECTION HELPERS (Change 4 support)
// ===========================================================
// Helper: Accumulate rewards for the day (primary grower for "Rewards today")
async accumulateRewards(rewardsDelta) {
try {
const stored = await new Promise(res => chrome.storage.local.get([this._accumulatedRewardsKey], r => res(r?.[this._accumulatedRewardsKey] || 0)));
const newTotal = stored + rewardsDelta;
await new Promise(res => chrome.storage.local.set({ [this._accumulatedRewardsKey]: newTotal }, res));
this.log(`Accumulated rewards updated: +${rewardsDelta} pts (total: ${newTotal})`);
return newTotal;
} catch (err) {
this.error('accumulateRewards failed:', err);
return 0; // Fallback on error
}
}
// Helper: Validate and correct on mismatch (re-calc as fallback if accumulation wrong)
async validateAndCorrect(accumulatedToday, reCalcToday, lastPeriodDelta = 0, retryCount = 0) {
try {
const difference = Math.abs(accumulatedToday - reCalcToday);
if (difference <= this._mismatchTolerance) {
this.log('Validation passed: Accumulated matches re-calc within tolerance.');
return accumulatedToday; // Stick with accumulation as primary
}
// Validation failed: Log and attempt re-calc correction
const modelCount = Object.keys(this.previousValues?.models || {}).length;
this.warn(`Validation failed: Accumulated (${accumulatedToday}) vs re-calc (${reCalcToday}) by ${difference} pts. Period delta: ${lastPeriodDelta}. Models: ${modelCount}. Retry: ${retryCount}.`);
// Correction: Re-scrape/re-calc up to 2 times to get accurate re-calc
for (let i = 0; i < 2; i++) {
try {
this.log('Correcting: Re-scraping and re-calculating...');
await this.autoScrollToFullBottom();
await new Promise(resolve => setTimeout(resolve, 3000)); // DOM settle
const refreshedValues = await this.getCurrentValues();
// CRITICAL: Preserve the original previousValues.timestamp before updating
const originalTimestamp = this.previousValues?.timestamp;
this.log(`[TIMESTAMP DIAGNOSTIC] In validateAndCorrect correction loop. Setting previousValues to refreshedValues. New timestamp: ${new Date(refreshedValues.timestamp).toLocaleString('en-US', { month: '2-digit', day: '2-digit', year: 'numeric', hour: 'numeric', minute: '2-digit' })}`);
this.previousValues = refreshedValues;
if (originalTimestamp) {
this.previousValues.timestamp = originalTimestamp;
this.log(`[TIMESTAMP DIAGNOSTIC] Preserved original timestamp: ${new Date(originalTimestamp).toLocaleString('en-US', { month: '2-digit', day: '2-digit', year: 'numeric', hour: 'numeric', minute: '2-digit' })}`);
}
const reCalcCorrected = (await this.computeRewardsSinceBaseline()).rewardPointsTotal;
const newDifference = Math.abs(accumulatedToday - reCalcCorrected);
if (newDifference <= this._mismatchTolerance) {
this.log(`Correction success: Re-calc adjusted to match (diff now ${newDifference}). Using accumulated.`);
return accumulatedToday; // Accumulated validated
}
} catch (err) {
this.warn(`Correction re-tap ${i + 1} failed: ${err}`);
}
}
// If retries fail, just log and continue (Option B: graceful degradation)
this.log('Correction: Validation failed to recover after retries. Continuing with accumulated value.');
return accumulatedToday; // Use accumulated despite mismatch
} catch (err) {
this.error('validateAndCorrect failed:', err);
return accumulatedToday; // Fallback: use accumulated
}
}
// ===========================================================
// START OF CHUNK 8 — Locking
// ===========================================================
// storage lock helpers
async acquireDailyLock(timeoutMs = this._dailyLockTimeoutMs) {
// Use a per-day lock key to avoid cross-day collisions
const today = new Date().toISOString().slice(0,10);
const lockKey = `${this._dailyLockBaseKey}_${today}`;
const now = Date.now();
return new Promise(resolve => chrome.storage.local.get([lockKey], res => {
const lock = res?.[lockKey] || null;
if (!lock || (now - lock.ts) > timeoutMs) {
if (lock && (now - lock.ts) > timeoutMs) {
chrome.storage.local.remove([lockKey], () => {
const newLock = { ts: now, owner: this._instanceId };
chrome.storage.local.set({ [lockKey]: newLock }, () => {
chrome.storage.local.get([lockKey], r2 => {
const confirmed = r2?.[lockKey]?.owner === this._instanceId;
this.log('acquireDailyLock (force unlock) result', { confirmed, owner: r2?.[lockKey]?.owner, instance: this._instanceId });
resolve(confirmed);
});
});
});
} else {
const newLock = { ts: now, owner: this._instanceId };
chrome.storage.local.set({ [lockKey]: newLock }, () => {
chrome.storage.local.get([lockKey], r2 => {
const confirmed = r2?.[lockKey]?.owner === this._instanceId;
this.log('acquireDailyLock result', { confirmed, owner: r2?.[lockKey]?.owner, instance: this._instanceId });
resolve(confirmed);
});
});
}
} else { this.log('acquireDailyLock failed, existing lock', lock); resolve(false); }
}));
}
async releaseDailyLock() {
const today = new Date().toISOString().slice(0,10);
const lockKey = `${this._dailyLockBaseKey}_${today}`;
return new Promise(resolve => chrome.storage.local.get([lockKey], res => {
const lock = res?.[lockKey] || null;
if (lock && lock.owner === this._instanceId) {
chrome.storage.local.remove([lockKey], () => { this.log('releaseDailyLock: released by', this._instanceId); resolve(true); });
} else resolve(false);
}));
}
// processing lock helpers (shorter timeout) to avoid race between periodic checks and daily summary
async acquireProcessingLock(timeoutMs = this._processingLockTimeoutMs) {
const now = Date.now();
return new Promise(resolve => chrome.storage.local.get([this._processingLockKey], res => {
const lock = res?.[this._processingLockKey] || null;
if (!lock || (now - lock.ts) > timeoutMs) {
const newLock = { ts: now, owner: this._instanceId };
chrome.storage.local.set({ [this._processingLockKey]: newLock }, () => {
chrome.storage.local.get([this._processingLockKey], r2 => {
const confirmed = r2?.[this._processingLockKey]?.owner === this._instanceId;
this.log('acquireProcessingLock result', { confirmed, owner: r2?.[this._processingLockKey]?.owner, instance: this._instanceId });
resolve(confirmed);
});
});
} else { this.log('acquireProcessingLock failed, existing lock', lock); resolve(false); }
}));
}
async releaseProcessingLock() {
return new Promise(resolve => chrome.storage.local.get([this._processingLockKey], res => {
const lock = res?.[this._processingLockKey] || null;
if (lock && lock.owner === this._instanceId) {
chrome.storage.local.remove([this._processingLockKey], () => { this.log('releaseProcessingLock: released by', this._instanceId); resolve(true); });
} else resolve(false);
}));
}
// ===========================================================
// END OF CHUNK 8 — Locking
// ===========================================================
// ===========================================================
// START OF CHUNK 9 — ReloadGuard
// ===========================================================
// ---- BEGIN ADDITION: ReloadGuard helpers (insert into ValueMonitor class) ----
// Count models within 2 downloads (downloads + 2*prints) of the next reward
_countCloseToAward(models) {
if (!models) return 0;
let close = 0;
for (const m of Object.values(models)) {
const downloads = Number(m.downloads || 0);
const prints = Number(m.prints || 0);
const total = this.calculateDownloadsEquivalent(downloads, prints);
const next = this.nextRewardDownloads(total);
const remaining = Math.max(0, next - total);
if (remaining <= 2) close++;
}
return close;
}
/*
Detect incomplete load using 2-way check you specified.
prevModels: object (previous snapshot.models)
currModels: object (current snapshot.models)
awardedCount: number (models that actually received an award in this run)
Returns:
{ suspect: bool, details: { prevTotal, currTotal, prevClose, currClose,
adjustedCurrClose, awardedCount, totalDrop, closeDrop } }
*/
_detectIncompleteLoadChecks(prevModels = {}, currModels = {}, awardedCount = 0) {
const prevTotal = Object.keys(prevModels || {}).length;
const currTotal = Object.keys(currModels || {}).length;
const prevClose = this._countCloseToAward(prevModels || {});
const currClose = this._countCloseToAward(currModels || {});
const adjustedCurrClose = currClose + (Number(awardedCount) || 0);
const totalDrop = prevTotal - currTotal;
const closeDrop = prevClose - adjustedCurrClose;
const suspect = (totalDrop >= 4) && (closeDrop >= 2);
return { suspect, details: { prevTotal, currTotal, prevClose,
currClose, adjustedCurrClose, awardedCount, totalDrop, closeDrop } };
}
// Soft re-scrape: scroll to bottom (auto-scroll repeated short steps) then re-run the DOM scrape method
async _rescrapeSoft({ step = 600, delay = 250, stableChecks = 3 } = {}) {
try {
// auto-scroll to bottom and wait for the DOM to settle
let lastHeight = document.body.scrollHeight;
let stable = 0;
while (true) {
window.scrollBy(0, step);
await new Promise(r => setTimeout(r, delay));
const h = document.body.scrollHeight;
const atBottom = (window.innerHeight + window.scrollY) >= (h - 2);
if (h === lastHeight) stable++; else { stable = 0; lastHeight = h; }
if (atBottom && stable >= stableChecks) break;
}
// small settle delay
await new Promise(r => setTimeout(r, 300));
// use your existing scrape function (e.g., getCurrentValues() or _scrapeData())
const newValues = this.getCurrentValues ? (await this.getCurrentValues()) : (await this._scrapeData());
return newValues;
} catch (err) {
console.warn('[ReloadGuard] _rescrapeSoft failed', err);
return null;
}
}
// Per-day reload cap & cooldown
async _shouldReloadToday() {
const now = Date.now();
const today = new Date().toISOString().slice(0,10);
return new Promise(res => {
chrome.storage.local.get(['reloadCountDate','reloadCount','lastReloadAt'], r => {
const storedDate = r.reloadCountDate;
const reloadCount = r.reloadCount || 0;
const lastReloadAt = r.lastReloadAt || 0;
// reset if different day
if (storedDate !== today) {
chrome.storage.local.set({ reloadCountDate: today, reloadCount: 0 }, () => res(true));
return;
}
const cooldownMs = 60 * 1000; // 1 minute cooldown
const cap = 3; // 3 reloads per day
if (reloadCount >= cap) return res(false);
if ((now - lastReloadAt) < cooldownMs) return res(false);
return res(true);
});
});
}
async _incrementReloadCount() {
const today = new Date().toISOString().slice(0,10);
return new Promise(res => {
chrome.storage.local.get(['reloadCountDate','reloadCount'], r => {
const storedDate = r.reloadCountDate;
let reloadCount = r.reloadCount || 0;
if (storedDate !== today) {
reloadCount = 1;
chrome.storage.local.set({ reloadCountDate: today, reloadCount,
lastReloadAt: Date.now() }, () => res({ reloadCount, today }));
} else {
reloadCount += 1;
chrome.storage.local.set({ reloadCount, lastReloadAt: Date.now() }, () => res({ reloadCount, today }));
}
});
});
}
// ---- END ADDITION ----
// ===========================================================
// END OF CHUNK 9 — ReloadGuard
// ===========================================================
// ===========================================================
// START OF CHUNK 10 — Daily Pre-Send Checks
// ===========================================================
// pre-send check to avoid duplicate daily sends
async preSendCheckAndMaybeWait(startTime) {
for (let attempt = 0; attempt < this._dailyMaxPreSendRetries; attempt++) {
const latest = await new Promise(res => chrome.storage.local.get([this._dailyStatsKey], r => res(r?.[this._dailyStatsKey] || null)));
if (latest && latest.timestamp >= startTime) { this.log('preSendCheck: found newer dailyStats, aborting send', { latestTs: new Date(latest.timestamp).toISOString(), startTime: new Date(startTime).toISOString() }); return false; }
const backoff = this._dailyPreSendBaseBackoffMs + Math.floor(Math.random()*700);
await new Promise(r => setTimeout(r, backoff));
}
return true;
}
// ===========================================================
// END OF CHUNK 10 — Daily Pre-Send Checks
// ===========================================================
// ===========================================================
// START OF CHUNK 11 — computeRewardsSinceBaseline
// ===========================================================
// side-effect-free computation of rewards since DAILY baseline (not periodic)
// This is the single source of truth for daily summary metrics
async computeRewardsSinceBaseline() {
await this.autoScrollToFullBottom();
const currentValues = await this.getCurrentValues();
if (!currentValues) {
this.error('Unable to get current values for compute');
return { rewardPointsTotal: 0, dailyDownloads: 0, dailyPrints: 0, dailyBoosts: 0, points: 0, pointsGained: 0, modelChanges: {}, rewardsEarned: [] /* add other fields as needed */ };
}
// STEP 1: Try to get the daily baseline (established at day rollover)
let dailyBaseline = await new Promise(res => chrome.storage.local.get([this._dailyBaselineKey], r => res(r?.[this._dailyBaselineKey] || null)));
this.log(`DIAGNOSTIC [STEP 1]: Retrieved _dailyBaselineKey. Exists: ${!!dailyBaseline}. Current values have ${Object.keys(currentValues.models || {}).length} models, ${currentValues.points} points.`);
// STEP 2: Use existing baseline regardless of day key.
// We stop discarding old baselines here because Chunk 5 now handles
// the reset only AFTER the report is safely sent.
if (dailyBaseline) {
this.log(`DIAGNOSTIC [STEP 2]: Using existing baseline from ${dailyBaseline.dayKey}.`);
}
// STEP 3: Fallback if baseline is completely missing
if (!dailyBaseline) {
this.log(`DIAGNOSTIC [STEP 3]: No baseline found at all. Creating an emergency baseline.`);
const currentDay = await this.getReportBasedDayKey();
dailyBaseline = {
models: currentValues.models || {},
points: currentValues.points || 0,
timestamp: Date.now(),
dayKey: currentDay
};
// We do NOT save it to storage here anymore; we just use it for this calculation.
} else {
this.log(`DIAGNOSTIC [STEP 3]: Using existing baseline from ${dailyBaseline.dayKey} to ensure data is not lost.`);
}
// STEP 4: Compute daily metrics STRICTLY from daily baseline to current
const modelChanges = {};
for (const [id, current] of Object.entries(currentValues.models || {})) {
// Find baseline for this model by ID first, then fallback to permalink/name matching
let previous = dailyBaseline?.models?.[id] || null;
if (!previous && current.permalink) {
previous = Object.values(dailyBaseline.models || {}).find(m => m?.permalink === current.permalink) || null;
}
if (!previous && current.name) {
const norm = current.name.trim().toLowerCase();
previous = Object.values(dailyBaseline.models || {}).find(m => m?.name?.trim().toLowerCase() === norm) || null;
}
if (!previous) {
// Model is new since baseline (e.g., added during the day)
// Treat current values as baseline for this model
previous = {
downloads: current.downloads,
prints: current.prints,
boosts: current.boosts
};
this.log(`Model ${current.name} not in daily baseline; treating as new with current as baseline for today.`);
}
const prevDownloads = Number(previous.downloads || 0);
const prevPrints = Number(previous.prints || 0);
const prevBoosts = Number(previous.boosts || 0);
const currDownloads = Number(current.downloads || 0);
const currPrints = Number(current.prints || 0);
const currBoosts = Number(current.boosts || 0);
const downloadsGained = currDownloads - prevDownloads;
const printsGained = currPrints - prevPrints;
const boostsGained = currBoosts - prevBoosts;
// Only include models with activity
if (downloadsGained <= 0 && printsGained <= 0 && boostsGained <= 0) continue;
// Filter out suspicious deltas (error detection)
if (downloadsGained > this._suspiciousDeltaLimit || printsGained > this._suspiciousDeltaLimit) continue;
modelChanges[id] = {
id,
name: current.name,
downloadsGained,
printsGained,
boostsGained,
previousDownloads: prevDownloads,
previousPrints: prevPrints,
currentDownloads: currDownloads,
currentPrints: currPrints,
permalink: current.permalink || previous?.permalink || null,
isExclusive: current.isExclusive
};
}
// STEP 5: Calculate daily totals (strictly from gained amounts)
const dailyDownloads = Object.values(modelChanges).reduce((s, m) => s + m.downloadsGained, 0);
const dailyPrints = Object.values(modelChanges).reduce((s, m) => s + m.printsGained, 0);
const dailyBoosts = Object.values(modelChanges).reduce((s, m) => s + (m.boostsGained || 0), 0);
this.log(`DIAGNOSTIC [STEP 5]: modelChanges count: ${Object.keys(modelChanges).length}, dailyDownloads: ${dailyDownloads}, dailyPrints: ${dailyPrints}, dailyBoosts: ${dailyBoosts}`);
// STEP 6: Compute rewards earned today (only thresholds crossed TODAY)
const rewardsEarned = [];
let rewardPointsTotal = 0;
for (const m of Object.values(modelChanges)) {
const prevDownloadsTotal = this.calculateDownloadsEquivalent(m.previousDownloads, m.previousPrints);
const currentDownloadsTotal = this.calculateDownloadsEquivalent(m.currentDownloads, m.currentPrints);
let cursor = prevDownloadsTotal;
const thresholdsHit = [];
const maxThresholdsPerModel = 200;
let thresholdsCount = 0;
while (cursor < currentDownloadsTotal && thresholdsCount < maxThresholdsPerModel) {
const interval = this.getRewardInterval(cursor);
const mod = cursor % interval;
const nextThreshold = (cursor === 0 || mod === 0) ? cursor + interval : cursor + (interval - mod);
if (nextThreshold <= currentDownloadsTotal) {
const rewardPoints = this.getRewardPointsForDownloads(nextThreshold);
thresholdsHit.push({ threshold: nextThreshold, rewardPoints });
cursor = nextThreshold;
thresholdsCount++;
} else {
break;
}
}
// Apply 25% bonus for Exclusive models
let baseReward = thresholdsHit.reduce((s, t) => s + t.rewardPoints, 0);
if (m.isExclusive) {
baseReward *= 1.25;
}
const rewardPointsTotalForModel = baseReward; // Keep as float
if (thresholdsHit.length > 0) {
rewardsEarned.push({
id: m.id,
name: m.name,
isExclusive: !!m.isExclusive,
thresholds: thresholdsHit.map(t => t.threshold),
rewardPointsTotalForModel
});
}
rewardPointsTotal += rewardPointsTotalForModel;
}
// STEP 7: Compute time strings for logging
const fromStr = new Date(dailyBaseline.timestamp).toLocaleString('en-US', { month: '2-digit', day: '2-digit', year: 'numeric', hour: 'numeric', minute: '2-digit' });
const toStr = new Date().toLocaleString('en-US', { month: '2-digit', day: '2-digit', year: 'numeric', hour: 'numeric', minute: '2-digit' });
this.log(`Daily summary computed: ${dailyDownloads} downloads, ${dailyPrints} prints, ${dailyBoosts} boosts, ${rewardPointsTotal} reward points from ${Object.keys(rewardsEarned).length} models`);
this.log(`DIAGNOSTIC [FINAL]: rewardsEarned length: ${rewardsEarned.length}, rewardPointsTotal: ${rewardPointsTotal}`);
return {
dailyDownloads,
dailyPrints,
dailyBoosts,
points: currentValues.points,
pointsGained: currentValues.points - (dailyBaseline ? dailyBaseline.points : 0),
rewardsEarned,
rewardPointsTotal,
modelChanges,
from: fromStr,
to: toStr
};
}
// ===========================================================
// END OF CHUNK 11 — computeRewardsSinceBaseline
// ===========================================================
// ===========================================================
// START OF CHUNK 12 — getDailySummary
// ===========================================================