From 0546b96a5f03f90bc451e69da04948db7b5758d8 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 19 Jun 2026 14:00:26 +0300 Subject: [PATCH 1/4] test(views): reproduce weekly-unique over-count for repeat same-week views MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a views plugin test: one device views the same view in two sessions within a single ISO week. The view-data weekly-unique bucket (d.w.u) must be 1, but live ingestion currently records 2 — it counts the same user's repeat same-week view twice. Regeneration (EE drill) already de-dupes correctly to 1; this test pins the live over-count in core where it actually lives. Expected to FAIL until the live counting is fixed. Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/views/tests.js | 114 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 plugins/views/tests.js diff --git a/plugins/views/tests.js b/plugins/views/tests.js new file mode 100644 index 00000000000..8289b54e979 --- /dev/null +++ b/plugins/views/tests.js @@ -0,0 +1,114 @@ +var request = require('supertest'); +var should = require('should'); +var crypto = require('crypto'); +var moment = require('moment-timezone'); +var testUtils = require("../../test/testUtils"); +request = request(testUtils.url); + +var APP_KEY = ""; +var API_KEY_ADMIN = ""; +var APP_ID = ""; +var DEVICE_ID = "views_weekly_unique_user"; +var VIEW_NAME = "viewsWeeklyUniqueTest"; +var db; + +// Anchor both views to Wednesday 12:00 UTC of the PREVIOUS ISO week: safely in the +// past, and far enough from the Monday week boundary that both events land in the +// same ISO week for any reasonable app timezone (avoids boundary/timezone flakiness). +var anchor = moment.utc().subtract(1, 'week').startOf('isoWeek').add(2, 'days').add(12, 'hours').valueOf(); +var t1 = Math.floor(anchor / 1000); +var t2 = Math.floor((anchor + 3 * 60 * 60 * 1000) / 1000); // +3h, same Wednesday / same ISO week + +var viewEvent = JSON.stringify([{"key": "[CLY]_view", "count": 1, "segmentation": {"name": VIEW_NAME, "visit": 1, "start": 1, "platform": "Android"}}]); + +describe('Testing views weekly-unique counting', function() { + describe('Same user viewing the same view in two sessions within one ISO week', function() { + it('init', function(done) { + API_KEY_ADMIN = testUtils.get("API_KEY_ADMIN"); + APP_ID = testUtils.get("APP_ID"); + APP_KEY = testUtils.get("APP_KEY"); + db = testUtils.client.db("countly"); + done(); + }); + + it('records the first session view', function(done) { + request + .get('/i?begin_session=1&app_key=' + APP_KEY + '&device_id=' + DEVICE_ID + '×tamp=' + t1 + '&events=' + viewEvent) + .expect(200) + .end(function(err) { + if (err) { + return done(err); + } + setTimeout(done, testUtils.testWaitTimeForDrillEvents * testUtils.testScalingFactor); + }); + }); + + it('ends the first session', function(done) { + request + .get('/i?end_session=1&app_key=' + APP_KEY + '&device_id=' + DEVICE_ID + '×tamp=' + (t1 + 3)) + .expect(200) + .end(function(err) { + if (err) { + return done(err); + } + setTimeout(done, 100 * testUtils.testScalingFactor); + }); + }); + + it('records the second session view in the same week', function(done) { + request + .get('/i?begin_session=1&app_key=' + APP_KEY + '&device_id=' + DEVICE_ID + '×tamp=' + t2 + '&events=' + viewEvent) + .expect(200) + .end(function(err) { + if (err) { + return done(err); + } + setTimeout(done, testUtils.testWaitTimeForDrillEvents * testUtils.testScalingFactor); + }); + }); + + it('ends the second session', function(done) { + request + .get('/i?end_session=1&app_key=' + APP_KEY + '&device_id=' + DEVICE_ID + '×tamp=' + (t2 + 3)) + .expect(200) + .end(function(err) { + if (err) { + return done(err); + } + setTimeout(done, testUtils.testWaitTimeForDrillEvents * testUtils.testScalingFactor); + }); + }); + + it('counts the view exactly once per week (one device viewed it)', function(done) { + db.collection("app_viewsmeta" + APP_ID).findOne({"view": VIEW_NAME}, function(err, viewMeta) { + if (err) { + return done(err); + } + should.exist(viewMeta); + var viewIdStr = "" + viewMeta._id; + // no-segment view-data collection holds the per-year ("_:0") + // rollup docs where weekly unique lives under d.w.u + var colName = "app_viewdata" + crypto.createHash('sha1').update("" + APP_ID).digest('hex'); + db.collection(colName).find({"_id": {"$regex": "^" + viewIdStr + "_"}}).toArray(function(err2, docs) { + if (err2) { + return done(err2); + } + docs.should.not.be.empty(); + // Only DEVICE_ID viewed VIEW_NAME, so every weekly-unique bucket must be 1. + var offending = []; + docs.forEach(function(doc) { + if (doc.d) { + for (var key in doc.d) { + if (/^w[0-9]+$/.test(key) && doc.d[key] && typeof doc.d[key].u !== "undefined" && doc.d[key].u !== 1) { + offending.push(doc._id + "." + key + ".u=" + doc.d[key].u); + } + } + } + }); + offending.should.eql([]); + done(); + }); + }); + }); + }); +}); From 85fcaffa61f4a6637ed85f00d78e1a81c4797bd3 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 19 Jun 2026 15:09:11 +0300 Subject: [PATCH 2/4] fix(views): stop over-counting weekly unique for repeat same-week views The weekly-unique gate in recordMetrics built its comparison moment with new moment(lastViewTimestamp). lastViewTimestamp is a unix timestamp in seconds (as the surrounding day/month/year gates treat it), but moment(Number) interprets the value as milliseconds, placing it in 1970. isoWeek() of that 1970 date is a tiny number that is almost always less than the current weeklyISO, so the gate fired on every repeat view in the same week and incremented d.w.u again -> weekly unique counted 2 for a single user. Daily/monthly/yearly gates were unaffected (they use elapsed-seconds comparisons). Use moment.unix() so isoWeek() reflects the real week. Also harden plugins/views/tests.js to assert a weekly-unique bucket was actually inspected (guards against a vacuous pass if weekly data stops being written). Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/views/api/api.js | 8 +++++++- plugins/views/tests.js | 12 ++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/plugins/views/api/api.js b/plugins/views/api/api.js index e0313876a4f..41ca1c35c6b 100644 --- a/plugins/views/api/api.js +++ b/plugins/views/api/api.js @@ -2228,7 +2228,13 @@ const escapedViewSegments = { "name": true, "segment": true, "height": true, "wi secInYear = (60 * 60 * 24 * (common.getDOY(params.time.timestamp, params.appTimezone) - 1)) + secInHour; - var lastMoment = new moment(lastViewTimestamp); + // lastViewTimestamp is a unix timestamp in seconds (like the + // other gates below compare it). new moment(Number) treats the + // value as milliseconds, placing it in 1970, so isoWeek() was a + // tiny number that is almost always < weeklyISO — making the + // weekly-unique gate fire on every repeat view in the same week + // (over-counting weekly unique). Use moment.unix() for seconds. + var lastMoment = moment.unix(lastViewTimestamp); lastMoment.tz(params.appTimezone); if (lastViewTimestamp < (params.time.timestamp - secInMin)) { diff --git a/plugins/views/tests.js b/plugins/views/tests.js index 8289b54e979..9d2ee16cf87 100644 --- a/plugins/views/tests.js +++ b/plugins/views/tests.js @@ -96,15 +96,23 @@ describe('Testing views weekly-unique counting', function() { docs.should.not.be.empty(); // Only DEVICE_ID viewed VIEW_NAME, so every weekly-unique bucket must be 1. var offending = []; + var weeklyBucketsChecked = 0; docs.forEach(function(doc) { if (doc.d) { for (var key in doc.d) { - if (/^w[0-9]+$/.test(key) && doc.d[key] && typeof doc.d[key].u !== "undefined" && doc.d[key].u !== 1) { - offending.push(doc._id + "." + key + ".u=" + doc.d[key].u); + if (/^w[0-9]+$/.test(key) && doc.d[key] && typeof doc.d[key].u !== "undefined") { + weeklyBucketsChecked++; + if (doc.d[key].u !== 1) { + offending.push(doc._id + "." + key + ".u=" + doc.d[key].u); + } } } } }); + // Guard against a vacuous pass: a weekly-unique bucket must actually + // have been written and inspected, otherwise an empty offending list + // would "pass" even if weekly data stopped being recorded entirely. + weeklyBucketsChecked.should.be.above(0); offending.should.eql([]); done(); }); From 9f1352f97b8eda15bdd8e545e99e93ea5a40ada4 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 19 Jun 2026 15:52:14 +0300 Subject: [PATCH 3/4] test(views): relocate weekly-unique test into tests/ dir and url-encode events Address review feedback: - The test was named plugins/views/tests.js, which Node resolves in preference to the existing plugins/views/tests/ directory, shadowing the whole views suite (tests/index.js -> views.js). Move it to plugins/views/tests/weeklyUnique.js and require it from index.js so the existing suite keeps running. - URL-encode the events JSON query param (encodeURIComponent) instead of concatenating raw JSON into the URL. Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/views/tests.js | 122 ----------------------------------------- 1 file changed, 122 deletions(-) delete mode 100644 plugins/views/tests.js diff --git a/plugins/views/tests.js b/plugins/views/tests.js deleted file mode 100644 index 9d2ee16cf87..00000000000 --- a/plugins/views/tests.js +++ /dev/null @@ -1,122 +0,0 @@ -var request = require('supertest'); -var should = require('should'); -var crypto = require('crypto'); -var moment = require('moment-timezone'); -var testUtils = require("../../test/testUtils"); -request = request(testUtils.url); - -var APP_KEY = ""; -var API_KEY_ADMIN = ""; -var APP_ID = ""; -var DEVICE_ID = "views_weekly_unique_user"; -var VIEW_NAME = "viewsWeeklyUniqueTest"; -var db; - -// Anchor both views to Wednesday 12:00 UTC of the PREVIOUS ISO week: safely in the -// past, and far enough from the Monday week boundary that both events land in the -// same ISO week for any reasonable app timezone (avoids boundary/timezone flakiness). -var anchor = moment.utc().subtract(1, 'week').startOf('isoWeek').add(2, 'days').add(12, 'hours').valueOf(); -var t1 = Math.floor(anchor / 1000); -var t2 = Math.floor((anchor + 3 * 60 * 60 * 1000) / 1000); // +3h, same Wednesday / same ISO week - -var viewEvent = JSON.stringify([{"key": "[CLY]_view", "count": 1, "segmentation": {"name": VIEW_NAME, "visit": 1, "start": 1, "platform": "Android"}}]); - -describe('Testing views weekly-unique counting', function() { - describe('Same user viewing the same view in two sessions within one ISO week', function() { - it('init', function(done) { - API_KEY_ADMIN = testUtils.get("API_KEY_ADMIN"); - APP_ID = testUtils.get("APP_ID"); - APP_KEY = testUtils.get("APP_KEY"); - db = testUtils.client.db("countly"); - done(); - }); - - it('records the first session view', function(done) { - request - .get('/i?begin_session=1&app_key=' + APP_KEY + '&device_id=' + DEVICE_ID + '×tamp=' + t1 + '&events=' + viewEvent) - .expect(200) - .end(function(err) { - if (err) { - return done(err); - } - setTimeout(done, testUtils.testWaitTimeForDrillEvents * testUtils.testScalingFactor); - }); - }); - - it('ends the first session', function(done) { - request - .get('/i?end_session=1&app_key=' + APP_KEY + '&device_id=' + DEVICE_ID + '×tamp=' + (t1 + 3)) - .expect(200) - .end(function(err) { - if (err) { - return done(err); - } - setTimeout(done, 100 * testUtils.testScalingFactor); - }); - }); - - it('records the second session view in the same week', function(done) { - request - .get('/i?begin_session=1&app_key=' + APP_KEY + '&device_id=' + DEVICE_ID + '×tamp=' + t2 + '&events=' + viewEvent) - .expect(200) - .end(function(err) { - if (err) { - return done(err); - } - setTimeout(done, testUtils.testWaitTimeForDrillEvents * testUtils.testScalingFactor); - }); - }); - - it('ends the second session', function(done) { - request - .get('/i?end_session=1&app_key=' + APP_KEY + '&device_id=' + DEVICE_ID + '×tamp=' + (t2 + 3)) - .expect(200) - .end(function(err) { - if (err) { - return done(err); - } - setTimeout(done, testUtils.testWaitTimeForDrillEvents * testUtils.testScalingFactor); - }); - }); - - it('counts the view exactly once per week (one device viewed it)', function(done) { - db.collection("app_viewsmeta" + APP_ID).findOne({"view": VIEW_NAME}, function(err, viewMeta) { - if (err) { - return done(err); - } - should.exist(viewMeta); - var viewIdStr = "" + viewMeta._id; - // no-segment view-data collection holds the per-year ("_:0") - // rollup docs where weekly unique lives under d.w.u - var colName = "app_viewdata" + crypto.createHash('sha1').update("" + APP_ID).digest('hex'); - db.collection(colName).find({"_id": {"$regex": "^" + viewIdStr + "_"}}).toArray(function(err2, docs) { - if (err2) { - return done(err2); - } - docs.should.not.be.empty(); - // Only DEVICE_ID viewed VIEW_NAME, so every weekly-unique bucket must be 1. - var offending = []; - var weeklyBucketsChecked = 0; - docs.forEach(function(doc) { - if (doc.d) { - for (var key in doc.d) { - if (/^w[0-9]+$/.test(key) && doc.d[key] && typeof doc.d[key].u !== "undefined") { - weeklyBucketsChecked++; - if (doc.d[key].u !== 1) { - offending.push(doc._id + "." + key + ".u=" + doc.d[key].u); - } - } - } - } - }); - // Guard against a vacuous pass: a weekly-unique bucket must actually - // have been written and inspected, otherwise an empty offending list - // would "pass" even if weekly data stopped being recorded entirely. - weeklyBucketsChecked.should.be.above(0); - offending.should.eql([]); - done(); - }); - }); - }); - }); -}); From 2a07cee6d994e33e85f36d8abab3d890b5f94fcd Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 19 Jun 2026 15:52:49 +0300 Subject: [PATCH 4/4] test(views): add relocated weekly-unique test + wire into tests/index.js Completes the relocation: the new plugins/views/tests/weeklyUnique.js and its require in tests/index.js were missing from the previous commit (which only removed the shadowing plugins/views/tests.js). Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/views/tests/index.js | 1 + plugins/views/tests/weeklyUnique.js | 122 ++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 plugins/views/tests/weeklyUnique.js diff --git a/plugins/views/tests/index.js b/plugins/views/tests/index.js index c859552a2b3..1a81c7554a9 100644 --- a/plugins/views/tests/index.js +++ b/plugins/views/tests/index.js @@ -1,2 +1,3 @@ require('./views.js'); +require('./weeklyUnique.js'); //require('./heatmaps.js'); diff --git a/plugins/views/tests/weeklyUnique.js b/plugins/views/tests/weeklyUnique.js new file mode 100644 index 00000000000..94a4191f0f6 --- /dev/null +++ b/plugins/views/tests/weeklyUnique.js @@ -0,0 +1,122 @@ +var request = require('supertest'); +var should = require('should'); +var crypto = require('crypto'); +var moment = require('moment-timezone'); +var testUtils = require("../../../test/testUtils"); +request = request(testUtils.url); + +var APP_KEY = ""; +var API_KEY_ADMIN = ""; +var APP_ID = ""; +var DEVICE_ID = "views_weekly_unique_user"; +var VIEW_NAME = "viewsWeeklyUniqueTest"; +var db; + +// Anchor both views to Wednesday 12:00 UTC of the PREVIOUS ISO week: safely in the +// past, and far enough from the Monday week boundary that both events land in the +// same ISO week for any reasonable app timezone (avoids boundary/timezone flakiness). +var anchor = moment.utc().subtract(1, 'week').startOf('isoWeek').add(2, 'days').add(12, 'hours').valueOf(); +var t1 = Math.floor(anchor / 1000); +var t2 = Math.floor((anchor + 3 * 60 * 60 * 1000) / 1000); // +3h, same Wednesday / same ISO week + +var viewEvent = JSON.stringify([{"key": "[CLY]_view", "count": 1, "segmentation": {"name": VIEW_NAME, "visit": 1, "start": 1, "platform": "Android"}}]); + +describe('Testing views weekly-unique counting', function() { + describe('Same user viewing the same view in two sessions within one ISO week', function() { + it('init', function(done) { + API_KEY_ADMIN = testUtils.get("API_KEY_ADMIN"); + APP_ID = testUtils.get("APP_ID"); + APP_KEY = testUtils.get("APP_KEY"); + db = testUtils.client.db("countly"); + done(); + }); + + it('records the first session view', function(done) { + request + .get('/i?begin_session=1&app_key=' + APP_KEY + '&device_id=' + DEVICE_ID + '×tamp=' + t1 + '&events=' + encodeURIComponent(viewEvent)) + .expect(200) + .end(function(err) { + if (err) { + return done(err); + } + setTimeout(done, testUtils.testWaitTimeForDrillEvents * testUtils.testScalingFactor); + }); + }); + + it('ends the first session', function(done) { + request + .get('/i?end_session=1&app_key=' + APP_KEY + '&device_id=' + DEVICE_ID + '×tamp=' + (t1 + 3)) + .expect(200) + .end(function(err) { + if (err) { + return done(err); + } + setTimeout(done, 100 * testUtils.testScalingFactor); + }); + }); + + it('records the second session view in the same week', function(done) { + request + .get('/i?begin_session=1&app_key=' + APP_KEY + '&device_id=' + DEVICE_ID + '×tamp=' + t2 + '&events=' + encodeURIComponent(viewEvent)) + .expect(200) + .end(function(err) { + if (err) { + return done(err); + } + setTimeout(done, testUtils.testWaitTimeForDrillEvents * testUtils.testScalingFactor); + }); + }); + + it('ends the second session', function(done) { + request + .get('/i?end_session=1&app_key=' + APP_KEY + '&device_id=' + DEVICE_ID + '×tamp=' + (t2 + 3)) + .expect(200) + .end(function(err) { + if (err) { + return done(err); + } + setTimeout(done, testUtils.testWaitTimeForDrillEvents * testUtils.testScalingFactor); + }); + }); + + it('counts the view exactly once per week (one device viewed it)', function(done) { + db.collection("app_viewsmeta" + APP_ID).findOne({"view": VIEW_NAME}, function(err, viewMeta) { + if (err) { + return done(err); + } + should.exist(viewMeta); + var viewIdStr = "" + viewMeta._id; + // no-segment view-data collection holds the per-year ("_:0") + // rollup docs where weekly unique lives under d.w.u + var colName = "app_viewdata" + crypto.createHash('sha1').update("" + APP_ID).digest('hex'); + db.collection(colName).find({"_id": {"$regex": "^" + viewIdStr + "_"}}).toArray(function(err2, docs) { + if (err2) { + return done(err2); + } + docs.should.not.be.empty(); + // Only DEVICE_ID viewed VIEW_NAME, so every weekly-unique bucket must be 1. + var offending = []; + var weeklyBucketsChecked = 0; + docs.forEach(function(doc) { + if (doc.d) { + for (var key in doc.d) { + if (/^w[0-9]+$/.test(key) && doc.d[key] && typeof doc.d[key].u !== "undefined") { + weeklyBucketsChecked++; + if (doc.d[key].u !== 1) { + offending.push(doc._id + "." + key + ".u=" + doc.d[key].u); + } + } + } + } + }); + // Guard against a vacuous pass: a weekly-unique bucket must actually + // have been written and inspected, otherwise an empty offending list + // would "pass" even if weekly data stopped being recorded entirely. + weeklyBucketsChecked.should.be.above(0); + offending.should.eql([]); + done(); + }); + }); + }); + }); +});