From bb022ba46068306e7ec1abdb3b9a73d4f7b1210c Mon Sep 17 00:00:00 2001 From: JohanDC-1999 <82451675+JohanDC-1999@users.noreply.github.com> Date: Mon, 6 Oct 2025 18:18:47 +0200 Subject: [PATCH 1/4] Rename script.js to findDuplicatesBySingleField.js --- .../{script.js => findDuplicatesBySingleField.js} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Server-Side Components/Background Scripts/Duplicate Finder/{script.js => findDuplicatesBySingleField.js} (100%) diff --git a/Server-Side Components/Background Scripts/Duplicate Finder/script.js b/Server-Side Components/Background Scripts/Duplicate Finder/findDuplicatesBySingleField.js similarity index 100% rename from Server-Side Components/Background Scripts/Duplicate Finder/script.js rename to Server-Side Components/Background Scripts/Duplicate Finder/findDuplicatesBySingleField.js From ee88421548e9107b7a846a603be5ba8a56e9ac50 Mon Sep 17 00:00:00 2001 From: JohanDC-1999 <82451675+JohanDC-1999@users.noreply.github.com> Date: Mon, 6 Oct 2025 18:22:06 +0200 Subject: [PATCH 2/4] Create findDuplicatesByCombination.js --- .../findDuplicatesByCombination.js | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 Server-Side Components/Background Scripts/Duplicate Finder/findDuplicatesByCombination.js diff --git a/Server-Side Components/Background Scripts/Duplicate Finder/findDuplicatesByCombination.js b/Server-Side Components/Background Scripts/Duplicate Finder/findDuplicatesByCombination.js new file mode 100644 index 0000000000..21fb7b668e --- /dev/null +++ b/Server-Side Components/Background Scripts/Duplicate Finder/findDuplicatesByCombination.js @@ -0,0 +1,92 @@ +/** +* Finds and reports records that have duplicate values across a combination of specified fields instead of a single field +* Useful for finding for example duplicate items where unique key is not just 1 field +*/ + +// --- UPDATE ONLY THE VALUES BELOW --- +var tableName = 'cmdb_model'; // ADD: The table you want to check for duplicates. +var fieldNames = ['name', 'model_number', 'manufacturer']; // ADD: An array of fields that create the unique combination. + +// --- DO NOT EDIT BELOW THIS LINE --- +findDuplicateCombinations(tableName, fieldNames); + +function findDuplicateCombinations(tableName, fieldNames) { + /**************************************/ + /*** Basic Error Handling ***/ + /**************************************/ + + // Check if the table exists + if (!gs.tableExists(tableName)) { + gs.print('Error: Table "' + tableName + '" does not exist.'); + return; + } + + // Check if fieldNames is a valid, non-empty array + if (!Array.isArray(fieldNames) || fieldNames.length === 0) { + gs.print('Error: The "fieldNames" variable must be an array with at least one field.'); + return; + } + + // Check if all specified fields exist on the table + var gr = new GlideRecord(tableName); + gr.initialize(); + for (var i = 0; i < fieldNames.length; i++) { + var currentField = fieldNames[i]; + if (!gr.isValidField(currentField)) { + gs.print('Error: The field "' + currentField + '" does not exist on the "' + tableName + '" table.'); + return; + } + } + + /***************************************/ + /*** Find Duplicate Combinations ***/ + /***************************************/ + var duplicateJson = {}; // Stores the duplicate records and their counts + var duplicateGroupCount = 0; // Counts the number of groups of duplicates + + var duplicateAggregate = new GlideAggregate(tableName); + duplicateAggregate.addAggregate('COUNT'); + + // Loop through the array to group by each field, creating the combination + for (var j = 0; j < fieldNames.length; j++) { + var field = fieldNames[j]; + duplicateAggregate.groupBy(field); + duplicateAggregate.addNotNullQuery(field); // Ignore records where any part of the combination is empty + } + + duplicateAggregate.addHaving('COUNT', '>', 1); // More than 1 means it is a duplicate combination + duplicateAggregate.query(); + + while (duplicateAggregate.next()) { + duplicateGroupCount++; + var combinationDisplay = []; + + for (var k = 0; k < fieldNames.length; k++) { + var fieldName = fieldNames[k]; + var fieldValue = duplicateAggregate.getDisplayValue(fieldName) || duplicateAggregate.getValue(fieldName); + combinationDisplay.push(fieldName + ': "' + fieldValue + '"'); + } + + var displayKey = combinationDisplay.join(', '); + var countInGroup = duplicateAggregate.getAggregate('COUNT'); + duplicateJson[displayKey] = countInGroup; + } + + /***************************************/ + /*** Print the Results ***/ + /***************************************/ + + // No duplicates found + var fieldCombinationString = '"' + fieldNames.join('", "') + '"'; + if (Object.keys(duplicateJson).length === 0) { + gs.print('No duplicates found for the field combination [' + fieldCombinationString + '] on table "' + tableName + '".'); + return; + } + + // Duplicates were found + gs.print("Found " + duplicateGroupCount + " groups of duplicates based on the combination [" + fieldCombinationString + "]:"); + + for (var key in duplicateJson) { + gs.print('Combination {' + key + '} has ' + duplicateJson[key] + ' occurrences.'); + } +} From f12e855d610778ac25a42553221cecfc0dae8dfc Mon Sep 17 00:00:00 2001 From: JohanDC-1999 <82451675+JohanDC-1999@users.noreply.github.com> Date: Mon, 6 Oct 2025 18:44:48 +0200 Subject: [PATCH 3/4] Update readme.md, combine both scripts into 1 readme --- .../Duplicate Finder/readme.md | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/Server-Side Components/Background Scripts/Duplicate Finder/readme.md b/Server-Side Components/Background Scripts/Duplicate Finder/readme.md index 3ca613a445..1f9a1931f2 100644 --- a/Server-Side Components/Background Scripts/Duplicate Finder/readme.md +++ b/Server-Side Components/Background Scripts/Duplicate Finder/readme.md @@ -1,10 +1,11 @@ -## ServiceNow Duplicate Record Finder -A simple server-side script for ServiceNow that finds and reports on duplicate values for any field on any table. It uses an efficient GlideAggregate query and formats the results into a clean, readable report. - +# ServiceNow Duplicate Record Finder +A simple server-side script for ServiceNow that finds and reports on duplicate values for any field on any table. It uses an efficient `GlideAggregate` query. +## Find Duplicates by Single Field - `findDuplicatesBySingleField.js` +This finds duplicates based on a single field which matches ### How to Use This script is designed to be run in **Scripts - Background** or as a Fix Script. 1. Navigate: Go to **System Definition > Scripts - Background** (or type sys.scripts.do in the filter navigator). -2. Copy Script: Copy the entire contents of the script.js file. +2. Copy Script: Copy the entire contents of the `findDuplicatesBySingleField.js` file. 3. Paste and Configure: Paste the script into the "Run script" text box. Add the table to search in `tableName` and the field to search for duplicates in `fieldName` ```js // Update ONLY below values to find duplicates @@ -22,6 +23,25 @@ This script is designed to be run in **Scripts - Background** or as a Fix Script var fieldName = 'short_description'; ``` - 4. Run Script: Click the "Run script" button. The results will be displayed on the screen below the editor. +## Find Duplicates by Field Combination - `findDuplicatesByCombination.js` +This is a more powerful script that finds duplicate records based on a unique combination of one or more fields. + +### How to use +This script is also designed to be run in **Scripts - Background** +1. Navigate: Go to **System Definition > Scripts - Background** +2. Copy Script: Copy the entire contents of the `findDuplicatesByCombination.js` file. +3. Paste and Configure: Paste the script into the "Run script" text box. Update the `tableName` field and the `fieldNames` array. The `fieldNames` variable is an array, so even a single field must be enclosed in square brackets `[]` + +```js +// --- UPDATE ONLY THE VALUES BELOW --- +var tableName = ''; // The table you want to check. +var fieldNames = ['field_1', 'field_2']; // An array of fields for the unique combination. +``` + +For example, to find models with the same name, model number and manufacturer +```js +var tableName = 'cmdb_model'; +var fieldNames = ['name', 'model_number', 'manufacturer']; +``` From 3a16218146ac237b9ecd5c7b2ec31460e1b37af5 Mon Sep 17 00:00:00 2001 From: JohanDC-1999 <82451675+JohanDC-1999@users.noreply.github.com> Date: Sat, 11 Oct 2025 16:27:59 +0200 Subject: [PATCH 4/4] Update findDuplicatesByCombination.js to have a more readable output --- .../findDuplicatesByCombination.js | 62 +++++++++++++++---- 1 file changed, 51 insertions(+), 11 deletions(-) diff --git a/Server-Side Components/Background Scripts/Duplicate Finder/findDuplicatesByCombination.js b/Server-Side Components/Background Scripts/Duplicate Finder/findDuplicatesByCombination.js index 21fb7b668e..83d3684999 100644 --- a/Server-Side Components/Background Scripts/Duplicate Finder/findDuplicatesByCombination.js +++ b/Server-Side Components/Background Scripts/Duplicate Finder/findDuplicatesByCombination.js @@ -1,7 +1,7 @@ /** -* Finds and reports records that have duplicate values across a combination of specified fields instead of a single field -* Useful for finding for example duplicate items where unique key is not just 1 field -*/ + * Finds and reports records that have duplicate values across a combination of specified fields instead of a single field + * Useful for finding for example duplicate items where unique key is not just 1 field + */ // --- UPDATE ONLY THE VALUES BELOW --- var tableName = 'cmdb_model'; // ADD: The table you want to check for duplicates. @@ -60,33 +60,73 @@ function findDuplicateCombinations(tableName, fieldNames) { while (duplicateAggregate.next()) { duplicateGroupCount++; var combinationDisplay = []; + var queryParams = []; for (var k = 0; k < fieldNames.length; k++) { var fieldName = fieldNames[k]; var fieldValue = duplicateAggregate.getDisplayValue(fieldName) || duplicateAggregate.getValue(fieldName); + var actualValue = duplicateAggregate.getValue(fieldName); combinationDisplay.push(fieldName + ': "' + fieldValue + '"'); + queryParams.push({ + field: fieldName, + value: actualValue + }); } var displayKey = combinationDisplay.join(', '); var countInGroup = duplicateAggregate.getAggregate('COUNT'); - duplicateJson[displayKey] = countInGroup; + duplicateJson[displayKey] = { + count: countInGroup, + queryParams: queryParams + }; } - /***************************************/ - /*** Print the Results ***/ - /***************************************/ + /***************************************************/ + /*** Print the Results & Fetch sys_id's ***/ + /***************************************************/ - // No duplicates found var fieldCombinationString = '"' + fieldNames.join('", "') + '"'; + if (Object.keys(duplicateJson).length === 0) { gs.print('No duplicates found for the field combination [' + fieldCombinationString + '] on table "' + tableName + '".'); return; } - // Duplicates were found - gs.print("Found " + duplicateGroupCount + " groups of duplicates based on the combination [" + fieldCombinationString + "]:"); + gs.print(''); + gs.print('======================================================'); + gs.print('DUPLICATE COMBINATIONS REPORT'); + gs.print('------------------------------------------------------'); + gs.print('Table: ' + tableName); + gs.print('Found ' + duplicateGroupCount + ' groups of duplicate records.'); + gs.print('======================================================\n'); + var groupNumber = 1; for (var key in duplicateJson) { - gs.print('Combination {' + key + '} has ' + duplicateJson[key] + ' occurrences.'); + var groupData = duplicateJson[key]; + + gs.print('Group ' + groupNumber + ':'); + gs.print(' - Occurrences: ' + groupData.count); + gs.print(' - Combination:'); + + var fields = key.split(', '); + for (var i = 0; i < fields.length; i++) { + gs.print(' - ' + fields[i]); + } + + gs.print(" - Duplicate sys_id's:"); + + // Perform the second query to get the sys_id's for this specific group + var rec = new GlideRecord(tableName); + for (var q = 0; q < groupData.queryParams.length; q++) { + var query = groupData.queryParams[q]; + rec.addQuery(query.field, query.value); + } + rec.query(); + + while (rec.next()) { + gs.print(' - ' + rec.getUniqueValue()); + } + gs.print(''); + groupNumber++; } }