From b30a4a720d2122dc99fba2372c45ce6bb9852299 Mon Sep 17 00:00:00 2001
From: Daniel Wood <daniel.j.wood11@gmail.com>
Date: Wed, 10 Apr 2024 14:29:07 -0400
Subject: [PATCH] change getRemote() to match dg rig.

Hey there, I was using this code in our interactive rig. Was getting stuck in a do loop when the s3 bucket grew past 1000 items. Porting over the getRemote() code from the s3.js file in the dailygraphics rig fixed the problem. Basically, I don't think the program was correctly getting the next page of 1000 results.
---
 root/tasks/lib/s3.js | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/root/tasks/lib/s3.js b/root/tasks/lib/s3.js
index cf51630..eda2334 100644
--- a/root/tasks/lib/s3.js
+++ b/root/tasks/lib/s3.js
@@ -45,8 +45,11 @@ var download = async function (Bucket, Key) {
 };
 
 // get a single page of results from S3
-var getRemote = async function (Bucket, Prefix, Marker = null) {
-  var results = await s3.send(new ListObjectsV2Command({ Bucket, Prefix, Marker }));
+var getRemote = async function (Bucket, Prefix, next = undefined) {
+  var input = { Bucket, Prefix };
+  if (next) input.ContinuationToken = next;
+  var command = new ListObjectsV2Command(input);
+  var results = await s3.send(command);
   var items = (results.Contents || []).map(function (obj) {
     return {
       file: obj.Key.replace(/.*\/synced\//, ""),
@@ -55,7 +58,7 @@ var getRemote = async function (Bucket, Prefix, Marker = null) {
       mtime: obj.LastModified,
     };
   }).filter((i) => i.size);
-  var next = results.IsTruncated ? items[items.length - 1].key : null;
+  var next = results.IsTruncated ? results.NextContinuationToken : null;
   return { items, next };
 };