[wip] Prefixed listing stream

bin/s3prefixed.js

@@ -0,0 +1,17 @@
+#!/usr/bin/env node
+
+var keepalive = require('agentkeepalive');
+var agent = new keepalive.HttpsAgent({
+  keepAlive: true,
+  maxSockets: Math.ceil(require('os').cpus().length * 16),
+  keepAliveTimeout: 60000
+});
+var s3scan = require('..');
+
+var s3url = process.argv[2];
+if (!s3url) {
+  console.error('Usage: s3prefixed <s3url>');
+  process.exit(1);
+}
+
+s3scan.Prefixed(s3url, { agent: agent }).pipe(process.stdout);

index.js

@@ -1,6 +1,7 @@
 var s3urls = require('s3urls');
 var Split = require('split');
 var List = require('./lib/keys');
+var Prefixed = require('./lib/prefixed');
 var Get = require('./lib/get');
 var Delete = require('./lib/delete');
 
@@ -19,6 +20,21 @@ var Delete = require('./lib/delete');
  */
 module.exports.List = List;
 
+/**
+ * Provides a readable stream of keys beneath the provided S3 prefix, replacing
+ * `{prefix}` in the input url with 0-255 hex characters [0-9a-f]
+ *
+ * @name s3scan.Prefixed
+ * @param {string} s3url - an S3 uri of the type `s3://bucket/{prefix}/something`
+ * @param {object} [options] - options to provide to the readable stream
+ * @param {object} [options.agent] - an HTTPS agent to use for S3 requests
+ * @returns {object} a readable stream of line-delimited keys
+ * @example
+ * require('s3scan').Prefixed('s3://my-bucket/{prefix}/my-key')
+ *   .pipe(process.stdout);
+ */
+module.exports.Prefixed = Prefixed;
+
 /**
  * Provides a transform stream that expects you to write line-delimited S3 keys
  * into it, and transforms them into a readable stream of S3.getObject responses

lib/prefixed.js

@@ -0,0 +1,38 @@
+var stream = require('stream');
+var Keys = require('./keys');
+var s3urls = require('s3urls');
+
+module.exports = function(s3url, options) {
+  if (!/\{prefix\}/.test(s3url)) return Keys(s3url, options);
+  options = options || {};
+
+  s3url = s3urls.fromUrl(s3url);
+  var combiner = new stream.PassThrough(options);
+  combiner.setMaxListeners(Infinity);
+
+  for (var i = 0; i < 256; i++) {
+    var prefix = ('0' + i.toString(16)).slice(-2);
+    var url = s3urls.toUrl(s3url.Bucket, s3url.Key.replace('{prefix}', prefix)).s3;
+    connect(
+      Keys(url, options)
+        .on('end', finishedList)
+        .on('error', function(err) { combiner.emit('error', err); })
+    );
+  }
+
+  var running = 0;
+  function connect(source) {
+    // if (running > 0) return setImmediate(connect, source);
+    running++;
+    source.pipe(combiner, { end: false });
+  }
+
+  var completed = 0;
+  function finishedList() {
+    completed++;
+    running--;
+    if (completed === 256) combiner.end();
+  }
+
+  return combiner;
+};