diff --git a/README.md b/README.md index 30d9224..4ab6a45 100644 --- a/README.md +++ b/README.md @@ -57,29 +57,198 @@ These all match the parameters used with [`levelup`](https://github.com/rvagg/node-levelup). The default encoding for the database is set to `json`. -## --start <key-pattern> -Specify the start of the current range. You can also use `gt` or `gte`. +## --get <key> +Get a value +```sh +lev --get foo +``` -## --end <key-pattern> -Specify the end of the current range. You can also use `lt` and `lte`. +## --put <key> +Put a value +```sh +lev --put foo --value bar +``` + +## --del <key> +Delete a value +```sh +lev --del foo +``` + +## --batch <operations> +Put or delete several values, using [`levelup` batch syntax](https://github.com/Level/levelup#dbbatcharray-options-callback-array-form) +```sh +lev --batch '[ +{"type":"del","key":"father"}, +{"type":"put","key":"name","value":"Yuri Irsenovich Kim"}, +{"type":"put","key":"dob","value":"16 February 1941"}, +{"type":"put","key":"spouse","value":"Kim Young-sook"}, +{"type":"put","key":"occupation","value":"Clown"} +]' +``` +or from a file +```sh +# there should be one entry per line +# either as valid JSON +echo '[ +{"type":"del","key":"father"}, +{"type":"put","key":"name","value":"Yuri Irsenovich Kim"}, +{"type":"put","key":"dob","value":"16 February 1941"}, +{"type":"put","key":"spouse","value":"Kim Young-sook"}, +{"type":"put","key":"occupation","value":"Clown"} +]' > ops.json +# or as newline-delimited JSON +echo ' +{"type":"del","key":"father"} +{"type":"put","key":"name","value":"Yuri Irsenovich Kim"} +{"type":"put","key":"dob","value":"16 February 1941"} +{"type":"put","key":"spouse","value":"Kim Young-sook"} +{"type":"put","key":"occupation","value":"Clown"} +' > ops.json +lev --batch ./ops.json +``` + +### Import / Export +If the type is omitted, defaults to `put`, which allows to use the command to do imports/exports, in combination with [`--all`](#--all): +```sh +lev --all > leveldb.export +lev /tmp/my-new-db --batch leveldb.export +``` +If it's a large export, you can compress it on the fly +```sh +lev --all | gzip -9 > leveldb.export.gz +gzip -dk leveldb.export.gz +lev /tmp/my-new-db --batch leveldb.export +``` + +### Delete by range +The `--batch` option can also be used to delete key/values by range in 2 steps: +``` +# 1 - collect all the key/values to delete +lev --all --start 'foo' --end 'fooz' > ./to_delete +# 2 - pass the file as argument to the --batch option with a --del flag +lev --batch ./to_delete --del +``` + +## --keys +List all the keys in the current range. Will tabularize the output by default (see `--line`). +```sh +lev --keys +``` ## --values -Only list the all of the values in the current range. +List all the values in the current range. Emit as a new-line delimited stream of json. +```sh +lev --values +``` -## --keys -Only list all of the keys in the current range. Will tabularize the output. +## --all +List all the keys and values in the current range. +Emit as a new-line delimited stream of json. +```sh +lev --all +``` +It can be used to create an export of the database, to be imported with [`--batch`](#--batch) +```sh +lev --all > leveldb.export +lev /tmp/my-new-db --batch leveldb.export +``` -## --keyEncoding <string> -Specify the encoding for the keys. +## --start <key-pattern> +Specify the start of the current range. You can also use `gt` or `gte`. +```sh +# output all keys after 'foo' +lev --keys --start 'foo' +# which is equivalent to +lev --keys --gte 'foo' +# the same for values +lev --values --start 'foo' +``` -## --valueEncoding <string> -Specify the encoding for the values. +## --end <key-pattern> +Specify the end of the current range. You can also use `lt` and `lte`. +```sh +# output all keys before 'fooz' +lev --keys --end 'fooz' +# which is equivalent to +lev --keys --lte 'fooz' +# the same for values +lev --values --end 'fooz' +# output all keys between 'foo' and 'fooz' +lev --keys --start 'foo' --end 'fooz' +``` + +## --match <key-pattern> +Filter keys or values by a pattern applied on the key +```sh +lev --keys --match 'f*' +lev --values --match 'f*' +lev --all --match 'f*' +# Equivalent to +lev --match 'f*' +``` + +See [`minimatch` doc](https://github.com/isaacs/minimatch#readme) for patterns ## --limit <number> Limit the number of records emitted in the current range. +```sh +lev --keys --limit 10 +lev --values --start 'foo' --end 'fooz' --limit 100 +lev --match 'f*' --limit 10 +``` ## --reverse Reverse the stream. +```sh +lev --keys --reverse +lev --keys --start 'foo' --end 'fooz' --limit 100 --reverse +``` +## --line +Output one key per line (instead of the default tabularized output) +```sh +lev --keys --line +``` + +## --length +Output the length of the current range +```sh +# Count all the key/value pairs in the database +lev --length +# Counts the keys and values between 'foo' and 'fooz' +lev --start 'foo' --end 'fooz' --length +``` + +## --valueEncoding <string> +Specify the encoding for the values (Defaults to 'json'). +```sh +lev --values --valueEncoding buffer +``` + +## --location <string> +Specify the path to the LevelDB to use. Defaults to the current directory. +```sh +lev --location /tmp/test-db --keys +# Equivalent to +lev /tmp/test-db --keys +``` + +## --map <JS function string or path> +Pass streams results in a map function +* either inline +```sh +lev --keys --map 'key => key.split(":")[1]' +lev --all --map 'data => data.value.replace(data.key, "")' +``` +* or from a JS file that exports a function +```js +# in ./map_fn.js +module.exports = key => key.split(":")[1] +``` +```sh +lev --keys --map ./map_fn.js +``` +If the function, returns null or undefined, the result is filtered-out diff --git a/index.js b/index.js index 4886f63..0dc158e 100644 --- a/index.js +++ b/index.js @@ -12,7 +12,16 @@ module.exports = function(args) { // find where the location by examining the arguments // and create an instance to work with. // - locate(args); + locate(args, function (err) { + if (err) { + console.error(err); + return process.exit(1); + } + init(args); + }); +}; + +function init (args) { var db = getDB(args); // @@ -20,14 +29,14 @@ module.exports = function(args) { // than the program should not be run in REPL mode. // var cliCommands = [ - 'keys', 'values', 'get', 'match', - 'put', 'del', 'createReadStream', 'batch' + 'keys', 'values', 'get', 'match', 'put', 'del', + 'all', 'batch', 'length', 'start', 'end', 'limit', 'map' ]; - + var cliMode = Object.keys(args).some(function(cmd) { return cliCommands.indexOf(cmd) > -1; }); - + if (cliMode) { return cli(db, args); } @@ -39,6 +48,5 @@ module.exports = function(args) { history(repl, args); completion(repl, cache); - }; diff --git a/lib/batch_from_file.js b/lib/batch_from_file.js new file mode 100644 index 0000000..0557c70 --- /dev/null +++ b/lib/batch_from_file.js @@ -0,0 +1,52 @@ +var fs = require('fs'); +var path = require('path'); +var split = require('split'); +var printAndExit = require('./print_and_exit'); + +module.exports = function(db, file, del) { + var batch = []; + var total = 0; + + fs.createReadStream(path.resolve(file)) + .pipe(split()) + .on('data', function(line) { + if (line[0] !== '{') return + line = line.replace(/,$/, '') + + // + // add each line to the batch + // + var entry = JSON.parse(line); + entry.type = del ? 'del' : (entry.type || 'put'); + batch.push(entry) + + // + // run operations by batches of 10000 entries + // + if (batch.length >= 10000) { + var stream = this; + stream.pause(); + db.batch(batch, function(err, val) { + if (err) return printAndExit(err, val); + total += batch.length + console.log('ops run:', total) + batch = []; + stream.resume(); + }) + } + + }) + .on('close', function() { + if (batch.length > 0) { + db.batch(batch, function(err, val) { + if (err) return printAndExit(err, val); + total += batch.length; + console.log('total ops:', total) + }); + } + else { + console.log('total ops:', total) + } + }) +} + diff --git a/lib/cache.js b/lib/cache.js index b4e3aeb..584cd0b 100644 --- a/lib/cache.js +++ b/lib/cache.js @@ -1,6 +1,6 @@ /* * - * cahce.js + * cache.js * save the keys from the readstream into an * array so that they can be autocompleted and suggested. * diff --git a/lib/cli.js b/lib/cli.js index e6513c9..b626663 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -7,74 +7,140 @@ */ var Tabulate = require('tabulate'); var print = require('./print'); -var minimatch = require("minimatch"); +var printAndExit = require('./print_and_exit'); +var minimatch = require('minimatch'); +var path = require('path'); -var tabulate = Tabulate(process.stdout); +var hasStreamArg = function(args) { + return args.all || args.values || args.keys || args.match || args.length +} module.exports = function(db, args) { - if (args.values || args.keys) { + if (!hasStreamArg(args) && (args.start || args.end || args.limit || args.map)) { + args.all = true + } + + var mapFn + if (args.map) { + try { + mapFn = require(path.resolve(args.map)) + } + catch (err) { + if (err.code !== 'MODULE_NOT_FOUND') throw err + mapFn = eval(args.map) + } + } + + if (hasStreamArg(args)) { + + var valueEncoding = args.valueEncoding || 'json'; + if (args.start) args.gte = args.start; + if (args.end) args.lte = args.end; + + var values = args.values; + var keys = args.keys; + var limit = typeof args.limit === 'number' ? args.limit : Infinity; + var count = 0; + + if (args.match) { + delete args.values; + delete args.keys; + delete args.limit; + } + else if (args.map) { + delete args.limit; + } + else if (args.length) { + args.keys = true + } + + if (args.keys) args.values = false + if (args.values) args.keys = false - if (args.values) args.keys = false; - if (args.keys) args.values = false; - var items = []; + var tabulate; + if (keys && !args.line && !args.map) { + tabulate = Tabulate(process.stdout); + var items = []; + } - db - .createReadStream(args) + db.createReadStream(args) .on('data', function(data) { - if (args.keys) { - return items.push(data); + if (args.match && !minimatch(data.key, args.match)) return; + + if (++count > limit) { + count-- + return this.emit('end'); + } + + if (args.length) { + return + } + + if (args.match) { + if (keys) data = data.key; + if (values) data = data.value; + } + + if (args.map) { + data = mapFn(data) + if (data == null) { + count-- + return + } } - else if (!args.valueEncoding || - args.valueEncoding == 'json') { - process.stdout.write(JSON.stringify(data) + '\n'); + + if (tabulate) { + items.push(data); + } + else if (valueEncoding === 'json') { + if (typeof data !== 'string') data = JSON.stringify(data); + process.stdout.write(data + '\n'); } else { process.stdout.write(data); } }) + .on('error', print) .on('end', function() { - if (args.keys) { - process.stdout.write(tabulate.write(items)) + if (args.length){ + process.stdout.write(count + '\n'); } - process.exit(0); + else if (tabulate && items.length > 0) { + process.stdout.write(tabulate.write(items).trim() + '\n'); + } + var exitCode = count > 0 ? 0 : 1 + process.exit(exitCode); + }); + } + // Test args.batch before args.del to be able to pass a --del flag + // to the --batch option + else if (args.batch) { + if (args.batch[0] !== '[') { + require('./batch_from_file')(db, args.batch, args.del) + return + } + var batch = JSON.parse(args.batch); + if (args.del) { + batch = batch.map(op => { + op.type = 'del' + return op }); + } + db.batch(batch, printAndExit); } else if (args.put) { - db.put(args.key || args.put, args.value, print) + db.put(args.key || args.put, args.value, printAndExit); } else if (args.get) { - db.get(args.key || args.get, print) - } - else if (args.match) { - db.createReadStream(args) - .on('data', function(data) { - if (!minimatch(data.key, args.match)) return; - if (!args.valueEncoding || - args.valueEncoding == 'json') { - - return process.stdout.write( - JSON.stringify(data) + '\n' - ); - } - print(err, data.value); - }) - .on('end', process.exit); - } - else if (args.createReadStream) { - db.createReadStream(args) - .on('data', print) - .on('end', process.exit); - } - else if (args.batch) { - db.batch(args.batch, print) + db.get(args.key || args.get, printAndExit); } else if (args.del) { - db.del(args.key || args.del, print) + db.del(args.key || args.del, printAndExit); } else { - print(null, 'No valid command'); + printAndExit(new Error('No valid command')); } }; diff --git a/lib/db.js b/lib/db.js index bcc7b5f..8833bc8 100644 --- a/lib/db.js +++ b/lib/db.js @@ -11,6 +11,13 @@ module.exports = function(args) { //if (!args.keyEncoding) { // args.keyEncoding = bytewise //} - return level(args.path, args) + + // Ignore args that aren't meant to be database options + // Start by cloning the args object so that those changes + // don't propagate anyware else + dbArgs = JSON.parse(JSON.stringify(args)) + delete dbArgs.limit + + return level(dbArgs.path, dbArgs) } diff --git a/lib/location.js b/lib/location.js index 7477eb7..462773b 100644 --- a/lib/location.js +++ b/lib/location.js @@ -4,9 +4,52 @@ * tries to find the location of the database. * */ -module.exports = function (argv) { + + var prompt = require('cli-prompt') + +module.exports = function (argv, cb) { var location = typeof argv.location === 'string'; argv.path = location && argv.location || argv._[0] || process.cwd(); + + if (isDatabasePath(argv.path)) { + cb(); + } + else { + requestConfirmation(argv.path, cb); + } + + + if (!argv.path) { + if (cwdIsADatabase()) { + argv.path = process.cwd(); + } + else { + console.error('no database found'); + return process.exit(1); + }; + }; +}; + +function isDatabasePath (path) { + try { + var testFilePath = require('path').resolve(path) + '/CURRENT' + var CURRENT = require('fs').readFileSync(testFilePath).toString(); + return CURRENT.split('-')[0] === 'MANIFEST'; + } catch (err) { + if (err.code === 'ENOENT') return false + throw err + }; }; +function requestConfirmation (path, cb) { + prompt(`do you really want to create a new database in ${path}? [y/N]`, function (val) { + if (val.toLowerCase().trim() === 'y') { + cb(); + } + else { + process.exit(1); + } + }) +} + diff --git a/lib/print_and_exit.js b/lib/print_and_exit.js new file mode 100644 index 0000000..8a0dcdb --- /dev/null +++ b/lib/print_and_exit.js @@ -0,0 +1,14 @@ +/* + * + * print_and_exit.js + * A wrapper of print for the CLI that exits with a non-zero code + * in case of errors + * + */ + var print = require('./print'); + + module.exports = function printAndExit(err, val) { + print(err, val) + var exitCode = err == null ? 0 : 1 + process.exit(exitCode) +} diff --git a/package.json b/package.json index 113e730..b8a8e72 100644 --- a/package.json +++ b/package.json @@ -35,12 +35,14 @@ "homepage": "https://github.com/hij1nx/lev", "dependencies": { "bytewise": "^1.1.0", + "cli-prompt": "^0.6.0", "level": "^1.3.0", "level-party": "^3.0.4", - "minimatch": "^2.0.1", + "minimatch": "^3.0.4", "minimist": "^1.1.0", "multilevel": "^7.3.0", "rc": "^0.5.4", + "split": "^1.0.1", "tabulate": "^1.0.0", "xtend": "^4.0.0" }