WIP: Issues/42 db structure rework

app.js

@@ -1,60 +1,24 @@
 const express = require('express'),
   debug = require('debug')('explorer'),
   path = require('path'),
-  bitcoinapi = require('bitcoin-node-api'),
+  { api } = require('./lib/api'),
   favicon = require('static-favicon'),
   logger = require('morgan'),
   cookieParser = require('cookie-parser'),
   bodyParser = require('body-parser'),
-  request = require('request'),
   settings = require('./lib/settings'),
   routes = require('./routes/index'),
   lib = require('./lib/explorer'),
   db = require('./lib/database'),
   locale = require('./lib/locale'),
-  { promisify, spawnCmd, requestp } = require('./lib/util')
-
-const app = express();
-
-// set database update intervals
-spawnCmd('node', [ 'scripts/sync.js', 'index', settings.index.index_mode || 'update' ])
-setInterval(function () {
-  spawnCmd('node', [ 'scripts/sync.js', 'index', 'update' ])
-}, settings.sync_timeout)
-setInterval(function () {
-  spawnCmd('node', [ 'scripts/sync.js', 'market' ])
-}, settings.market_timeout)
-setInterval(function () {
-  spawnCmd('node', [ 'scripts/peers.js' ])
-}, settings.peer_timeout)
-
-const info = require('./info');
+  { requestp } = require('./lib/util'),
+  info = require('./info')
+
+const app = express()
 info(app)
+api.setCachers(db.rpc)
 
-// bitcoinapi
-bitcoinapi.setWalletDetails(settings.wallet);
-if (settings.heavy != true) {
-  bitcoinapi.setAccess('only', [ 'getinfo', 'getnetworkhashps', 'getmininginfo','getdifficulty', 'getconnectioncount',
-    'getblockcount', 'getblockhash', 'getblock', 'getrawtransaction', 'getpeerinfo', 'gettxoutsetinfo', 'getmempoolinfo', 'getrawmempool' ]);
-} else {
-  // enable additional heavy api calls
-  /*
-    getvote - Returns the current block reward vote setting.
-    getmaxvote - Returns the maximum allowed vote for the current phase of voting.
-    getphase - Returns the current voting phase ('Mint', 'Limit' or 'Sustain').
-    getreward - Returns the current block reward, which has been decided democratically in the previous round of block reward voting.
-    getnextrewardestimate - Returns an estimate for the next block reward based on the current state of decentralized voting.
-    getnextrewardwhenstr - Returns string describing how long until the votes are tallied and the next block reward is computed.
-    getnextrewardwhensec - Same as above, but returns integer seconds.
-    getsupply - Returns the current money supply.
-    getmaxmoney - Returns the maximum possible money supply.
-  */
-  bitcoinapi.setAccess('only', [ 'getinfo', 'getstakinginfo', 'getnetworkhashps', 'getdifficulty', 'getconnectioncount',
-    'getblockcount', 'getblockhash', 'getblock', 'getrawtransaction','getmaxmoney', 'getvote',
-    'getmaxvote', 'getphase', 'getreward', 'getnextrewardestimate', 'getnextrewardwhenstr',
-    'getnextrewardwhensec', 'getsupply', 'gettxoutsetinfo', 'getmempoolinfo', 'getrawmempool' ]);
-}
-// view engine setup
+  // view engine setup
 app.set('views', path.join(__dirname, 'views'));
 app.set('view engine', 'jade');
 
@@ -66,7 +30,7 @@ app.use(cookieParser());
 app.use(express.static(path.join(__dirname, 'public')));
 
 // routes
-app.use('/api', bitcoinapi.app);
+app.use('/api', api.app);
 app.use('/', routes);
 app.use('/ext/getmoneysupply', function(req,res){
   lib.get_supply(function(supply){
@@ -111,74 +75,35 @@ app.use('/ext/getdistribution', function(req,res){
   });
 });
 
-app.use('/ext/getlasttxs', function (req, res) {
-  return db.get_last_txs(parseInt(req.query.count), req.query.minAmount * 100000000, req.query.start).then(txs =>
-    res.send({data: txs})
-  ).catch(err => {
-    debug(err)
-    res.send({ error: `An error occurred: ${err}` })
-  })
-});
-
-app.use('/ext/getblocks/:start/:end', function (req, res) {
+app.use('/ext/getblocks/:start/:end', async function (req, res) {
   const endpoint = settings.address || `http://${req.headers.host}`
   const start = parseInt(req.param('start'))
   const end = parseInt(req.param('end'))
   const reverse = req.query.reverse && req.query.reverse.toLowerCase() === 'true'
-  const strip = req.query.strip && req.query.strip.toLowerCase() === 'true'
+  const flds = typeof req.query.flds === 'string' ? req.query.flds.split(',') : req.query.flds || []
 
   if (start > end) {
     res.send({ error: `End blockheight must be greater than or equal to the start blockheight.` })
     return
   }
 
+  const blockcount = await requestp(`${endpoint}/api/getblockcount`)
   let heights = Array(end - start + 1).fill(undefined).map((_, i) => start + i)
-  const txReq = () => Promise.all(heights.map(i =>
-    db.getTxs({ height: i }).then(txs => {
-      // sorts transactions from newest to oldest
-      txs.sort((a, b) => {
-        if (a.blockindex !== b.blockindex) return a.blockindex > b.blockindex ? -1 : 1
-        if (a.timestamp !== b.timestamp) return a.timestamp > b.timestamp ? -1 : 1
-        return a._id > b._id ? -1 : 1
-      })
-      // since reverse means to go from newest to oldest
-      return reverse ? txs : txs.reverse()
-    })
-  ))
-  const infoReq = (blockcount) => Promise.all(heights.map(i =>
-    promisify(lib.get_blockhash, i)
-      .then(hash =>
-        hash.includes('There was an error') ? null : lib.getBlock(hash, undefined, blockcount)
-      )
-  )).then(infos => strip ? infos.filter(info => info !== null) : infos)
-  const onErr = err => {
-    debug(err)
-    res.send({ error: `An error occurred: ${err}` })
+  if (reverse) heights = heights.map(h => blockcount - h + 1)
+
+  const searchFlds = flds[0] === 'summary'
+    ? { fulltx: 0, _id: 0 }
+    : flds ? flds.reduce((acc, fld) => ({ ...acc, [fld]: 1 }), { _id: 0, height: 1 }) : []
+  let blocks = await db.getBlocks(heights, searchFlds)
+  if (!blocks) {
+    blocks = await Promise.all(heights.map(h => lib.getRawRpc('getblockhash', [ h ]).then(hash => lib.getRawRpc('getblock', [ hash ]))))
+  } else {
+    blocks = blocks.sort((a, b) => (reverse ? -1 : 1) * (a.height <= b.height ? -1 : 1))
   }
-
-  promisify(request, `${endpoint}/api/getblockcount`, { json: true }).then(([ err, resp, height ]) => {
-    if (reverse) heights = heights.map(h => height - h + 1)
-    return height
-  }).then(blockcount => {
-    if (req.query.flds === 'summary') {
-      infoReq(blockcount).then(infos => res.send({ data: { blockcount, blocks: infos } })).catch(onErr)
-    } else if (req.query.flds && req.query.flds.length === 1 && req.query.flds[0] === 'tx') {
-      txReq().then(txs => res.send({ data: { blockcount, blocks: txs } })).catch(onErr)
-    } else {
-      Promise.all([ txReq(), infoReq(blockcount) ]).then(([ txs, infos ]) => {
-        res.send({
-          data: { blockcount, blocks: infos.map((info, i) => ({ ...info, tx: txs[i] })).map(block => {
-            if (req.query.flds && req.query.flds.length) {
-              Object.keys(block).forEach(key => {
-                if (!req.query.flds.includes(key)) delete block[key]
-              })
-            }
-            return block
-          }) }
-        })
-      }).catch(onErr)
-    }
-  }).catch(onErr)
+  blocks.forEach(block => {
+    if (!flds.includes('height') && flds[0] !== 'summary') delete block['height']
+  })
+  res.send({ data: { blockcount, blocks } })
 })
 
 app.use('/ext/connections', function(req,res){
@@ -200,7 +125,6 @@ app.set('googleplus', settings.googleplus);
 app.set('youtube', settings.youtube);
 app.set('genesis_block', settings.genesis_block);
 app.set('index', settings.index);
-app.set('heavy', settings.heavy);
 app.set('txcount', settings.txcount);
 app.set('nethash', settings.nethash);
 app.set('nethash_units', settings.nethash_units);

bin/cluster.js

@@ -3,7 +3,7 @@ const cluster = require('cluster')
 const fs = require('fs')
 const settings = require('../lib/settings')
 const db = require('../lib/database')
-const { prettyPrint } = require('../lib/util')
+const { prettyPrint, spawnCmd } = require('../lib/util')
 
 if (cluster.isMaster) {
   fs.writeFile('./tmp/cluster.pid', process.pid, function (err) {
@@ -12,27 +12,44 @@ if (cluster.isMaster) {
       process.exit(1)
     } else {
       debug('Starting cluster with pid: ' + process.pid)
+
+      const updateIntervals = []
+
       // ensure workers exit cleanly
       process.on('SIGINT', () => {
         debug('Cluster shutting down...')
         for (let worker of Object.values(cluster.workers)) {
           worker.kill()
         }
         // exit the master process
+        // updateIntervals.forEach(i => clearInterval(i))
         process.exit(0)
       })
 
       // ensure workers have a valid schema to serve before spawning them
       db.connect(settings.dbsettings).then(() =>
         db.setupSchema()
       ).then(() => {
+        // set database update intervals
+        spawnCmd('node', [ 'scripts/sync.js', 'index', settings.index.index_mode || 'update' ])
+        updateIntervals.push(setInterval(function () {
+          spawnCmd('node', [ 'scripts/sync.js', 'index', 'update' ])
+        }, settings.sync_timeout))
+        updateIntervals.push(setInterval(function () {
+          spawnCmd('node', [ 'scripts/sync.js', 'market' ])
+        }, settings.market_timeout))
+        updateIntervals.push(setInterval(function () {
+          spawnCmd('node', [ 'scripts/peers.js' ])
+        }, settings.peer_timeout))
+
         // spawn a worker for each cpu core
         require('os').cpus().forEach(_ => {
           cluster.fork()
         })
       }).catch(err => {
         debug(`An error occured setting up cluster: ${prettyPrint(err)}`)
         debug('Aborting...')
+        // updateIntervals.forEach(i => clearInterval(i))
         process.exit(1)
       })
 

bin/instance.js

@@ -1,6 +1,9 @@
 #!/usr/bin/env node
-const debug = require('debug')('explorer')
+
+// ensure the api singleton has been initialized before anything else
 const settings = require('../lib/settings')
+
+const debug = require('debug')('explorer')
 const db = require('../lib/database')
 const app = require('../app')
 const { promisify } = require('../lib/util')
@@ -9,9 +12,11 @@ app.set('port', process.env.PORT || settings.port)
 
 db.connect(settings.dbsettings).then(() =>
   promisify(db.get_stats, settings.coin)
-).then((stats) => {
+).then(stats => {
   app.locals.stats = stats
   const server = app.listen(app.get('port'), () => {
     debug('Express server listening on port ' + server.address().port)
   })
+}).catch(err => {
+  process.exit(1)
 })

config/default.json

@@ -1,14 +1,19 @@
 {
   "dbsettings": {
     "uri": "mongodb://localhost:27017/blockchain-explorer",
-    "options": { "useNewUrlParser": true },
+    "options": { "useNewUrlParser": true, "autoIndex": false },
     "benchmark_uri": "mongodb://localhost:27017/explorer-benchmark",
     "benchmark_options": { "useNewUrlParser": true }
   },
   "wallet": {
     "url": "http://localhost:18332",
     "username": "undefined",
-    "password": "undefined"
+    "password": "undefined",
+    "network": "testnet",
+    "ssl": {
+      "enabled": false,
+      "strict": false
+    }
   },
   "title": "Equibit",
   "address": "http://127.0.0.1:3001",
@@ -23,6 +28,7 @@
   "sync_timeout": 60000,
   "market_timeout": 120000,
   "peer_timeout": 240000,
+  "ui_interval": 30000,
   "confirmations": 40,
   "locale": "locale/en.json",
   "display": {
@@ -72,7 +78,6 @@
   "youtube": "UCBWEY89pt-DpPYyPedqGWEg",
   "genesis_tx": "ea914133c255e8b47fb99d26b8627f90e12f5a9c3bc86269652d474d9814aaca",
   "genesis_block": "0000e9b3a79f70fb0be95b38604636441323450731e9ee1b5ef410791dac7184",
-  "heavy": false,
   "txcount": 100,
   "show_sent_received": true,
   "supply": "COINBASE",