diff --git a/README.md b/README.md index 71fa60f..37c674e 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Requirements * NodeJS http://nodejs.org/ * MongoDB http://www.mongodb.org/ +* FFmpeg https://ffmpeg.org/ Setup @@ -21,7 +22,7 @@ Setup 2. Restore the DB using `AVnodeDB.zip` [mongorestore](http://docs.mongodb.org/manual/reference/program/mongorestore/) with `mongorestore --drop -d avnode ` 3. Request the file repository `/warehouse` to g.delgobbo@flyer.it (you don't need it to let the app starts) 4. Run `npm install && bower install` -5. Run `npm start` +5. Run `npm start` and `npm run start:videostranscoder` 6. Login with your FLxER user or use user: GianlucaDelGobbo password: GianlucaDelGobbo @@ -31,6 +32,12 @@ Contributing Want to contribute? Great!!! +Development +------------ + +For development we use `nodemon` to detect changes during developement. Ensure to start both scripts the main app with `npm run dev` and our videotranscoding queue worker with `npm run dev:videotranscoder`. + + ### Commands 1. Fork it. diff --git a/app/server/modules/filehandler.js b/app/server/modules/filehandler.js new file mode 100644 index 0000000..895f355 --- /dev/null +++ b/app/server/modules/filehandler.js @@ -0,0 +1,79 @@ +var FFmpeg = require('fluent-ffmpeg'), + config = require('getconfig'), + path = require('path'); + +// FIXME Move to config. +var directory = config.sitepath+'/warehouse/uploads/videos/'; + +var webPreset = function(command) { + command + .format('mp4') + //.audioCodec('libfaac') + .videoCodec('libx264') + .keepDAR(); +}; +var mobilePreset = function(command) { + command + .format('mp4') + //.audioCodec('libfaac') + .videoCodec('libx264') + .keepDAR() + .size('720x?'); +}; + +var fileName = function(file) { + return path.basename(file.name, path.extname(file.name)); +}; + +module.exports.createThumbnails = function(file, next) { + FFmpeg(directory + file.name) + .on('filenames', function(filenames) { + next(null, filenames); + }) + .screenshots({ + timemarks: ['10%', '25%'], + folder: directory, + filename: fileName(file) + '.png' + }); +}; + +module.exports.mobileVersion = function(file, next) { + var start = new Date().getTime(); + + //FIXME Check if file is actually a transcodable video file + var destination = directory + path.basename(file.name, path.extname(file.name)) + '-mobile.mp4'; + FFmpeg(directory + file.name) + .preset(mobilePreset) + .output(destination) + .on('end', function() { + var end = new Date().getTime(); + var seconds = ((end - start) / 60); + console.log('Mobile version took', seconds, ' seconds'); + next(null, file); + }) + .run(); +}; + +module.exports.transcode = function(file, next) { + var start = new Date().getTime(); + + //FIXME Check if file is actually a transcodable video file + var destination = directory + path.basename(file.name, path.extname(file.name)) + '.mp4'; + FFmpeg(directory + file.name) + .preset(webPreset) + .output(destination) + .on('end', function() { + var end = new Date().getTime(); + var seconds = ((end - start) / 60); + console.log('Transcoding took', seconds, ' seconds'); + next(null, file); + }) + .run(); +}; + +module.exports.info = function(file, next) { + FFmpeg.ffprobe(directory+file.name, function(err, metadata) { + if (err) return next(err, null); + next(null, metadata); + }); +}; diff --git a/app/server/modules/queue.js b/app/server/modules/queue.js new file mode 100644 index 0000000..4a06f32 --- /dev/null +++ b/app/server/modules/queue.js @@ -0,0 +1,35 @@ +var config = require('getconfig'), + mongodb = require('mongodb'), + mongoDbQueue = require('mongodb-queue'); + +var queue = null; +var con = config.mongo; + +module.exports.connect = function (done) { + mongodb.MongoClient.connect(con, function(err, db) { + queue = mongoDbQueue(db, 'queue'); + done(err); + }); +}; + +module.exports.get = function(handler) { + queue.get(handler); +}; + +module.exports.ping = function(ack, done) { + queue.ping(ack, done); +}; + +module.exports.remove = function(ack, done) { + queue.ack(ack, done); +}; + +module.exports.add = function(job) { + if (queue) { + queue.add(job, function(err) { + if (err) throw err; + console.log('job added', job); + }); + } +}; + diff --git a/app/server/routes/api.js b/app/server/routes/api.js index 1363310..5a25650 100644 --- a/app/server/routes/api.js +++ b/app/server/routes/api.js @@ -1,12 +1,12 @@ var express = require('express'); var router = express.Router(); +var config = require('getconfig'); var fs = require('fs'); -var process = require('process'); var path = require('path'); var multer = require('multer'); -var upload = multer({ dest: process.cwd() + '/warehouse/tmp/' }); +var upload = multer({ dest: config.sitepath + '/warehouse/tmp/' }); var mime = require('mime'); var sha1 = require('sha1'); @@ -16,8 +16,6 @@ var _ = require('lodash'); var validateParams = require('../validation.js').validateParams; var Joi = require('joi'); -var config = require('getconfig'); - var multipart = require('connect-multiparty'); var resumable = require('../modules/resumable.js')('/tmp/avnode-uploads/'); var uuid = require('uuid'); @@ -28,7 +26,7 @@ router.post('/upload/image', upload.single('image'), function (req, res) { var extension = mime.extension(req.file.mimetype); if (extension === 'png' || extension === 'jpeg') { response = '/warehouse/uploads/' + sha1(req.file.originalname) + '.' + extension; - var destAbsolute = process.cwd() + response; + var destAbsolute = config.sitepath + response; fs.createReadStream(req.file.path).pipe(fs.createWriteStream(destAbsolute)); fs.unlink(req.file.path); } @@ -56,14 +54,15 @@ router.get( ); router.post('/upload/files', function(req, res){ - var destination = process.cwd() + '/warehouse/uploads/videos/'; + var destination = config.sitepath + '/warehouse/uploads/videos/'; if (!fs.existsSync(destination)){ fs.mkdirSync(destination); } resumable.post(req, function(status, filename, original_filename, identifier){ if (status === 'done') { // FIXME Path.extname can be something different than the acutal file extension. - var uniqueFileName = uuid.v4() + path.extname(filename); + var id = uuid.v4(); + var uniqueFileName = id + path.extname(filename); //when all chunks uploaded, then createWriteStream to /uploads folder with filename var stream = fs.createWriteStream(destination + uniqueFileName); //stitches the file chunks back together to create the original file. @@ -75,6 +74,7 @@ router.post('/upload/files', function(req, res){ }); } res.send({ + id: id, fileName: uniqueFileName, status: status }); @@ -189,4 +189,28 @@ router.get( }); } ); +router.get('/video/:id/poster', function(req,res) { + try { + res.sendFile(config.sitepath + '/warehouse/uploads/videos/' + req.params.id); + } + catch (e) { + res.sendStatus(404); + } +}); +router.get('/video/:id', function(req,res) { + try { + res.sendFile(config.sitepath + '/warehouse/uploads/videos/' + req.params.id + '.mp4'); + } + catch (e) { + res.sendStatus(404); + } +}); +router.get('/video/:id/mobile', function(req,res) { + try { + res.sendFile(config.sitepath + '/warehouse/uploads/videos/' + req.params.id + '-mobile.mp4'); + } + catch (e) { + res.sendStatus(404); + } +}); module.exports = router; diff --git a/app/server/routes/controlpanel/footages.js b/app/server/routes/controlpanel/footages.js index 11f2626..dde27ce 100644 --- a/app/server/routes/controlpanel/footages.js +++ b/app/server/routes/controlpanel/footages.js @@ -4,16 +4,20 @@ var User = require('../../models/user'); var config = require('getconfig'); var mongoose = require('mongoose'); var _ = require('lodash'); +var Queue = require('../../modules/queue'); exports.listGet = function get(req, res) { + // In case a new one will be created + var footageId = mongoose.Types.ObjectId(); User.findOne({_id: req.user._id}) .populate('footages') .exec(function(err, resolvedUser) { res.render('controlpanel/footages/list', { config: config, user: req.user, - footages: resolvedUser.footages + footages: resolvedUser.footages, + newFootageId: footageId }); }); }; @@ -26,19 +30,28 @@ exports.createPost = function post(req, res) { // TODO Display error/alert, because permalink isn't unique res.redirect('/controlpanel/footage'); } else { + var footageId = mongoose.Types.ObjectId(); if (req.body.file) { var file = JSON.parse(req.body.file); var attachment = new File({ _id: mongoose.Types.ObjectId(), - file: file.id, + uuid: file.uuid, + name: file.name, original_name: file.original_name, size: file.size, mimetype: file.type, - duration: 129831, - encoded: false + }); + Queue.add({ + type: 'thumbnail', + file: file, + footage: footageId + }); + Queue.add({ + type: 'transcode', + file: file, + footage: footageId }); } - var footageId = mongoose.Types.ObjectId(); new Footage({ _id: footageId, title: req.body.title, @@ -72,12 +85,11 @@ exports.updatePost = function(req, res) { } else if (file !== null) { attachment = new File({ _id: mongoose.Types.ObjectId(), - file: file.id, + uuid: file.uuid, + name: file.name, original_name: file.original_name, size: file.size, mimetype: file.type, - duration: 129831, - encoded: false }); } @@ -115,8 +127,9 @@ exports.editGet = function(req, res) { }; exports.filePost = function(req, res) { - var file = req.body.file; - res.status(200).json(file); + var file = JSON.parse(req.body.file); + console.log('>>>>>>>',file); + res.status(200).json(JSON.stringify(file)); }; exports.deleteReq = function post(req,res) { diff --git a/app/server/schema/file.js b/app/server/schema/file.js index 3196682..eed60f1 100644 --- a/app/server/schema/file.js +++ b/app/server/schema/file.js @@ -1,11 +1,17 @@ var Schema = require('mongoose').Schema; module.exports = new Schema({ - file: String, + uuid: String, + name: String, original_name: String, mimetype: String, - preview: String, size: Number, - duration: Number, - encoded:Boolean + duration: {type: Number, default: 0}, + metadata: Object, + previews: Array, + status: { + preview: {type: Boolean, default: false}, + transcoded: {type: Boolean, default: false}, + mobile: {type: Boolean, default: false} + } }); diff --git a/app/server/schema/footage.js b/app/server/schema/footage.js index 6aeb66e..10103cc 100644 --- a/app/server/schema/footage.js +++ b/app/server/schema/footage.js @@ -19,7 +19,6 @@ module.exports = new Schema({ text: {}, is_public: Boolean, file: File, //always one - preview_file: String, //FIXME put it inside file tags: [Tag], stats: { visits: Number, diff --git a/app/server/schema/gallery.js b/app/server/schema/gallery.js index 9f17498..c34273b 100644 --- a/app/server/schema/gallery.js +++ b/app/server/schema/gallery.js @@ -12,7 +12,6 @@ var text = {}; config.locales.forEach(function(locale) { text[locale] = String; - }); module.exports = new Schema({ diff --git a/app/server/schema/tvshow.js b/app/server/schema/tvshow.js index 1f0825f..2ed0395 100644 --- a/app/server/schema/tvshow.js +++ b/app/server/schema/tvshow.js @@ -13,10 +13,6 @@ config.locales.forEach(function (locale) { text[locale] = String; }); - - - - module.exports = new Schema({ old_id: String, creation_date: Date, diff --git a/app/server/views/controlpanel/footages/list.jade b/app/server/views/controlpanel/footages/list.jade index 59e81cd..670fd49 100644 --- a/app/server/views/controlpanel/footages/list.jade +++ b/app/server/views/controlpanel/footages/list.jade @@ -18,7 +18,8 @@ block inner-content input(name="permalink", type="text", onblur='validatePermalink(this)', placeholder="permalink", required).form-control div.col-xs-12.col-md-6 - input(name="file", type="hidden", required) + input(name="id", type="hidden", value="#{newFootageId}", required) + input(name="file", type="hidden", required)#file div.form-group label | Title @@ -70,20 +71,52 @@ block inner-content div.panel.panel-default div.panel-body = f.title - span= f.file - div.row - div.col-xs-12.col-md-6 - a.btn.btn-block.btn-primary(href="/controlpanel/footage/edit/#{f._id}") + if f.file && f.file.status.preview && !f.file.status.transcoded + img(src="/api/video/#{f.file.previews[0]}/poster") + if f.file && f.file.status.preview && f.file.status.transcoded + video(poster="/api/video/#{f.file.previews[0]}/poster", controls, style="max-width: 100%; height: auto;") + source(src="/api/video/#{f.file.uuid}") + if f.file && !f.file.status.preview && !f.file.status.transcoded + div.text-center + i.fa.fa-spinner.fa-3x.fa-spin + div + | grenerating preview + hr + div.pull-left + if f.file.status.transcoded === false + span.label.label-info + i.fa.fa-circle-o-notch.fa-spin + |   web version + else + span.label.label-success + i.fa.fa-check + |   web version + br + if f.file.status.mobile === false + span.label.label-info + i.fa.fa-circle-o-notch.fa-spin + |   mobile version + else + span.label.label-success + i.fa.fa-check + |   mobile version + + + div.pull-right + div.btn-group + a.btn.btn-primary(href="/controlpanel/footage/edit/#{f._id}") i.fa.fa-pencil - | edit - div.col-xs-12.col-md-6 - a.btn.btn-block.btn-danger(onclick='deleteFootage(this, "#{f._id}")') + a.btn.btn-danger(onclick='deleteFootage(this, "#{f._id}")') i.fa.fa-trash - | delete + div.clearfix script(src="//cdnjs.cloudflare.com/ajax/libs/resumable.js/1.0.2/resumable.min.js") script. + + // Footage id + var newFootageId ="#{newFootageId}"; + function deleteFootage(el, id) { $.ajax({ type: "DELETE", @@ -119,7 +152,7 @@ block inner-content r.assignBrowse($('.resumable-browse')[0]); // Handle file add event r.on('fileAdded', function(file){ - $('.resumable-drop').hide(); + //$('.resumable-drop').hide(); // Show progress bar $('.resumable-progress, .resumable-list').show(); // Show pause, hide resume @@ -145,12 +178,14 @@ block inner-content $('.resumable-file-'+result.uniqueIdentifier+' .resumable-file-progress').html('(completed)'); var message = JSON.parse(message); var file = { - id: result.uniqueIdentifier, + uuid: message.id, name: message.fileName, original_name: result.file.name, size: result.file.size, - type: result.file.type + type: result.file.type, + footageId: newFootageId }; + console.log(file); $.ajax({ type: "POST", url: "/controlpanel/footage/file", @@ -159,9 +194,7 @@ block inner-content file: JSON.stringify(file) }, success: function(data) { - // FIXME $('input[name="file"]').attr('value', data); - console.log(data); console.log($('input[name="file"]')); } }); diff --git a/config/default.json b/config/default.json index ae4ec82..8d43a63 100644 --- a/config/default.json +++ b/config/default.json @@ -17,6 +17,10 @@ "regex": { "permalink": "^[\\w-_]+$" }, + "redis": { + "host": "127.0.0.1", + "port": 6379 + }, "amazon": { "key": "", "secret": "" diff --git a/init.js b/init.js index ea6ce84..b31ca87 100644 --- a/init.js +++ b/init.js @@ -1,16 +1,19 @@ var config = require('getconfig'); var express = require('express'); var mongoose = require('mongoose'); - +var queue = require('./app/server/modules/queue'); module.exports = function(ready) { var app = express(); + queue.connect(function(err) { + if (err) throw err; + console.log('Queue connection established'); + }); app.root = __dirname; require('./app/server/setup')(app, express); require('./app/server/router')(app); - var server = null; mongoose.connect(config.mongo); mongoose.connection.on('error', function(error) { diff --git a/package.json b/package.json index 90c7ea0..6814239 100644 --- a/package.json +++ b/package.json @@ -18,12 +18,14 @@ "express-favicon": "^1.0.1", "express-session": "^1.12.1", "flat": "^1.6.0", + "fluent-ffmpeg": "^2.1.0", "formidable": "^1.0.17", "getconfig": "^2.2.0", "i18n": "^0.5.0", "imagemagick": "^0.1.3", "jade": "^1.11.0", "joi": "^7.2.2", + "kue": "^0.11.1", "lodash": "^3.10.1", "mailchimp": "^1.1.3", "mailchimp-api": "^2.0.7", @@ -34,6 +36,7 @@ "moment": "^2.10.6", "mongo-connect": "0.0.6", "mongodb": "^2.0.49", + "mongodb-queue": "^2.1.0", "mongoose": "^4.3.1", "morgan": "^1.6.1", "multer": "^1.1.0", @@ -46,6 +49,8 @@ "passport-local": "^1.0.0", "passport-remember-me": "0.0.1", "passport-twitter": "^1.0.2", + "pm2": "^1.1.3", + "redis": "^2.6.2", "request": "^2.67.0", "rimraf": "^2.4.4", "sha1": "^1.1.1", @@ -62,7 +67,12 @@ }, "scripts": { "start": "node app.js", + "start:videotranscoder": "node videotranscoder.js", "dev": "nodemon app.js", + "dev:videotranscoder": "nodemon videotranscoder.js", + "pm2-start": "./node_modules/pm2/bin/pm2 start pm2.json", + "pm2-stop": "./node_modules/pm2/bin/pm2 delete pm2.json", + "pm2-logs": "./node_modules/pm2/bin/pm2 logs", "test": "./node_modules/.bin/mocha", "pretest": "eslint -c .eslintrc.json --ignore-path .gitignore --ignore-path .eslintignore . && stylint app/public/css/style.styl" }, diff --git a/pm2.json b/pm2.json new file mode 100644 index 0000000..09bfc20 --- /dev/null +++ b/pm2.json @@ -0,0 +1,17 @@ +{ + apps: [{ + name: "App", + script: "./app.js", + watch: "./app", + ignore_watch: [ + "app/public" + ] + },{ + name: "VideoTranscoder", + script: "./videotranscoder.js", + watch: "./videotranscoder.js", + ignore_watch: [ + "app/public" + ] + }] +} diff --git a/test/spec/models/footage.test.js b/test/spec/models/footage.test.js index 1cc5443..5ffee10 100644 --- a/test/spec/models/footage.test.js +++ b/test/spec/models/footage.test.js @@ -14,16 +14,16 @@ describe('Footage Model', function() { ] }; }); - it('isowner should return false if footage id is not present', function() { + it('isOwner should return false if footage id is not present', function() { expect(isOwner(user, '1111')).toBe(false); }); it('isOwner should return true if user has footage id', function() { expect(isOwner(user, '57c411ca3bbefc0e37b877e9')).toBe(true); }); - it('isOwner should return true if valid id is string', function() { + it('isOwner should return true if valid id is of type string', function() { expect(isOwner(user, '57c411ca3bbefc0e37b877e9')).toBe(true); }); - it('isOwner should return true if valid id is ObjectID', function() { + it('isOwner should return true if valid id is an ObjectId', function() { var id = mongoose.Types.ObjectId('57c411cf2819c01d377b54e8'); expect(isOwner(user, id)).toBe(true); }); diff --git a/videotranscoder.js b/videotranscoder.js new file mode 100644 index 0000000..41d363d --- /dev/null +++ b/videotranscoder.js @@ -0,0 +1,109 @@ +var queue = require('./app/server/modules/queue'), + config = require('getconfig'), + Footage = require('./app/server/models/footage'), + Filehandler = require('./app/server/modules/filehandler'), + _ = require('lodash'), + mongoose = require('mongoose'); + +mongoose.connect(config.mongo); +mongoose.connection.on('error', function(error) { + console.log('MONGOOSE ERROR', error); +}); +mongoose.connection.once('open', function() { + mongoose.set('debug', true); + console.log(config.mongo); +}); +queue.connect(function(err) { + if (err) throw err; + processQueue(); +}); + +var processQueue = function() { + queue.get(function(err, job) { + if (err) throw err; + if (job) { + handleJob(job, processQueue); + } else { + setTimeout(processQueue, 1000); + } + }); +}; + +var handleJob = function(job, next) { + switch(job.payload.type) { + case 'thumbnail': + thumbnail(job, function(err, job) { + queue.remove(job.ack, function(err) { + if (err) throw err; + console.log('thumbnail job finished', job.id); + next(); + }); + }); + break; + case 'transcode': + handleFile(job, function(err, job) { + if (err) throw err; + if (job) { + queue.remove(job.ack, function(err) { + if (err) throw err; + console.log('transcode job finished', job.id); + next(); + }); + } + }); + break; + default: + console.log('Unknown type:', job.payload.type); + setTimeout(next, 1000); + } +}; + +var handleFile = function(job, next) { + var interval = setInterval(function() { + queue.ping(job.ack, function(err) { + if (err) console.log(err); + }); + }, 5000); + Filehandler.info(job.payload.file, function(err, metadata) { + Footage.findByIdAndUpdate(job.payload.footage, {$set: { + 'file.metadata': metadata, + 'file.duration': metadata.format.duration + }}, function (err) { + if (err) throw err; + Filehandler.transcode(job.payload.file, function(err) { + if (err) throw err; + Footage.findByIdAndUpdate(job.payload.footage, {$set: {'file.status.transcoded': true}}, function (err) { + if (err) throw err; + Filehandler.mobileVersion(job.payload.file, function(err) { + if (err) throw err; + Footage.findByIdAndUpdate(job.payload.footage, {$set: {'file.status.mobile': true}}, function (err) { + if (err) throw err; + clearInterval(interval); + next(null, job); + }); + }); + }); + }); + }); + }); +}; + +var thumbnail = function(job, next) { + var interval = setInterval(function() { + queue.ping(job.ack, function(err) { + if (err) console.log(err); + }); + }, 5000); + Filehandler.createThumbnails(job.payload.file, function(err, result) { + if (err) throw err; + var previews = result; + if (!_.isArray(result)) { + previews = [result]; + } + Footage.findByIdAndUpdate(job.payload.footage, {$set: {'file.previews': previews, 'file.status.preview': true}}, {}, function (err) { + if (err) throw err; + clearInterval(interval); + next(null, job); + }); + }); +};