From cdb977eff9f14db641ba4a09a3a2d33f74ff3f4f Mon Sep 17 00:00:00 2001 From: Patrick Finkbeiner Date: Wed, 7 Sep 2016 12:52:04 +0200 Subject: [PATCH 1/8] Transcode videos and generate mobile versions. --- app/server/modules/filehandler.js | 79 +++++++++ app/server/modules/queue.js | 35 ++++ app/server/routes/api.js | 38 +++- app/server/routes/controlpanel/footages.js | 46 +++-- app/server/schema/file.js | 14 +- app/server/schema/footage.js | 1 - app/server/schema/gallery.js | 1 - app/server/schema/tvshow.js | 4 - .../views/controlpanel/footages/list.jade | 60 +++++-- config/default.json | 4 + init.js | 7 +- package.json | 7 +- pm2.json | 17 ++ videotranscoder.js | 166 ++++++++++++++++++ 14 files changed, 436 insertions(+), 43 deletions(-) create mode 100644 app/server/modules/filehandler.js create mode 100644 app/server/modules/queue.js create mode 100644 pm2.json create mode 100644 videotranscoder.js diff --git a/app/server/modules/filehandler.js b/app/server/modules/filehandler.js new file mode 100644 index 0000000..d9a5edc --- /dev/null +++ b/app/server/modules/filehandler.js @@ -0,0 +1,79 @@ +var FFmpeg = require('fluent-ffmpeg'), + config = require('getconfig'), + path = require('path'); + +// FIXME Move to config. +var directory = config.sitepath+'/warehouse/uploads/videos/'; + +function webPreset(command) { + command + .format('mp4') + //.audioCodec('libfaac') + .videoCodec('libx264') + .keepDAR(); +} +function mobilePreset(command) { + command + .format('mp4') + //.audioCodec('libfaac') + .videoCodec('libx264') + .keepDAR() + .size('720x?'); +} + +var fileName = function(file) { + return path.basename(file.name, path.extname(file.name)); +}; + +module.exports.createThumbnails = function(file, next) { + FFmpeg(directory + file.name) + .on('filenames', function(filenames) { + next(null, filenames); + }) + .screenshots({ + timemarks: ['10%', '25%'], + folder: directory, + filename: fileName(file) + '.png' + }); +}; + +module.exports.mobileVersion = function(file, next) { + var start = new Date().getTime(); + + //FIXME Check if file is actually a transcodable video file + var destination = directory + path.basename(file.name, path.extname(file.name)) + '-mobile.mp4'; + FFmpeg(directory + file.name) + .preset(mobilePreset) + .output(destination) + .on('end', function() { + var end = new Date().getTime(); + var seconds = ((end - start) / 60); + console.log('Mobile version took', seconds, ' seconds'); + next(null, file); + }) + .run(); +}; + +module.exports.transcode = function(file, next) { + var start = new Date().getTime(); + + //FIXME Check if file is actually a transcodable video file + var destination = directory + path.basename(file.name, path.extname(file.name)) + '.mp4'; + FFmpeg(directory + file.name) + .preset(webPreset) + .output(destination) + .on('end', function() { + var end = new Date().getTime(); + var seconds = ((end - start) / 60); + console.log('Transcoding took', seconds, ' seconds'); + next(null, file); + }) + .run(); +}; + +module.exports.info = function(file, next) { + FFmpeg.ffprobe(directory+file.name, function(err, metadata) { + if (err) return next(err, null); + next(null, metadata); + }); +}; diff --git a/app/server/modules/queue.js b/app/server/modules/queue.js new file mode 100644 index 0000000..4a06f32 --- /dev/null +++ b/app/server/modules/queue.js @@ -0,0 +1,35 @@ +var config = require('getconfig'), + mongodb = require('mongodb'), + mongoDbQueue = require('mongodb-queue'); + +var queue = null; +var con = config.mongo; + +module.exports.connect = function (done) { + mongodb.MongoClient.connect(con, function(err, db) { + queue = mongoDbQueue(db, 'queue'); + done(err); + }); +}; + +module.exports.get = function(handler) { + queue.get(handler); +}; + +module.exports.ping = function(ack, done) { + queue.ping(ack, done); +}; + +module.exports.remove = function(ack, done) { + queue.ack(ack, done); +}; + +module.exports.add = function(job) { + if (queue) { + queue.add(job, function(err) { + if (err) throw err; + console.log('job added', job); + }); + } +}; + diff --git a/app/server/routes/api.js b/app/server/routes/api.js index 1363310..5a25650 100644 --- a/app/server/routes/api.js +++ b/app/server/routes/api.js @@ -1,12 +1,12 @@ var express = require('express'); var router = express.Router(); +var config = require('getconfig'); var fs = require('fs'); -var process = require('process'); var path = require('path'); var multer = require('multer'); -var upload = multer({ dest: process.cwd() + '/warehouse/tmp/' }); +var upload = multer({ dest: config.sitepath + '/warehouse/tmp/' }); var mime = require('mime'); var sha1 = require('sha1'); @@ -16,8 +16,6 @@ var _ = require('lodash'); var validateParams = require('../validation.js').validateParams; var Joi = require('joi'); -var config = require('getconfig'); - var multipart = require('connect-multiparty'); var resumable = require('../modules/resumable.js')('/tmp/avnode-uploads/'); var uuid = require('uuid'); @@ -28,7 +26,7 @@ router.post('/upload/image', upload.single('image'), function (req, res) { var extension = mime.extension(req.file.mimetype); if (extension === 'png' || extension === 'jpeg') { response = '/warehouse/uploads/' + sha1(req.file.originalname) + '.' + extension; - var destAbsolute = process.cwd() + response; + var destAbsolute = config.sitepath + response; fs.createReadStream(req.file.path).pipe(fs.createWriteStream(destAbsolute)); fs.unlink(req.file.path); } @@ -56,14 +54,15 @@ router.get( ); router.post('/upload/files', function(req, res){ - var destination = process.cwd() + '/warehouse/uploads/videos/'; + var destination = config.sitepath + '/warehouse/uploads/videos/'; if (!fs.existsSync(destination)){ fs.mkdirSync(destination); } resumable.post(req, function(status, filename, original_filename, identifier){ if (status === 'done') { // FIXME Path.extname can be something different than the acutal file extension. - var uniqueFileName = uuid.v4() + path.extname(filename); + var id = uuid.v4(); + var uniqueFileName = id + path.extname(filename); //when all chunks uploaded, then createWriteStream to /uploads folder with filename var stream = fs.createWriteStream(destination + uniqueFileName); //stitches the file chunks back together to create the original file. @@ -75,6 +74,7 @@ router.post('/upload/files', function(req, res){ }); } res.send({ + id: id, fileName: uniqueFileName, status: status }); @@ -189,4 +189,28 @@ router.get( }); } ); +router.get('/video/:id/poster', function(req,res) { + try { + res.sendFile(config.sitepath + '/warehouse/uploads/videos/' + req.params.id); + } + catch (e) { + res.sendStatus(404); + } +}); +router.get('/video/:id', function(req,res) { + try { + res.sendFile(config.sitepath + '/warehouse/uploads/videos/' + req.params.id + '.mp4'); + } + catch (e) { + res.sendStatus(404); + } +}); +router.get('/video/:id/mobile', function(req,res) { + try { + res.sendFile(config.sitepath + '/warehouse/uploads/videos/' + req.params.id + '-mobile.mp4'); + } + catch (e) { + res.sendStatus(404); + } +}); module.exports = router; diff --git a/app/server/routes/controlpanel/footages.js b/app/server/routes/controlpanel/footages.js index f6b1acc..ad37bb2 100644 --- a/app/server/routes/controlpanel/footages.js +++ b/app/server/routes/controlpanel/footages.js @@ -3,35 +3,60 @@ var File = require('../../models/file'); var User = require('../../models/user'); var config = require('getconfig'); var mongoose = require('mongoose'); +var mime = require('mime'); var _ = require('lodash'); +var Queue = require('../../modules/queue'); exports.listGet = function get(req, res) { - console.log(Footage); + // In case a new one will be created + var footageId = mongoose.Types.ObjectId(); User.findOne({_id: req.user._id}) .populate('footages') .exec(function(err, resolvedUser) { res.render('controlpanel/footages/list', { config: config, user: req.user, - footages: resolvedUser.footages + footages: resolvedUser.footages, + newFootageId: footageId }); }); }; exports.createPost = function post(req, res) { + var footageId = mongoose.Types.ObjectId(req.body.id); if (req.body.file) { var file = JSON.parse(req.body.file); var attachment = new File({ _id: mongoose.Types.ObjectId(), - file: file.id, + uuid: file.uuid, + name: file.name, original_name: file.original_name, size: file.size, - mimetype: file.type, - duration: 129831, - encoded: false + mimetype: mime.lookup(file.name), + }); + + Queue.add({ + type: 'thumbnail', + file: file, + footage: footageId }); + Queue.add({ + type: 'footageFile', + file: file, + footage: footageId + }); + +// Queue.add({ +// type: 'metadata', +// file: file, +// footage: footageId +// }); +// Queue.add({ +// type: 'transcode', +// file: file, +// footage: footageId +// }); } - var footageId = mongoose.Types.ObjectId(); new Footage({ _id: footageId, title: req.body.title, @@ -59,7 +84,8 @@ exports.updatePost = function(req, res) { } else if (file !== null) { attachment = new File({ _id: mongoose.Types.ObjectId(), - file: file.id, + uuid: file.uuid, + name: file.name, original_name: file.original_name, size: file.size, mimetype: file.type, @@ -98,8 +124,8 @@ exports.editGet = function(req, res) { }; exports.filePost = function(req, res) { - var file = req.body.file; - res.status(200).json(file); + var file = JSON.parse(req.body.file); + res.status(200).json(JSON.stringify(file)); }; exports.deleteReq = function post(req,res) { diff --git a/app/server/schema/file.js b/app/server/schema/file.js index 3196682..eed60f1 100644 --- a/app/server/schema/file.js +++ b/app/server/schema/file.js @@ -1,11 +1,17 @@ var Schema = require('mongoose').Schema; module.exports = new Schema({ - file: String, + uuid: String, + name: String, original_name: String, mimetype: String, - preview: String, size: Number, - duration: Number, - encoded:Boolean + duration: {type: Number, default: 0}, + metadata: Object, + previews: Array, + status: { + preview: {type: Boolean, default: false}, + transcoded: {type: Boolean, default: false}, + mobile: {type: Boolean, default: false} + } }); diff --git a/app/server/schema/footage.js b/app/server/schema/footage.js index 3a54f31..73b7b75 100644 --- a/app/server/schema/footage.js +++ b/app/server/schema/footage.js @@ -18,7 +18,6 @@ module.exports = new Schema({ text: {}, is_public: Boolean, file: File, //always one - preview_file: String, //FIXME put it inside file tags: [Tag], stats: { visits: Number, diff --git a/app/server/schema/gallery.js b/app/server/schema/gallery.js index 9f17498..c34273b 100644 --- a/app/server/schema/gallery.js +++ b/app/server/schema/gallery.js @@ -12,7 +12,6 @@ var text = {}; config.locales.forEach(function(locale) { text[locale] = String; - }); module.exports = new Schema({ diff --git a/app/server/schema/tvshow.js b/app/server/schema/tvshow.js index 1f0825f..2ed0395 100644 --- a/app/server/schema/tvshow.js +++ b/app/server/schema/tvshow.js @@ -13,10 +13,6 @@ config.locales.forEach(function (locale) { text[locale] = String; }); - - - - module.exports = new Schema({ old_id: String, creation_date: Date, diff --git a/app/server/views/controlpanel/footages/list.jade b/app/server/views/controlpanel/footages/list.jade index 9634733..6ba14e3 100644 --- a/app/server/views/controlpanel/footages/list.jade +++ b/app/server/views/controlpanel/footages/list.jade @@ -8,6 +8,7 @@ block inner-content div.row form.form-horizontal(method="post", action="/controlpanel/footage/create", data-parsley-validate) div.col-xs-12.col-md-6 + input(name="id", type="hidden", value="#{newFootageId}", required) input(name="file", type="hidden", required)#file div.form-group label @@ -31,6 +32,7 @@ block inner-content ul.resumable-list input.btn.btn-primary.btn-block(type="submit", value="create") + h2= newFootageId if footages div.footages @@ -40,20 +42,52 @@ block inner-content div.panel.panel-default div.panel-body = f.title - span= f.file - div.row - div.col-xs-12.col-md-6 - a.btn.btn-block.btn-primary(href="/controlpanel/footage/edit/#{f._id}") + if f.file && f.file.status.preview && !f.file.status.transcoded + img(src="/api/video/#{f.file.previews[0]}/poster") + if f.file && f.file.status.preview && f.file.status.transcoded + video(poster="/api/video/#{f.file.previews[0]}/poster", controls, style="max-width: 100%; height: auto;") + source(src="/api/video/#{f.file.uuid}") + if f.file && !f.file.status.preview && !f.file.status.transcoded + div.text-center + i.fa.fa-spinner.fa-3x.fa-spin + div + | grenerating preview + hr + div.pull-left + if f.file.status.transcoded === false + span.label.label-info + i.fa.fa-circle-o-notch.fa-spin + |   web version + else + span.label.label-success + i.fa.fa-check + |   web version + br + if f.file.status.mobile === false + span.label.label-info + i.fa.fa-circle-o-notch.fa-spin + |   mobile version + else + span.label.label-success + i.fa.fa-check + |   mobile version + + + div.pull-right + div.btn-group + a.btn.btn-primary(href="/controlpanel/footage/edit/#{f._id}") i.fa.fa-pencil - | edit - div.col-xs-12.col-md-6 - a.btn.btn-block.btn-danger(onclick='deleteFootage(this, "#{f._id}")') + a.btn.btn-danger(onclick='deleteFootage(this, "#{f._id}")') i.fa.fa-trash - | delete + div.clearfix script(src="//cdnjs.cloudflare.com/ajax/libs/resumable.js/1.0.2/resumable.min.js") script. + + // Footage id + var newFootageId ="#{newFootageId}"; + function deleteFootage(el, id) { $.ajax({ type: "DELETE", @@ -83,7 +117,7 @@ block inner-content r.assignBrowse($('.resumable-browse')[0]); // Handle file add event r.on('fileAdded', function(file){ - $('.resumable-drop').hide(); + //$('.resumable-drop').hide(); // Show progress bar $('.resumable-progress, .resumable-list').show(); // Show pause, hide resume @@ -109,12 +143,14 @@ block inner-content $('.resumable-file-'+result.uniqueIdentifier+' .resumable-file-progress').html('(completed)'); var message = JSON.parse(message); var file = { - id: result.uniqueIdentifier, + uuid: message.id, name: message.fileName, original_name: result.file.name, size: result.file.size, - type: result.file.type + type: result.file.type, + footageId: newFootageId }; + console.log(file); $.ajax({ type: "POST", url: "/controlpanel/footage/file", @@ -123,9 +159,7 @@ block inner-content file: JSON.stringify(file) }, success: function(data) { - // FIXME $('input[name="file"]').attr('value', data); - console.log(data); console.log($('input[name="file"]')); } }); diff --git a/config/default.json b/config/default.json index ae4ec82..8d43a63 100644 --- a/config/default.json +++ b/config/default.json @@ -17,6 +17,10 @@ "regex": { "permalink": "^[\\w-_]+$" }, + "redis": { + "host": "127.0.0.1", + "port": 6379 + }, "amazon": { "key": "", "secret": "" diff --git a/init.js b/init.js index ea6ce84..b31ca87 100644 --- a/init.js +++ b/init.js @@ -1,16 +1,19 @@ var config = require('getconfig'); var express = require('express'); var mongoose = require('mongoose'); - +var queue = require('./app/server/modules/queue'); module.exports = function(ready) { var app = express(); + queue.connect(function(err) { + if (err) throw err; + console.log('Queue connection established'); + }); app.root = __dirname; require('./app/server/setup')(app, express); require('./app/server/router')(app); - var server = null; mongoose.connect(config.mongo); mongoose.connection.on('error', function(error) { diff --git a/package.json b/package.json index 90c7ea0..11baa1b 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "imagemagick": "^0.1.3", "jade": "^1.11.0", "joi": "^7.2.2", + "kue": "^0.11.1", "lodash": "^3.10.1", "mailchimp": "^1.1.3", "mailchimp-api": "^2.0.7", @@ -46,6 +47,8 @@ "passport-local": "^1.0.0", "passport-remember-me": "0.0.1", "passport-twitter": "^1.0.2", + "pm2": "^1.1.3", + "redis": "^2.6.2", "request": "^2.67.0", "rimraf": "^2.4.4", "sha1": "^1.1.1", @@ -62,7 +65,9 @@ }, "scripts": { "start": "node app.js", - "dev": "nodemon app.js", + "dev": "./node_modules/pm2/bin/pm2 start pm2.json", + "stop": "./node_modules/pm2/bin/pm2 delete pm2.json", + "logs": "./node_modules/pm2/bin/pm2 logs", "test": "./node_modules/.bin/mocha", "pretest": "eslint -c .eslintrc.json --ignore-path .gitignore --ignore-path .eslintignore . && stylint app/public/css/style.styl" }, diff --git a/pm2.json b/pm2.json new file mode 100644 index 0000000..09bfc20 --- /dev/null +++ b/pm2.json @@ -0,0 +1,17 @@ +{ + apps: [{ + name: "App", + script: "./app.js", + watch: "./app", + ignore_watch: [ + "app/public" + ] + },{ + name: "VideoTranscoder", + script: "./videotranscoder.js", + watch: "./videotranscoder.js", + ignore_watch: [ + "app/public" + ] + }] +} diff --git a/videotranscoder.js b/videotranscoder.js new file mode 100644 index 0000000..d840363 --- /dev/null +++ b/videotranscoder.js @@ -0,0 +1,166 @@ +var queue = require('./app/server/modules/queue'), + config = require('getconfig'), + Footage = require('./app/server/models/footage'), + Filehandler = require('./app/server/modules/filehandler'), + _ = require('lodash'), + mongoose = require('mongoose'); + +mongoose.connect(config.mongo); +mongoose.connection.on('error', function(error) { + console.log('MONGOOSE ERROR', error); +}); +mongoose.connection.once('open', function() { + mongoose.set('debug', true); + console.log(config.mongo); +}); +queue.connect(function(err) { + if (err) throw err; + processQueue(); +}); + +var processQueue = function() { + queue.get(function(err, job) { + if (err) throw err; + if (job) { + handleJob(job, processQueue); + } else { + setTimeout(processQueue, 1000); + } + }); +}; + +var handleJob = function(job, next) { + console.log('handle job', job.payload.type); + switch(job.payload.type) { + case 'metadata': + metadata(job, function(err, job) { + if (err) throw err; + queue.remove(job.ack, function(err) { + if (err) throw err; + console.log('metadata job finished', job.id); + next(); + }); + }); + break; + case 'thumbnail': + console.log('get thumbnail jooooob'); + thumbnail(job, function(err, job) { + queue.remove(job.ack, function(err) { + if (err) throw err; + console.log('thumbnail job finished', job.id); + next(); + }); + }); + break; + case 'transcode': + transcode(job, function(err, job) { + if (err) throw err; + queue.remove(job.ack, function(err) { + if (err) throw err; + console.log('transcode job finished', job.id); + next(); + }); + }); + break; + case 'footageFile': + handleFile(job, function(err, job) { + if (err) throw err; + if (job) { + queue.remove(job.ack, function(err) { + if (err) throw err; + console.log('transcode job finished', job.id); + next(); + }); + } + }); + break; + default: + console.log('Unknown type:', job.payload.type); + setTimeout(next, 1000); + } +}; + +var handleFile = function(job, next) { + var interval = setInterval(function() { + queue.ping(job.ack, function(err) { + if (err) console.log(err); + }); + }, 5000); + + Filehandler.info(job.payload.file, function(err, metadata) { + Footage.findByIdAndUpdate(job.payload.footage, {$set: { + 'file.metadata': metadata, + 'file.duration': metadata.format.duration + }}, function (err) { + if (err) throw err; + Filehandler.transcode(job.payload.file, function(err) { + if (err) throw err; + Footage.findByIdAndUpdate(job.payload.footage, {$set: {'file.status.transcoded': true}}, function (err) { + if (err) throw err; + Filehandler.mobileVersion(job.payload.file, function(err) { + if (err) throw err; + Footage.findByIdAndUpdate(job.payload.footage, {$set: {'file.status.mobile': true}}, function (err) { + if (err) throw err; + clearInterval(interval); + next(null, job); + }); + }); + }); + }); + }); + }); +}; + +var thumbnail = function(job, next) { + var interval = setInterval(function() { + queue.ping(job.ack, function(err) { + if (err) console.log(err); + }); + }, 5000); + Filehandler.createThumbnails(job.payload.file, function(err, result) { + if (err) throw err; + var previews = result; + if (!_.isArray(result)) { + previews = [result]; + } + Footage.findByIdAndUpdate(job.payload.footage, {$set: {'file.previews': previews, 'file.status.preview': true}}, {}, function (err) { + if (err) throw err; + clearInterval(interval); + next(null, job); + }); + }); +}; + +var transcode = function(job, next) { + var interval = setInterval(function() { + queue.ping(job.ack, function(err) { + if (err) console.log(err); + }); + }, 5000); + Filehandler.transcode(job.payload.file, function(err) { + if (err) throw err; + Footage.findbyidandupdate(job.payload.footage, {$set: {'file.status.transcoded': true}}, function (err) { + if (err) throw err; + clearInterval(interval); + next(null, job); + }); + }); +}; + +var metadata = function(job, next) { + var interval = setInterval(function() { + queue.ping(job.ack, function(err) { + if (err) console.log(err); + }); + }, 5000); + Filehandler.info(job.payload.file, function(err, metadata){ + Footage.findByIdAndUpdate(job.payload.footage, {$set: { + 'file.metadata': metadata, + 'file.duration': metadata.format.duration + }}, function (err) { + if (err) throw err; + clearInterval(interval); + next(null, job); + }); + }); +}; From a89bfa4caa042c7ac4f13447794b65eb059ca7ef Mon Sep 17 00:00:00 2001 From: Patrick Finkbeiner Date: Tue, 13 Sep 2016 10:33:13 +0200 Subject: [PATCH 2/8] Save some packages and rewrite script commands. --- package.json | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 11baa1b..e6e2155 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "express-favicon": "^1.0.1", "express-session": "^1.12.1", "flat": "^1.6.0", + "fluent-ffmpeg": "^2.1.0", "formidable": "^1.0.17", "getconfig": "^2.2.0", "i18n": "^0.5.0", @@ -35,6 +36,7 @@ "moment": "^2.10.6", "mongo-connect": "0.0.6", "mongodb": "^2.0.49", + "mongodb-queue": "^2.1.0", "mongoose": "^4.3.1", "morgan": "^1.6.1", "multer": "^1.1.0", @@ -65,9 +67,10 @@ }, "scripts": { "start": "node app.js", - "dev": "./node_modules/pm2/bin/pm2 start pm2.json", - "stop": "./node_modules/pm2/bin/pm2 delete pm2.json", - "logs": "./node_modules/pm2/bin/pm2 logs", + "start:videotranscoder": "node videotranscoder.js", + "pm2-start": "./node_modules/pm2/bin/pm2 start pm2.json", + "pm2-stop": "./node_modules/pm2/bin/pm2 delete pm2.json", + "pm2-logs": "./node_modules/pm2/bin/pm2 logs", "test": "./node_modules/.bin/mocha", "pretest": "eslint -c .eslintrc.json --ignore-path .gitignore --ignore-path .eslintignore . && stylint app/public/css/style.styl" }, From d7b29f25f4b0c306b7eecc68461b7769f11b5063 Mon Sep 17 00:00:00 2001 From: Patrick Finkbeiner Date: Tue, 13 Sep 2016 10:58:25 +0200 Subject: [PATCH 3/8] Add dev & dev:videotranscoder to run scripts. --- package.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/package.json b/package.json index e6e2155..6814239 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,8 @@ "scripts": { "start": "node app.js", "start:videotranscoder": "node videotranscoder.js", + "dev": "nodemon app.js", + "dev:videotranscoder": "nodemon videotranscoder.js", "pm2-start": "./node_modules/pm2/bin/pm2 start pm2.json", "pm2-stop": "./node_modules/pm2/bin/pm2 delete pm2.json", "pm2-logs": "./node_modules/pm2/bin/pm2 logs", From c9a96cc666e9f26e60defbe8e0fdfcdaf0a63b60 Mon Sep 17 00:00:00 2001 From: Patrick Finkbeiner Date: Tue, 13 Sep 2016 11:04:13 +0200 Subject: [PATCH 4/8] Remove some console log and simplify queue worker. --- app/server/routes/controlpanel/footages.js | 13 +---- videotranscoder.js | 57 ---------------------- 2 files changed, 1 insertion(+), 69 deletions(-) diff --git a/app/server/routes/controlpanel/footages.js b/app/server/routes/controlpanel/footages.js index ad37bb2..e3f35f3 100644 --- a/app/server/routes/controlpanel/footages.js +++ b/app/server/routes/controlpanel/footages.js @@ -41,21 +41,10 @@ exports.createPost = function post(req, res) { footage: footageId }); Queue.add({ - type: 'footageFile', + type: 'transcode', file: file, footage: footageId }); - -// Queue.add({ -// type: 'metadata', -// file: file, -// footage: footageId -// }); -// Queue.add({ -// type: 'transcode', -// file: file, -// footage: footageId -// }); } new Footage({ _id: footageId, diff --git a/videotranscoder.js b/videotranscoder.js index d840363..41d363d 100644 --- a/videotranscoder.js +++ b/videotranscoder.js @@ -30,20 +30,8 @@ var processQueue = function() { }; var handleJob = function(job, next) { - console.log('handle job', job.payload.type); switch(job.payload.type) { - case 'metadata': - metadata(job, function(err, job) { - if (err) throw err; - queue.remove(job.ack, function(err) { - if (err) throw err; - console.log('metadata job finished', job.id); - next(); - }); - }); - break; case 'thumbnail': - console.log('get thumbnail jooooob'); thumbnail(job, function(err, job) { queue.remove(job.ack, function(err) { if (err) throw err; @@ -53,16 +41,6 @@ var handleJob = function(job, next) { }); break; case 'transcode': - transcode(job, function(err, job) { - if (err) throw err; - queue.remove(job.ack, function(err) { - if (err) throw err; - console.log('transcode job finished', job.id); - next(); - }); - }); - break; - case 'footageFile': handleFile(job, function(err, job) { if (err) throw err; if (job) { @@ -86,7 +64,6 @@ var handleFile = function(job, next) { if (err) console.log(err); }); }, 5000); - Filehandler.info(job.payload.file, function(err, metadata) { Footage.findByIdAndUpdate(job.payload.footage, {$set: { 'file.metadata': metadata, @@ -130,37 +107,3 @@ var thumbnail = function(job, next) { }); }); }; - -var transcode = function(job, next) { - var interval = setInterval(function() { - queue.ping(job.ack, function(err) { - if (err) console.log(err); - }); - }, 5000); - Filehandler.transcode(job.payload.file, function(err) { - if (err) throw err; - Footage.findbyidandupdate(job.payload.footage, {$set: {'file.status.transcoded': true}}, function (err) { - if (err) throw err; - clearInterval(interval); - next(null, job); - }); - }); -}; - -var metadata = function(job, next) { - var interval = setInterval(function() { - queue.ping(job.ack, function(err) { - if (err) console.log(err); - }); - }, 5000); - Filehandler.info(job.payload.file, function(err, metadata){ - Footage.findByIdAndUpdate(job.payload.footage, {$set: { - 'file.metadata': metadata, - 'file.duration': metadata.format.duration - }}, function (err) { - if (err) throw err; - clearInterval(interval); - next(null, job); - }); - }); -}; From 7105efdd042f115b28eb801b06f54b6e916897d1 Mon Sep 17 00:00:00 2001 From: Patrick Finkbeiner Date: Tue, 13 Sep 2016 11:45:36 +0200 Subject: [PATCH 5/8] Update README.md --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 71fa60f..37c674e 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Requirements * NodeJS http://nodejs.org/ * MongoDB http://www.mongodb.org/ +* FFmpeg https://ffmpeg.org/ Setup @@ -21,7 +22,7 @@ Setup 2. Restore the DB using `AVnodeDB.zip` [mongorestore](http://docs.mongodb.org/manual/reference/program/mongorestore/) with `mongorestore --drop -d avnode ` 3. Request the file repository `/warehouse` to g.delgobbo@flyer.it (you don't need it to let the app starts) 4. Run `npm install && bower install` -5. Run `npm start` +5. Run `npm start` and `npm run start:videostranscoder` 6. Login with your FLxER user or use user: GianlucaDelGobbo password: GianlucaDelGobbo @@ -31,6 +32,12 @@ Contributing Want to contribute? Great!!! +Development +------------ + +For development we use `nodemon` to detect changes during developement. Ensure to start both scripts the main app with `npm run dev` and our videotranscoding queue worker with `npm run dev:videotranscoder`. + + ### Commands 1. Fork it. From b3eaf850ce4634c65f1ad742a33153870e3c59bd Mon Sep 17 00:00:00 2001 From: Patrick Finkbeiner Date: Tue, 13 Sep 2016 11:47:00 +0200 Subject: [PATCH 6/8] =?UTF-8?q?Resolves=20linting=20issue=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/server/routes/controlpanel/footages.js | 1 - 1 file changed, 1 deletion(-) diff --git a/app/server/routes/controlpanel/footages.js b/app/server/routes/controlpanel/footages.js index ca54718..dde27ce 100644 --- a/app/server/routes/controlpanel/footages.js +++ b/app/server/routes/controlpanel/footages.js @@ -3,7 +3,6 @@ var File = require('../../models/file'); var User = require('../../models/user'); var config = require('getconfig'); var mongoose = require('mongoose'); -var mime = require('mime'); var _ = require('lodash'); var Queue = require('../../modules/queue'); From ea751bdd1fd1ceb83b108de34934dc7d6945ea6e Mon Sep 17 00:00:00 2001 From: Patrick Finkbeiner Date: Tue, 13 Sep 2016 11:49:04 +0200 Subject: [PATCH 7/8] =?UTF-8?q?Unifiy=20function=20writing=20style?= =?UTF-8?q?=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/server/modules/filehandler.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/server/modules/filehandler.js b/app/server/modules/filehandler.js index d9a5edc..895f355 100644 --- a/app/server/modules/filehandler.js +++ b/app/server/modules/filehandler.js @@ -5,21 +5,21 @@ var FFmpeg = require('fluent-ffmpeg'), // FIXME Move to config. var directory = config.sitepath+'/warehouse/uploads/videos/'; -function webPreset(command) { +var webPreset = function(command) { command .format('mp4') //.audioCodec('libfaac') .videoCodec('libx264') .keepDAR(); -} -function mobilePreset(command) { +}; +var mobilePreset = function(command) { command .format('mp4') //.audioCodec('libfaac') .videoCodec('libx264') .keepDAR() .size('720x?'); -} +}; var fileName = function(file) { return path.basename(file.name, path.extname(file.name)); From 466d9fa1201c4e9d4a34d55870684ae7dd9a1fa5 Mon Sep 17 00:00:00 2001 From: Patrick Finkbeiner Date: Wed, 14 Sep 2016 10:31:46 +0200 Subject: [PATCH 8/8] =?UTF-8?q?Adjust=20test=20cases=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/spec/models/footage.test.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/spec/models/footage.test.js b/test/spec/models/footage.test.js index 1cc5443..5ffee10 100644 --- a/test/spec/models/footage.test.js +++ b/test/spec/models/footage.test.js @@ -14,16 +14,16 @@ describe('Footage Model', function() { ] }; }); - it('isowner should return false if footage id is not present', function() { + it('isOwner should return false if footage id is not present', function() { expect(isOwner(user, '1111')).toBe(false); }); it('isOwner should return true if user has footage id', function() { expect(isOwner(user, '57c411ca3bbefc0e37b877e9')).toBe(true); }); - it('isOwner should return true if valid id is string', function() { + it('isOwner should return true if valid id is of type string', function() { expect(isOwner(user, '57c411ca3bbefc0e37b877e9')).toBe(true); }); - it('isOwner should return true if valid id is ObjectID', function() { + it('isOwner should return true if valid id is an ObjectId', function() { var id = mongoose.Types.ObjectId('57c411cf2819c01d377b54e8'); expect(isOwner(user, id)).toBe(true); });