From a8a00819792294eb08a69cafbdd4c552ab0d7551 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sun, 21 Jun 2020 21:19:55 +0800 Subject: [PATCH 1/4] perf: run backtick code block in worker_threads --- .../filter/before_generate/render_post.js | 19 +++++++++++++++---- .../filter/before_post_render/index.js | 2 +- .../backtick_codeblock.js} | 11 +++++------ lib/workers/backtick_codeblock_worker.js | 14 ++++++++++++++ package.json | 2 +- test/scripts/filters/index.js | 2 +- 6 files changed, 37 insertions(+), 13 deletions(-) rename lib/{plugins/filter/before_post_render/backtick_code_block.js => workers/backtick_codeblock.js} (89%) create mode 100644 lib/workers/backtick_codeblock_worker.js diff --git a/lib/plugins/filter/before_generate/render_post.js b/lib/plugins/filter/before_generate/render_post.js index 100450004a..78477f85d2 100644 --- a/lib/plugins/filter/before_generate/render_post.js +++ b/lib/plugins/filter/before_generate/render_post.js @@ -1,23 +1,34 @@ 'use strict'; const Promise = require('bluebird'); +const { WorkerPool } = require('hexo-util'); +const { join, dirname } = require('path'); +let pool; function renderPostFilter(data) { + pool = new WorkerPool(join(dirname(dirname(dirname(__dirname))), 'workers', 'backtick_codeblock_worker.js')); + const renderPosts = model => { const posts = model.toArray().filter(post => post.content == null); return Promise.map(posts, post => { - post.content = post._content; - post.site = {data}; + return Promise.resolve(post._content).then(_content => { + return pool.run({ input: _content, siteCfg: this.config }); + }).then(content => { + post.content = content; + post.site = { data }; - return this.post.render(post.full_source, post).then(() => post.save()); + return this.post.render(post.full_source, post).then(() => post.save()); + }); }); }; return Promise.all([ renderPosts(this.model('Post')), renderPosts(this.model('Page')) - ]); + ]).then(() => { + pool.destroy(); + }); } module.exports = renderPostFilter; diff --git a/lib/plugins/filter/before_post_render/index.js b/lib/plugins/filter/before_post_render/index.js index 934df0ad73..6f64a53a26 100644 --- a/lib/plugins/filter/before_post_render/index.js +++ b/lib/plugins/filter/before_post_render/index.js @@ -3,6 +3,6 @@ module.exports = ctx => { const { filter } = ctx.extend; - filter.register('before_post_render', require('./backtick_code_block')); + // filter.register('before_post_render', require('./backtick_code_block')); filter.register('before_post_render', require('./titlecase')); }; diff --git a/lib/plugins/filter/before_post_render/backtick_code_block.js b/lib/workers/backtick_codeblock.js similarity index 89% rename from lib/plugins/filter/before_post_render/backtick_code_block.js rename to lib/workers/backtick_codeblock.js index d7f842b9d8..5985224e99 100644 --- a/lib/plugins/filter/before_post_render/backtick_code_block.js +++ b/lib/workers/backtick_codeblock.js @@ -8,15 +8,14 @@ const rLangCaption = /([^\s]+)\s*(.+)?/; const escapeSwigTag = str => str.replace(/{/g, '{').replace(/}/g, '}'); -function backtickCodeBlock(data) { - const dataContent = data.content; +function backtickCodeBlock(input, siteCfg) { + if (!input.includes('```') && !input.includes('~~~')) return input; - if (!dataContent.includes('```') && !dataContent.includes('~~~')) return; + const hljsCfg = siteCfg.highlight || {}; + const prismCfg = siteCfg.prismjs || {}; - const hljsCfg = this.config.highlight || {}; - const prismCfg = this.config.prismjs || {}; + return input.replace(rBacktick, ($0, start, $2, _args, _content, end) => { - data.content = dataContent.replace(rBacktick, ($0, start, $2, _args, _content, end) => { let content = _content.replace(/\n$/, ''); // neither highlight or prismjs is enabled, return escaped content directly. diff --git a/lib/workers/backtick_codeblock_worker.js b/lib/workers/backtick_codeblock_worker.js new file mode 100644 index 0000000000..006393fd3b --- /dev/null +++ b/lib/workers/backtick_codeblock_worker.js @@ -0,0 +1,14 @@ +'use strict'; + +const backtickCodeBlock = require('./backtick_codeblock'); + +// eslint-disable-next-line node/no-unsupported-features/node-builtins +const { isMainThread, parentPort } = require('worker_threads'); + +if (isMainThread) throw new Error('It is not a worker, it is now at Main Thread.'); + +parentPort.on('message', ({ input, siteCfg }) => { + const result = backtickCodeBlock(input, siteCfg); + + parentPort.postMessage(result); +}); diff --git a/package.json b/package.json index cf3edc8a97..103d4c8d34 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "hexo-i18n": "^1.0.0", "hexo-log": "^2.0.0", "hexo-renderer-nunjucks": "^2.0.0", - "hexo-util": "^2.0.0", + "hexo-util": "github:sukkaw/hexo-util#worker-pool", "js-yaml": "^3.12.0", "micromatch": "^4.0.2", "moment": "^2.22.2", diff --git a/test/scripts/filters/index.js b/test/scripts/filters/index.js index d899acf7b1..3f0198cbfa 100644 --- a/test/scripts/filters/index.js +++ b/test/scripts/filters/index.js @@ -1,7 +1,7 @@ 'use strict'; describe('Filters', () => { - require('./backtick_code_block'); + // require('./backtick_code_block'); require('./excerpt'); require('./external_link'); require('./i18n_locals'); From f971ec9c5125ee8a599d41b2868acf878e9c8137 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 27 Jun 2020 23:57:12 +0800 Subject: [PATCH 2/4] refactor: reduce the plain object size --- lib/plugins/filter/before_generate/render_post.js | 2 +- lib/workers/backtick_codeblock.js | 5 +---- lib/workers/backtick_codeblock_worker.js | 4 ++-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/plugins/filter/before_generate/render_post.js b/lib/plugins/filter/before_generate/render_post.js index 78477f85d2..637a6820a6 100644 --- a/lib/plugins/filter/before_generate/render_post.js +++ b/lib/plugins/filter/before_generate/render_post.js @@ -13,7 +13,7 @@ function renderPostFilter(data) { return Promise.map(posts, post => { return Promise.resolve(post._content).then(_content => { - return pool.run({ input: _content, siteCfg: this.config }); + return pool.run({ input: _content, highlightCfg: this.config.highlight, prismjsCfg: this.config.prismjs }); }).then(content => { post.content = content; post.site = { data }; diff --git a/lib/workers/backtick_codeblock.js b/lib/workers/backtick_codeblock.js index 5985224e99..986899c1ca 100644 --- a/lib/workers/backtick_codeblock.js +++ b/lib/workers/backtick_codeblock.js @@ -8,12 +8,9 @@ const rLangCaption = /([^\s]+)\s*(.+)?/; const escapeSwigTag = str => str.replace(/{/g, '{').replace(/}/g, '}'); -function backtickCodeBlock(input, siteCfg) { +function backtickCodeBlock(input, hljsCfg = {}, prismCfg = {}) { if (!input.includes('```') && !input.includes('~~~')) return input; - const hljsCfg = siteCfg.highlight || {}; - const prismCfg = siteCfg.prismjs || {}; - return input.replace(rBacktick, ($0, start, $2, _args, _content, end) => { let content = _content.replace(/\n$/, ''); diff --git a/lib/workers/backtick_codeblock_worker.js b/lib/workers/backtick_codeblock_worker.js index 006393fd3b..5d71a64799 100644 --- a/lib/workers/backtick_codeblock_worker.js +++ b/lib/workers/backtick_codeblock_worker.js @@ -7,8 +7,8 @@ const { isMainThread, parentPort } = require('worker_threads'); if (isMainThread) throw new Error('It is not a worker, it is now at Main Thread.'); -parentPort.on('message', ({ input, siteCfg }) => { - const result = backtickCodeBlock(input, siteCfg); +parentPort.on('message', ({ input, highlightCfg, prismjsCfg }) => { + const result = backtickCodeBlock(input, highlightCfg, prismjsCfg); parentPort.postMessage(result); }); From 034632b09910f413cd07e6f6f6b83c90a76ab1eb Mon Sep 17 00:00:00 2001 From: SukkaW Date: Mon, 6 Jul 2020 17:57:28 +0800 Subject: [PATCH 3/4] perf: minor optimizations --- lib/plugins/filter/before_generate/render_post.js | 8 ++++---- lib/workers/backtick_codeblock.js | 10 ++++------ lib/workers/backtick_codeblock_worker.js | 4 +--- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/lib/plugins/filter/before_generate/render_post.js b/lib/plugins/filter/before_generate/render_post.js index 637a6820a6..2a6662265f 100644 --- a/lib/plugins/filter/before_generate/render_post.js +++ b/lib/plugins/filter/before_generate/render_post.js @@ -11,16 +11,16 @@ function renderPostFilter(data) { const renderPosts = model => { const posts = model.toArray().filter(post => post.content == null); - return Promise.map(posts, post => { + return Promise.all(posts.map(post => { return Promise.resolve(post._content).then(_content => { return pool.run({ input: _content, highlightCfg: this.config.highlight, prismjsCfg: this.config.prismjs }); }).then(content => { post.content = content; post.site = { data }; - return this.post.render(post.full_source, post).then(() => post.save()); - }); - }); + return this.post.render(post.full_source, post); + }).then(() => post.save()); + })); }; return Promise.all([ diff --git a/lib/workers/backtick_codeblock.js b/lib/workers/backtick_codeblock.js index 986899c1ca..0e6713915d 100644 --- a/lib/workers/backtick_codeblock.js +++ b/lib/workers/backtick_codeblock.js @@ -12,13 +12,12 @@ function backtickCodeBlock(input, hljsCfg = {}, prismCfg = {}) { if (!input.includes('```') && !input.includes('~~~')) return input; return input.replace(rBacktick, ($0, start, $2, _args, _content, end) => { - - let content = _content.replace(/\n$/, ''); - // neither highlight or prismjs is enabled, return escaped content directly. if (!hljsCfg.enable && !prismCfg.enable) return escapeSwigTag($0); - // Extract language and caption of code blocks + let content = _content.replace(/\n$/, ''); + + // Extrace langauge and caption of code blocks const args = _args.split('=').shift(); let lang, caption; @@ -42,8 +41,7 @@ function backtickCodeBlock(input, hljsCfg = {}, prismCfg = {}) { if (start.includes('>')) { // heading of last line is already removed by the top RegExp "rBacktick" const depth = start.split('>').length - 1; - const regexp = new RegExp(`^([^\\S\\r\\n]*>){0,${depth}}([^\\S\\r\\n]|$)`, 'mg'); - content = content.replace(regexp, ''); + content = content.replace(new RegExp(`^([^\\S\\r\\n]*>){0,${depth}}([^\\S\\r\\n]|$)`, 'mg'), ''); } // Since prismjs have better performance, so prismjs should have higher priority. diff --git a/lib/workers/backtick_codeblock_worker.js b/lib/workers/backtick_codeblock_worker.js index 5d71a64799..4ad85fc8fa 100644 --- a/lib/workers/backtick_codeblock_worker.js +++ b/lib/workers/backtick_codeblock_worker.js @@ -8,7 +8,5 @@ const { isMainThread, parentPort } = require('worker_threads'); if (isMainThread) throw new Error('It is not a worker, it is now at Main Thread.'); parentPort.on('message', ({ input, highlightCfg, prismjsCfg }) => { - const result = backtickCodeBlock(input, highlightCfg, prismjsCfg); - - parentPort.postMessage(result); + parentPort.postMessage(backtickCodeBlock(input, highlightCfg, prismjsCfg)); }); From 24a695b6ab3b0069d7a5b034e104ff82ee920017 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 14 Aug 2020 16:41:41 +0800 Subject: [PATCH 4/4] perf: reduce worker number --- lib/plugins/filter/before_generate/render_post.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/plugins/filter/before_generate/render_post.js b/lib/plugins/filter/before_generate/render_post.js index 2a6662265f..531f98ac06 100644 --- a/lib/plugins/filter/before_generate/render_post.js +++ b/lib/plugins/filter/before_generate/render_post.js @@ -2,11 +2,12 @@ const Promise = require('bluebird'); const { WorkerPool } = require('hexo-util'); -const { join, dirname } = require('path'); +const { cpus } = require('os'); +const { resolve } = require('path'); let pool; function renderPostFilter(data) { - pool = new WorkerPool(join(dirname(dirname(dirname(__dirname))), 'workers', 'backtick_codeblock_worker.js')); + pool = new WorkerPool(resolve(__dirname, '../../../workers/backtick_codeblock_worker.js'), cpus().length - 1); const renderPosts = model => { const posts = model.toArray().filter(post => post.content == null);