From 1ab93374416ab756059c0a3c722013d8ae9b18a6 Mon Sep 17 00:00:00 2001 From: Stephan Balmer Date: Mon, 25 Sep 2017 14:28:44 +0200 Subject: [PATCH 1/6] run extraction on build --- packages/core/buildPlugin.js | 30 -- packages/core/package.js | 5 - packages/extract/extract.js | 544 ++++++++++++++--------------------- packages/extract/package.js | 14 +- 4 files changed, 218 insertions(+), 375 deletions(-) delete mode 100644 packages/core/buildPlugin.js diff --git a/packages/core/buildPlugin.js b/packages/core/buildPlugin.js deleted file mode 100644 index 62bd9e0..0000000 --- a/packages/core/buildPlugin.js +++ /dev/null @@ -1,30 +0,0 @@ -// currently this file only handles extracts, we could do other stuff in the futre - -var fs = Npm.require('fs'); -var EXTRACTS_FILE = 'server/extracts.msgfmt~'; - -/* - * So what's going on here? msgfmt:extracts creates a file with a - * tilde ("~") suffix, to prevent Meteor from reloading when it's updated. - * That file itself is only created on reload, so that would cause a double - * reload and be a total pain. But, Meteor doesn't bundle ~ files either. - * So we use a build plugin, on the presence of a similarly named file, - * to look for this file, and bundle it. This happens on load, so will - * catch it only on the next reload, but that's fine, since the file is - * only used in production. -*/ - -function msgfmtHandler(compileStep) { - if (fs.existsSync(EXTRACTS_FILE)) { - var contents = fs.readFileSync(EXTRACTS_FILE).toString('utf8'); - - compileStep.addJavaScript({ - path: 'server/extracts.msgfmt.js', - sourcePath: process.cwd() + '/' + EXTRACTS_FILE, - //data: 'console.log("NODE_ENV="+process.env.NODE_ENV); if (1 || process.env.NODE_ENV === "production") msgfmt.addNative.apply(msgfmt, ' + contents + ');' - data: 'if (process.env.NODE_ENV === "production") msgfmt.addNative.apply(msgfmt, ' + contents + ');' - }); - } -} - -Plugin.registerSourceHandler('msgfmt', msgfmtHandler); diff --git a/packages/core/package.js b/packages/core/package.js index 09be223..48f1846 100644 --- a/packages/core/package.js +++ b/packages/core/package.js @@ -6,11 +6,6 @@ Package.describe({ documentation: 'README.md' }); -Package.registerBuildPlugin({ - name: 'msgfmt', - sources: [ 'buildPlugin.js' ] -}); - var client = 'client'; var server = 'server'; var both = [ client, server ]; diff --git a/packages/extract/extract.js b/packages/extract/extract.js index 5349b70..94e584a 100644 --- a/packages/extract/extract.js +++ b/packages/extract/extract.js @@ -1,7 +1,4 @@ -/* - * Recall for everything in this file that it is only run in development with - * a local database. - */ +"use strict"; var EXTRACTS_FILE = 'server/extracts.msgfmt~'; @@ -9,253 +6,182 @@ var fs = Npm.require('fs'); var path = Npm.require('path'); var walk = Npm.require('walk'); -var efiles = mfPkg.mfExtractFiles = new Meteor.Collection('mfExtractFiles'); - -var relUp = path.join('..','..','..','..','..'); -var extractsFile = path.join.apply(null, [relUp].concat(EXTRACTS_FILE.split('/'))); +/* + * So what's going on here? msgfmt:extracts creates a file with a + * tilde ("~") suffix, to prevent Meteor from reloading when it's updated. + * That file itself is only created on reload, so that would cause a double + * reload and be a total pain. But, Meteor doesn't bundle ~ files either. + * to look for this file, and bundle it. This happens on load, so will + * catch it only on the next reload, but that's fine, since the file is + * only used in production. +*/ -var toDict = function(array) { - var out = {}; - for (var i=0; i < array.length; i++) { - out[array[i]._id] = array[i]; - delete out[array[i]._id]._id; - } - return out; +function msgfmtHandler(compileStep) { + const root = process.cwd(); + const extracted = extract(root); + compileStep.addJavaScript({ + path: 'server/extracts.msgfmt.js', + sourcePath: root + '/' + EXTRACTS_FILE, + data: 'msgfmt.addNative.apply(msgfmt, ' + extracted + ');' + }); } -var checkForUpdates = function(m, force) { - // https://github.com/meteor/meteor/pull/3704/files - if (m && !m.refresh) - return; - - log.debug('Checking for changed files...'); - - var startTime = Date.now(); - var lastTime = startTime; - - var oldFilesInfo = toDict(efiles.find().fetch()); - - log.debug('Retrieved old file info from database (in a fiber) in ' + - (Date.now() - lastTime) + 'ms'); - lastTime = Date.now(); - - var walker = walk.walk(relUp, { - followLinks: false, - filters: [ - /\/\.[^\.]+\/|node_modules/ // skip .directories (hidden) & node_modules - ] - }); - - var changedFiles = []; - var upserts = {}; - - walker.on('file', function(root, stat, next) { - // Add this file to the list of files (skip .dirs) - if (stat.name.match(/\.(html|js|coffee|jsx|jade)$/)) { - var prettyDir = root.substr(relUp.length-1); - var file = path.join(prettyDir, stat.name); - - var oldFileInfo = oldFilesInfo[file]; - if (force || !oldFileInfo || oldFileInfo.mtime < stat.mtime) { - upserts[file] = { mtime: stat.mtime }; - changedFiles[file] = { - fromCwd: path.join(root, stat.name), - mtime: stat.mtime - }; - delete oldFilesInfo[file]; - } else { - // We check later any files not mentioned so we can remove from DB - delete oldFilesInfo[file]; - } - } - next(); - }); +Plugin.registerSourceHandler('msgfmt', msgfmtHandler); - walker.on('end', Meteor.bindEnvironment(function() { - var id, key, name; +var extract = (root) => { + const cacheFile = path.join.apply(null, [root].concat(EXTRACTS_FILE.split('/'))); - log.debug('Finished walking files (non-fiber async) in ' + - (Date.now() - lastTime) + 'ms'); - lastTime = Date.now(); - - if (!Object.keys(changedFiles).length) { - log.debug("No changed files, nothing to do"); - return; - } - - var newStrings = {}; - var oldStrings = {}; - var unchangedStrings = {}; - var nativeStrings = mfPkg.strings[mfPkg.native]; - var saveData; - - for (name in changedFiles) { - var file = changedFiles[name]; - var content = fs.readFileSync(file.fromCwd, 'utf8'); - var mtime = new Date(file.mtime).getTime(); - log.debug('Extracting from ' + name); - handlers[path.extname(name).substr(1)](name, content, mtime, newStrings); + let cacheJSON; + try { + cacheJSON = fs.readFileSync(cacheFile, 'utf8'); + } catch(e) { + // Ok, we rebuild all } - // Only compare oldStrings from the files we're looking at - for (key in nativeStrings) { - if (upserts[nativeStrings[key].file] || oldFilesInfo[nativeStrings[key].file]) - oldStrings[key] = nativeStrings[key]; - } - - log.debug('Finished processing ' + - Object.keys(changedFiles).length + ' file(s) (blocking) in ' + - (Date.now() - lastTime) + 'ms'); - lastTime = Date.now(); - - var changeCount = 0, newCount = 0, removeCount = 0; - for (key in newStrings) { - if (oldStrings[key]) { - if (newStrings[key].text === oldStrings[key].text) { - if (oldStrings[key].removed) { - // Basically a new key - newCount++; - newStrings[key].ctime = newStrings[key].mtime; - } else { - // No change - unchangedStrings[key] = newStrings[key]; - delete newStrings[key]; - } - } else { - changeCount++; + let validCache = false; + if (cacheJSON) { + try { + const cacheRead = JSON.parse(cacheJSON); + if (cacheRead && cacheRead.extracts) { + validCache = cacheRead; + } + } catch(e) { + // Ok, we rebuild all } - delete oldStrings[key]; - } else { - newCount++; - newStrings[key].ctime = newStrings[key].mtime; - } - } - - // if a key existed before but not anymore, mark as removed - for (key in oldStrings) { - if (oldStrings[key].removed) { - unchangedStrings[key] = oldStrings[key]; - continue; - } - log.trace('Marking "' + key + '" as removed.'); - newStrings[key] = oldStrings[key]; - newStrings[key].removed = true; - newStrings[key].mtime = Date.now(); - removeCount++; - } - - log.debug('Finished comparing strings in ' + - (Date.now() - lastTime) + 'ms'); - lastTime = Date.now(); - - if (newCount || changeCount || removeCount) - log.info(newCount + ' string(s) added, ' + - changeCount + ' changed, and ' + - removeCount + ' marked as removed.'); - else - log.debug('0 string(s) added, 0 changed, and 0 marked as removed.'); - - // console.log(newStrings); - - if (Object.keys(newStrings).length) { - // console.log(newStrings); - var max = _.max(newStrings, function(s) { return s.mtime; }).mtime; - - mfPkg.addNative(newStrings, { - extractedAt: Date.now(), - updatedAt: max - }); } + const cache = validCache ? validCache : { extracts: {} }; + + // Track whether files changed, this means we have to write the cache file. + let filesChanged = false; + + // Track whether strings in any of the observed files have changed, this + // means we have to update the DB + let stringsChanged = false; + + const extractsPerFile = {}; + const listeners = { + 'file': function(dir, stat, next) { + try { + const ext = path.extname(stat.name).substr(1); + if (!ext) { next(); return; } + + const handler = handlers[ext]; + if (!handler) { next(); return; } + + const stamp = new Date(stat.mtime).getTime(); + const filePath = path.join(dir.substr(root.length+1), stat.name); + const oldInfo = cache.extracts[filePath]; + + // Use cache for files that have the same mtime from the last run + if (oldInfo && oldInfo.mtime === stamp) { + extractsPerFile[filePath] = oldInfo; + next(); return; + } + filesChanged = true; + + const fileExtracts = []; + const track = function(entry) { + fileExtracts.push(entry); + }; + + const content = fs.readFileSync(path.join(dir, stat.name), 'utf8'); + + handler(content, track); + + let changed = fileExtracts.length > 0; + if (oldInfo) { + changed = JSON.stringify(fileExtracts) !== JSON.stringify(oldInfo.entries); + } + + if (changed) { + stringsChanged = true; + } + + extractsPerFile[filePath] = + { entries: fileExtracts + , mtime: stamp + }; + } catch(e) { + console.log("Error extracting msgfmt strings from filePath ", e); + } + next(); + } + }; - if (Object.keys(newStrings).length || force) { - // was: msgfmt.strings[msgfmt.native] ? - var unorderedStrings = _.extend({}, unchangedStrings, newStrings); - var orderedStrings = {}; - _.each(Object.keys(unorderedStrings).sort(), function(key) { - orderedStrings[key] = unorderedStrings[key]; - }); - - saveData = JSON.stringify([ - orderedStrings, - { - extractedAt: Date.now(), - updatedAt: max || msgfmt.meta[msgfmt.native].updatedAt || Date.now() + walk.walkSync(root, + { followLinks: false + , listeners: listeners + , filters: + [ /\/\./ // Skip dotfiles + , 'node_modules' + ] + } + ); + + const extracts = {}; + let currentCache = cache; + + try { + // Detect when files with entries were were deleted + Object.keys(cache.extracts).forEach((filePath) => { + if (!extractsPerFile[filePath]) { + filesChanged = true; + if (cache.extracts[filePath].entries.length > 0) { + stringsChanged = true; + } + } + }); + + if (filesChanged) { + let update = new Date().getTime(); + if (!stringsChanged && cache) { + update = cache.update; + } + // Update cache + currentCache = + { extracts: extractsPerFile + , update: update + }; + const cacheStr = JSON.stringify(currentCache, null, 2); + fs.writeFile(cacheFile, cacheStr); // Fire and forget } - ]); - } - log.debug('Finished mfPkg.addNative in ' + (Date.now() - lastTime) + 'ms'); - lastTime = Date.now(); - - // Update changed files - for (id in upserts) - efiles.upsert(id, { $set: upserts[id] }); - - // Remove files that no longer exist - for (id in oldFilesInfo) - efiles.remove(id); - - log.debug('Finished updating database (in a fiber) in ' + - (Date.now() - lastTime) + 'ms'); - lastTime = Date.now(); - - // Report Back - log.debug(Object.keys(upserts).length + ' file(s) changed and ' + - Object.keys(oldFilesInfo).length + ' file(s) removed.'); - if (Object.keys(upserts).length) - log.debug('Changed files: ' + _.keys(upserts).join(', ')); - if (Object.keys(oldFilesInfo).length) - log.debug('Removed files: ' + _.keys(oldFilesInfo).join(', ')); - - if (saveData) { - log.trace('Writing ' + EXTRACTS_FILE + '...'); - fs.writeFile(extractsFile, saveData, function() { - log.trace('Finished writing ' + EXTRACTS_FILE); - }); + Object.keys(extractsPerFile).forEach((filePath) => { + const fileExtracts = extractsPerFile[filePath]; + for (let fileExtract of fileExtracts.entries) { + // Record entry in extracts dict but don't overwrite already + // existing entries. + const b = Object.assign({}, fileExtract); + b.mtime = fileExtracts.mtime; + b.file = filePath; + const a = extracts[b.key]; + if (a) { + // Consider this a conflict only when the text differs. + if (b.text && a.text != b.text) { + console.log("Msgfmt conflict on key " + a.key + ". Using the first."); + console.log(a); + console.log(b); + } + return; + } + extracts[b.key] = b; + } + }); + } catch(e) { + console.log("Error building extracts.msgfmt~ ", e); } - })); -} -msgfmt.forceExtract = function() { - checkForUpdates(null, true /* force */); - return 'Forcing (re)-extract of all files and keys. See results in main console ' + - 'log, according to your default debug level'; + const now = new Date().getTime(); + return JSON.stringify( + [ extracts + , { extractedAt: now + , updatedAt: currentCache.update + } + ] + ); }; -var log; -var boundCheck = Meteor.bindEnvironment(checkForUpdates); - -// https://github.com/meteor/meteor/pull/3704/files -process.on('SIGUSR2', boundCheck); // Meteor < 1.0.4 -process.on('SIGHUP', boundCheck); // Meteor >= 1.0.4 -process.on('message', boundCheck); // Meteor >= 1.0.4 - -// No reason to block startup, we can do update gradually asyncronously -Meteor.startup(function() { - log = new Logger('msgfmt:extracts'); - Logger.setLevel('msgfmt:extracts', msgfmt.initOptions.extractLogLevel || 'info'); - - var dir = path.dirname(extractsFile); - fs.exists(dir, function(exists) { - if (exists) { - var triggerFile = extractsFile.replace(/~$/,''); - fs.exists(triggerFile, function(exists) { - if (!exists) - fs.writeFile(triggerFile, '# Used by ' + EXTRACTS_FILE + ', do not delete.\n'); - }); - } else { - log.trace('Creating ' + path.dirname(EXTRACTS_FILE) + ' in app root...'); - fs.mkdir(dir, function(err) { - if (err) throw err; - log.trace('Created ' + path.dirname(EXTRACTS_FILE) + ' in app root.') - }); - } - }); - - checkForUpdates(); -}); -/* handler helpers */ function attrDict(string) { var result, out = {}, re = /(\w+)=(['"])(.*?)\2/g; @@ -265,30 +191,12 @@ function attrDict(string) { return out; } -var lastFile = null; -function logKey(key, text, file, line, strings) { - if (strings[key] && strings[key].text != text) - log.warn('{ ' + key + ': "' + text + '" } in ' - + file + ':' + line + ' replaces DUP_KEY\n { ' - + key + ': "' + strings[key].text + '" } in ' - + strings[key].file + ':' + strings[key].line); - - if (!log) - return; - - if (file != lastFile) { - lastFile = file; - log.trace(file); - } - - log.trace('* ' + key + ': "' + text.replace(/\s+/g, ' ') + '"'); -} /* handlers */ var handlers = {}; -handlers.html = function(file, data, mtime, strings) { +handlers.html = function(data, track) { // XXX TODO, escaped quotes var result, re; @@ -296,84 +204,72 @@ handlers.html = function(file, data, mtime, strings) { // or attribute=(mf "key" 'text' attr1=val1 attr2=val2 etc) re = /(?:{{|=\()[\s]?mf (['"])(.*?)\1 ?(["'])(.*?)\3(.*?)(?:}}|\))/g; while (result = re.exec(data)) { - var key = result[2], text = result[4], attributes = attrDict(result[5]); - var tpl = /