diff --git a/common/models/templates/package.json b/common/models/templates/package.json index d4eafa6f0cf..3cc7ecebac0 100644 --- a/common/models/templates/package.json +++ b/common/models/templates/package.json @@ -33,7 +33,7 @@ }, "scripts": { "build": "tsc", - "pretest": "npm run build", + "pretest": "tsc -b tsconfig.bundled.json", "test": "mocha -r test/helpers.js" }, "bugs": { @@ -47,7 +47,7 @@ "@types/node": "^14.0.4", "chai": "^4.3.4", "mocha": "^8.4.0", - "typescript": "^3.8.3" + "typescript": "^4.5.4" }, "dependencies": { "@keymanapp/models-wordbreakers": "*" diff --git a/common/models/templates/test/helpers.js b/common/models/templates/test/helpers.js index e3931d648ac..268e24b7ddc 100644 --- a/common/models/templates/test/helpers.js +++ b/common/models/templates/test/helpers.js @@ -7,8 +7,10 @@ var _ = global; // TODO: then mocha invocation is as follows: // TODO: mocha -r @keymanapp/models-test-helpers test/ -// KMW string must be included, so do it here: -require('@keymanapp/web-utils'); +// Ensure that we can successfully load the module & apply kmwLength, as it's +// needed for some of the unit tests. +require('../build/index.bundled.js'); + assert.ok('๐Ÿ’ฉ'.kmwLength); /** @@ -21,7 +23,7 @@ _.jsonFixture = function (name) { /** * Returns the Context of an empty buffer; no text, at both the start and * end of the buffer. - * + * * @returns {Context} */ _.emptyContext = function emptyContext() { diff --git a/common/models/templates/test/test-common.js b/common/models/templates/test/test-common.js index 299cb254435..7dfc565369a 100644 --- a/common/models/templates/test/test-common.js +++ b/common/models/templates/test/test-common.js @@ -3,7 +3,7 @@ */ var assert = require('chai').assert; -var models = require('../').models; +var models = require('../build/index.bundled.js').models; describe('Common utility functions', function() { // TODO: unit tests for other common utility functions @@ -97,7 +97,7 @@ describe('Common utility functions', function() { let apple = { insert: 'apple', deleteLeft: 0, - deleteRight: 2 + deleteRight: 2 }; let banana = { @@ -143,7 +143,7 @@ describe('Common utility functions', function() { let apple = { insert: 'apple', deleteLeft: 0, - deleteRight: 2 + deleteRight: 2 }; let banana = { @@ -167,7 +167,7 @@ describe('Common utility functions', function() { let apple = { insert: 'apple', deleteLeft: 0, - deleteRight: 2 + deleteRight: 2 }; let banana = { diff --git a/common/models/templates/test/test-priority-queue.js b/common/models/templates/test/test-priority-queue.js index 9f2e48e1d29..b004027d84a 100644 --- a/common/models/templates/test/test-priority-queue.js +++ b/common/models/templates/test/test-priority-queue.js @@ -3,12 +3,12 @@ */ var assert = require('chai').assert; -var PriorityQueue = require('../').models.PriorityQueue; +var PriorityQueue = require('../build/index.bundled.js').models.PriorityQueue; describe('Priority queue', function() { it('can act as a min-heap', function () { let input = [1, 10, 2, 9, 3, 8, 4, 7, 5, 6]; - + let queue = new PriorityQueue((a, b) => a - b); input.forEach((input) => queue.enqueue(input)); @@ -41,7 +41,7 @@ describe('Priority queue', function() { it('can act as a max-heap', function () { let input = [1, 10, 2, 9, 3, 8, 4, 7, 5, 6]; - + let queue = new PriorityQueue((a, b) => b - a); input.forEach((input) => queue.enqueue(input)); diff --git a/common/models/templates/test/test-quote-behavior.js b/common/models/templates/test/test-quote-behavior.js index eb169424be6..dd5408f6a96 100644 --- a/common/models/templates/test/test-quote-behavior.js +++ b/common/models/templates/test/test-quote-behavior.js @@ -3,7 +3,7 @@ */ var assert = require('chai').assert; -var QuoteBehavior = require('../').models.QuoteBehavior; +var QuoteBehavior = require('../build/index.bundled.js').models.QuoteBehavior; describe('Quote behaviors', function() { describe('Script directionality', function() { @@ -144,15 +144,15 @@ describe('Quote behaviors', function() { quotesForKeepSuggestion: { open: `โ€œ`, close: `โ€`}, insertAfterWord: " " } - + assert.throws(function() { QuoteBehavior.apply(QuoteBehavior.default, "hello", englishPunctuation, QuoteBehavior.default); }); - + assert.throws(function() { QuoteBehavior.apply(QuoteBehavior.useQuotes, "hello", englishPunctuation, QuoteBehavior.default); }); - + assert.throws(function() { QuoteBehavior.apply(QuoteBehavior.noQuotes, "hello", englishPunctuation, QuoteBehavior.default); }); diff --git a/common/models/templates/test/test-tokenization.js b/common/models/templates/test/test-tokenization.js index 22b975a69c2..4775064b38d 100644 --- a/common/models/templates/test/test-tokenization.js +++ b/common/models/templates/test/test-tokenization.js @@ -3,7 +3,7 @@ */ var assert = require('chai').assert; -var models = require('../').models; +var models = require('../build/index.bundled.js').models; var wordBreakers = require('@keymanapp/models-wordbreakers').wordBreakers; describe('Tokenization functions', function() { @@ -132,7 +132,7 @@ describe('Tokenization functions', function() { left: '', startOfBuffer: true, right: '', endOfBuffer: true }; - + let tokenization = models.tokenize(wordBreakers.default, context); let expectedResult = { @@ -140,7 +140,7 @@ describe('Tokenization functions', function() { right: [], caretSplitsToken: false }; - + assert.deepEqual(tokenization, expectedResult); }); @@ -153,7 +153,7 @@ describe('Tokenization functions', function() { right: [], caretSplitsToken: false }; - + assert.deepEqual(tokenization, expectedResult); }); @@ -163,7 +163,7 @@ describe('Tokenization functions', function() { left: ' ', startOfBuffer: true, right: '', endOfBuffer: true }; - + let tokenization = models.tokenize(wordBreakers.default, context); let expectedResult = { @@ -171,7 +171,7 @@ describe('Tokenization functions', function() { right: [], caretSplitsToken: false }; - + assert.deepEqual(tokenization, expectedResult); }); @@ -233,7 +233,7 @@ describe('Tokenization functions', function() { case 'แžŸแŸ’แžšแžปแž€แžแŸ’แž˜แŸ‚แžš': return [srok, shiftSpan(khmer, srok.length)]; // array of the two. case 'แž€แžแŸ’แž˜แŸ‚แžš': - // I'd admittedly be at least somewhat surprised if a real wordbreaker got this + // I'd admittedly be at least somewhat surprised if a real wordbreaker got this // and similar situations perfectly right... but at least it gives us what // we need for a test. return [k, shiftSpan(khmer, k.length)]; diff --git a/common/models/templates/test/test-trie-model.js b/common/models/templates/test/test-trie-model.js index 36a5944b840..b48dc254ecc 100644 --- a/common/models/templates/test/test-trie-model.js +++ b/common/models/templates/test/test-trie-model.js @@ -3,7 +3,7 @@ */ var assert = require('chai').assert; -var TrieModel = require('../').models.TrieModel; +var TrieModel = require('../build/index.bundled.js').models.TrieModel; describe('LMLayerWorker trie model for word lists', function() { describe('instantiation', function () { @@ -20,7 +20,7 @@ describe('LMLayerWorker trie model for word lists', function() { } } }) - + assert.equal(model.punctuation.insertAfterWord, spaceMark); assert.equal(model.punctuation.quotesForKeepSuggestion.open, openQuote); assert.equal(model.punctuation.quotesForKeepSuggestion.close, closeQuote); @@ -176,7 +176,7 @@ describe('LMLayerWorker trie model for word lists', function() { ); }); }); - + describe('The default key function', function () { it('uses the default key function', function () { var model = new TrieModel(jsonFixture('tries/accented')); diff --git a/common/models/templates/test/test-trie-traversal.js b/common/models/templates/test/test-trie-traversal.js index 09d1361525b..31f52f854a3 100644 --- a/common/models/templates/test/test-trie-traversal.js +++ b/common/models/templates/test/test-trie-traversal.js @@ -3,7 +3,7 @@ */ var assert = require('chai').assert; -var TrieModel = require('../').models.TrieModel; +var TrieModel = require('../build/index.bundled.js').models.TrieModel; // Useful for tests related to strings with supplementary pairs. var smpForUnicode = function(code){ @@ -52,7 +52,7 @@ describe('Trie traversal abstractions', function() { assert.isEmpty(child.traversal().entries); for(tChild of traversalInner1.children()) { - if(tChild.char == 'h') { + if(tChild.char == 'h') { hSuccess = true; let traversalInner2 = tChild.traversal(); assert.isDefined(traversalInner2); @@ -64,7 +64,7 @@ describe('Trie traversal abstractions', function() { eSuccess = true; let traversalInner3 = hChild.traversal(); assert.isDefined(traversalInner3); - + assert.isDefined(traversalInner3.entries); assert.equal(traversalInner3.entries[0], "the"); @@ -104,7 +104,7 @@ describe('Trie traversal abstractions', function() { assert.isEmpty(child.traversal().entries); for(tChild of traversalInner1.children()) { - if(tChild.char == 'r') { + if(tChild.char == 'r') { let traversalInner2 = tChild.traversal(); assert.isDefined(traversalInner2); assert.isArray(tChild.traversal().entries); @@ -185,7 +185,7 @@ describe('Trie traversal abstractions', function() { assert.isEmpty(child.traversal().entries); for(aChild of traversalInner1.children()) { - if(aChild.char == smpP) { + if(aChild.char == smpP) { pSuccess = true; let traversalInner2 = aChild.traversal(); assert.isDefined(traversalInner2); diff --git a/common/models/templates/tsconfig.bundled.json b/common/models/templates/tsconfig.bundled.json new file mode 100644 index 00000000000..fd8dc5f65a5 --- /dev/null +++ b/common/models/templates/tsconfig.bundled.json @@ -0,0 +1,21 @@ +{ + // This variant of the tsconfig.json exists to create a 'leaf', 'bundled' + // version of the models/templates build product. The same reference + // cannot be prepended twice in a composite tsc build, posing problems + // for certain down-line builds if the two tsconfigs are not differentiated. + "extends": "./tsconfig.json", + "compilerOptions": { + "outFile": "build/index.bundled.js", + }, + "references": [ + { "path": "../../web/keyman-version", "prepend": true}, + { "path": "../../web/utils", "prepend": true }, + { "path": "../types" } + ], + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "test" + ] +} diff --git a/common/models/wordbreakers/package.json b/common/models/wordbreakers/package.json index 38cf3a01756..a7fd87a1f0b 100644 --- a/common/models/wordbreakers/package.json +++ b/common/models/wordbreakers/package.json @@ -14,7 +14,7 @@ ], "homepage": "https://github.com/keymanapp/keyman", "license": "MIT", - "main": "index.js", + "main": "build/index.js", "directories": { "lib": "lib", "test": "test" diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index c587ca08de9..357372b2280 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -3,8 +3,8 @@ namespace wordBreakers { /** * Word breaker based on Unicode Standard Annex #29, Section 4.1: - * Default Word Boundary Specification. - * + * Default Word Boundary Specification. + * * @see http://unicode.org/reports/tr29/#Word_Boundaries * @see https://github.com/eddieantonio/unicode-default-word-boundary/tree/v12.0.0 */ @@ -349,6 +349,12 @@ namespace wordBreakers { /** * Binary search for the word break property of a given CODE POINT. + * + * The auto-generated data.ts master array defines a **character range** + * lookup table. If a character's codepoint is equal to or greater than + * the I.Start value for an entry and exclusively less than the next entry, + * it falls in the first entry's range bucket and is classified accordingly + * by this method. */ function searchForProperty(codePoint: number, left: number, right: number): WordBreakProperty { // All items that are not found in the array are assigned the 'Other' property. diff --git a/common/models/wordbreakers/test/test-ascii-word-breaker.js b/common/models/wordbreakers/test/test-ascii-word-breaker.js index 598780ad959..55b9c3b8ba0 100644 --- a/common/models/wordbreakers/test/test-ascii-word-breaker.js +++ b/common/models/wordbreakers/test/test-ascii-word-breaker.js @@ -1,5 +1,5 @@ var assert = require('chai').assert; -var breakASCIIWords = require('..').wordBreakers['ascii']; +var breakASCIIWords = require('../build').wordBreakers['ascii']; describe('The ASCII word breaker', function () { it('should break simple English sentences', function () { diff --git a/common/models/wordbreakers/test/test-default-word-breaker.js b/common/models/wordbreakers/test/test-default-word-breaker.js index 6cf23626956..7898723de21 100644 --- a/common/models/wordbreakers/test/test-default-word-breaker.js +++ b/common/models/wordbreakers/test/test-default-word-breaker.js @@ -3,9 +3,9 @@ */ const assert = require('chai').assert; -const breakWords = require('..').wordBreakers['default']; +const breakWords = require('../build').wordBreakers['default']; -const SHY = '\u00AD'; +const SHY = '\u00AD'; // Other, Format. The "Soft HYphen" - usually invisible unless needed for word-wrapping. describe('The default word breaker', function () { it('should break multilingual text', function () { @@ -20,4 +20,16 @@ describe('The default word breaker', function () { "let's", 'eat', 'phแปŸ', '!', '๐Ÿฅฃ' ]); }); + + it('handles heavily-punctuated English text', function() { + // This test case brought to you by http://unicode.org/reports/tr29/#Word_Boundaries, Figure 1. + let breaks = breakWords( + `The quick ("brown") fox can't jump 32.3 feet, right?` + ); + let words = breaks.map(span => span.text); + assert.deepEqual(words, [ + 'The', 'quick', '(', '"', 'brown', '"', ')', 'fox', "can't", + 'jump', '32.3', 'feet', ',', 'right', '?' + ]); + }); }); diff --git a/common/models/wordbreakers/test/test-placeholder-word-breaker.js b/common/models/wordbreakers/test/test-placeholder-word-breaker.js index b23fd5d6bb8..dbc1aef34e6 100644 --- a/common/models/wordbreakers/test/test-placeholder-word-breaker.js +++ b/common/models/wordbreakers/test/test-placeholder-word-breaker.js @@ -1,5 +1,5 @@ const assert = require('chai').assert; -const breakWords = require('..').wordBreakers['placeholder']; +const breakWords = require('../build').wordBreakers['placeholder']; describe('The placeholder word breaker', function () { diff --git a/common/predictive-text/unit_tests/test.sh b/common/predictive-text/unit_tests/test.sh index ed8c787c3e5..8a30532771c 100755 --- a/common/predictive-text/unit_tests/test.sh +++ b/common/predictive-text/unit_tests/test.sh @@ -33,6 +33,18 @@ test-headless ( ) { _FLAGS="$_FLAGS --reporter mocha-teamcity-reporter" fi + pushd "$KEYMAN_ROOT/common/models/wordbreakers" + npm run test || fail "models/wordbreakers tests failed" + popd + + pushd "$KEYMAN_ROOT/common/models/templates" + npm run test || fail "models/templates tests failed" + popd + + pushd "$KEYMAN_ROOT/common/models/types" + npm run test || fail "models/types tests failed" + popd + npm run mocha -- --recursive $_FLAGS ./unit_tests/headless/*.js ./unit_tests/headless/**/*.js } diff --git a/common/web/utils/package.json b/common/web/utils/package.json index 680231d3b3e..eee6e450407 100644 --- a/common/web/utils/package.json +++ b/common/web/utils/package.json @@ -1,8 +1,8 @@ { "name": "@keymanapp/web-utils", "description": "Common utility functions used throughout other Keyman packages", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", + "main": "./build/index.js", + "types": "./build/index.d.ts", "scripts": { "build": "gosh ./build.sh", "tsc": "tsc" diff --git a/package-lock.json b/package-lock.json index 03dd8ee2b05..32b64610651 100644 --- a/package-lock.json +++ b/package-lock.json @@ -117,7 +117,7 @@ "@types/node": "^14.0.4", "chai": "^4.3.4", "mocha": "^8.4.0", - "typescript": "^3.8.3" + "typescript": "^4.5.4" } }, "common/models/templates/node_modules/@types/node": { @@ -126,19 +126,6 @@ "integrity": "sha512-q4jlIR71hUpWTnGhXWcakgkZeHa3CCjcQcnuzU8M891BAWA2jHiziiWEPEkdS5pFsz7H9HJiy8BrK7tBRNrY7A==", "dev": true }, - "common/models/templates/node_modules/typescript": { - "version": "3.9.10", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.10.tgz", - "integrity": "sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q==", - "dev": true, - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=4.2.0" - } - }, "common/models/types": { "name": "@keymanapp/models-types", "license": "MIT", @@ -7791,7 +7778,7 @@ "@types/node": "^14.0.4", "chai": "^4.3.4", "mocha": "^8.4.0", - "typescript": "^3.8.3" + "typescript": "^4.5.4" }, "dependencies": { "@types/node": { @@ -7799,12 +7786,6 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.12.tgz", "integrity": "sha512-q4jlIR71hUpWTnGhXWcakgkZeHa3CCjcQcnuzU8M891BAWA2jHiziiWEPEkdS5pFsz7H9HJiy8BrK7tBRNrY7A==", "dev": true - }, - "typescript": { - "version": "3.9.10", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.10.tgz", - "integrity": "sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q==", - "dev": true } } },