Support alternative file extensions (#1231)

Author: dcodeIO

cli/asc.d.ts

@@ -137,6 +137,8 @@ interface CompilerOptions {
   printrtti?: boolean;
   /** Disables terminal colors. */
   noColors?: boolean;
+  /** Specifies an alternative file extension. */
+  extension?: string;
 }
 
 /** Compiler API options. */

cli/asc.js

@@ -42,6 +42,21 @@ const EOL = process.platform === "win32" ? "\r\n" : "\n";
 const SEP = process.platform === "win32" ? "\\" : "/";
 const binaryen = global.binaryen || (global.binaryen = require("binaryen"));
 
+// Sets up an extension with its definition counterpart and relevant regexes.
+function setupExtension(extension) {
+  if (!extension.startsWith(".")) extension = "." + extension;
+  return {
+    ext: extension,
+    ext_d: ".d" + extension,
+    re: new RegExp("\\" + extension + "$"),
+    re_d: new RegExp("\\.d\\" + extension + "$"),
+    re_except_d: new RegExp("^(?!.*\\.d\\" + extension + "$).*\\" + extension + "$"),
+    re_index: new RegExp("(?:^|[\\\\\\/])index\\" + extension + "$")
+  };
+}
+
+const defaultExtension = setupExtension(".ts");
+
 // Proxy Binaryen's ready event
 Object.defineProperty(exports, "ready", {
   get: function() { return binaryen.ready; }
@@ -109,23 +124,23 @@ exports.defaultShrinkLevel = 1;
 exports.libraryFiles = exports.isBundle ? BUNDLE_LIBRARY : (() => { // set up if not a bundle
   const libDir = path.join(__dirname, "..", "std", "assembly");
   const bundled = {};
-  find.files(libDir, find.TS_EXCEPT_DTS)
-      .forEach(file => bundled[file.replace(/\.ts$/, "")] = fs.readFileSync(path.join(libDir, file), "utf8" ));
+  find.files(libDir, defaultExtension.re_except_d)
+      .forEach(file => bundled[file.replace(defaultExtension.re, "")] = fs.readFileSync(path.join(libDir, file), "utf8" ));
   return bundled;
 })();
 
 /** Bundled definition files. */
 exports.definitionFiles = exports.isBundle ? BUNDLE_DEFINITIONS : (() => { // set up if not a bundle
   const stdDir = path.join(__dirname, "..", "std");
   return {
-    "assembly": fs.readFileSync(path.join(stdDir, "assembly", "index.d.ts"), "utf8"),
-    "portable": fs.readFileSync(path.join(stdDir, "portable", "index.d.ts"), "utf8")
+    "assembly": fs.readFileSync(path.join(stdDir, "assembly", "index" + defaultExtension.ext_d), "utf8"),
+    "portable": fs.readFileSync(path.join(stdDir, "portable", "index" + defaultExtension.ext_d), "utf8")
   };
 })();
 
 /** Convenience function that parses and compiles source strings directly. */
 exports.compileString = (sources, options) => {
-  if (typeof sources === "string") sources = { "input.ts": sources };
+  if (typeof sources === "string") sources = { ["input" + defaultExtension.ext]: sources };
   const output = Object.create({
     stdout: createMemoryStream(),
     stderr: createMemoryStream()
@@ -169,6 +184,7 @@ exports.main = function main(argv, options, callback) {
   const writeFile = options.writeFile || writeFileNode;
   const listFiles = options.listFiles || listFilesNode;
   const stats = options.stats || createStats();
+  let extension = defaultExtension;
 
   // Output must be specified if not present in the environment
   if (!stdout) throw Error("'options.stdout' must be specified");
@@ -213,6 +229,15 @@ exports.main = function main(argv, options, callback) {
     return callback(null);
   }
 
+  // Use another extension if requested
+  if (typeof args.extension === "string") {
+    if (/^\.?[0-9a-zA-Z]{1,14}$/.test(args.extension)) {
+      extension = setupExtension(args.extension);
+    } else {
+      return callback(Error("Invalid extension: " + args.extension));
+    }
+  }
+
   // Print the help message if requested or no source files are provided
   if (args.help || !argv.length) {
     var out = args.help ? stdout : stderr;
@@ -222,9 +247,9 @@ exports.main = function main(argv, options, callback) {
       "  " + color.cyan("asc") + " [entryFile ...] [options]",
       "",
       color.white("EXAMPLES"),
-      "  " + color.cyan("asc") + " hello.ts",
-      "  " + color.cyan("asc") + " hello.ts -b hello.wasm -t hello.wat",
-      "  " + color.cyan("asc") + " hello1.ts hello2.ts -b -O > hello.wasm",
+      "  " + color.cyan("asc") + " hello" + extension.ext,
+      "  " + color.cyan("asc") + " hello" + extension.ext + " -b hello.wasm -t hello.wat",
+      "  " + color.cyan("asc") + " hello1" + extension.ext + " hello2" + extension.ext + " -b -O > hello.wasm",
       "",
       color.white("OPTIONS"),
     ].concat(
@@ -319,7 +344,7 @@ exports.main = function main(argv, options, callback) {
     let transformArgs = args.transform;
     for (let i = 0, k = transformArgs.length; i < k; ++i) {
       let filename = transformArgs[i].trim();
-      if (!tsNodeRegistered && filename.endsWith('.ts')) {
+      if (!tsNodeRegistered && filename.endsWith(".ts")) { // ts-node requires .ts specifically
         require("ts-node").register({ transpileOnly: true, skipProject: true, compilerOptions: { target: "ES2016" } });
         tsNodeRegistered = true;
       }
@@ -363,7 +388,7 @@ exports.main = function main(argv, options, callback) {
     if (libPath.indexOf("/") >= 0) return; // in sub-directory: imported on demand
     stats.parseCount++;
     stats.parseTime += measure(() => {
-      assemblyscript.parse(program, exports.libraryFiles[libPath], exports.libraryPrefix + libPath + ".ts", false);
+      assemblyscript.parse(program, exports.libraryFiles[libPath], exports.libraryPrefix + libPath + extension.ext, false);
     });
   });
   const customLibDirs = [];
@@ -374,7 +399,7 @@ exports.main = function main(argv, options, callback) {
     for (let i = 0, k = customLibDirs.length; i < k; ++i) { // custom
       let libDir = customLibDirs[i];
       let libFiles;
-      if (libDir.endsWith(".ts")) {
+      if (libDir.endsWith(extension.ext)) {
         libFiles = [ path.basename(libDir) ];
         libDir = path.dirname(libDir);
       } else {
@@ -385,7 +410,7 @@ exports.main = function main(argv, options, callback) {
         let libText = readFile(libPath, libDir);
         if (libText === null) return callback(Error("Library file '" + libPath + "' not found."));
         stats.parseCount++;
-        exports.libraryFiles[libPath.replace(/\.ts$/, "")] = libText;
+        exports.libraryFiles[libPath.replace(extension.re, "")] = libText;
         stats.parseTime += measure(() => {
           assemblyscript.parse(program, libText, exports.libraryPrefix + libPath, false);
         });
@@ -406,13 +431,13 @@ exports.main = function main(argv, options, callback) {
     const libraryPrefix = exports.libraryPrefix;
     const libraryFiles = exports.libraryFiles;
 
-    // Try file.ts, file/index.ts, file.d.ts
+    // Try file.ext, file/index.ext, file.d.ext
     if (!internalPath.startsWith(libraryPrefix)) {
-      if ((sourceText = readFile(sourcePath = internalPath + ".ts", baseDir)) == null) {
-        if ((sourceText = readFile(sourcePath = internalPath + "/index.ts", baseDir)) == null) {
-          // portable d.ts: uses the .js file next to it in JS or becomes an import in Wasm
-          sourcePath = internalPath + ".ts";
-          sourceText = readFile(internalPath + ".d.ts", baseDir);
+      if ((sourceText = readFile(sourcePath = internalPath + extension.ext, baseDir)) == null) {
+        if ((sourceText = readFile(sourcePath = internalPath + "/index" + extension.ext, baseDir)) == null) {
+          // portable d.ext: uses the .js file next to it in JS or becomes an import in Wasm
+          sourcePath = internalPath + extension.ext;
+          sourceText = readFile(internalPath + extension.ext_d, baseDir);
         }
       }
 
@@ -422,18 +447,18 @@ exports.main = function main(argv, options, callback) {
       const indexName = plainName + "/index";
       if (libraryFiles.hasOwnProperty(plainName)) {
         sourceText = libraryFiles[plainName];
-        sourcePath = libraryPrefix + plainName + ".ts";
+        sourcePath = libraryPrefix + plainName + extension.ext;
       } else if (libraryFiles.hasOwnProperty(indexName)) {
         sourceText = libraryFiles[indexName];
-        sourcePath = libraryPrefix + indexName + ".ts";
+        sourcePath = libraryPrefix + indexName + extension.ext;
       } else { // custom lib dirs
         for (const libDir of customLibDirs) {
-          if ((sourceText = readFile(plainName + ".ts", libDir)) != null) {
-            sourcePath = libraryPrefix + plainName + ".ts";
+          if ((sourceText = readFile(plainName + extension.ext, libDir)) != null) {
+            sourcePath = libraryPrefix + plainName + extension.ext;
             break;
           } else {
-            if ((sourceText = readFile(indexName + ".ts", libDir)) != null) {
-              sourcePath = libraryPrefix + indexName + ".ts";
+            if ((sourceText = readFile(indexName + extension.ext, libDir)) != null) {
+              sourcePath = libraryPrefix + indexName + extension.ext;
               break;
             }
           }
@@ -463,25 +488,25 @@ exports.main = function main(argv, options, callback) {
                   try {
                     let json = JSON.parse(jsonText);
                     if (typeof json.ascMain === "string") {
-                      mainPath = json.ascMain.replace(/[\/\\]index\.ts$/, "");
+                      mainPath = json.ascMain.replace(extension.re_index, "");
                       packageMains.set(packageName, mainPath);
                     }
                   } catch (e) { }
                 }
               }
               const mainDir = path.join(currentPath, packageName, mainPath);
               const plainName = filePath;
-              if ((sourceText = readFile(path.join(mainDir, plainName + ".ts"), baseDir)) != null) {
-                sourcePath = libraryPrefix + packageName + "/" + plainName + ".ts";
-                packageBases.set(sourcePath.replace(/\.ts$/, ""), path.join(currentPath, packageName));
-                if (args.traceResolution) stderr.write("  -> " + path.join(mainDir, plainName + ".ts") + EOL);
+              if ((sourceText = readFile(path.join(mainDir, plainName + extension.ext), baseDir)) != null) {
+                sourcePath = libraryPrefix + packageName + "/" + plainName + extension.ext;
+                packageBases.set(sourcePath.replace(extension.re, ""), path.join(currentPath, packageName));
+                if (args.traceResolution) stderr.write("  -> " + path.join(mainDir, plainName + extension.ext) + EOL);
                 break;
               } else if (!isPackageRoot) {
                 const indexName = filePath + "/index";
-                if ((sourceText = readFile(path.join(mainDir, indexName + ".ts"), baseDir)) !== null) {
-                  sourcePath = libraryPrefix + packageName + "/" + indexName + ".ts";
-                  packageBases.set(sourcePath.replace(/\.ts$/, ""), path.join(currentPath, packageName));
-                  if (args.traceResolution) stderr.write("  -> " + path.join(mainDir, indexName + ".ts") + EOL);
+                if ((sourceText = readFile(path.join(mainDir, indexName + extension.ext), baseDir)) !== null) {
+                  sourcePath = libraryPrefix + packageName + "/" + indexName + extension.ext;
+                  packageBases.set(sourcePath.replace(extension.re, ""), path.join(currentPath, packageName));
+                  if (args.traceResolution) stderr.write("  -> " + path.join(mainDir, indexName + extension.ext) + EOL);
                   break;
                 }
               }
@@ -519,33 +544,34 @@ exports.main = function main(argv, options, callback) {
     let runtimeText = exports.libraryFiles[runtimePath];
     if (runtimeText == null) {
       runtimePath = runtimeName;
-      runtimeText = readFile(runtimePath + ".ts", baseDir);
+      runtimeText = readFile(runtimePath + extension.ext, baseDir);
       if (runtimeText == null) return callback(Error("Runtime '" + runtimeName + "' not found."));
     } else {
       runtimePath = "~lib/" + runtimePath;
     }
     stats.parseCount++;
     stats.parseTime += measure(() => {
-      assemblyscript.parse(program, runtimeText, runtimePath, true);
+      assemblyscript.parse(program, runtimeText, runtimePath + extension.ext, true);
     });
   }
 
   // Include entry files
   for (let i = 0, k = argv.length; i < k; ++i) {
     const filename = argv[i];
 
-    let sourcePath = String(filename).replace(/\\/g, "/").replace(/(\.ts|\/)$/, "");
+    let sourcePath = String(filename).replace(/\\/g, "/").replace(extension.re, "").replace(/[\\\/]$/, "");
+    
     // Setting the path to relative path
     sourcePath = path.isAbsolute(sourcePath) ? path.relative(baseDir, sourcePath) : sourcePath;
 
-    // Try entryPath.ts, then entryPath/index.ts
-    let sourceText = readFile(sourcePath + ".ts", baseDir);
+    // Try entryPath.ext, then entryPath/index.ext
+    let sourceText = readFile(sourcePath + extension.ext, baseDir);
     if (sourceText == null) {
-      sourceText = readFile(sourcePath + "/index.ts", baseDir);
-      if (sourceText == null) return callback(Error("Entry file '" + sourcePath + ".ts' not found."));
-      sourcePath += "/index.ts";
+      sourceText = readFile(sourcePath + "/index" + extension.ext, baseDir);
+      if (sourceText == null) return callback(Error("Entry file '" + sourcePath + extension.ext + "' not found."));
+      sourcePath += "/index" + extension.ext;
     } else {
-      sourcePath += ".ts";
+      sourcePath += extension.ext;
     }
 
     stats.parseCount++;
@@ -741,7 +767,7 @@ exports.main = function main(argv, options, callback) {
           map.sourceRoot = "./" + basename;
           let contents = [];
           map.sources.forEach((name, index) => {
-            let text = assemblyscript.getSource(program, name.replace(/\.ts$/, ""));
+            let text = assemblyscript.getSource(program, name.replace(extension.re, ""));
             if (text == null) return callback(Error("Source of file '" + name + "' not found."));
             contents[index] = text;
           });
@@ -882,7 +908,7 @@ exports.main = function main(argv, options, callback) {
     var files;
     try {
       stats.readTime += measure(() => {
-        files = fs.readdirSync(path.join(baseDir, dirname)).filter(file => /^(?!.*\.d\.ts$).*\.ts$/.test(file));
+        files = fs.readdirSync(path.join(baseDir, dirname)).filter(file => extension.re_except_d.test(file))
       });
       return files;
     } catch (e) {

cli/asc.json

@@ -252,6 +252,10 @@
     "type": "s",
     "default": "."
   },
+  "extension": {
+    "description": "Specifies an alternative file extension to use.",
+    "type": "s"
+  },
   "noUnsafe": {
     "description": [
       "Disallows the use of unsafe features in user code.",

cli/util/find.d.ts

@@ -4,5 +4,3 @@
  */
 
 export function files(dirname: string, filter?: ((name: string) => bool) | RegExp): string[];
-export const TS: RegExp;
-export const TS_EXCEPT_DTS: RegExp;

cli/util/find.js

@@ -19,6 +19,3 @@ function findFiles(dirname, filter) {
 }
 
 exports.files = findFiles;
-
-exports.TS = /\.ts$/;
-exports.TS_EXCEPT_DTS = /(?:(?!\.d).{2}|^.{0,1})\.ts$/;

package.json

@@ -55,10 +55,11 @@
     "check": "npm run check:config && npm run check:compiler && tsc --noEmit --target ESNEXT --module commonjs --experimentalDecorators tests/require/index",
     "check:config": "tsc --noEmit -p src --diagnostics --listFiles",
     "check:compiler": "tslint -c tslint.json --project src --formatters-dir lib/lint/formatters --format as",
-    "test": "npm run test:parser && npm run test:compiler && npm run test:packages",
+    "test": "npm run test:parser && npm run test:compiler && npm run test:packages && npm run test:extension",
     "test:parser": "node tests/parser",
     "test:compiler": "node tests/compiler",
     "test:packages": "cd tests/packages && npm run test",
+    "test:extension": "cd tests/extension && npm run test",
     "make": "npm run clean && npm test && npm run build && npm test",
     "all": "npm run check && npm run make",
     "docs": "typedoc --tsconfig tsconfig-docs.json --mode modules --name \"AssemblyScript Compiler API\" --out ./docs/api --ignoreCompilerErrors --excludeNotExported --excludePrivate --excludeExternals --exclude **/std/** --includeDeclarations --readme src/README.md",

src/ast.ts

@@ -29,7 +29,8 @@ import {
 import {
   normalizePath,
   resolvePath,
-  CharCode
+  CharCode,
+  isTrivialAlphanum
 } from "./util";
 
 /** Indicates the kind of a node. */
@@ -716,7 +717,7 @@ export abstract class Node {
       } else { // absolute
         if (!normalizedPath.startsWith(LIBRARY_PREFIX)) normalizedPath = LIBRARY_PREFIX + normalizedPath;
       }
-      node.internalPath = mangleInternalPath(normalizedPath);
+      node.internalPath = normalizedPath;
     } else {
       node.internalPath = null;
     }
@@ -804,7 +805,7 @@ export abstract class Node {
     } else { // absolute in library
       if (!normalizedPath.startsWith(LIBRARY_PREFIX)) normalizedPath = LIBRARY_PREFIX + normalizedPath;
     }
-    node.internalPath = mangleInternalPath(normalizedPath);
+    node.internalPath = normalizedPath;
     return node;
   }
 
@@ -825,7 +826,7 @@ export abstract class Node {
     } else {
       if (!normalizedPath.startsWith(LIBRARY_PREFIX)) normalizedPath = LIBRARY_PREFIX + normalizedPath;
     }
-    node.internalPath = mangleInternalPath(normalizedPath);
+    node.internalPath = normalizedPath;
     return node;
   }
 
@@ -2088,7 +2089,19 @@ export function findDecorator(kind: DecoratorKind, decorators: DecoratorNode[] |
 
 /** Mangles an external to an internal path. */
 export function mangleInternalPath(path: string): string {
-  if (path.endsWith(".ts")) path = path.substring(0, path.length - 3);
+  var pos = path.lastIndexOf(".");
+  var len = path.length;
+  if (pos >= 0 && len - pos >= 2) { // at least one char plus dot
+    let cur = pos;
+    while (++cur < len) {
+      if (!isTrivialAlphanum(path.charCodeAt(cur))) {
+        assert(false); // not a valid external path
+        return path;
+      }
+    }
+    return path.substring(0, pos);
+  }
+  assert(false); // not an external path
   return path;
 }
 

src/parser.ts

@@ -125,7 +125,7 @@ export class Parser extends DiagnosticEmitter {
     /** Whether this is an entry file. */
     isEntry: bool
   ): void {
-    // the frontend gives us paths with .ts endings
+    // the frontend gives us paths with file extensions
     var normalizedPath = normalizePath(path);
     var internalPath = mangleInternalPath(normalizedPath);
     // check if already processed

src/program.ts

@@ -532,7 +532,7 @@ export class Program extends DiagnosticEmitter {
   ) {
     super(diagnostics);
     this.options = options;
-    var nativeSource = new Source(LIBRARY_SUBST, "[native code]", SourceKind.LIBRARY_ENTRY);
+    var nativeSource = new Source(LIBRARY_SUBST + ".wasm", "[native code]", SourceKind.LIBRARY_ENTRY);
     this.nativeSource = nativeSource;
     var nativeFile = new File(this, nativeSource);
     this.nativeFile = nativeFile;