Merge pull request #11 from digimuza/Add-ability-to-configurate-encodings

Add ability to configure encoding

Author: digimuza

api-extractor.json

@@ -0,0 +1,38 @@
+{
+	"$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json",
+	"mainEntryPointFilePath": "<projectFolder>lib/index.d.ts",
+	"bundledPackages": [],
+	"compiler": {},
+	"apiReport": {
+		"enabled": false
+	},
+	"docModel": {
+		"enabled": true,
+		"apiJsonFilePath": "<projectFolder>/temp/<unscopedPackageName>.api.json"
+	},
+	"dtsRollup": {
+		"enabled": true,
+		"untrimmedFilePath": "<projectFolder>/temp/<unscopedPackageName>.d.ts"
+	},
+	"tsdocMetadata": {},
+	"messages": {
+		"compilerMessageReporting": {
+			"default": {
+				"logLevel": "none"
+			}
+		},
+		"extractorMessageReporting": {
+			"default": {
+				"logLevel": "none"
+			},
+			"ae-forgotten-export": {
+				"logLevel": "none"
+			}
+		},
+		"tsdocMessageReporting": {
+			"default": {
+				"logLevel": "none"
+			}
+		}
+	}
+}

package.json

@@ -26,7 +26,7 @@
     "json-parser"
   ],
   "scripts": {
-    "build": "rm -rf ./lib && tsc && yarn ts-docflux",
+    "build": "rm -rf ./lib && tsc",
     "watch": "yarn link && tsc --watch",
     "test": "cd ./benchmarks && yarn && yarn test"
   },
@@ -61,6 +61,6 @@
     "papaparse": "^5.2.0",
     "pretty-ms": "^7.0.1",
     "tmp": "^0.2.1",
-    "ts-prime": "0.3.15"
+    "ts-prime": "^1.0.0"
   }
 }

src/cache.ts

@@ -65,28 +65,28 @@ export function _cacheIter<T>(data: AnyIterable<T>, options: CacheIterOptions):
         const lockFile = resolve(options.cacheFolder, ".lock")
         if (existsSync(lockFile)) {
             options.logger?.info('Lock file exist. This usually means that iterator was not cached fully. Cache folder will be deleted and recreated with new iterator')
-            rmdirSync(options.cacheFolder, { recursive: true })
+            rmdirSync(options.cacheFolder)
         }
         if (!existsSync(metaFile)) {
             options.logger?.info('Meta file does not exists. Deleting cache folder...')
-            rmdirSync(options.cacheFolder, { recursive: true })
+            removeIfExists(options);
         }
 
         if (existsSync(metaFile)) {
             const meta: { referenceId: string, createdAt: string, iteratableId: string, format: CacheIterOptions['nice'] } = await readJSON(metaFile)
             if (meta.referenceId !== referenceId) {
                 options.logger?.info('Reference id changed. Deleting cache folder...')
-                rmdirSync(options.cacheFolder, { recursive: true })
+                removeIfExists(options);
             }
 
             if (!P.equals(meta.format, options.nice)) {
                 options.logger?.info('Cache format changed. Deleting cache folder...')
-                rmdirSync(options.cacheFolder, { recursive: true })
+                removeIfExists(options);
             }
 
             if (!P.equals(meta.iteratableId, iteratableId)) {
                 options.logger?.info('Source iterator structure changed. Deleting cache folder...')
-                rmdirSync(options.cacheFolder, { recursive: true })
+                removeIfExists(options);
             }
         }
 
@@ -106,7 +106,7 @@ export function _cacheIter<T>(data: AnyIterable<T>, options: CacheIterOptions):
         const cache = getCache()
         if (cache) return cache
         if (existsSync(options.cacheFolder)) {
-            rmdirSync(options.cacheFolder, { recursive: true })
+            removeIfExists(options)
         }
         await ensureFile(lockFile)
 
@@ -135,6 +135,12 @@ export function _cacheIter<T>(data: AnyIterable<T>, options: CacheIterOptions):
     })
 }
 
+function removeIfExists(options: CacheIterOptions) {
+    if (existsSync(options.cacheFolder)) {
+        rmdirSync(options.cacheFolder);
+    }
+}
+
 /**
  * Cache iterator output to file.
  * Useful when we need develop complex iterator pipelines.

src/xml.ts

@@ -86,6 +86,7 @@ export interface XMLReadOptions extends ProgressReportOptions, FastXMLParser, Fi
      * <Person>...</Person>
      */
     nodeName: string
+    encoding?: "utf8" | "utf16le" | "ascii" | "base64" | "binary"
 }
 export interface FastXMLWriteOptions {
     /**
@@ -250,7 +251,7 @@ export function xmlRead<T>(options: XMLReadOptions): IX<T> {
         arrayMode: false, //"strict"
     }
     let count = 0
-    const elMatch = new RegExp(`<${options.nodeName}( .*)?>`, 'gm')
+    const elMatch = new RegExp(`<${options.nodeName}( ?.*)?>`, 'gm')
     async function* iter() {
         for await (const buffer of bufferRead({
             ...options,
@@ -261,7 +262,10 @@ export function xmlRead<T>(options: XMLReadOptions): IX<T> {
                 options.progress?.(q)
             }
         })) {
-            const full = `${last}${buffer.toString()}`
+            const full = `${last}${buffer.toString(options.encoding)}`
+            if (!full.match(elMatch)) {
+                throw new Error(`Failed to find node: ${options.nodeName}`)
+            }
             const beef = full.replace(elMatch, `!@###@!<${options.nodeName}>`).split(`!@###@!`)
             last = beef.pop() || ''
             for (const qwe of beef) {
@@ -273,7 +277,7 @@ export function xmlRead<T>(options: XMLReadOptions): IX<T> {
                     ...options
 
                 })[options.nodeName]
-                if (parsedResult == null) continue 
+                if (parsedResult == null) continue
                 yield parsedResult
                 count++
             }

yarn.lock

@@ -600,16 +600,16 @@ ts-docflux@^0.0.13:
     xml-flow "^1.0.4"
     zod "^1.11.11"
 
-ts-prime@0.3.15:
-  version "0.3.15"
-  resolved "https://registry.yarnpkg.com/ts-prime/-/ts-prime-0.3.15.tgz#75fc8777b6618ae3136a3d599d339f52617ad2ec"
-  integrity sha512-ADiQE/4hGlzy/AkVCE+Vt5ETe8QOUSCbbrL9Vo9lGps/fUKIYykje9hLj/6Mbbcq684u872G3Ul7D84vM0+/xw==
-
 ts-prime@^0.3.7:
   version "0.3.12"
   resolved "https://registry.yarnpkg.com/ts-prime/-/ts-prime-0.3.12.tgz#0b755d8dc366ecb49d4b337864b289d2e5c2161a"
   integrity sha512-HsaCT3zW5+HaSMXYVC7nC8fI7R2MqGOg6yvjmhY3OiMG5lETVDq1shmuL1/+t5jmHDI2XHDlR8UyH4aXFEKz8g==
 
+ts-prime@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/ts-prime/-/ts-prime-1.0.0.tgz#573f73e67f30478a0c3317da47cab9bf053d1056"
+  integrity sha512-+vHlwnDWLaSDcLPBMBFjnSbbY/HZwg5eju9/pmNsGiXL6N/Gwd/d6w0PGCbnRm7j+mMISQrRUiVVvExTS6GJxA==
+
 tslib@^1.9.3:
   version "1.14.1"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00"