fix: add automatic encoding detection

frontend/package.json

@@ -45,6 +45,7 @@
         "html2canvas": "^1.4.1",
         "i18next": "^23.11.5",
         "js-sha256": "^0.11.0",
+        "jschardet": "^3.1.4",
         "jszip": "^3.10.1",
         "lodash": "^4.17.21",
         "lucide-react": "^0.399.0",

frontend/src/modules/core/helpers/files.ts

@@ -1,4 +1,5 @@
 import JSZip from "jszip";
+import jschardet from "jschardet";
 
 export const FASTA_EXTENSIONS = [".fa", ".mpfa", ".fna", ".fsa", ".fasta"];
 
@@ -34,15 +35,30 @@ export const downloadFileFromUrl = (url: string) => {
  * @param file - the file to read
  * @returns a promise that resolves with the file's text content
  */
-export const readFileAsText = (file: Blob): Promise<string> => {
+export const readFileAsText = (file: File | Blob): Promise<string> => {
     return new Promise((resolve, reject) => {
         try {
             const reader = new FileReader();
-            reader.readAsText(file);
-            reader.onloadend = () => resolve(reader.result as string);
+            reader.onload = () => {
+                const arrayBuffer = reader.result as ArrayBuffer;
+                const uint8Array = new Uint8Array(arrayBuffer);
+                // Convert Uint8Array to binary string for encoding detection
+                const binaryString = Array.from(uint8Array)
+                    .map(byte => String.fromCharCode(byte))
+                    .join("");
+                // Detect encoding as export sources are not deterministic
+                const detected = jschardet.detect(binaryString);
+                const encoding = detected.encoding || "UTF-8";
+                // Decode using TextDecoder with detected encoding
+                const decoder = new TextDecoder(encoding);
+                const text = decoder.decode(uint8Array);
+                resolve(text);
+            };
             reader.onerror = reject;
+            reader.readAsArrayBuffer(file);
         } catch (error) {
-            console.log(error)
+            console.log(error);
+            reject(error);
         }
     });
 };
@@ -53,7 +69,7 @@ export const readFileAsText = (file: Blob): Promise<string> => {
  * @returns a promise that resolves with an array of the files' text content
  */
 export const readFilesAsText = (files: File[]): Promise<string[]> => {
-    const filePromises = files.map((file) => readFileAsText(file));
+    const filePromises = files.map(file => readFileAsText(file));
     return Promise.all(filePromises);
 };
 
@@ -86,7 +102,7 @@ export const formatData = (
                 // for bacterial uploads:
                 // fasta file name contains fasta id
                 // content contains assembly
-                fastaSequencesArray.push({fastaId: file.filename.split(".")[0], sequence: file.content});
+                fastaSequencesArray.push({ fastaId: file.filename.split(".")[0], sequence: file.content });
             }
         }
         return fastaSequencesArray;
@@ -99,7 +115,7 @@ export const formatData = (
         } else {
             let rows = Object.values(fileReaderResult)[0].split("\n");
             // filter empty lines to prevent empty cells
-            rows = rows.filter((line) => line !== "");
+            rows = rows.filter(line => line !== "");
             const columns: string[] = rows[0]
                 .replace(/["'\n]/g, "")
                 .split(";")
@@ -118,7 +134,7 @@ export const formatData = (
                 });
                 rowData.push(rowObject);
             }
-            return {columns: columns, rows: rowData};
+            return { columns: columns, rows: rowData };
         }
     }
 };
@@ -143,7 +159,7 @@ export const collectFastaIdsAndSequences = (fastaSequences: Array<string>) => {
         }
 
         if (fastaId) {
-            fastaSequencesArray.push({fastaId: fastaId, sequence: genome});
+            fastaSequencesArray.push({ fastaId: fastaId, sequence: genome });
         }
     }
     return fastaSequencesArray;

frontend/src/modules/core/tests/unit/helpers/files.test.ts

@@ -146,7 +146,7 @@ describe("FilesHelper", () => {
             window.URL.createObjectURL = vi.fn(() => ":object_url:");
             // mock methods that are not yet supported by jsdom
             const spyOnCreateElement = vi.spyOn(document, "createElement").mockImplementation(() => link);
-            const spyOnLinkClick = vi.spyOn(link, "click");
+            const spyOnLinkClick = vi.spyOn(link, "click").mockImplementation(() => {}); // Prevent actual navigation
             downloadFile(blob, ":file_name:");
 
             expect(spyOnCreateElement).toHaveBeenCalledOnce();