From 6ed56aebc53023bf6c9a7286a96c89c3630bdfaa Mon Sep 17 00:00:00 2001 From: Raunak Raj <71929976+bajrangCoder@users.noreply.github.com> Date: Wed, 17 Jun 2026 22:43:24 +0530 Subject: [PATCH 1/2] feat: do file-listing and index on java for file and saf --- src/lib/fileList.js | 102 +- src/plugins/sdcard/index.d.ts | 49 + src/plugins/sdcard/plugin.xml | 3 +- src/plugins/sdcard/src/android/SDcard.java | 27 + .../sdcard/src/android/WorkspaceIndex.java | 1243 +++++++++++++++++ src/plugins/sdcard/www/plugin.js | 17 +- src/sidebarApps/searchInFiles/index.js | 392 +++--- src/utils/binaryExtensions.js | 58 + 8 files changed, 1703 insertions(+), 188 deletions(-) create mode 100644 src/plugins/sdcard/src/android/WorkspaceIndex.java diff --git a/src/lib/fileList.js b/src/lib/fileList.js index fb773335d..d24fe3c7e 100644 --- a/src/lib/fileList.js +++ b/src/lib/fileList.js @@ -13,6 +13,7 @@ const filesTree = {}; const pendingScans = new Set(); const events = { "add-file": [], + "push-file": [], "remove-file": [], "add-folder": [], "remove-folder": [], @@ -125,7 +126,7 @@ export default function files(dir) { } /** - * @typedef {'add-file'|'remove-file'|'add-folder'|'remove-folder'|'refresh'} FileListEvent + * @typedef {'add-file'|'push-file'|'remove-file'|'add-folder'|'remove-folder'|'refresh'} FileListEvent */ /** @@ -227,7 +228,7 @@ export async function addRoot({ url, name }) { const tree = await Tree.createRoot(url, name); filesTree[url] = tree; - trackScan(getAllFiles(tree)); + trackScan(getAllFiles(tree, null, { indexContent: false })); emit("add-folder", tree); } catch (error) { // ignore @@ -251,10 +252,14 @@ function onRemoveFolder({ url }) { * @param {Tree} parent - An array to store files * @param {Tree} [root] - Root path */ -async function getAllFiles(parent, root) { +async function getAllFiles(parent, root, options = {}) { root = root || parent.root; if (!parent.children || !root.isConnected) return; + if (supportsNativeWorkspace(root.url)) { + return getAllFilesNative(parent, root, options); + } + try { const entries = await fsOperation(parent.url).lsDir(); const promises = []; @@ -276,11 +281,97 @@ async function getAllFiles(parent, root) { // why not outside? because parent may be removed if (!root.isConnected) return; parent.children.length = 0; - getAllFiles(parent); + getAllFiles(parent, root, options); }, 3000); } } +function supportsNativeWorkspace(url = "") { + return ( + typeof sdcard !== "undefined" && + typeof sdcard.workspaceScan === "function" && + (/^file:/.test(url) || /^content:/.test(url)) + ); +} + +async function getAllFilesNative(parent, root, options = {}) { + const id = `scan-${Date.now()}-${Math.random().toString(36).slice(2)}`; + + return new Promise((resolve, reject) => { + let settled = false; + + const finish = (fn, value) => { + if (settled) return; + settled = true; + fn(value); + }; + + const cancelIfDisconnected = () => { + if (root.isConnected) return false; + try { + sdcard.workspaceCancel(id); + } catch (_) { + // ignore cancellation failures + } + finish(resolve); + return true; + }; + + sdcard.workspaceScan( + { + id, + rootUrl: parent.url, + title: parent.name, + excludeFolders: settings.value.excludeFolders, + showHiddenFiles: !!settings.value.fileBrowser?.showHiddenFiles, + defaultEncoding: settings.value.defaultFileEncoding, + indexContent: !!options.indexContent, + }, + (event) => { + if (cancelIfDisconnected()) return; + switch (event?.type || event?.action) { + case "batch": + addNativeEntries(root, event.entries || []); + break; + case "done": + finish(resolve); + break; + case "error": + finish(reject, new Error(event.error || "Native scan failed")); + break; + } + }, + (error) => { + finish(reject, error); + }, + ); + }); +} + +function addNativeEntries(root, entries) { + for (const item of entries) { + const parentUrl = item.parentUrl || item.parent; + const parentTree = parentUrl === root.url ? root : getTree([root], parentUrl); + if (!parentTree?.children) continue; + if (parentTree.children.find(({ url }) => url === item.url)) continue; + + const file = new Tree( + item.name, + item.url, + item.isDirectory, + item.mime || item.type, + item.size, + item.modifiedDate, + ); + parentTree.children.push(file); + + if (!file.children) { + emit("push-file", file); + emit("add-file", file); + } + } +} + /** * Emit an event * @param {string} event @@ -307,7 +398,7 @@ function trackScan(scan) { async function createChildTree(parent, item, root) { if (!root.isConnected) return; const { name, url, isDirectory, mime, type, size, modifiedDate } = item; - const exists = parent.children.findIndex(({ value }) => value === url); + const exists = parent.children.findIndex((child) => child.url === url); if (exists > -1) { return; } @@ -344,6 +435,7 @@ async function createChildTree(parent, item, root) { } emit("push-file", file); + emit("add-file", file); } export class Tree { diff --git a/src/plugins/sdcard/index.d.ts b/src/plugins/sdcard/index.d.ts index 1933dd305..f726ac263 100644 --- a/src/plugins/sdcard/index.d.ts +++ b/src/plugins/sdcard/index.d.ts @@ -39,6 +39,30 @@ interface DocumentFile { uri: string; } +interface WorkspaceFileEntry { + rootUrl: string; + parent: string; + parentUrl: string; + name: string; + path: string; + url: string; + uri: string; + mime?: string; + type?: string; + isDirectory: boolean; + isFile: boolean; + size: number; + modifiedDate: number; +} + +type WorkspaceEvent = + | { id: string; type: 'status'; action: 'status'; state: string; message: string; progress: number } + | { id: string; type: 'batch'; action: 'batch'; entries: WorkspaceFileEntry[] } + | { id: string; type: 'search-result'; action: 'search-result'; data: any } + | { id: string; type: 'replace-result'; action: 'replace-result'; file: WorkspaceFileEntry; text: string } + | { id: string; type: 'progress'; action: 'progress'; data: number } + | { id: string; type: 'done' | 'done-searching' | 'done-replacing' | 'error'; action: string; [key: string]: any }; + interface SDcard { /** * Copy file/directory to given destination @@ -257,6 +281,31 @@ interface SDcard { ): { unwatch: () => void; }; + workspaceScan( + options: any, + onEvent: (event: WorkspaceEvent) => void, + onFail: (err: any) => void, + ): void; + workspaceSearch( + options: any, + onEvent: (event: WorkspaceEvent) => void, + onFail: (err: any) => void, + ): void; + workspaceCancel( + id: string, + onSuccess?: (res: 'OK') => void, + onFail?: (err: any) => void, + ): void; + workspaceMarkDirty( + urls: string[], + onSuccess?: (res: 'OK') => void, + onFail?: (err: any) => void, + ): void; + workspaceClear( + roots: string[], + onSuccess?: (res: 'OK') => void, + onFail?: (err: any) => void, + ): void; } declare var sdcard: SDcard; diff --git a/src/plugins/sdcard/plugin.xml b/src/plugins/sdcard/plugin.xml index 19dda5cf5..85de5d63a 100644 --- a/src/plugins/sdcard/plugin.xml +++ b/src/plugins/sdcard/plugin.xml @@ -21,6 +21,7 @@ + - \ No newline at end of file + diff --git a/src/plugins/sdcard/src/android/SDcard.java b/src/plugins/sdcard/src/android/SDcard.java index 01f14070f..059776472 100644 --- a/src/plugins/sdcard/src/android/SDcard.java +++ b/src/plugins/sdcard/src/android/SDcard.java @@ -58,12 +58,15 @@ public class SDcard extends CordovaPlugin { private DocumentFile originalRootFile; private CallbackContext activityResultCallback; private HashMap fileObservers = new HashMap(); + private WorkspaceIndex workspaceIndex; public void initialize(CordovaInterface cordova, CordovaWebView webView) { super.initialize(cordova, webView); this.REQUEST_CODE = this.ACCESS_INTENT; this.context = cordova.getContext(); this.activity = cordova.getActivity(); + this.contentResolver = this.context.getContentResolver(); + this.workspaceIndex = new WorkspaceIndex(this.context); this.storageManager = (StorageManager) this.activity.getSystemService( Context.STORAGE_SERVICE ); @@ -153,6 +156,30 @@ public boolean execute( case "unwatch file": unwatchFile(arg1); break; + case "workspace scan": + workspaceIndex.scan( + args.optJSONObject(0) == null ? new JSONObject() : args.optJSONObject(0), + callback + ); + break; + case "workspace search": + workspaceIndex.search( + args.optJSONObject(0) == null ? new JSONObject() : args.optJSONObject(0), + callback + ); + break; + case "workspace cancel": + workspaceIndex.cancel(arg1); + callback.success("OK"); + break; + case "workspace mark dirty": + workspaceIndex.markDirty(args.optJSONArray(0)); + callback.success("OK"); + break; + case "workspace clear": + workspaceIndex.clear(args.optJSONArray(0)); + callback.success("OK"); + break; default: return false; } diff --git a/src/plugins/sdcard/src/android/WorkspaceIndex.java b/src/plugins/sdcard/src/android/WorkspaceIndex.java new file mode 100644 index 000000000..d780b4531 --- /dev/null +++ b/src/plugins/sdcard/src/android/WorkspaceIndex.java @@ -0,0 +1,1243 @@ +package com.foxdebug.sdcard; + +import android.content.ContentResolver; +import android.content.ContentValues; +import android.content.Context; +import android.database.Cursor; +import android.database.sqlite.SQLiteDatabase; +import android.database.sqlite.SQLiteOpenHelper; +import android.net.Uri; +import android.provider.DocumentsContract; +import android.provider.DocumentsContract.Document; +import android.util.Log; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; +import org.apache.cordova.CallbackContext; +import org.apache.cordova.PluginResult; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +class WorkspaceIndex { + private static final String TAG = "WorkspaceIndex"; + private static final String SEPARATOR = "::"; + private static final int DB_VERSION = 1; + private static final int BATCH_SIZE = 200; + private static final int MAX_INDEXED_CHARS = 512 * 1024; + private static final int READ_LIMIT_BYTES = MAX_INDEXED_CHARS * 4; + + private static final Set BINARY_EXTENSIONS = new HashSet<>(); + private static final Set TEXT_EXTENSIONS = new HashSet<>(); + private static final Set BINARY_MIME_TYPES = new HashSet<>(); + private static final Set TEXT_MIME_TYPES = new HashSet<>(); + + static { + Collections.addAll( + BINARY_EXTENSIONS, + "3gp", + "7z", + "aab", + "aac", + "apk", + "avi", + "bin", + "bmp", + "class", + "db", + "dex", + "dll", + "doc", + "docx", + "eot", + "exe", + "flac", + "gif", + "gz", + "heic", + "ico", + "jar", + "jpeg", + "jpg", + "keystore", + "m4a", + "m4v", + "mkv", + "mov", + "mp3", + "mp4", + "o", + "odt", + "ogg", + "otf", + "pdf", + "png", + "ppt", + "pptx", + "pyc", + "rar", + "so", + "sqlite", + "sqlite3", + "tar", + "tgz", + "ttf", + "wav", + "webm", + "webp", + "woff", + "woff2", + "xls", + "xlsx", + "xz", + "zip" + ); + Collections.addAll( + TEXT_EXTENSIONS, + "astro", + "c", + "cc", + "cfg", + "conf", + "cpp", + "cs", + "css", + "csv", + "cxx", + "dart", + "env", + "go", + "graphql", + "h", + "hpp", + "htm", + "html", + "java", + "js", + "json", + "jsx", + "kt", + "kts", + "less", + "lua", + "md", + "mjs", + "php", + "properties", + "py", + "rb", + "rs", + "sass", + "scss", + "sh", + "sql", + "svg", + "swift", + "toml", + "ts", + "tsx", + "txt", + "vue", + "xml", + "yaml", + "yml" + ); + Collections.addAll( + BINARY_MIME_TYPES, + "application/java-archive", + "application/java-vm", + "application/octet-stream", + "application/pdf", + "application/vnd.android.package-archive", + "application/zip", + "application/x-7z-compressed", + "application/x-rar-compressed", + "application/x-sqlite3", + "application/x-tar", + "application/x-xz" + ); + Collections.addAll( + TEXT_MIME_TYPES, + "application/javascript", + "application/json", + "application/ld+json", + "application/sql", + "application/typescript", + "application/x-javascript", + "application/x-php", + "application/x-sh", + "application/x-yaml", + "application/xhtml+xml", + "application/xml", + "image/svg+xml" + ); + } + + private final Context context; + private final ContentResolver resolver; + private final ExecutorService executor = Executors.newFixedThreadPool(2); + private final Map jobs = new ConcurrentHashMap<>(); + private final DB db; + + WorkspaceIndex(Context context) { + this.context = context.getApplicationContext(); + this.resolver = context.getContentResolver(); + this.db = new DB(this.context); + } + + void scan(JSONObject options, CallbackContext callback) { + final String id = options.optString("id", UUID.randomUUID().toString()); + final Job job = new Job(id); + jobs.put(id, job); + + executor.execute( + () -> { + try { + runScan(job, options, callback); + } catch (Exception error) { + sendError(callback, id, error); + } finally { + jobs.remove(id); + } + } + ); + } + + void search(JSONObject options, CallbackContext callback) { + final String id = options.optString("id", UUID.randomUUID().toString()); + final Job job = new Job(id); + jobs.put(id, job); + + executor.execute( + () -> { + try { + runSearch(job, options, callback); + } catch (Exception error) { + sendError(callback, id, error); + } finally { + jobs.remove(id); + } + } + ); + } + + void cancel(String id) { + Job job = jobs.get(id); + if (job != null) job.cancelled = true; + } + + void markDirty(JSONArray urls) { + SQLiteDatabase writable = db.getWritableDatabase(); + for (int i = 0; i < urls.length(); i++) { + String url = urls.optString(i, null); + if (url == null || url.length() == 0) continue; + writable.delete("content", "url = ?", new String[] { url }); + } + } + + void clear(JSONArray roots) { + SQLiteDatabase writable = db.getWritableDatabase(); + if (roots == null || roots.length() == 0) { + writable.delete("content", null, null); + writable.delete("files", null, null); + writable.delete("workspaces", null, null); + return; + } + + for (int i = 0; i < roots.length(); i++) { + String root = roots.optString(i, null); + if (root == null || root.length() == 0) continue; + writable.delete("content", "url IN (SELECT url FROM files WHERE root_url = ?)", new String[] { root }); + writable.delete("files", "root_url = ?", new String[] { root }); + writable.delete("workspaces", "root_url = ?", new String[] { root }); + } + } + + private void runScan(Job job, JSONObject options, CallbackContext callback) + throws Exception { + String rootUrl = options.getString("rootUrl"); + String title = options.optString("title", basename(rootUrl)); + JSONArray exclude = options.optJSONArray("excludeFolders"); + boolean showHiddenFiles = options.optBoolean("showHiddenFiles", false); + String defaultEncoding = options.optString("defaultEncoding", "UTF-8"); + boolean indexContent = options.optBoolean("indexContent", false); + + ContentValues workspace = new ContentValues(); + workspace.put("root_url", rootUrl); + workspace.put("title", title); + workspace.put("indexed_at", System.currentTimeMillis()); + workspace.put("options_hash", String.valueOf(options.toString().hashCode())); + db.getWritableDatabase().replace("workspaces", null, workspace); + db.getWritableDatabase().delete("files", "root_url = ?", new String[] { rootUrl }); + + JSONArray batch = new JSONArray(); + ScanStats stats = new ScanStats(); + + sendStatus(callback, job.id, "scanning", "Scanning project files", 0, true); + + if (isSafUrl(rootUrl)) { + SafUrl rootSafUrl = parseSafUrl(rootUrl); + scanSafDir( + job, + callback, + rootSafUrl.treeUrl, + rootUrl, + rootSafUrl.docId, + rootUrl, + title, + title, + exclude, + showHiddenFiles, + defaultEncoding, + indexContent, + batch, + stats + ); + } else { + File rootFile = fileFromUrl(rootUrl); + scanFileDir( + job, + callback, + rootUrl, + rootFile, + rootUrl, + title, + title, + exclude, + showHiddenFiles, + defaultEncoding, + indexContent, + batch, + stats + ); + } + + flushBatch(callback, job.id, batch); + JSONObject done = baseEvent(job.id, "done"); + done.put("files", stats.files); + done.put("dirs", stats.dirs); + done.put("indexed", stats.indexed); + send(callback, done, false); + } + + private void scanFileDir( + Job job, + CallbackContext callback, + String rootUrl, + File dir, + String parentUrl, + String parentPath, + String title, + JSONArray exclude, + boolean showHiddenFiles, + String defaultEncoding, + boolean indexContent, + JSONArray batch, + ScanStats stats + ) throws Exception { + if (job.cancelled || dir == null) return; + File[] children = dir.listFiles(); + if (children == null) return; + + for (File child : children) { + if (job.cancelled) return; + String name = child.getName(); + if (!showHiddenFiles && name.startsWith(".")) continue; + + boolean isDir = child.isDirectory(); + String url = Uri.fromFile(child).toString(); + String path = joinPath(parentPath, name); + String mime = isDir ? Document.MIME_TYPE_DIR : normalizeMime(name, guessMime(name)); + FileEntry entry = new FileEntry( + rootUrl, + parentUrl, + url, + name, + path, + mime, + isDir, + child.length(), + child.lastModified() + ); + + addEntry(callback, job.id, batch, entry, stats); + if (isDir) { + if (isExcluded(path + "/", exclude)) continue; + scanFileDir( + job, + callback, + rootUrl, + child, + url, + path, + title, + exclude, + showHiddenFiles, + defaultEncoding, + indexContent, + batch, + stats + ); + } else { + if (indexContent) { + indexFile(entry, defaultEncoding); + stats.indexed += 1; + } + } + } + } + + private void scanSafDir( + Job job, + CallbackContext callback, + String treeUrl, + String rootUrl, + String parentDocId, + String parentUrl, + String parentPath, + String title, + JSONArray exclude, + boolean showHiddenFiles, + String defaultEncoding, + boolean indexContent, + JSONArray batch, + ScanStats stats + ) throws Exception { + if (job.cancelled) return; + Uri childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree( + Uri.parse(treeUrl), + parentDocId + ); + Cursor cursor = null; + try { + cursor = + resolver.query( + childrenUri, + new String[] { + Document.COLUMN_DOCUMENT_ID, + Document.COLUMN_DISPLAY_NAME, + Document.COLUMN_MIME_TYPE, + Document.COLUMN_SIZE, + Document.COLUMN_LAST_MODIFIED, + }, + null, + null, + null + ); + if (cursor == null) return; + + while (cursor.moveToNext()) { + if (job.cancelled) return; + String docId = cursor.getString(0); + String name = cursor.getString(1); + String mime = normalizeMime(name, cursor.getString(2)); + long size = safeLong(cursor, 3); + long modified = safeLong(cursor, 4); + + if (!showHiddenFiles && name != null && name.startsWith(".")) continue; + + boolean isDir = Document.MIME_TYPE_DIR.equals(mime); + String url = treeUrl + SEPARATOR + docId; + String path = joinPath(parentPath, name); + FileEntry entry = new FileEntry( + rootUrl, + parentUrl, + url, + name, + path, + mime, + isDir, + size, + modified + ); + + addEntry(callback, job.id, batch, entry, stats); + if (isDir) { + if (isExcluded(path + "/", exclude)) continue; + scanSafDir( + job, + callback, + treeUrl, + rootUrl, + docId, + url, + path, + title, + exclude, + showHiddenFiles, + defaultEncoding, + indexContent, + batch, + stats + ); + } else { + if (indexContent) { + indexFile(entry, defaultEncoding); + stats.indexed += 1; + } + } + } + } finally { + if (cursor != null) cursor.close(); + } + } + + private void addEntry( + CallbackContext callback, + String id, + JSONArray batch, + FileEntry entry, + ScanStats stats + ) throws Exception { + saveFile(entry); + batch.put(entry.toJSON()); + if (entry.isDirectory) stats.dirs += 1; else stats.files += 1; + if (batch.length() >= BATCH_SIZE) { + flushBatch(callback, id, batch); + } + } + + private void flushBatch(CallbackContext callback, String id, JSONArray batch) + throws JSONException { + if (batch.length() == 0) return; + JSONObject event = baseEvent(id, "batch"); + event.put("entries", new JSONArray(batch.toString())); + send(callback, event, true); + while (batch.length() > 0) batch.remove(0); + } + + private void runSearch(Job job, JSONObject options, CallbackContext callback) + throws Exception { + JSONArray files = options.optJSONArray("files"); + if (files == null) files = new JSONArray(); + String search = options.optString("search", ""); + String replace = options.optString("replace", null); + String mode = options.optString("mode", "search"); + JSONObject searchOptions = options.optJSONObject("options"); + if (searchOptions == null) searchOptions = new JSONObject(); + JSONObject overlays = options.optJSONObject("overlays"); + if (overlays == null) overlays = new JSONObject(); + String defaultEncoding = options.optString("defaultEncoding", "UTF-8"); + boolean useIndex = options.optBoolean("useIndex", false); + + Pattern pattern = compileSearchPattern(search, searchOptions); + int total = files.length(); + int processed = 0; + + sendStatus(callback, job.id, "searching", "Searching files", 0, true); + + for (int i = 0; i < total; i++) { + if (job.cancelled) return; + JSONObject file = files.getJSONObject(i); + if (!isSupportedUrl(file.optString("url"))) { + processed += 1; + continue; + } + if (shouldSkipSearchFile(file, searchOptions)) { + processed += 1; + continue; + } + + String content = getFileContent(file, overlays, defaultEncoding, useIndex); + if (content == null) { + processed += 1; + continue; + } + + if ("replace".equals(mode)) { + String text = pattern.matcher(content).replaceAll(replace == null ? "" : replace); + JSONObject result = baseEvent(job.id, "replace-result"); + result.put("file", file); + result.put("text", text); + send(callback, result, true); + } else { + JSONObject result = searchInContent(job.id, file, content, pattern); + if (result != null) send(callback, result, true); + } + + processed += 1; + if (processed % 10 == 0 || processed == total) { + sendProgress(callback, job.id, total == 0 ? 100 : (processed * 100) / total); + } + } + + sendProgress(callback, job.id, 100); + send(callback, baseEvent(job.id, "replace".equals(mode) ? "done-replacing" : "done-searching"), false); + } + + private Pattern compileSearchPattern(String search, JSONObject options) + throws PatternSyntaxException { + boolean regExp = options.optBoolean("regExp", false); + boolean wholeWord = options.optBoolean("wholeWord", false); + boolean caseSensitive = options.optBoolean("caseSensitive", false); + + String pattern = regExp ? search : Pattern.quote(search); + if (wholeWord) pattern = "\\b" + pattern + "\\b"; + + int flags = Pattern.MULTILINE; + if (!caseSensitive) flags |= Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE; + return Pattern.compile(pattern, flags); + } + + private JSONObject searchInContent( + String id, + JSONObject file, + String content, + Pattern pattern + ) throws JSONException { + Matcher matcher = pattern.matcher(content); + JSONArray matches = new JSONArray(); + StringBuilder text = new StringBuilder(file.optString("name")); + if (text.length() > 30) { + text = new StringBuilder("..." + text.substring(text.length() - 30)); + } + + while (matcher.find()) { + String word = matcher.group(); + int start = matcher.start(); + int end = matcher.end(); + String[] surrounding = getSurrounding(content, word, start, end); + JSONObject match = new JSONObject(); + match.put("match", word); + match.put("renderText", surrounding[1]); + match.put("position", position(content, start, end)); + matches.put(match); + text.append("\n\t").append(surrounding[0].trim()); + } + + if (matches.length() == 0) return null; + + JSONObject data = new JSONObject(); + data.put("file", file); + data.put("matches", matches); + data.put("text", text.toString()); + + JSONObject event = baseEvent(id, "search-result"); + event.put("data", data); + return event; + } + + private String getFileContent( + JSONObject file, + JSONObject overlays, + String defaultEncoding, + boolean useIndex + ) throws Exception { + String url = file.optString("url"); + if (overlays.has(url)) return overlays.optString(url, ""); + + long size = file.optLong("size", 0); + long modified = normalizeModified(file); + + if (useIndex) { + SQLiteDatabase readable = db.getReadableDatabase(); + Cursor cursor = null; + try { + cursor = + readable.query( + "content", + new String[] { "text", "size", "modified_date" }, + "url = ?", + new String[] { url }, + null, + null, + null + ); + if (cursor != null && cursor.moveToFirst()) { + long cachedSize = cursor.getLong(1); + long cachedModified = cursor.getLong(2); + if ((size == 0 || cachedSize == size) && (modified == 0 || cachedModified == modified)) { + return cursor.getString(0); + } + } + } finally { + if (cursor != null) cursor.close(); + } + } + + FileEntry entry = FileEntry.fromJSON(file); + if (useIndex) return indexFile(entry, defaultEncoding); + return readFileText(entry, defaultEncoding); + } + + private String indexFile(FileEntry entry, String defaultEncoding) { + if (entry.isDirectory || isBinary(entry) || entry.size > READ_LIMIT_BYTES) { + return null; + } + + try { + String text = readFileText(entry, defaultEncoding); + if (text == null) return null; + String encoding = normalizeEncoding(defaultEncoding); + + ContentValues values = new ContentValues(); + values.put("url", entry.url); + values.put("size", entry.size); + values.put("modified_date", entry.modifiedDate); + values.put("encoding", encoding); + values.put("text", text); + values.put("lower_text", text.toLowerCase(Locale.ROOT)); + values.put("indexed_at", System.currentTimeMillis()); + db.getWritableDatabase().replace("content", null, values); + return text; + } catch (Exception error) { + Log.d(TAG, "Unable to index " + entry.url, error); + return null; + } + } + + private String readFileText(FileEntry entry, String defaultEncoding) + throws Exception { + if (entry.isDirectory || isBinary(entry) || entry.size > READ_LIMIT_BYTES) { + return null; + } + + byte[] bytes = readBytes(entry.url, READ_LIMIT_BYTES + 1); + if (bytes.length > READ_LIMIT_BYTES) return null; + String encoding = detectEncoding(bytes, defaultEncoding); + String text = new String(bytes, Charset.forName(encoding)); + if (text.length() > MAX_INDEXED_CHARS || hasBinaryChars(text)) return null; + return text; + } + + private byte[] readBytes(String url, int limit) throws Exception { + InputStream input = null; + try { + if (isSafUrl(url)) { + input = resolver.openInputStream(formatSafUri(url)); + } else { + input = new FileInputStream(fileFromUrl(url)); + } + if (input == null) return new byte[0]; + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read; + int total = 0; + while ((read = input.read(buffer)) != -1) { + total += read; + if (total > limit) { + output.write(buffer, 0, read - (total - limit)); + break; + } + output.write(buffer, 0, read); + } + return output.toByteArray(); + } finally { + if (input != null) input.close(); + } + } + + private void saveFile(FileEntry entry) { + ContentValues values = new ContentValues(); + values.put("url", entry.url); + values.put("root_url", entry.rootUrl); + values.put("parent_url", entry.parentUrl); + values.put("path", entry.path); + values.put("name", entry.name); + values.put("mime", entry.mime); + values.put("is_directory", entry.isDirectory ? 1 : 0); + values.put("size", entry.size); + values.put("modified_date", entry.modifiedDate); + values.put("indexed_at", System.currentTimeMillis()); + values.put("skipped_reason", isBinary(entry) ? "binary" : null); + db.getWritableDatabase().replace("files", null, values); + } + + private boolean shouldSkipSearchFile(JSONObject file, JSONObject options) { + if (FileEntry.fromJSON(file).isBinary()) return true; + String path = file.optString("path", ""); + if (path.length() == 0) return false; + + JSONArray excludes = splitPatterns(options.optString("exclude", "")); + JSONArray includes = splitPatterns(options.optString("include", "")); + if (includes.length() == 0) includes.put("**"); + + return matchesAny(path, excludes) || !matchesAny(path, includes); + } + + private JSONArray splitPatterns(String value) { + JSONArray result = new JSONArray(); + if (value == null || value.trim().length() == 0) return result; + String[] patterns = value.split(","); + for (String pattern : patterns) { + String item = pattern.trim(); + if (item.length() > 0) result.put(item); + } + return result; + } + + private boolean isExcluded(String path, JSONArray excludes) { + return matchesAny(path, excludes); + } + + private boolean matchesAny(String path, JSONArray patterns) { + if (patterns == null) return false; + for (int i = 0; i < patterns.length(); i++) { + String pattern = patterns.optString(i, ""); + if (globMatches(path, pattern)) return true; + } + return false; + } + + private boolean globMatches(String path, String pattern) { + if (pattern == null || pattern.length() == 0) return false; + String normalizedPath = path.replace('\\', '/'); + String normalizedPattern = pattern.replace('\\', '/'); + if ("**".equals(normalizedPattern)) return true; + + String regex = globToRegex(normalizedPattern); + if (Pattern.compile(regex).matcher(normalizedPath).matches()) return true; + + int slash = normalizedPath.lastIndexOf('/'); + String basename = slash >= 0 ? normalizedPath.substring(slash + 1) : normalizedPath; + return Pattern.compile(regex).matcher(basename).matches(); + } + + private String globToRegex(String glob) { + StringBuilder regex = new StringBuilder("^"); + for (int i = 0; i < glob.length(); i++) { + char ch = glob.charAt(i); + if (ch == '*') { + boolean doublestar = i + 1 < glob.length() && glob.charAt(i + 1) == '*'; + if (doublestar) { + regex.append(".*"); + i++; + } else { + regex.append("[^/]*"); + } + } else if (ch == '?') { + regex.append('.'); + } else if ("\\.[]{}()+-^$|".indexOf(ch) >= 0) { + regex.append('\\').append(ch); + } else { + regex.append(ch); + } + } + regex.append('$'); + return regex.toString(); + } + + private boolean isBinary(FileEntry entry) { + return entry.isBinary(); + } + + private String detectEncoding(byte[] bytes, String defaultEncoding) { + if (bytes.length >= 3 && (bytes[0] & 0xff) == 0xef && (bytes[1] & 0xff) == 0xbb && (bytes[2] & 0xff) == 0xbf) { + return "UTF-8"; + } + if (bytes.length >= 2 && (bytes[0] & 0xff) == 0xff && (bytes[1] & 0xff) == 0xfe) { + return "UTF-16LE"; + } + if (bytes.length >= 2 && (bytes[0] & 0xff) == 0xfe && (bytes[1] & 0xff) == 0xff) { + return "UTF-16BE"; + } + return normalizeEncoding(defaultEncoding); + } + + private String normalizeEncoding(String defaultEncoding) { + if (defaultEncoding == null || defaultEncoding.equals("auto") || defaultEncoding.length() == 0) { + return "UTF-8"; + } + try { + if (Charset.isSupported(defaultEncoding)) return defaultEncoding; + } catch (Exception ignored) {} + return "UTF-8"; + } + + private boolean hasBinaryChars(String text) { + int len = Math.min(text.length(), 2048); + for (int i = 0; i < len; i++) { + char ch = text.charAt(i); + if ( + (ch >= 0 && ch <= 8) || + ch == 11 || + ch == 12 || + (ch >= 14 && ch <= 31) || + ch == 127 + ) { + return true; + } + } + return false; + } + + private JSONObject position(String content, int start, int end) + throws JSONException { + JSONObject position = new JSONObject(); + position.put("start", lineColumn(content, start)); + position.put("end", lineColumn(content, end)); + return position; + } + + private JSONObject lineColumn(String content, int offset) throws JSONException { + int row = 0; + int column = 0; + int max = Math.min(offset, content.length()); + for (int i = 0; i < max; i++) { + if (content.charAt(i) == '\n') { + row += 1; + column = 0; + } else { + column += 1; + } + } + JSONObject result = new JSONObject(); + result.put("row", row); + result.put("column", column); + return result; + } + + private String[] getSurrounding(String content, String word, int start, int end) { + int max = 50; + int remaining = max - (end - start); + String line; + String renderText; + + if (remaining <= 0) { + renderText = word.length() > max ? word.substring(word.length() - max) : word; + line = "..." + renderText; + } else { + int left = remaining / 2; + int right = left; + int leftStart = Math.max(0, start - left); + int rightEnd = Math.min(content.length(), end + right); + line = content.substring(leftStart, start) + word + content.substring(end, rightEnd); + renderText = word; + } + + return new String[] { + line.replaceAll("[\\r\\n]+", " ⏎ "), + renderText.replaceAll("[\\r\\n]+", " ⏎ "), + }; + } + + private void sendStatus( + CallbackContext callback, + String id, + String state, + String message, + int progress, + boolean keep + ) throws JSONException { + JSONObject event = baseEvent(id, "status"); + event.put("state", state); + event.put("message", message); + event.put("progress", progress); + send(callback, event, keep); + } + + private void sendProgress(CallbackContext callback, String id, int progress) + throws JSONException { + JSONObject event = baseEvent(id, "progress"); + event.put("data", progress); + send(callback, event, true); + } + + private JSONObject baseEvent(String id, String type) throws JSONException { + JSONObject event = new JSONObject(); + event.put("id", id); + event.put("type", type); + event.put("action", type); + return event; + } + + private void send(CallbackContext callback, JSONObject event, boolean keep) { + PluginResult result = new PluginResult(PluginResult.Status.OK, event); + result.setKeepCallback(keep); + callback.sendPluginResult(result); + } + + private void sendError(CallbackContext callback, String id, Exception error) { + try { + JSONObject event = baseEvent(id, "error"); + event.put("error", error.getMessage() == null ? error.toString() : error.getMessage()); + send(callback, event, false); + } catch (JSONException jsonError) { + callback.error(error.getMessage()); + } + } + + private boolean isSupportedUrl(String url) { + return url != null && (url.startsWith("file:") || url.startsWith("content:")); + } + + private boolean isSafUrl(String url) { + return url != null && url.startsWith("content:"); + } + + private Uri formatSafUri(String url) { + SafUrl safUrl = parseSafUrl(url); + if (!url.contains(SEPARATOR)) { + return DocumentsContract.buildDocumentUriUsingTree( + Uri.parse(safUrl.treeUrl), + safUrl.docId + ); + } + return DocumentsContract.buildDocumentUriUsingTree( + Uri.parse(safUrl.treeUrl), + safUrl.docId + ); + } + + private SafUrl parseSafUrl(String url) { + if (url.contains(SEPARATOR)) { + String[] parts = url.split(SEPARATOR, 2); + return new SafUrl(parts[0], parts[1]); + } + Uri treeUri = Uri.parse(url); + return new SafUrl(url, DocumentsContract.getTreeDocumentId(treeUri)); + } + + private File fileFromUrl(String url) { + Uri uri = Uri.parse(url); + return new File(uri.getPath()); + } + + private long safeLong(Cursor cursor, int index) { + try { + if (cursor.isNull(index)) return 0; + return cursor.getLong(index); + } catch (Exception ignored) { + return 0; + } + } + + private long normalizeModified(JSONObject file) { + long modified = file.optLong("modifiedDate", 0); + if (modified == 0) modified = file.optLong("lastModified", 0); + return modified; + } + + private String joinPath(String parent, String name) { + if (parent == null || parent.length() == 0) return name; + if (name == null || name.length() == 0) return parent; + if (parent.endsWith("/")) return parent + name; + return parent + "/" + name; + } + + private String basename(String url) { + if (url == null || url.length() == 0) return ""; + int slash = url.lastIndexOf('/'); + return slash >= 0 ? url.substring(slash + 1) : url; + } + + private String guessMime(String name) { + if (name == null) return null; + String lower = name.toLowerCase(Locale.ROOT); + if (lower.endsWith(".js")) return "application/javascript"; + if (lower.endsWith(".ts") || lower.endsWith(".tsx")) return "application/typescript"; + if (lower.endsWith(".jsx")) return "application/javascript"; + if (lower.endsWith(".json")) return "application/json"; + if (lower.endsWith(".svg")) return "image/svg+xml"; + if (lower.endsWith(".html") || lower.endsWith(".htm")) return "text/html"; + if (lower.endsWith(".css")) return "text/css"; + if (lower.endsWith(".md")) return "text/markdown"; + if (lower.endsWith(".txt")) return "text/plain"; + if (lower.endsWith(".xml")) return "application/xml"; + return null; + } + + private String normalizeMime(String name, String mime) { + String guessed = guessMime(name); + if (guessed != null && (mime == null || mime.length() == 0 || "application/octet-stream".equals(mime))) { + return guessed; + } + return mime; + } + + private static class Job { + final String id; + volatile boolean cancelled = false; + + Job(String id) { + this.id = id; + } + } + + private static class ScanStats { + int files = 0; + int dirs = 0; + int indexed = 0; + } + + private static class SafUrl { + final String treeUrl; + final String docId; + + SafUrl(String treeUrl, String docId) { + this.treeUrl = treeUrl; + this.docId = docId; + } + } + + private static class FileEntry { + final String rootUrl; + final String parentUrl; + final String url; + final String name; + final String path; + final String mime; + final boolean isDirectory; + final long size; + final long modifiedDate; + + FileEntry( + String rootUrl, + String parentUrl, + String url, + String name, + String path, + String mime, + boolean isDirectory, + long size, + long modifiedDate + ) { + this.rootUrl = rootUrl; + this.parentUrl = parentUrl; + this.url = url; + this.name = name == null ? "" : name; + this.path = path == null ? this.name : path; + this.mime = mime; + this.isDirectory = isDirectory; + this.size = Math.max(0, size); + this.modifiedDate = Math.max(0, modifiedDate); + } + + JSONObject toJSON() throws JSONException { + JSONObject json = new JSONObject(); + json.put("rootUrl", rootUrl); + json.put("parent", parentUrl); + json.put("parentUrl", parentUrl); + json.put("url", url); + json.put("uri", url); + json.put("name", name); + json.put("path", path); + json.put("mime", mime); + json.put("type", mime); + json.put("isDirectory", isDirectory); + json.put("isFile", !isDirectory); + json.put("size", size); + json.put("modifiedDate", modifiedDate); + return json; + } + + static FileEntry fromJSON(JSONObject json) { + return new FileEntry( + json.optString("rootUrl", ""), + json.optString("parentUrl", json.optString("parent", "")), + json.optString("url", ""), + json.optString("name", ""), + json.optString("path", ""), + json.optString("mime", json.optString("type", null)), + json.optBoolean("isDirectory", false), + json.optLong("size", 0), + json.optLong("modifiedDate", json.optLong("lastModified", 0)) + ); + } + + boolean isBinary() { + if (isDirectory) return false; + String normalizedMime = mime == null ? "" : mime.toLowerCase(Locale.ROOT).split(";")[0].trim(); + if (normalizedMime.startsWith("text/") || TEXT_MIME_TYPES.contains(normalizedMime)) return false; + if (isTextExtension(name)) return false; + if ( + normalizedMime.startsWith("audio/") || + normalizedMime.startsWith("font/") || + normalizedMime.startsWith("image/") || + normalizedMime.startsWith("model/") || + normalizedMime.startsWith("video/") || + BINARY_MIME_TYPES.contains(normalizedMime) + ) { + return true; + } + + String lowerName = name.toLowerCase(Locale.ROOT); + int dot = lowerName.lastIndexOf('.'); + if (dot < 0) return false; + String ext = lowerName.substring(dot + 1); + if (BINARY_EXTENSIONS.contains(ext)) return true; + int previousDot = lowerName.lastIndexOf('.', dot - 1); + return previousDot >= 0 && BINARY_EXTENSIONS.contains(lowerName.substring(previousDot + 1)); + } + + private boolean isTextExtension(String filename) { + String lowerName = filename == null ? "" : filename.toLowerCase(Locale.ROOT); + int dot = lowerName.lastIndexOf('.'); + return dot >= 0 && TEXT_EXTENSIONS.contains(lowerName.substring(dot + 1)); + } + } + + private static class DB extends SQLiteOpenHelper { + DB(Context context) { + super(context, "acode_workspace_index.db", null, DB_VERSION); + } + + @Override + public void onCreate(SQLiteDatabase db) { + db.execSQL( + "CREATE TABLE IF NOT EXISTS workspaces (" + + "root_url TEXT PRIMARY KEY, " + + "title TEXT, " + + "indexed_at INTEGER, " + + "options_hash TEXT" + + ")" + ); + db.execSQL( + "CREATE TABLE IF NOT EXISTS files (" + + "url TEXT PRIMARY KEY, " + + "root_url TEXT, " + + "parent_url TEXT, " + + "path TEXT, " + + "name TEXT, " + + "mime TEXT, " + + "is_directory INTEGER, " + + "size INTEGER, " + + "modified_date INTEGER, " + + "indexed_at INTEGER, " + + "skipped_reason TEXT" + + ")" + ); + db.execSQL( + "CREATE TABLE IF NOT EXISTS content (" + + "url TEXT PRIMARY KEY, " + + "size INTEGER, " + + "modified_date INTEGER, " + + "encoding TEXT, " + + "text TEXT, " + + "lower_text TEXT, " + + "indexed_at INTEGER" + + ")" + ); + db.execSQL("CREATE INDEX IF NOT EXISTS idx_files_root ON files(root_url)"); + db.execSQL("CREATE INDEX IF NOT EXISTS idx_files_parent ON files(parent_url)"); + } + + @Override + public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) { + db.execSQL("DROP TABLE IF EXISTS content"); + db.execSQL("DROP TABLE IF EXISTS files"); + db.execSQL("DROP TABLE IF EXISTS workspaces"); + onCreate(db); + } + } +} diff --git a/src/plugins/sdcard/www/plugin.js b/src/plugins/sdcard/www/plugin.js index ed6761e71..d93d75f4c 100644 --- a/src/plugins/sdcard/www/plugin.js +++ b/src/plugins/sdcard/www/plugin.js @@ -62,5 +62,20 @@ module.exports = { }, listEncodings: function (onSuccess, onFail) { cordova.exec(onSuccess, onFail, 'SDcard', 'list encodings', []); + }, + workspaceScan: function (options, onEvent, onFail) { + cordova.exec(onEvent, onFail, 'SDcard', 'workspace scan', [options || {}]); + }, + workspaceSearch: function (options, onEvent, onFail) { + cordova.exec(onEvent, onFail, 'SDcard', 'workspace search', [options || {}]); + }, + workspaceCancel: function (id, onSuccess, onFail) { + cordova.exec(onSuccess, onFail, 'SDcard', 'workspace cancel', [id]); + }, + workspaceMarkDirty: function (urls, onSuccess, onFail) { + cordova.exec(onSuccess, onFail, 'SDcard', 'workspace mark dirty', [urls || []]); + }, + workspaceClear: function (roots, onSuccess, onFail) { + cordova.exec(onSuccess, onFail, 'SDcard', 'workspace clear', [roots || []]); } -}; \ No newline at end of file +}; diff --git a/src/sidebarApps/searchInFiles/index.js b/src/sidebarApps/searchInFiles/index.js index 00a06587b..0d2e06b73 100644 --- a/src/sidebarApps/searchInFiles/index.js +++ b/src/sidebarApps/searchInFiles/index.js @@ -12,7 +12,6 @@ import openFile from "lib/openFile"; import settings from "lib/settings"; import helpers from "utils/helpers"; import { createSearchResultView } from "./cmResultView"; -import { createSearchIndex } from "./searchIndex"; // Local highlight sources const words = []; @@ -32,6 +31,7 @@ const $exclude = Ref(); const $include = Ref(); const $wholeWord = Ref(); const $caseSensitive = Ref(); +const $useIndex = Ref(); const $btnReplaceAll = Ref(); const $resultOverview = Ref(); const $error = Reactive(); @@ -39,9 +39,6 @@ const $progress = Reactive(); const $indexStatus = Reactive(""); const FILE_LIST_WAIT_TIMEOUT = 250; -const INDEX_QUERY_TIMEOUT = 120; -const INDEX_SYNC_DELAY = 700; -const IDLE_INDEX_RETRY_DELAY = 10000; const SEARCH_WORKER_COUNT = 1; const resultOverview = { @@ -60,6 +57,7 @@ const WHOLE_WORD = "search-in-files-whole-word"; const REG_EXP = "search-in-files-reg-exp"; const EXCLUDE = "search-in-files-exclude"; const INCLUDE = "search-in-files-include"; +const USE_INDEX = "search-in-files-use-native-index"; const store = { get caseSensitive() { @@ -92,6 +90,12 @@ const store = { set include(value) { return localStorage.setItem(INCLUDE, value); }, + get useIndex() { + return localStorage.getItem(USE_INDEX) === "true"; + }, + set useIndex(value) { + localStorage.setItem(USE_INDEX, value); + }, }; const debounceSearch = helpers.debounce(searchAll, 500); @@ -106,20 +110,16 @@ let replacing = false; let newFiles = 0; let searching = false; let searchVersion = 0; -let lastIndexSyncKey = ""; -let pendingIndexFiles = null; -let indexSyncTimer = null; let pendingResultText = ""; let pendingResultFlush = 0; - -const searchIndex = createSearchIndex({ - readFile: readSearchFileContent, - onStatus: updateIndexStatus, -}); +let nativeSearchId = null; +let activeSearchTasks = 0; +let activeReplaceTasks = 0; addEventListener($regExp, "change", onInput); addEventListener($wholeWord, "change", onInput); addEventListener($caseSensitive, "change", onInput); +addEventListener($useIndex, "change", onInput); addEventListener($search, "input", onInput); addEventListener($include, "input", onInput); addEventListener($exclude, "input", onInput); @@ -216,6 +216,12 @@ export default [ text=".*" ref={$regExp} /> +
@@ -270,7 +276,6 @@ export default [ >
); - scheduleAutomaticIndex(); }, false, // show as first item () => {}, @@ -309,45 +314,7 @@ async function onWorkerMessage(e) { } case "search-result": { - const { file, matches, text } = data; - - if (!matches.length) return; - if (filesSearched.includes(file)) return; - - filesSearched.push(Tree.fromJSON(file)); - // Clear any ghost text on first result - if (filesSearched.length === 1) { - searchResult.setValue(""); - } - resultOverview.filesCount += 1; - resultOverview.matchesCount += matches.length; - $resultOverview.innerHTML = searchResultText( - resultOverview.filesCount, - resultOverview.matchesCount, - ); - - const index = filesSearched.length - 1; - results.push({ - file: index, - match: null, - position: null, - }); - - fileNames.push({ name: file.name, path: file.path }); - for (const result of matches) { - result.file = index; - results.push(result); - if (words.length < MAX_HL_WORDS) { - const token = escapeStringRegexp(result.renderText); - if (!words.includes(token)) words.push(token); - } - } - - if (fileNames.length > 1) { - appendSearchResultText(`\n${text}`); - } else { - appendSearchResultText(text); - } + appendSearchResult(data); break; } @@ -364,14 +331,8 @@ async function onWorkerMessage(e) { case "done-replacing": { e.target.doneReplacing = true; - if (workers.find((worker) => worker.started && !worker.doneReplacing)) { - break; - } - - await helpers.showInterstitialIfReady(); - terminateWorker(false); - replacing = false; + await finishReplaceTask(); break; } @@ -382,17 +343,8 @@ async function onWorkerMessage(e) { break; } - const showAd = results.length > 100; - if (showAd) { - await helpers.showInterstitialIfReady(); - } - - if (!results.length) { - searchResult.setGhostText(strings["no result"], { row: 0, column: 0 }); - } - - searching = false; terminateWorker(false); + await finishSearchTask(); break; } @@ -412,6 +364,72 @@ async function onWorkerMessage(e) { } } +function appendSearchResult(data) { + const { file, matches, text } = data; + + if (!matches.length) return; + if (filesSearched.find((item) => item.url === file.url)) return; + + filesSearched.push(Tree.fromJSON(file)); + if (filesSearched.length === 1) { + searchResult.setValue(""); + } + resultOverview.filesCount += 1; + resultOverview.matchesCount += matches.length; + $resultOverview.innerHTML = searchResultText( + resultOverview.filesCount, + resultOverview.matchesCount, + ); + + const index = filesSearched.length - 1; + results.push({ + file: index, + match: null, + position: null, + }); + + fileNames.push({ name: file.name, path: file.path }); + for (const result of matches) { + result.file = index; + results.push(result); + if (words.length < MAX_HL_WORDS) { + const token = escapeStringRegexp(result.renderText); + if (!words.includes(token)) words.push(token); + } + } + + if (fileNames.length > 1) { + appendSearchResultText(`\n${text}`); + } else { + appendSearchResultText(text); + } +} + +async function finishSearchTask() { + activeSearchTasks = Math.max(0, activeSearchTasks - 1); + if (activeSearchTasks > 0) return; + + const showAd = results.length > 100; + if (showAd) { + await helpers.showInterstitialIfReady(); + } + + if (!results.length) { + searchResult.setGhostText(strings["no result"], { row: 0, column: 0 }); + } + + searching = false; + nativeSearchId = null; +} + +async function finishReplaceTask() { + activeReplaceTasks = Math.max(0, activeReplaceTasks - 1); + if (activeReplaceTasks > 0) return; + await helpers.showInterstitialIfReady(); + replacing = false; + nativeSearchId = null; +} + /** * On input event handler * @param {InputEvent} e @@ -434,6 +452,10 @@ function onInput(e) { store.regExp = $regExp.el.checked; } + if (target === $useIndex.el) { + store.useIndex = $useIndex.el.checked; + } + if (target === $exclude.el) { store.exclude = $exclude.el.value; } @@ -443,9 +465,12 @@ function onInput(e) { } terminateWorker(); - stopSearchIndex(); + cancelNativeSearch(); + $indexStatus.value = ""; searchVersion += 1; searching = false; + activeSearchTasks = 0; + activeReplaceTasks = 0; newFiles = 0; $error.value = ""; results.length = 0; @@ -456,7 +481,6 @@ function onInput(e) { searchResult.setGhostText(strings["searching..."], { row: 0, column: 0 }); clearPendingResultText(); removeEvents(); - scheduleAutomaticIndex(IDLE_INDEX_RETRY_DELAY); debounceSearch(); } @@ -491,23 +515,7 @@ async function searchAll() { allFiles.push(new Tree(file.name, file.uri, false)); }); - const allFileJson = allFiles.map((file) => file.toJSON()); - pendingIndexFiles = allFileJson; - - let filesToSearch = allFiles; - try { - const indexResult = await withTimeout( - searchIndex.query(allFileJson, search, options, forceUrls), - INDEX_QUERY_TIMEOUT, - ); - if (version !== searchVersion) return; - filesToSearch = getIndexedFiles(allFiles, indexResult); - } catch (error) { - console.warn( - "Search index query failed. Falling back to full scan.", - error, - ); - } + const filesToSearch = allFiles; if (!filesToSearch.length) { searchResult.setGhostText(strings["no result"], { row: 0, column: 0 }); @@ -520,7 +528,17 @@ async function searchAll() { fileNames.length = 0; currentSearchRegex = regex; searchResult.setGhostText(strings["searching..."], { row: 0, column: 0 }); - sendMessage("search-files", filesToSearch, regex, options); + const nativeFiles = filesToSearch.filter((file) => supportsNativeSearch(file.url)); + const workerFiles = filesToSearch.filter((file) => !supportsNativeSearch(file.url)); + activeSearchTasks = 0; + if (nativeFiles.length) { + activeSearchTasks += 1; + sendNativeSearch("search", nativeFiles, search, options); + } + if (workerFiles.length) { + activeSearchTasks += 1; + sendMessage("search-files", workerFiles, regex, options); + } } async function readSearchFileContent(uri) { @@ -538,21 +556,92 @@ async function readSearchFileContent(uri) { return fsOperation(uri).readFile(settings.value.defaultFileEncoding); } -function getIndexedFiles(allFiles, indexResult) { - if (!indexResult?.supported || !Array.isArray(indexResult.urls)) - return allFiles; - - const indexedUrls = new Set(indexResult.urls); - return allFiles.filter((file) => indexedUrls.has(file.url)); +function supportsNativeSearch(url = "") { + return ( + typeof sdcard !== "undefined" && + typeof sdcard.workspaceSearch === "function" && + (/^file:/.test(url) || /^content:/.test(url)) + ); } -function syncSearchIndex(files) { - const syncKey = getIndexSyncKey(files); - if (syncKey === lastIndexSyncKey) return; +function cancelNativeSearch() { + if (!nativeSearchId || typeof sdcard === "undefined") return; + try { + sdcard.workspaceCancel(nativeSearchId); + } catch (_) { + // ignore cancellation failures + } + nativeSearchId = null; +} + +function sendNativeSearch(mode, searchFiles, search, options, replace) { + const id = `search-${Date.now()}-${Math.random().toString(36).slice(2)}`; + nativeSearchId = id; + sdcard.workspaceSearch( + { + id, + mode, + files: searchFiles.map((file) => file.toJSON()), + search, + replace, + options, + overlays: getOpenFileOverlays(searchFiles), + defaultEncoding: settings.value.defaultFileEncoding, + useIndex: store.useIndex, + }, + async (event) => { + if (!event || event.id !== id) return; + switch (event.type || event.action) { + case "status": + $indexStatus.value = event.message || ""; + break; + case "progress": + $progress.value = event.data || 0; + break; + case "search-result": + appendSearchResult(event.data); + break; + case "replace-result": + filesReplaced.push(event.file); + openFile(event.file.url, { + render: filesSearched.length === filesReplaced.length, + text: event.text, + }); + break; + case "done-searching": + await finishSearchTask(); + break; + case "done-replacing": + await finishReplaceTask(); + break; + case "error": + console.error(event.error); + $error.value = event.error || "Native search failed"; + await (mode === "replace" ? finishReplaceTask() : finishSearchTask()); + break; + } + }, + async (error) => { + console.error(error); + $error.value = error?.message || String(error); + await (mode === "replace" ? finishReplaceTask() : finishSearchTask()); + }, + ); +} - lastIndexSyncKey = syncKey; - $indexStatus.value = "Search index queued"; - searchIndex.sync(files); +function getOpenFileOverlays(searchFiles) { + const supportedUrls = new Set(searchFiles.map(({ url }) => url)); + const overlays = {}; + editorManager.files.forEach((file) => { + if (!file.uri || !supportedUrls.has(file.uri)) return; + if (!file.session?.doc) return; + try { + overlays[file.uri] = file.session.doc.toString() || ""; + } catch (_) { + // ignore invalid editor docs + } + }); + return overlays; } async function waitForFileListIfReady() { @@ -562,87 +651,17 @@ async function waitForFileListIfReady() { } } -function scheduleAutomaticIndex(delay = INDEX_SYNC_DELAY) { - clearTimeout(indexSyncTimer); - - indexSyncTimer = setTimeout(() => { - prepareAutomaticIndex(); - }, delay); -} - -function prepareAutomaticIndex() { - waitForFileList().then(() => { - if (searching || replacing) { - scheduleAutomaticIndex(IDLE_INDEX_RETRY_DELAY); - return; - } - - const allFiles = files().filter((file) => !helpers.isBinary(file)); - if (!allFiles.length) return; - pendingIndexFiles = allFiles.map((file) => file.toJSON()); - scheduleSearchIndexSync(); - }); -} - -function scheduleSearchIndexSync(delay = INDEX_SYNC_DELAY) { - clearTimeout(indexSyncTimer); - - indexSyncTimer = setTimeout(() => { - if (searching || replacing || !pendingIndexFiles) return; - - const files = pendingIndexFiles; - pendingIndexFiles = null; - syncSearchIndex(files); - }, delay); -} - -function stopSearchIndex() { - clearTimeout(indexSyncTimer); - indexSyncTimer = null; - $indexStatus.value = ""; - searchIndex.stop(); -} - -function getIndexSyncKey(files) { - return files - .map( - ({ url, size = 0, modifiedDate = 0 }) => - `${url}:${size}:${modifiedDate || 0}`, - ) - .join("\n"); -} - function markIndexDirty(urls) { - lastIndexSyncKey = ""; - searchIndex.markDirty(urls); -} - -function updateIndexStatus(status = {}) { - if (status.state === "indexing" && status.total) { - $indexStatus.value = - status.message || `Indexing ${status.indexed}/${status.total}`; - return; - } - - if (status.state === "error") { - $indexStatus.value = status.message || "Search index unavailable"; - return; - } - - if (status.state === "queued") { - $indexStatus.value = status.message || "Search index queued"; - return; - } - - if (status.state === "ready") { - $indexStatus.value = status.message || "Search index ready"; - setTimeout(() => { - if ($indexStatus.value === status.message) $indexStatus.value = ""; - }, 2500); - return; + if ( + typeof sdcard !== "undefined" && + typeof sdcard.workspaceMarkDirty === "function" + ) { + try { + sdcard.workspaceMarkDirty(urls); + } catch (_) { + // ignore native dirty-mark failures + } } - - $indexStatus.value = ""; } function appendSearchResultText(text) { @@ -692,7 +711,18 @@ async function replaceAll() { if (!regex) return; replacing = true; - sendMessage("replace-files", filesSearched, regex, options, replace); + activeReplaceTasks = 0; + const nativeFiles = filesSearched.filter((file) => supportsNativeSearch(file.url)); + const workerFiles = filesSearched.filter((file) => !supportsNativeSearch(file.url)); + if (nativeFiles.length) { + activeReplaceTasks += 1; + sendNativeSearch("replace", nativeFiles, search, options, replace); + } + if (workerFiles.length) { + activeReplaceTasks += 1; + sendMessage("replace-files", workerFiles, regex, options, replace); + } + if (!activeReplaceTasks) replacing = false; } /** diff --git a/src/utils/binaryExtensions.js b/src/utils/binaryExtensions.js index b35b82d8c..a255ed6db 100644 --- a/src/utils/binaryExtensions.js +++ b/src/utils/binaryExtensions.js @@ -200,6 +200,55 @@ export const binaryExtensions = [ ]; const binaryExtensionSet = new Set(binaryExtensions); +const textExtensionSet = new Set([ + "astro", + "c", + "cc", + "cfg", + "conf", + "cpp", + "cs", + "css", + "csv", + "cxx", + "dart", + "env", + "go", + "graphql", + "h", + "hpp", + "htm", + "html", + "java", + "js", + "json", + "jsx", + "kt", + "kts", + "less", + "lua", + "md", + "mjs", + "php", + "properties", + "py", + "rb", + "rs", + "sass", + "scss", + "sh", + "sql", + "svg", + "swift", + "toml", + "ts", + "tsx", + "txt", + "vue", + "xml", + "yaml", + "yml", +]); const binaryMimePrefixes = ["audio/", "font/", "image/", "model/", "video/"]; @@ -280,11 +329,20 @@ export function isBinaryFile(file) { const mime = file.mime || file.type; if (isTextMime(mime)) return false; + if (isTextPath(file.url || file.path || file.name)) return false; if (isBinaryMime(mime)) return true; return isBinaryPath(file.url || file.path || file.name); } +export function isTextPath(file) { + const path = String(file ?? "").split(/[?#]/)[0]; + const basename = path.split(/[\\/]/).pop()?.toLowerCase() || ""; + const lastDot = basename.lastIndexOf("."); + if (lastDot === -1) return false; + return textExtensionSet.has(basename.slice(lastDot + 1)); +} + export function isBinaryPath(file) { const path = String(file ?? "").split(/[?#]/)[0]; const basename = path.split(/[\\/]/).pop()?.toLowerCase() || ""; From 3356ec61d0fff764e64f4cd8a080c01a86ae1234 Mon Sep 17 00:00:00 2001 From: Raunak Raj <71929976+bajrangCoder@users.noreply.github.com> Date: Wed, 17 Jun 2026 22:45:12 +0530 Subject: [PATCH 2/2] format --- src/lib/fileList.js | 3 ++- src/sidebarApps/searchInFiles/index.js | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/lib/fileList.js b/src/lib/fileList.js index d24fe3c7e..387bb7d8a 100644 --- a/src/lib/fileList.js +++ b/src/lib/fileList.js @@ -351,7 +351,8 @@ async function getAllFilesNative(parent, root, options = {}) { function addNativeEntries(root, entries) { for (const item of entries) { const parentUrl = item.parentUrl || item.parent; - const parentTree = parentUrl === root.url ? root : getTree([root], parentUrl); + const parentTree = + parentUrl === root.url ? root : getTree([root], parentUrl); if (!parentTree?.children) continue; if (parentTree.children.find(({ url }) => url === item.url)) continue; diff --git a/src/sidebarApps/searchInFiles/index.js b/src/sidebarApps/searchInFiles/index.js index 0d2e06b73..7ba5ac7db 100644 --- a/src/sidebarApps/searchInFiles/index.js +++ b/src/sidebarApps/searchInFiles/index.js @@ -528,8 +528,12 @@ async function searchAll() { fileNames.length = 0; currentSearchRegex = regex; searchResult.setGhostText(strings["searching..."], { row: 0, column: 0 }); - const nativeFiles = filesToSearch.filter((file) => supportsNativeSearch(file.url)); - const workerFiles = filesToSearch.filter((file) => !supportsNativeSearch(file.url)); + const nativeFiles = filesToSearch.filter((file) => + supportsNativeSearch(file.url), + ); + const workerFiles = filesToSearch.filter( + (file) => !supportsNativeSearch(file.url), + ); activeSearchTasks = 0; if (nativeFiles.length) { activeSearchTasks += 1; @@ -712,8 +716,12 @@ async function replaceAll() { replacing = true; activeReplaceTasks = 0; - const nativeFiles = filesSearched.filter((file) => supportsNativeSearch(file.url)); - const workerFiles = filesSearched.filter((file) => !supportsNativeSearch(file.url)); + const nativeFiles = filesSearched.filter((file) => + supportsNativeSearch(file.url), + ); + const workerFiles = filesSearched.filter( + (file) => !supportsNativeSearch(file.url), + ); if (nativeFiles.length) { activeReplaceTasks += 1; sendNativeSearch("replace", nativeFiles, search, options, replace);