Add support for lazy texture loading

src/data_win.c

@@ -2000,7 +2000,7 @@ static void parseSTRG(BinaryReader* reader, DataWin* dw) {
     free(ptrs);
 }
 
-static void parseTXTR(BinaryReader* reader, DataWin* dw, size_t chunkEnd) {
+static void parseTXTR(BinaryReader* reader, DataWin* dw, size_t chunkEnd, bool loadTextureDataLazily) {
     Txtr* t = &dw->txtr;
 
     uint32_t count;
@@ -2076,9 +2076,61 @@ static void parseTXTR(BinaryReader* reader, DataWin* dw, size_t chunkEnd) {
     }
 
     // Load blob data into owned buffers
-    repeat(count, i) {
-        if (t->textures[i].blobOffset == 0 || t->textures[i].blobSize == 0) continue;
-        t->textures[i].blobData = BinaryReader_readBytesAt(reader, t->textures[i].blobOffset, t->textures[i].blobSize);
+    if (!loadTextureDataLazily) {
+        repeat(count, i) {
+            if (t->textures[i].blobOffset == 0 || t->textures[i].blobSize == 0) continue;
+            t->textures[i].blobData = BinaryReader_readBytesAt(reader, t->textures[i].blobOffset, t->textures[i].blobSize);
+        }
+    }
+}
+
+void DataWin_loadTxtrIfNeeded(DataWin* dw, uint32_t textureId) {
+    Txtr* t = &dw->txtr;
+    Texture* tex = &t->textures[textureId];
+
+    if (tex->blobOffset == 0 || tex->blobSize == 0) return;
+    if (tex->blobData != nullptr) return;
+
+    if (!dw->lazyLoadFile) {
+        fprintf(stderr, "%s: called without a lazy load file.\n", __func__);
+        return;
+    }
+
+    while (true) {
+        tex->blobData = malloc(tex->blobSize);
+
+        if (!tex->blobData) {
+            fprintf(stderr, "%s: failed to allocate texture data. trying to free a texture.\n", __func__);
+
+            // TODO: use an LRU cache instead
+            bool found = false;
+            for (uint32_t i = 0; i < t->count; i++) {
+                if (t->textures[i].blobData) {
+                    fprintf(stderr, "%s: evicting texture %u to free up memory\n", __func__, i);
+                    free(t->textures[i].blobData);
+                    t->textures[i].blobData = nullptr;
+                    found = true;
+                    break;
+                }
+            }
+
+            if (!found) {
+                fprintf(stderr, "%s: no textures to free. sorry bucko!\n", __func__);
+            }
+        } else {
+            memset(tex->blobData, 0, tex->blobSize);
+            long old_seek = ftell(dw->lazyLoadFile);
+            fseek(dw->lazyLoadFile, tex->blobOffset, SEEK_SET);
+            size_t read = fread(tex->blobData, 1, tex->blobSize, dw->lazyLoadFile);
+            fseek(dw->lazyLoadFile, old_seek, SEEK_SET);
+
+            if (read != tex->blobSize) {
+                fprintf(stderr, "%s: couldn't read %u bytes to load a texture.\n", __func__, tex->blobSize);
+            }
+
+	        fprintf(stderr, "%s: loaded texture data for page %u\n", __func__, textureId);
+            break;
+        }
     }
 }
 
@@ -2247,13 +2299,15 @@ DataWin* DataWin_parse(const char* filePath, DataWinParserOptions options) {
         // Bulk-read the chunk data into memory for fast parsing
         uint8_t* chunkBuffer = nullptr;
         if (shouldParse && chunkLength > 0) {
-            chunkBuffer = safeMalloc(chunkLength);
-            size_t read = fread(chunkBuffer, 1, chunkLength, reader.file);
-            if (read != chunkLength) {
-                fprintf(stderr, "DataWin: short read on chunk %.4s (expected %u, got %zu)\n", chunkName, chunkLength, read);
-                exit(1);
+            chunkBuffer = malloc(chunkLength);
+            if (chunkBuffer) {
+                size_t read = fread(chunkBuffer, 1, chunkLength, reader.file);
+                if (read != chunkLength) {
+                    fprintf(stderr, "DataWin: short read on chunk %.4s (expected %u, got %zu)\n", chunkName, chunkLength, read);
+                    exit(1);
+                }
+                BinaryReader_setBuffer(&reader, chunkBuffer, chunkDataStart, chunkLength);
             }
-            BinaryReader_setBuffer(&reader, chunkBuffer, chunkDataStart, chunkLength);
         }
 
         if (options.parseGen8 && memcmp(chunkName, "GEN8", 4) == 0) {
@@ -2318,7 +2372,7 @@ DataWin* DataWin_parse(const char* filePath, DataWinParserOptions options) {
         } else if (options.parseStrg && memcmp(chunkName, "STRG", 4) == 0) {
             parseSTRG(&reader, dw);
         } else if (options.parseTxtr && memcmp(chunkName, "TXTR", 4) == 0) {
-            parseTXTR(&reader, dw, chunkEnd);
+            parseTXTR(&reader, dw, chunkEnd, options.lazyLoadTextures);
         } else if (options.parseAudo && memcmp(chunkName, "AUDO", 4) == 0) {
             parseAUDO(&reader, dw);
         } else {
@@ -2347,7 +2401,8 @@ DataWin* DataWin_parse(const char* filePath, DataWinParserOptions options) {
 
     // If lazy-loading rooms, keep the file handle open for DataWin_loadRoomPayload, otherwise close it now
     dw->lazyLoadRooms = options.lazyLoadRooms;
-    if (options.lazyLoadRooms) {
+    dw->lazyLoadTextures = options.lazyLoadTextures;
+    if (options.lazyLoadRooms || options.lazyLoadTextures) {
         dw->lazyLoadFile = file;
         dw->lazyLoadFilePath = safeStrdup(filePath);
         dw->fileSize = (size_t) fileSize;
@@ -2565,7 +2620,7 @@ void DataWin_free(DataWin* dw) {
     free(dw->strgBuffer);
     free(dw->bytecodeBuffer);
 
-    // Close the lazy-load file handle (only open when lazyLoadRooms was enabled)
+    // Close the lazy-load file handle (only open when lazyLoadRooms/lazyLoadTextures was enabled)
     if (dw->lazyLoadFile != nullptr) {
         fclose(dw->lazyLoadFile);
         dw->lazyLoadFile = nullptr;

src/data_win.h

@@ -41,6 +41,8 @@ typedef struct {
 
     // If true, Room payloads (backgrounds, views, gameObjects, tiles, layers) are parsed on demand via DataWin_loadRoomPayload during gameplay.
     bool lazyLoadRooms;
+    // If true, TXTR objects will be loaded on demand via DataWin_loadTxtrIfNeeded, and unloaded if memory is tight.
+    bool lazyLoadTextures;
 
     // When lazyLoadRooms is true, this list indicates which rooms should be loaded during load time instead of demand. They will also not be freed.
     StringBooleanEntry* eagerlyLoadedRooms;
@@ -910,6 +912,7 @@ struct DataWin {
     char* lazyLoadFilePath; // owned strdup of the original file path, for diagnostics
     size_t fileSize; // cached size of the DataWin, captured at parse time. Used for platforms where fseek(SEEK_END)+ftell is unreliable due to buffering (like the PlayStation 2).
     bool lazyLoadRooms; // mirrors the parser option so Runner can branch without re-reading options
+    bool lazyLoadTextures; // ditto
 };
 
 DataWin* DataWin_parse(const char* filePath, DataWinParserOptions options);
@@ -929,3 +932,4 @@ bool DataWin_isVersionAtLeast(const DataWin* dw, uint32_t major, uint32_t minor,
 void DataWin_bumpVersionTo(DataWin* dw, uint32_t major, uint32_t minor, uint32_t release, uint32_t build);
 void GamePath_computeInternal(GamePath* path);
 PathPositionResult GamePath_getPosition(GamePath* path, float t);
+void DataWin_loadTxtrIfNeeded(DataWin* dw, uint32_t textureId);

src/gl/gl_renderer.c

@@ -429,6 +429,8 @@ static bool ensureTextureLoaded(GLRenderer* gl, uint32_t pageId) {
     DataWin* dw = gl->base.dataWin;
     Texture* txtr = &dw->txtr.textures[pageId];
 
+    DataWin_loadTxtrIfNeeded(dw, pageId);
+
     int w, h;
     bool gm2022_5 = DataWin_isVersionAtLeast(dw, 2022, 5, 0, 0);
     uint8_t* pixels = ImageDecoder_decodeToRgba(txtr->blobData, (size_t) txtr->blobSize, gm2022_5, &w, &h);

src/gl_legacy/gl_legacy_renderer.c

@@ -297,6 +297,8 @@ static bool ensureTextureLoaded(GLLegacyRenderer* gl, uint32_t pageId) {
     DataWin* dw = gl->base.dataWin;
     Texture* txtr = &dw->txtr.textures[pageId];
 
+    DataWin_loadTxtrIfNeeded(dw, pageId);
+
     bool gm2022_5 = DataWin_isVersionAtLeast(dw, 2022, 5, 0, 0);
     uint8_t* pixels = ImageDecoder_decodeToRgba(txtr->blobData, (size_t) txtr->blobSize, gm2022_5, &w, &h);
     if (pixels == nullptr) {

src/sdl/main.c

@@ -93,6 +93,7 @@ typedef struct {
     const char* renderer;
     YoYoOperatingSystem osType;
     bool lazyRooms;
+    bool lazyTextures;
     StringBooleanEntry* eagerRooms; // stb_ds string-keyed set of room names
     int profilerFramesBetween; // 0 = disabled
 #ifdef ENABLE_VM_OPCODE_PROFILER
@@ -189,6 +190,7 @@ static void parseCommandLineArgs(CommandLineArgs* args, int argc, char* argv[])
         {"playback-inputs", required_argument, nullptr, 'P'},
         {"renderer", required_argument, nullptr, 'g'},
         {"lazy-rooms", no_argument, nullptr, 'z'},
+        {"lazy-textures", no_argument, nullptr, 't'},
         {"eager-room", required_argument, nullptr, 'G'},
         {"os-type", required_argument, nullptr, 'O'},
         {"profile-gml-scripts", required_argument, nullptr, 'q'},
@@ -359,6 +361,9 @@ static void parseCommandLineArgs(CommandLineArgs* args, int argc, char* argv[])
             case 'z':
                 args->lazyRooms = true;
                 break;
+            case 't':
+                args->lazyTextures = true;
+                break;
             case 'G':
                 shput(args->eagerRooms, optarg, true);
                 break;
@@ -680,6 +685,7 @@ int main(int argc, char* argv[]) {
             .parseAudo = true,
             .skipLoadingPreciseMasksForNonPreciseSprites = true,
             .lazyLoadRooms = args.lazyRooms,
+            .lazyLoadTextures = args.lazyTextures,
             .eagerlyLoadedRooms = args.eagerRooms
         }
     );