diff --git a/app/schemas/app.gamenative.db.PluviaDatabase/21.json b/app/schemas/app.gamenative.db.PluviaDatabase/21.json new file mode 100644 index 0000000000..ee7c29f906 --- /dev/null +++ b/app/schemas/app.gamenative.db.PluviaDatabase/21.json @@ -0,0 +1,1310 @@ +{ + "formatVersion": 1, + "database": { + "version": 21, + "identityHash": "6a4c899dae37cdecb5f1c92ca5a916e4", + "entities": [ + { + "tableName": "app_info", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `is_downloaded` INTEGER NOT NULL, `downloaded_depots` TEXT NOT NULL, `dlc_depots` TEXT NOT NULL, `branch` TEXT NOT NULL DEFAULT 'public', `recovered_install_size_bytes` INTEGER NOT NULL DEFAULT 0, `custom_install_path` TEXT NOT NULL DEFAULT '', PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isDownloaded", + "columnName": "is_downloaded", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "downloadedDepots", + "columnName": "downloaded_depots", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "dlcDepots", + "columnName": "dlc_depots", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "branch", + "columnName": "branch", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "'public'" + }, + { + "fieldPath": "recoveredInstallSizeBytes", + "columnName": "recovered_install_size_bytes", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "customInstallPath", + "columnName": "custom_install_path", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "cached_license", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `license_json` TEXT NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "licenseJson", + "columnName": "license_json", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "app_change_numbers", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`appId` INTEGER, `changeNumber` INTEGER, PRIMARY KEY(`appId`))", + "fields": [ + { + "fieldPath": "appId", + "columnName": "appId", + "affinity": "INTEGER" + }, + { + "fieldPath": "changeNumber", + "columnName": "changeNumber", + "affinity": "INTEGER" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "appId" + ] + } + }, + { + "tableName": "encrypted_app_ticket", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`app_id` INTEGER NOT NULL, `result` INTEGER NOT NULL, `ticket_version_no` INTEGER NOT NULL, `crc_encrypted_ticket` INTEGER NOT NULL, `cb_encrypted_user_data` INTEGER NOT NULL, `cb_encrypted_app_ownership_ticket` INTEGER NOT NULL, `encrypted_ticket` BLOB NOT NULL, `timestamp` INTEGER NOT NULL, PRIMARY KEY(`app_id`))", + "fields": [ + { + "fieldPath": "appId", + "columnName": "app_id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "result", + "columnName": "result", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "ticketVersionNo", + "columnName": "ticket_version_no", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "crcEncryptedTicket", + "columnName": "crc_encrypted_ticket", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "cbEncryptedUserData", + "columnName": "cb_encrypted_user_data", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "cbEncryptedAppOwnershipTicket", + "columnName": "cb_encrypted_app_ownership_ticket", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "encryptedTicket", + "columnName": "encrypted_ticket", + "affinity": "BLOB", + "notNull": true + }, + { + "fieldPath": "timestamp", + "columnName": "timestamp", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "app_id" + ] + } + }, + { + "tableName": "app_file_change_lists", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`appId` INTEGER, `userFileInfo` TEXT NOT NULL, PRIMARY KEY(`appId`))", + "fields": [ + { + "fieldPath": "appId", + "columnName": "appId", + "affinity": "INTEGER" + }, + { + "fieldPath": "userFileInfo", + "columnName": "userFileInfo", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "appId" + ] + } + }, + { + "tableName": "steam_app", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `package_id` INTEGER NOT NULL, `owner_account_id` TEXT NOT NULL, `license_flags` INTEGER NOT NULL, `received_pics` INTEGER NOT NULL, `last_change_number` INTEGER NOT NULL, `ufs_parse_version` INTEGER NOT NULL DEFAULT 0, `depots` TEXT NOT NULL, `branches` TEXT NOT NULL, `name` TEXT NOT NULL, `type` INTEGER NOT NULL, `os_list` INTEGER NOT NULL, `release_state` INTEGER NOT NULL, `release_date` INTEGER NOT NULL, `metacritic_score` INTEGER NOT NULL, `metacritic_full_url` TEXT NOT NULL, `logo_hash` TEXT NOT NULL, `logo_small_hash` TEXT NOT NULL, `icon_hash` TEXT NOT NULL, `client_icon_hash` TEXT NOT NULL, `client_tga_hash` TEXT NOT NULL, `small_capsule` TEXT NOT NULL, `header_image` TEXT NOT NULL, `library_assets` TEXT NOT NULL, `primary_genre` INTEGER NOT NULL, `review_score` INTEGER NOT NULL, `review_percentage` INTEGER NOT NULL, `controller_support` INTEGER NOT NULL, `demo_of_app_id` INTEGER NOT NULL, `developer` TEXT NOT NULL, `publisher` TEXT NOT NULL, `homepage_url` TEXT NOT NULL, `game_manual_url` TEXT NOT NULL, `load_all_before_launch` INTEGER NOT NULL, `dlc_app_ids` TEXT NOT NULL, `is_free_app` INTEGER NOT NULL, `dlc_for_app_id` INTEGER NOT NULL, `must_own_app_to_purchase` INTEGER NOT NULL, `dlc_available_on_store` INTEGER NOT NULL, `optional_dlc` INTEGER NOT NULL, `game_dir` TEXT NOT NULL, `install_script` TEXT NOT NULL, `no_servers` INTEGER NOT NULL, `order` INTEGER NOT NULL, `primary_cache` INTEGER NOT NULL, `valid_os_list` INTEGER NOT NULL, `third_party_cd_key` INTEGER NOT NULL, `visible_only_when_installed` INTEGER NOT NULL, `visible_only_when_subscribed` INTEGER NOT NULL, `launch_eula_url` TEXT NOT NULL, `require_default_install_folder` INTEGER NOT NULL, `content_type` INTEGER NOT NULL, `install_dir` TEXT NOT NULL, `use_launch_cmd_line` INTEGER NOT NULL, `launch_without_workshop_updates` INTEGER NOT NULL, `use_mms` INTEGER NOT NULL, `install_script_signature` TEXT NOT NULL, `install_script_override` INTEGER NOT NULL, `config` TEXT NOT NULL, `ufs` TEXT NOT NULL, `workshop_mods` INTEGER NOT NULL DEFAULT 0, `enabled_workshop_item_ids` TEXT NOT NULL DEFAULT '', `workshop_download_pending` INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "packageId", + "columnName": "package_id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "ownerAccountId", + "columnName": "owner_account_id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "licenseFlags", + "columnName": "license_flags", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "receivedPICS", + "columnName": "received_pics", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastChangeNumber", + "columnName": "last_change_number", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "ufsParseVersion", + "columnName": "ufs_parse_version", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "depots", + "columnName": "depots", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "branches", + "columnName": "branches", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "type", + "columnName": "type", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "osList", + "columnName": "os_list", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "releaseState", + "columnName": "release_state", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "releaseDate", + "columnName": "release_date", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "metacriticScore", + "columnName": "metacritic_score", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "metacriticFullUrl", + "columnName": "metacritic_full_url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "logoHash", + "columnName": "logo_hash", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "logoSmallHash", + "columnName": "logo_small_hash", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "iconHash", + "columnName": "icon_hash", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "clientIconHash", + "columnName": "client_icon_hash", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "clientTgaHash", + "columnName": "client_tga_hash", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "smallCapsule", + "columnName": "small_capsule", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "headerImage", + "columnName": "header_image", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "libraryAssets", + "columnName": "library_assets", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "primaryGenre", + "columnName": "primary_genre", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reviewScore", + "columnName": "review_score", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reviewPercentage", + "columnName": "review_percentage", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "controllerSupport", + "columnName": "controller_support", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "demoOfAppId", + "columnName": "demo_of_app_id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "developer", + "columnName": "developer", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "publisher", + "columnName": "publisher", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "homepageUrl", + "columnName": "homepage_url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "gameManualUrl", + "columnName": "game_manual_url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "loadAllBeforeLaunch", + "columnName": "load_all_before_launch", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "dlcAppIds", + "columnName": "dlc_app_ids", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "isFreeApp", + "columnName": "is_free_app", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "dlcForAppId", + "columnName": "dlc_for_app_id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "mustOwnAppToPurchase", + "columnName": "must_own_app_to_purchase", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "dlcAvailableOnStore", + "columnName": "dlc_available_on_store", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "optionalDlc", + "columnName": "optional_dlc", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "gameDir", + "columnName": "game_dir", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "installScript", + "columnName": "install_script", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "noServers", + "columnName": "no_servers", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "order", + "columnName": "order", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "primaryCache", + "columnName": "primary_cache", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "validOSList", + "columnName": "valid_os_list", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "thirdPartyCdKey", + "columnName": "third_party_cd_key", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "visibleOnlyWhenInstalled", + "columnName": "visible_only_when_installed", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "visibleOnlyWhenSubscribed", + "columnName": "visible_only_when_subscribed", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "launchEulaUrl", + "columnName": "launch_eula_url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "requireDefaultInstallFolder", + "columnName": "require_default_install_folder", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "contentType", + "columnName": "content_type", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "installDir", + "columnName": "install_dir", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "useLaunchCmdLine", + "columnName": "use_launch_cmd_line", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "launchWithoutWorkshopUpdates", + "columnName": "launch_without_workshop_updates", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "useMms", + "columnName": "use_mms", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "installScriptSignature", + "columnName": "install_script_signature", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "installScriptOverride", + "columnName": "install_script_override", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "config", + "columnName": "config", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "ufs", + "columnName": "ufs", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "workshopMods", + "columnName": "workshop_mods", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "enabledWorkshopItemIds", + "columnName": "enabled_workshop_item_ids", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + }, + { + "fieldPath": "workshopDownloadPending", + "columnName": "workshop_download_pending", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "steam_file_hash_cache", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`appId` INTEGER NOT NULL, `absPath` TEXT NOT NULL, `sizeBytes` INTEGER NOT NULL, `mtimeMillis` INTEGER NOT NULL, `sha` BLOB NOT NULL, PRIMARY KEY(`appId`, `absPath`))", + "fields": [ + { + "fieldPath": "appId", + "columnName": "appId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "absPath", + "columnName": "absPath", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "sizeBytes", + "columnName": "sizeBytes", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "mtimeMillis", + "columnName": "mtimeMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "sha", + "columnName": "sha", + "affinity": "BLOB", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "appId", + "absPath" + ] + } + }, + { + "tableName": "steam_license", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`packageId` INTEGER NOT NULL, `last_change_number` INTEGER NOT NULL, `time_created` INTEGER NOT NULL, `time_next_process` INTEGER NOT NULL, `minute_limit` INTEGER NOT NULL, `minutes_used` INTEGER NOT NULL, `payment_method` INTEGER NOT NULL, `license_flags` INTEGER NOT NULL, `purchase_code` TEXT NOT NULL, `license_type` INTEGER NOT NULL, `territory_code` INTEGER NOT NULL, `access_token` INTEGER NOT NULL, `owner_account_id` TEXT NOT NULL, `master_package_id` INTEGER NOT NULL, `app_ids` TEXT NOT NULL, `depot_ids` TEXT NOT NULL, PRIMARY KEY(`packageId`))", + "fields": [ + { + "fieldPath": "packageId", + "columnName": "packageId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastChangeNumber", + "columnName": "last_change_number", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "timeCreated", + "columnName": "time_created", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "timeNextProcess", + "columnName": "time_next_process", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "minuteLimit", + "columnName": "minute_limit", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "minutesUsed", + "columnName": "minutes_used", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "paymentMethod", + "columnName": "payment_method", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "licenseFlags", + "columnName": "license_flags", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "purchaseCode", + "columnName": "purchase_code", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "licenseType", + "columnName": "license_type", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "territoryCode", + "columnName": "territory_code", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "accessToken", + "columnName": "access_token", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "ownerAccountId", + "columnName": "owner_account_id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "masterPackageID", + "columnName": "master_package_id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "appIds", + "columnName": "app_ids", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "depotIds", + "columnName": "depot_ids", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "packageId" + ] + } + }, + { + "tableName": "gog_games", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `title` TEXT NOT NULL, `slug` TEXT NOT NULL, `download_size` INTEGER NOT NULL, `install_size` INTEGER NOT NULL, `is_installed` INTEGER NOT NULL, `install_path` TEXT NOT NULL, `image_url` TEXT NOT NULL, `icon_url` TEXT NOT NULL, `background_url` TEXT NOT NULL DEFAULT '', `description` TEXT NOT NULL, `release_date` TEXT NOT NULL, `developer` TEXT NOT NULL, `publisher` TEXT NOT NULL, `genres` TEXT NOT NULL, `languages` TEXT NOT NULL, `last_played` INTEGER NOT NULL, `play_time` INTEGER NOT NULL, `type` INTEGER NOT NULL, `exclude` INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "slug", + "columnName": "slug", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "downloadSize", + "columnName": "download_size", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "installSize", + "columnName": "install_size", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isInstalled", + "columnName": "is_installed", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "installPath", + "columnName": "install_path", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "imageUrl", + "columnName": "image_url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "iconUrl", + "columnName": "icon_url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "backgroundUrl", + "columnName": "background_url", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "releaseDate", + "columnName": "release_date", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "developer", + "columnName": "developer", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "publisher", + "columnName": "publisher", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "genres", + "columnName": "genres", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "languages", + "columnName": "languages", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "lastPlayed", + "columnName": "last_played", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "playTime", + "columnName": "play_time", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "type", + "columnName": "type", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "exclude", + "columnName": "exclude", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "epic_games", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `catalog_id` TEXT NOT NULL, `app_name` TEXT NOT NULL, `title` TEXT NOT NULL, `namespace` TEXT NOT NULL, `developer` TEXT NOT NULL, `publisher` TEXT NOT NULL, `is_installed` INTEGER NOT NULL, `install_path` TEXT NOT NULL, `platform` TEXT NOT NULL, `version` TEXT NOT NULL, `executable` TEXT NOT NULL, `install_size` INTEGER NOT NULL, `download_size` INTEGER NOT NULL, `art_cover` TEXT NOT NULL, `art_square` TEXT NOT NULL, `art_logo` TEXT NOT NULL, `art_portrait` TEXT NOT NULL, `can_run_offline` INTEGER NOT NULL, `requires_ot` INTEGER NOT NULL, `cloud_save_enabled` INTEGER NOT NULL, `save_folder` TEXT NOT NULL, `third_party_managed_app` TEXT NOT NULL, `is_ea_managed` INTEGER NOT NULL, `is_dlc` INTEGER NOT NULL, `base_game_app_name` TEXT NOT NULL, `description` TEXT NOT NULL, `release_date` TEXT NOT NULL, `genres` TEXT NOT NULL, `tags` TEXT NOT NULL, `last_played` INTEGER NOT NULL, `play_time` INTEGER NOT NULL, `type` INTEGER NOT NULL, `eos_catalog_item_id` TEXT NOT NULL, `eos_app_id` TEXT NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "catalogId", + "columnName": "catalog_id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "appName", + "columnName": "app_name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "namespace", + "columnName": "namespace", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "developer", + "columnName": "developer", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "publisher", + "columnName": "publisher", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "isInstalled", + "columnName": "is_installed", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "installPath", + "columnName": "install_path", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "platform", + "columnName": "platform", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "version", + "columnName": "version", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "executable", + "columnName": "executable", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "installSize", + "columnName": "install_size", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "downloadSize", + "columnName": "download_size", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "artCover", + "columnName": "art_cover", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "artSquare", + "columnName": "art_square", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "artLogo", + "columnName": "art_logo", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "artPortrait", + "columnName": "art_portrait", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "canRunOffline", + "columnName": "can_run_offline", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "requiresOT", + "columnName": "requires_ot", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "cloudSaveEnabled", + "columnName": "cloud_save_enabled", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "saveFolder", + "columnName": "save_folder", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "thirdPartyManagedApp", + "columnName": "third_party_managed_app", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "isEAManaged", + "columnName": "is_ea_managed", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isDLC", + "columnName": "is_dlc", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "baseGameAppName", + "columnName": "base_game_app_name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "releaseDate", + "columnName": "release_date", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "genres", + "columnName": "genres", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "tags", + "columnName": "tags", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "lastPlayed", + "columnName": "last_played", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "playTime", + "columnName": "play_time", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "type", + "columnName": "type", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "eosCatalogItemId", + "columnName": "eos_catalog_item_id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "eosAppId", + "columnName": "eos_app_id", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "amazon_games", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`app_id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `product_id` TEXT NOT NULL, `entitlement_id` TEXT NOT NULL DEFAULT '', `title` TEXT NOT NULL, `is_installed` INTEGER NOT NULL, `install_path` TEXT NOT NULL, `art_url` TEXT NOT NULL, `hero_url` TEXT NOT NULL DEFAULT '', `purchased_date` TEXT NOT NULL, `developer` TEXT NOT NULL DEFAULT '', `publisher` TEXT NOT NULL DEFAULT '', `release_date` TEXT NOT NULL DEFAULT '', `download_size` INTEGER NOT NULL DEFAULT 0, `install_size` INTEGER NOT NULL DEFAULT 0, `version_id` TEXT NOT NULL DEFAULT '', `product_sku` TEXT NOT NULL DEFAULT '', `last_played` INTEGER NOT NULL DEFAULT 0, `play_time_minutes` INTEGER NOT NULL DEFAULT 0, `product_json` TEXT NOT NULL)", + "fields": [ + { + "fieldPath": "appId", + "columnName": "app_id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "productId", + "columnName": "product_id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "entitlementId", + "columnName": "entitlement_id", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "isInstalled", + "columnName": "is_installed", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "installPath", + "columnName": "install_path", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "artUrl", + "columnName": "art_url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "heroUrl", + "columnName": "hero_url", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + }, + { + "fieldPath": "purchasedDate", + "columnName": "purchased_date", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "developer", + "columnName": "developer", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + }, + { + "fieldPath": "publisher", + "columnName": "publisher", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + }, + { + "fieldPath": "releaseDate", + "columnName": "release_date", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + }, + { + "fieldPath": "downloadSize", + "columnName": "download_size", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "installSize", + "columnName": "install_size", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "versionId", + "columnName": "version_id", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + }, + { + "fieldPath": "productSku", + "columnName": "product_sku", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + }, + { + "fieldPath": "lastPlayed", + "columnName": "last_played", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "playTimeMinutes", + "columnName": "play_time_minutes", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "productJson", + "columnName": "product_json", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "app_id" + ] + }, + "indices": [ + { + "name": "index_amazon_games_product_id", + "unique": false, + "columnNames": [ + "product_id" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_amazon_games_product_id` ON `${TABLE_NAME}` (`product_id`)" + } + ] + }, + { + "tableName": "downloading_app_info", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`appId` INTEGER NOT NULL, `dlcAppIds` TEXT NOT NULL, `branch` TEXT NOT NULL DEFAULT 'public', PRIMARY KEY(`appId`))", + "fields": [ + { + "fieldPath": "appId", + "columnName": "appId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "dlcAppIds", + "columnName": "dlcAppIds", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "branch", + "columnName": "branch", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "'public'" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "appId" + ] + } + }, + { + "tableName": "steam_unlocked_branch", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`appId` INTEGER NOT NULL, `branchName` TEXT NOT NULL, `password` TEXT NOT NULL, PRIMARY KEY(`appId`, `branchName`))", + "fields": [ + { + "fieldPath": "appId", + "columnName": "appId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "branchName", + "columnName": "branchName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "password", + "columnName": "password", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "appId", + "branchName" + ] + } + } + ], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '6a4c899dae37cdecb5f1c92ca5a916e4')" + ] + } +} \ No newline at end of file diff --git a/app/src/main/java/app/gamenative/data/PostSyncInfo.kt b/app/src/main/java/app/gamenative/data/PostSyncInfo.kt index fdb1b57d15..46689576ff 100644 --- a/app/src/main/java/app/gamenative/data/PostSyncInfo.kt +++ b/app/src/main/java/app/gamenative/data/PostSyncInfo.kt @@ -14,6 +14,8 @@ data class PostSyncInfo( val filesDownloaded: Int = 0, val filesDeleted: Int = 0, val filesManaged: Int = 0, + val hashCacheHits: Int = 0, + val hashCacheMisses: Int = 0, val bytesUploaded: Long = 0L, val bytesDownloaded: Long = 0L, val microsecTotal: Long = 0L, diff --git a/app/src/main/java/app/gamenative/data/SteamFileHashCache.kt b/app/src/main/java/app/gamenative/data/SteamFileHashCache.kt new file mode 100644 index 0000000000..1f1b63e2fc --- /dev/null +++ b/app/src/main/java/app/gamenative/data/SteamFileHashCache.kt @@ -0,0 +1,32 @@ +package app.gamenative.data + +import androidx.room.Entity + +@Entity( + tableName = "steam_file_hash_cache", + primaryKeys = ["appId", "absPath"], +) +data class SteamFileHashCache( + val appId: Int, + val absPath: String, + val sizeBytes: Long, + val mtimeMillis: Long, + val sha: ByteArray, +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is SteamFileHashCache) return false + return appId == other.appId && absPath == other.absPath && + sizeBytes == other.sizeBytes && mtimeMillis == other.mtimeMillis && + sha.contentEquals(other.sha) + } + + override fun hashCode(): Int { + var result = appId + result = 31 * result + absPath.hashCode() + result = 31 * result + sizeBytes.hashCode() + result = 31 * result + mtimeMillis.hashCode() + result = 31 * result + sha.contentHashCode() + return result + } +} diff --git a/app/src/main/java/app/gamenative/db/PluviaDatabase.kt b/app/src/main/java/app/gamenative/db/PluviaDatabase.kt index ec09f22a9e..b4a6d39a46 100644 --- a/app/src/main/java/app/gamenative/db/PluviaDatabase.kt +++ b/app/src/main/java/app/gamenative/db/PluviaDatabase.kt @@ -8,6 +8,7 @@ import app.gamenative.data.ChangeNumbers import app.gamenative.data.AppInfo import app.gamenative.data.FileChangeLists import app.gamenative.data.SteamApp +import app.gamenative.data.SteamFileHashCache import app.gamenative.data.SteamLicense import app.gamenative.data.CachedLicense import app.gamenative.data.DownloadingAppInfo @@ -26,6 +27,7 @@ import app.gamenative.db.converters.GOGConverter import app.gamenative.db.dao.ChangeNumbersDao import app.gamenative.db.dao.FileChangeListsDao import app.gamenative.db.dao.SteamAppDao +import app.gamenative.db.dao.SteamFileHashCacheDao import app.gamenative.db.dao.SteamLicenseDao import app.gamenative.db.dao.AppInfoDao import app.gamenative.db.dao.CachedLicenseDao @@ -46,6 +48,7 @@ const val DATABASE_NAME = "pluvia.db" EncryptedAppTicket::class, FileChangeLists::class, SteamApp::class, + SteamFileHashCache::class, SteamLicense::class, GOGGame::class, EpicGame::class, @@ -53,7 +56,7 @@ const val DATABASE_NAME = "pluvia.db" DownloadingAppInfo::class, SteamUnlockedBranch::class, ], - version = 20, + version = 21, // For db migration, visit https://developer.android.com/training/data-storage/room/migrating-db-versions for more information exportSchema = true, // It is better to handle db changes carefully, as GN is getting much more users. autoMigrations = [ @@ -73,6 +76,7 @@ const val DATABASE_NAME = "pluvia.db" AutoMigration(from = 17, to = 18), // Added workshop_mods, enabled_workshop_item_ids, workshop_download_pending to steam_app AutoMigration(from = 18, to = 19), // Added recovered_install_size_bytes to app_info AutoMigration(from = 19, to = 20), // Added custom_install_path to app_info + AutoMigration(from = 20, to = 21), // Added steam_file_hash_cache table ] ) @TypeConverters( @@ -90,6 +94,8 @@ abstract class PluviaDatabase : RoomDatabase() { abstract fun steamAppDao(): SteamAppDao + abstract fun steamFileHashCacheDao(): SteamFileHashCacheDao + abstract fun appChangeNumbersDao(): ChangeNumbersDao abstract fun appFileChangeListsDao(): FileChangeListsDao diff --git a/app/src/main/java/app/gamenative/db/dao/SteamFileHashCacheDao.kt b/app/src/main/java/app/gamenative/db/dao/SteamFileHashCacheDao.kt new file mode 100644 index 0000000000..f8b3d5e48d --- /dev/null +++ b/app/src/main/java/app/gamenative/db/dao/SteamFileHashCacheDao.kt @@ -0,0 +1,19 @@ +package app.gamenative.db.dao + +import androidx.room.Dao +import androidx.room.Insert +import androidx.room.OnConflictStrategy +import androidx.room.Query +import app.gamenative.data.SteamFileHashCache + +@Dao +interface SteamFileHashCacheDao { + @Insert(onConflict = OnConflictStrategy.REPLACE) + suspend fun insert(entry: SteamFileHashCache) + + @Query("SELECT * FROM steam_file_hash_cache WHERE appId = :appId AND absPath = :absPath") + suspend fun getByAppIdAndPath(appId: Int, absPath: String): SteamFileHashCache? + + @Query("DELETE FROM steam_file_hash_cache WHERE appId = :appId") + suspend fun deleteByAppId(appId: Int) +} diff --git a/app/src/main/java/app/gamenative/di/DatabaseModule.kt b/app/src/main/java/app/gamenative/di/DatabaseModule.kt index 5fba10d10b..f4a7f56630 100644 --- a/app/src/main/java/app/gamenative/di/DatabaseModule.kt +++ b/app/src/main/java/app/gamenative/di/DatabaseModule.kt @@ -41,6 +41,10 @@ class DatabaseModule { @Singleton fun provideSteamAppDao(db: PluviaDatabase) = db.steamAppDao() + @Provides + @Singleton + fun provideSteamFileHashCacheDao(db: PluviaDatabase) = db.steamFileHashCacheDao() + @Provides @Singleton fun provideAppChangeNumbersDao(db: PluviaDatabase) = db.appChangeNumbersDao() diff --git a/app/src/main/java/app/gamenative/service/SteamAutoCloud.kt b/app/src/main/java/app/gamenative/service/SteamAutoCloud.kt index 3e1c90ddc0..a8e2ae59d9 100644 --- a/app/src/main/java/app/gamenative/service/SteamAutoCloud.kt +++ b/app/src/main/java/app/gamenative/service/SteamAutoCloud.kt @@ -6,9 +6,11 @@ import app.gamenative.R import app.gamenative.data.PostSyncInfo import app.gamenative.data.SaveFilePattern import app.gamenative.data.SteamApp +import app.gamenative.data.SteamFileHashCache import app.gamenative.data.UserFileInfo import app.gamenative.data.UserFilesDownloadResult import app.gamenative.data.UserFilesUploadResult +import app.gamenative.db.dao.SteamFileHashCacheDao import app.gamenative.enums.PathType import app.gamenative.enums.SaveLocation import app.gamenative.enums.SyncResult @@ -32,6 +34,7 @@ import java.nio.file.FileSystemException import java.nio.file.Path import java.nio.file.Paths import java.util.Date +import java.util.concurrent.atomic.AtomicInteger import java.util.stream.Collectors import java.util.zip.ZipInputStream import kotlin.io.path.name @@ -57,7 +60,6 @@ import java.io.IOException import java.io.OutputStream import java.net.SocketTimeoutException import java.nio.file.attribute.FileTime -import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.atomic.AtomicLong /** @@ -67,6 +69,11 @@ object SteamAutoCloud { private const val MAX_USER_FILE_RETRIES = 3 + internal data class HashLookupResult( + val sha: ByteArray, + val wasCacheHit: Boolean, + ) + /** Computes SHA-1 hash by streaming the file in chunks to avoid OOM on large files. */ private fun streamingShaHash(path: Path): ByteArray { val digest = MessageDigest.getInstance("SHA-1") @@ -100,6 +107,39 @@ object SteamAutoCloud { return total } + internal suspend fun getCachedShaOrHash( + appId: Int, + path: Path, + hashCacheDao: SteamFileHashCacheDao, + ): HashLookupResult { + val absPath = path.pathString + val sizeBytes = Files.size(path) + val mtimeMillis = Files.getLastModifiedTime(path).toMillis() + val cached = hashCacheDao.getByAppIdAndPath(appId, absPath) + + if (cached != null && cached.sizeBytes == sizeBytes && cached.mtimeMillis == mtimeMillis) { + return HashLookupResult( + sha = cached.sha, + wasCacheHit = true, + ) + } + + val sha = streamingShaHash(path) + hashCacheDao.insert( + SteamFileHashCache( + appId = appId, + absPath = absPath, + sizeBytes = sizeBytes, + mtimeMillis = mtimeMillis, + sha = sha, + ), + ) + return HashLookupResult( + sha = sha, + wasCacheHit = false, + ) + } + fun syncUserFiles( appInfo: SteamApp, clientId: Long, @@ -202,6 +242,10 @@ object SteamAutoCloud { Paths.get(getFilePrefix(file, fileList), file.filename).pathString } + val hashCacheHits = AtomicInteger(0) + val hashCacheMisses = AtomicInteger(0) + val hashCacheDao = steamInstance.db.steamFileHashCacheDao() + val getFullFilePath: (AppFileInfo, AppFileChangeList) -> Path = getFullFilePath@{ file, fileList -> val gameInstallPrefix = "%${PathType.GameInstall.name}%" if (file.filename.startsWith(gameInstallPrefix)) { @@ -278,7 +322,7 @@ object SteamAutoCloud { } } - val getLocalUserFilesAsPrefixMap: () -> Map> = { + val getLocalUserFilesAsPrefixMap: suspend () -> Map> = { val savePatterns = appInfo.ufs.saveFilePatterns.filter { userFile -> userFile.root.isWindows } val result = mutableMapOf>() @@ -294,27 +338,40 @@ object SteamAutoCloud { Timber.i("Looking for saves in $basePath with pattern ${userFile.pattern} (prefix ${userFile.prefix})") - val files = FileUtils.findFilesRecursive( + val filePaths = FileUtils.findFilesRecursive( rootPath = basePath, pattern = userFile.pattern, maxDepth = 5, - ).map { - val sha = streamingShaHash(it) + ).collect(Collectors.toList()) + val files = buildList { + for (path in filePaths) { + val hashLookup = getCachedShaOrHash( + appId = appInfo.id, + path = path, + hashCacheDao = hashCacheDao, + ) + if (hashLookup.wasCacheHit) { + hashCacheHits.incrementAndGet() + } else { + hashCacheMisses.incrementAndGet() + } + val sha = hashLookup.sha - Timber.i("Found ${it.pathString}\n\tin ${userFile.prefix}\n\twith sha [${sha.joinToString(", ")}]") + Timber.i("Found ${path.pathString}\n\tin ${userFile.prefix}\n\twith sha [${sha.joinToString(", ")}]") - val relativePath = basePath.relativize(it).pathString + val relativePath = basePath.relativize(path).pathString - UserFileInfo( - root = userFile.root, - path = userFile.substitutedPath, - filename = relativePath, - timestamp = Files.getLastModifiedTime(it).toMillis(), - sha = sha, - cloudRoot = userFile.uploadRoot, - cloudPath = userFile.uploadPath - ) - }.collect(Collectors.toList()) + add(UserFileInfo( + root = userFile.root, + path = userFile.substitutedPath, + filename = relativePath, + timestamp = Files.getLastModifiedTime(path).toMillis(), + sha = sha, + cloudRoot = userFile.uploadRoot, + cloudPath = userFile.uploadPath + )) + } + } Timber.i("Found ${files.size} file(s) in $basePath for pattern ${userFile.pattern}") @@ -329,28 +386,41 @@ object SteamAutoCloud { Timber.i("Scanning $basePath recursively (depth 5) under ${rootType.name}") - val files = FileUtils.findFilesRecursive( + val steamUserDataPaths = FileUtils.findFilesRecursive( rootPath = basePath, pattern = "*", maxDepth = 5, - ).map { - val sha = streamingShaHash(it) - - val relativePath = basePath.relativize(it).pathString - - Timber.i("Found ${it.pathString}\n\tin %${rootType.name}%\n\twith sha [${sha.joinToString(", ")}]") - - // Store relative path in filename; empty path component - UserFileInfo( - root = rootType, - path = "", - filename = relativePath, - timestamp = Files.getLastModifiedTime(it).toMillis(), - sha = sha, - cloudRoot = rootType, - cloudPath = "" - ) - }.collect(Collectors.toList()) + ).collect(Collectors.toList()) + val files = buildList { + for (path in steamUserDataPaths) { + val hashLookup = getCachedShaOrHash( + appId = appInfo.id, + path = path, + hashCacheDao = hashCacheDao, + ) + if (hashLookup.wasCacheHit) { + hashCacheHits.incrementAndGet() + } else { + hashCacheMisses.incrementAndGet() + } + val sha = hashLookup.sha + + val relativePath = basePath.relativize(path).pathString + + Timber.i("Found ${path.pathString}\n\tin %${rootType.name}%\n\twith sha [${sha.joinToString(", ")}]") + + // Store relative path in filename; empty path component + add(UserFileInfo( + root = rootType, + path = "", + filename = relativePath, + timestamp = Files.getLastModifiedTime(path).toMillis(), + sha = sha, + cloudRoot = rootType, + cloudPath = "" + )) + } + } Timber.i("Found ${files.size} file(s) in $basePath") @@ -361,6 +431,11 @@ object SteamAutoCloud { result.getOrPut(prefixKey) { mutableListOf() }.addAll(files) } + Timber.i( + "Local save hash cache stats for ${appInfo.id} (${appInfo.name}): " + + "hits=${hashCacheHits.get()}, misses=${hashCacheMisses.get()}, files=${hashCacheHits.get() + hashCacheMisses.get()}", + ) + result } @@ -423,6 +498,7 @@ object SteamAutoCloud { val result = downloadSingleFile( appInfo = appInfo, steamCloud = steamCloud, + hashCacheDao = hashCacheDao, file = file, fileList = fileList, getFilePrefixPath = getFilePrefixPath, @@ -966,6 +1042,8 @@ object SteamAutoCloud { filesDownloaded = filesDownloaded, filesDeleted = filesDeleted, filesManaged = filesManaged, + hashCacheHits = hashCacheHits.get(), + hashCacheMisses = hashCacheMisses.get(), bytesUploaded = bytesUploaded, bytesDownloaded = bytesDownloaded, microsecTotal = microsecTotal, @@ -986,6 +1064,7 @@ object SteamAutoCloud { private suspend fun downloadSingleFile( appInfo: SteamApp, steamCloud: SteamCloud, + hashCacheDao: SteamFileHashCacheDao, file: AppFileInfo, fileList: AppFileChangeList, getFilePrefixPath: (AppFileInfo, AppFileChangeList) -> String, @@ -1136,6 +1215,21 @@ object SteamAutoCloud { return null } + val actualSize = Files.size(actualFilePath) + if (actualSize != totalFileSize) { + Timber.w("Downloaded size for $prefixedPath was $actualSize, expected $totalFileSize - skipping cache seed") + } else { + hashCacheDao.insert( + SteamFileHashCache( + appId = appInfo.id, + absPath = actualFilePath.pathString, + sizeBytes = actualSize, + mtimeMillis = Files.getLastModifiedTime(actualFilePath).toMillis(), + sha = streamingShaHash(actualFilePath), + ), + ) + } + val finishedFiles = completedFiles.incrementAndGet() val finalProgress = if (totalRawBytes > 0L) { (downloadedRawBytes.get().toFloat() / totalRawBytes).coerceIn(0f, 1f) diff --git a/app/src/main/java/app/gamenative/service/SteamService.kt b/app/src/main/java/app/gamenative/service/SteamService.kt index eb689e4df2..7120e7c797 100644 --- a/app/src/main/java/app/gamenative/service/SteamService.kt +++ b/app/src/main/java/app/gamenative/service/SteamService.kt @@ -38,6 +38,7 @@ import app.gamenative.db.dao.ChangeNumbersDao import app.gamenative.db.dao.EncryptedAppTicketDao import app.gamenative.db.dao.FileChangeListsDao import app.gamenative.db.dao.SteamAppDao +import app.gamenative.db.dao.SteamFileHashCacheDao import app.gamenative.db.dao.SteamLicenseDao import app.gamenative.enums.LoginResult import app.gamenative.enums.Marker @@ -212,6 +213,9 @@ class SteamService : Service(), IChallengeUrlChanged { @Inject lateinit var fileChangeListsDao: FileChangeListsDao + @Inject + lateinit var steamFileHashCacheDao: SteamFileHashCacheDao + @Inject lateinit var cachedLicenseDao: CachedLicenseDao @@ -1247,6 +1251,7 @@ class SteamService : Service(), IChallengeUrlChanged { appInfoDao.deleteApp(appId) changeNumbersDao.deleteByAppId(appId) fileChangeListsDao.deleteByAppId(appId) + steamFileHashCacheDao.deleteByAppId(appId) downloadingAppInfoDao.deleteApp(appId) appDao.clearWorkshopState(appId) @@ -1255,6 +1260,7 @@ class SteamService : Service(), IChallengeUrlChanged { appInfoDao.deleteApp(dlcAppId) changeNumbersDao.deleteByAppId(dlcAppId) fileChangeListsDao.deleteByAppId(dlcAppId) + steamFileHashCacheDao.deleteByAppId(dlcAppId) } } } diff --git a/app/src/test/java/app/gamenative/service/SteamAutoCloudTest.kt b/app/src/test/java/app/gamenative/service/SteamAutoCloudTest.kt index 054d238840..117f6a5325 100644 --- a/app/src/test/java/app/gamenative/service/SteamAutoCloudTest.kt +++ b/app/src/test/java/app/gamenative/service/SteamAutoCloudTest.kt @@ -8,6 +8,7 @@ import app.gamenative.data.FileChangeLists import app.gamenative.data.PostSyncInfo import app.gamenative.data.SaveFilePattern import app.gamenative.data.SteamApp +import app.gamenative.data.SteamFileHashCache import app.gamenative.data.UFS import app.gamenative.db.PluviaDatabase import app.gamenative.enums.AppType @@ -34,6 +35,7 @@ import okhttp3.Protocol import okhttp3.Response import okhttp3.ResponseBody import java.util.Date +import java.nio.file.Files import org.junit.After import org.junit.Assert.* import org.junit.Before @@ -54,6 +56,7 @@ import org.robolectric.RobolectricTestRunner import java.io.File import java.io.IOException import java.lang.reflect.Field +import java.nio.file.Path import java.util.EnumSet import java.util.concurrent.CompletableFuture @@ -330,6 +333,80 @@ class SteamAutoCloudTest { assertEquals("Should have 5 files managed", 5, result.filesManaged) } + @Test + fun getCachedShaOrHash_reusesCachedShaWhenMetadataMatches() = runBlocking { + val hashFile = File(saveFilesDir, "cached_hash_test.sav") + hashFile.writeBytes("cache me".toByteArray()) + val path = hashFile.toPath() + val sizeBytes = Files.size(path) + val mtimeMillis = Files.getLastModifiedTime(path).toMillis() + val cachedSha = ByteArray(20) { 7 } + + db.steamFileHashCacheDao().insert( + SteamFileHashCache( + appId = steamAppId, + absPath = path.toString(), + sizeBytes = sizeBytes, + mtimeMillis = mtimeMillis, + sha = cachedSha, + ), + ) + + val sha = SteamAutoCloud.getCachedShaOrHash( + appId = steamAppId, + path = path, + hashCacheDao = db.steamFileHashCacheDao(), + ) + + assertTrue("Should report cache hit", sha.wasCacheHit) + assertArrayEquals("Should reuse cached SHA when size and mtime match", cachedSha, sha.sha) + } + + @Test + fun getCachedShaOrHash_rehashesWhenMetadataChanges() = runBlocking { + val hashFile = File(saveFilesDir, "rehash_test.sav") + hashFile.writeBytes("old-data".toByteArray()) + val path = hashFile.toPath() + val originalSizeBytes = Files.size(path) + val originalMtimeMillis = Files.getLastModifiedTime(path).toMillis() + val cachedSha = ByteArray(20) { 3 } + + db.steamFileHashCacheDao().insert( + SteamFileHashCache( + appId = steamAppId, + absPath = path.toString(), + sizeBytes = originalSizeBytes, + mtimeMillis = originalMtimeMillis, + sha = cachedSha, + ), + ) + + hashFile.writeBytes("new-data-with-different-size".toByteArray()) + + val sha = SteamAutoCloud.getCachedShaOrHash( + appId = steamAppId, + path = path, + hashCacheDao = db.steamFileHashCacheDao(), + ) + + assertFalse("Should report cache miss when metadata changes", sha.wasCacheHit) + assertFalse("Should not reuse cached SHA when metadata changes", cachedSha.contentEquals(sha.sha)) + val cachedEntry = db.steamFileHashCacheDao().getByAppIdAndPath(steamAppId, path.toString()) + assertNotNull("Cache entry should be updated", cachedEntry) + assertArrayEquals("Updated cache should store new SHA", sha.sha, cachedEntry!!.sha) + } + + @Test(expected = java.nio.file.NoSuchFileException::class) + fun getCachedShaOrHash_throwsWhenFileDoesNotExist() = runBlocking { + val nonExistent = File(saveFilesDir, "does_not_exist.sav").toPath() + SteamAutoCloud.getCachedShaOrHash( + appId = steamAppId, + path = nonExistent, + hashCacheDao = db.steamFileHashCacheDao(), + ) + Unit + } + // @Test fun testDownloadCloudSavesOnFirstBoot() = runBlocking { // Clear existing files and database state @@ -526,6 +603,17 @@ class SteamAutoCloudTest { assertEquals("File 2 content should match", cloudFile2Content.contentToString(), expectedFile2.readBytes().contentToString()) assertEquals("File 3 content should match", cloudFile3Content.contentToString(), expectedFile3.readBytes().contentToString()) + val cacheEntry1 = db.steamFileHashCacheDao().getByAppIdAndPath(steamAppId, expectedFile1.toPath().toString()) + val cacheEntry2 = db.steamFileHashCacheDao().getByAppIdAndPath(steamAppId, expectedFile2.toPath().toString()) + val cacheEntry3 = db.steamFileHashCacheDao().getByAppIdAndPath(steamAppId, expectedFile3.toPath().toString()) + + assertNotNull("File 1 cache entry should exist", cacheEntry1) + assertNotNull("File 2 cache entry should exist", cacheEntry2) + assertNotNull("File 3 cache entry should exist", cacheEntry3) + assertArrayEquals("File 1 cache SHA should match cloud SHA", cloudFile1Sha, cacheEntry1!!.sha) + assertArrayEquals("File 2 cache SHA should match cloud SHA", cloudFile2Sha, cacheEntry2!!.sha) + assertArrayEquals("File 3 cache SHA should match cloud SHA", cloudFile3Sha, cacheEntry3!!.sha) + // Verify database change number was updated val changeNumber = db.appChangeNumbersDao().getByAppId(steamAppId) assertNotNull("Change number should exist", changeNumber) @@ -1861,6 +1949,17 @@ class SteamAutoCloudTest { assertEquals("File 2 content", file2Content.contentToString(), expected2.readBytes().contentToString()) assertEquals("File 3 content", file3Content.contentToString(), expected3.readBytes().contentToString()) + // verify downloads seeded the hash cache with the correct SHA + val cache1 = db.steamFileHashCacheDao().getByAppIdAndPath(steamAppId, expected1.toPath().toString()) + val cache2 = db.steamFileHashCacheDao().getByAppIdAndPath(steamAppId, expected2.toPath().toString()) + val cache3 = db.steamFileHashCacheDao().getByAppIdAndPath(steamAppId, expected3.toPath().toString()) + assertNotNull("Cache entry for file 1 should exist", cache1) + assertNotNull("Cache entry for file 2 should exist", cache2) + assertNotNull("Cache entry for file 3 should exist", cache3) + assertArrayEquals("File 1 cache SHA should match file content", sha1(file1Content), cache1!!.sha) + assertArrayEquals("File 2 cache SHA should match file content", sha1(file2Content), cache2!!.sha) + assertArrayEquals("File 3 cache SHA should match file content", sha1(file3Content), cache3!!.sha) + // verify DB change number updated val cn = db.appChangeNumbersDao().getByAppId(steamAppId) assertNotNull("Change number should exist", cn) @@ -2314,7 +2413,6 @@ class SteamAutoCloudTest { assertEquals(SyncResult.Success, result!!.syncResult) assertTrue("Should have downloaded files", result.filesDownloaded > 0) } - @Test fun synced_cloudAdvanced_metadataFailure_doesNotCancelSiblingDownloads() = runBlocking { val localCn = 5L