Skip to content

feat: Cache Steam save file hashes between syncs #1227

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
utkarshdalal merged 10 commits into from
May 17, 2026
Merged

feat: Cache Steam save file hashes between syncs #1227

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
9711e87
Cache Steam save file hashes between syncs
kiequoo Apr 15, 2026
521a53b
Address code review comments on steam-save-hash-cache
kiequoo Apr 15, 2026
fc8b4e4
Seed Steam save hash cache after downloads
kiequoo Apr 15, 2026
be44649
Fix runBlocking in hash cache lookups and hoist DAO reference
kiequoo Apr 15, 2026
2cb8495
Verify download size before seeding hash cache
kiequoo Apr 15, 2026
e1d85a2
Add cache-seeding assertions to firstBoot_multipleCloudFiles_download…
kiequoo Apr 15, 2026
a50af79
Clear hash cache for DLC app IDs when deleting an app
kiequoo Apr 17, 2026
2efd32f
fix: repair SteamAutoCloud build after rebase
kiequoo Apr 20, 2026
20fe228
refactor: inject Steam file hash cache dao
kiequoo Apr 20, 2026
9ed4d59
chore: add Room schema export for database v21
kiequoo Apr 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,310 changes: 1,310 additions & 0 deletions app/schemas/app.gamenative.db.PluviaDatabase/21.json
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions app/src/main/java/app/gamenative/data/PostSyncInfo.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ data class PostSyncInfo(
val filesDownloaded: Int = 0,
val filesDeleted: Int = 0,
val filesManaged: Int = 0,
val hashCacheHits: Int = 0,
val hashCacheMisses: Int = 0,
val bytesUploaded: Long = 0L,
val bytesDownloaded: Long = 0L,
val microsecTotal: Long = 0L,
Expand Down
32 changes: 32 additions & 0 deletions app/src/main/java/app/gamenative/data/SteamFileHashCache.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package app.gamenative.data

import androidx.room.Entity

@Entity(
tableName = "steam_file_hash_cache",
primaryKeys = ["appId", "absPath"],
)
data class SteamFileHashCache(
val appId: Int,
val absPath: String,
val sizeBytes: Long,
val mtimeMillis: Long,
val sha: ByteArray,
) {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other !is SteamFileHashCache) return false
return appId == other.appId && absPath == other.absPath &&
sizeBytes == other.sizeBytes && mtimeMillis == other.mtimeMillis &&
sha.contentEquals(other.sha)
}

override fun hashCode(): Int {
var result = appId
result = 31 * result + absPath.hashCode()
result = 31 * result + sizeBytes.hashCode()
result = 31 * result + mtimeMillis.hashCode()
result = 31 * result + sha.contentHashCode()
return result
}
}
8 changes: 7 additions & 1 deletion app/src/main/java/app/gamenative/db/PluviaDatabase.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import app.gamenative.data.ChangeNumbers
import app.gamenative.data.AppInfo
import app.gamenative.data.FileChangeLists
import app.gamenative.data.SteamApp
import app.gamenative.data.SteamFileHashCache
import app.gamenative.data.SteamLicense
import app.gamenative.data.CachedLicense
import app.gamenative.data.DownloadingAppInfo
Expand All @@ -26,6 +27,7 @@ import app.gamenative.db.converters.GOGConverter
import app.gamenative.db.dao.ChangeNumbersDao
import app.gamenative.db.dao.FileChangeListsDao
import app.gamenative.db.dao.SteamAppDao
import app.gamenative.db.dao.SteamFileHashCacheDao
import app.gamenative.db.dao.SteamLicenseDao
import app.gamenative.db.dao.AppInfoDao
import app.gamenative.db.dao.CachedLicenseDao
Expand All @@ -46,14 +48,15 @@ const val DATABASE_NAME = "pluvia.db"
EncryptedAppTicket::class,
FileChangeLists::class,
SteamApp::class,
SteamFileHashCache::class,
SteamLicense::class,
GOGGame::class,
EpicGame::class,
AmazonGame::class,
DownloadingAppInfo::class,
SteamUnlockedBranch::class,
],
version = 20,
version = 21,
// For db migration, visit https://developer.android.com/training/data-storage/room/migrating-db-versions for more information
exportSchema = true, // It is better to handle db changes carefully, as GN is getting much more users.
autoMigrations = [
Expand All @@ -73,6 +76,7 @@ const val DATABASE_NAME = "pluvia.db"
AutoMigration(from = 17, to = 18), // Added workshop_mods, enabled_workshop_item_ids, workshop_download_pending to steam_app
AutoMigration(from = 18, to = 19), // Added recovered_install_size_bytes to app_info
AutoMigration(from = 19, to = 20), // Added custom_install_path to app_info
AutoMigration(from = 20, to = 21), // Added steam_file_hash_cache table
]
)
@TypeConverters(
Expand All @@ -90,6 +94,8 @@ abstract class PluviaDatabase : RoomDatabase() {

abstract fun steamAppDao(): SteamAppDao

abstract fun steamFileHashCacheDao(): SteamFileHashCacheDao

abstract fun appChangeNumbersDao(): ChangeNumbersDao

abstract fun appFileChangeListsDao(): FileChangeListsDao
Expand Down
19 changes: 19 additions & 0 deletions app/src/main/java/app/gamenative/db/dao/SteamFileHashCacheDao.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package app.gamenative.db.dao

import androidx.room.Dao
import androidx.room.Insert
import androidx.room.OnConflictStrategy
import androidx.room.Query
import app.gamenative.data.SteamFileHashCache

@Dao
interface SteamFileHashCacheDao {
@Insert(onConflict = OnConflictStrategy.REPLACE)
suspend fun insert(entry: SteamFileHashCache)

@Query("SELECT * FROM steam_file_hash_cache WHERE appId = :appId AND absPath = :absPath")
suspend fun getByAppIdAndPath(appId: Int, absPath: String): SteamFileHashCache?

@Query("DELETE FROM steam_file_hash_cache WHERE appId = :appId")
suspend fun deleteByAppId(appId: Int)
}
4 changes: 4 additions & 0 deletions app/src/main/java/app/gamenative/di/DatabaseModule.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,10 @@ class DatabaseModule {
@Singleton
fun provideSteamAppDao(db: PluviaDatabase) = db.steamAppDao()

@Provides
@Singleton
fun provideSteamFileHashCacheDao(db: PluviaDatabase) = db.steamFileHashCacheDao()

@Provides
@Singleton
fun provideAppChangeNumbersDao(db: PluviaDatabase) = db.appChangeNumbersDao()
Expand Down
166 changes: 130 additions & 36 deletions app/src/main/java/app/gamenative/service/SteamAutoCloud.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@ import app.gamenative.R
import app.gamenative.data.PostSyncInfo
import app.gamenative.data.SaveFilePattern
import app.gamenative.data.SteamApp
import app.gamenative.data.SteamFileHashCache
import app.gamenative.data.UserFileInfo
import app.gamenative.data.UserFilesDownloadResult
import app.gamenative.data.UserFilesUploadResult
import app.gamenative.db.dao.SteamFileHashCacheDao
import app.gamenative.enums.PathType
import app.gamenative.enums.SaveLocation
import app.gamenative.enums.SyncResult
Expand All @@ -32,6 +34,7 @@ import java.nio.file.FileSystemException
import java.nio.file.Path
import java.nio.file.Paths
import java.util.Date
import java.util.concurrent.atomic.AtomicInteger
import java.util.stream.Collectors
import java.util.zip.ZipInputStream
import kotlin.io.path.name
Expand All @@ -57,7 +60,6 @@ import java.io.IOException
import java.io.OutputStream
import java.net.SocketTimeoutException
import java.nio.file.attribute.FileTime
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicLong

/**
Expand All @@ -67,6 +69,11 @@ object SteamAutoCloud {

private const val MAX_USER_FILE_RETRIES = 3

internal data class HashLookupResult(
val sha: ByteArray,
val wasCacheHit: Boolean,
)

/** Computes SHA-1 hash by streaming the file in chunks to avoid OOM on large files. */
private fun streamingShaHash(path: Path): ByteArray {
val digest = MessageDigest.getInstance("SHA-1")
Expand Down Expand Up @@ -100,6 +107,39 @@ object SteamAutoCloud {
return total
}

internal suspend fun getCachedShaOrHash(
appId: Int,
path: Path,
hashCacheDao: SteamFileHashCacheDao,
): HashLookupResult {
val absPath = path.pathString
val sizeBytes = Files.size(path)
val mtimeMillis = Files.getLastModifiedTime(path).toMillis()
val cached = hashCacheDao.getByAppIdAndPath(appId, absPath)

if (cached != null && cached.sizeBytes == sizeBytes && cached.mtimeMillis == mtimeMillis) {
return HashLookupResult(
sha = cached.sha,
wasCacheHit = true,
)
}

val sha = streamingShaHash(path)
hashCacheDao.insert(
SteamFileHashCache(
appId = appId,
absPath = absPath,
sizeBytes = sizeBytes,
mtimeMillis = mtimeMillis,
sha = sha,
),
)
return HashLookupResult(
sha = sha,
wasCacheHit = false,
)
}

fun syncUserFiles(
appInfo: SteamApp,
clientId: Long,
Expand Down Expand Up @@ -202,6 +242,10 @@ object SteamAutoCloud {
Paths.get(getFilePrefix(file, fileList), file.filename).pathString
}

val hashCacheHits = AtomicInteger(0)
val hashCacheMisses = AtomicInteger(0)
val hashCacheDao = steamInstance.db.steamFileHashCacheDao()

val getFullFilePath: (AppFileInfo, AppFileChangeList) -> Path = getFullFilePath@{ file, fileList ->
val gameInstallPrefix = "%${PathType.GameInstall.name}%"
if (file.filename.startsWith(gameInstallPrefix)) {
Expand Down Expand Up @@ -278,7 +322,7 @@ object SteamAutoCloud {
}
}

val getLocalUserFilesAsPrefixMap: () -> Map<String, List<UserFileInfo>> = {
val getLocalUserFilesAsPrefixMap: suspend () -> Map<String, List<UserFileInfo>> = {
val savePatterns = appInfo.ufs.saveFilePatterns.filter { userFile -> userFile.root.isWindows }

val result = mutableMapOf<String, MutableList<UserFileInfo>>()
Expand All @@ -294,27 +338,40 @@ object SteamAutoCloud {

Timber.i("Looking for saves in $basePath with pattern ${userFile.pattern} (prefix ${userFile.prefix})")

val files = FileUtils.findFilesRecursive(
val filePaths = FileUtils.findFilesRecursive(
rootPath = basePath,
pattern = userFile.pattern,
maxDepth = 5,
).map {
val sha = streamingShaHash(it)
).collect(Collectors.toList())
val files = buildList {
for (path in filePaths) {
val hashLookup = getCachedShaOrHash(
appId = appInfo.id,
path = path,
hashCacheDao = hashCacheDao,
)
if (hashLookup.wasCacheHit) {
hashCacheHits.incrementAndGet()
} else {
hashCacheMisses.incrementAndGet()
}
val sha = hashLookup.sha

Timber.i("Found ${it.pathString}\n\tin ${userFile.prefix}\n\twith sha [${sha.joinToString(", ")}]")
Timber.i("Found ${path.pathString}\n\tin ${userFile.prefix}\n\twith sha [${sha.joinToString(", ")}]")

val relativePath = basePath.relativize(it).pathString
val relativePath = basePath.relativize(path).pathString

UserFileInfo(
root = userFile.root,
path = userFile.substitutedPath,
filename = relativePath,
timestamp = Files.getLastModifiedTime(it).toMillis(),
sha = sha,
cloudRoot = userFile.uploadRoot,
cloudPath = userFile.uploadPath
)
}.collect(Collectors.toList())
add(UserFileInfo(
root = userFile.root,
path = userFile.substitutedPath,
filename = relativePath,
timestamp = Files.getLastModifiedTime(path).toMillis(),
sha = sha,
cloudRoot = userFile.uploadRoot,
cloudPath = userFile.uploadPath
))
}
}

Timber.i("Found ${files.size} file(s) in $basePath for pattern ${userFile.pattern}")

Expand All @@ -329,28 +386,41 @@ object SteamAutoCloud {

Timber.i("Scanning $basePath recursively (depth 5) under ${rootType.name}")

val files = FileUtils.findFilesRecursive(
val steamUserDataPaths = FileUtils.findFilesRecursive(
rootPath = basePath,
pattern = "*",
maxDepth = 5,
).map {
val sha = streamingShaHash(it)

val relativePath = basePath.relativize(it).pathString

Timber.i("Found ${it.pathString}\n\tin %${rootType.name}%\n\twith sha [${sha.joinToString(", ")}]")

// Store relative path in filename; empty path component
UserFileInfo(
root = rootType,
path = "",
filename = relativePath,
timestamp = Files.getLastModifiedTime(it).toMillis(),
sha = sha,
cloudRoot = rootType,
cloudPath = ""
)
}.collect(Collectors.toList())
).collect(Collectors.toList())
val files = buildList {
for (path in steamUserDataPaths) {
val hashLookup = getCachedShaOrHash(
appId = appInfo.id,
path = path,
hashCacheDao = hashCacheDao,
)
if (hashLookup.wasCacheHit) {
hashCacheHits.incrementAndGet()
} else {
hashCacheMisses.incrementAndGet()
}
val sha = hashLookup.sha

val relativePath = basePath.relativize(path).pathString

Timber.i("Found ${path.pathString}\n\tin %${rootType.name}%\n\twith sha [${sha.joinToString(", ")}]")

// Store relative path in filename; empty path component
add(UserFileInfo(
root = rootType,
path = "",
filename = relativePath,
timestamp = Files.getLastModifiedTime(path).toMillis(),
sha = sha,
cloudRoot = rootType,
cloudPath = ""
))
}
}

Timber.i("Found ${files.size} file(s) in $basePath")

Expand All @@ -361,6 +431,11 @@ object SteamAutoCloud {
result.getOrPut(prefixKey) { mutableListOf() }.addAll(files)
}

Timber.i(
"Local save hash cache stats for ${appInfo.id} (${appInfo.name}): " +
"hits=${hashCacheHits.get()}, misses=${hashCacheMisses.get()}, files=${hashCacheHits.get() + hashCacheMisses.get()}",
)
Comment on lines +434 to +437
Copy link
Copy Markdown
Contributor

@coderabbitai coderabbitai Bot Apr 28, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Log cache stats once per sync, not once per scan.

getLocalUserFilesAsPrefixMap() runs more than once in some sync paths, while hashCacheHits/hashCacheMisses are shared across the whole sync. Logging here emits partial totals on the first scan and cumulative totals on later rescans, so the advertised per-sync metric is misleading.

🤖 Prompt for AI Agents 
Verify each finding against the current code and only fix it if needed.

In `@app/src/main/java/app/gamenative/service/SteamAutoCloud.kt` around lines 434
- 437, The current Timber.i inside getLocalUserFilesAsPrefixMap() logs
hashCacheHits/hashCacheMisses per scan but those Atomic counters are shared
across the whole sync, producing misleading partial totals; fix by logging once
per sync instead: remove or stop logging from getLocalUserFilesAsPrefixMap() and
instead emit the Timber.i after the full sync completes (in the method that
orchestrates the sync), or alternatively snapshot and reset
hashCacheHits/hashCacheMisses at the start of each
getLocalUserFilesAsPrefixMap() invocation so the logged values represent only
that scan; reference getLocalUserFilesAsPrefixMap(), hashCacheHits,
hashCacheMisses, and the sync orchestration method to relocate the log.


result
}

Expand Down Expand Up @@ -423,6 +498,7 @@ object SteamAutoCloud {
val result = downloadSingleFile(
appInfo = appInfo,
steamCloud = steamCloud,
hashCacheDao = hashCacheDao,
file = file,
fileList = fileList,
getFilePrefixPath = getFilePrefixPath,
Expand Down Expand Up @@ -966,6 +1042,8 @@ object SteamAutoCloud {
filesDownloaded = filesDownloaded,
filesDeleted = filesDeleted,
filesManaged = filesManaged,
hashCacheHits = hashCacheHits.get(),
hashCacheMisses = hashCacheMisses.get(),
bytesUploaded = bytesUploaded,
bytesDownloaded = bytesDownloaded,
microsecTotal = microsecTotal,
Expand All @@ -986,6 +1064,7 @@ object SteamAutoCloud {
private suspend fun downloadSingleFile(
appInfo: SteamApp,
steamCloud: SteamCloud,
hashCacheDao: SteamFileHashCacheDao,
file: AppFileInfo,
fileList: AppFileChangeList,
getFilePrefixPath: (AppFileInfo, AppFileChangeList) -> String,
Expand Down Expand Up @@ -1136,6 +1215,21 @@ object SteamAutoCloud {
return null
}

val actualSize = Files.size(actualFilePath)
if (actualSize != totalFileSize) {
Timber.w("Downloaded size for $prefixedPath was $actualSize, expected $totalFileSize - skipping cache seed")
} else {
hashCacheDao.insert(
SteamFileHashCache(
appId = appInfo.id,
absPath = actualFilePath.pathString,
sizeBytes = actualSize,
mtimeMillis = Files.getLastModifiedTime(actualFilePath).toMillis(),
sha = streamingShaHash(actualFilePath),
),
)
}

val finishedFiles = completedFiles.incrementAndGet()
val finalProgress = if (totalRawBytes > 0L) {
(downloadedRawBytes.get().toFloat() / totalRawBytes).coerceIn(0f, 1f)
Expand Down
Loading
Loading