fix(auto-upload): file detection

app/src/main/java/com/nextcloud/client/database/dao/FileSystemDao.kt

@@ -8,6 +8,7 @@
 package com.nextcloud.client.database.dao
 
 import androidx.room.Dao
+import androidx.room.Delete
 import androidx.room.Insert
 import androidx.room.OnConflictStrategy
 import androidx.room.Query
@@ -19,6 +20,9 @@ interface FileSystemDao {
     @Insert(onConflict = OnConflictStrategy.REPLACE)
     fun insertOrReplace(filesystemEntity: FilesystemEntity)
 
+    @Delete
+    fun delete(entity: FilesystemEntity)
+
     @Query(
         """
         DELETE FROM ${ProviderMeta.ProviderTableMeta.FILESYSTEM_TABLE_NAME}

app/src/main/java/com/nextcloud/client/database/dao/UploadDao.kt

@@ -41,7 +41,7 @@ interface UploadDao {
             "WHERE ${ProviderTableMeta.UPLOADS_ACCOUNT_NAME} = :accountName " +
             "AND ${ProviderTableMeta.UPLOADS_REMOTE_PATH} = :remotePath"
     )
-    fun deleteByAccountAndRemotePath(remotePath: String, accountName: String)
+    fun deleteByRemotePathAndAccountName(remotePath: String, accountName: String)
 
     @Query(
         "SELECT * FROM " + ProviderTableMeta.UPLOADS_TABLE_NAME +
@@ -51,7 +51,7 @@ interface UploadDao {
     )
     fun getUploadById(id: Long, accountName: String): UploadEntity?
 
-    @Insert(onConflict = OnConflictStrategy.Companion.REPLACE)
+    @Insert(onConflict = OnConflictStrategy.REPLACE)
     fun insertOrReplace(entity: UploadEntity): Long
 
     @Query(

app/src/main/java/com/nextcloud/client/jobs/BackgroundJobFactory.kt

@@ -179,7 +179,7 @@ class BackgroundJobFactory @Inject constructor(
         powerManagementService = powerManagementService,
         syncedFolderProvider = syncedFolderProvider,
         backgroundJobManager = backgroundJobManager.get(),
-        repository = FileSystemRepository(dao = database.fileSystemDao(), context),
+        repository = FileSystemRepository(dao = database.fileSystemDao(), uploadsStorageManager, context),
         viewThemeUtils = viewThemeUtils.get(),
         localBroadcastManager = localBroadcastManager.get()
     )

app/src/main/java/com/nextcloud/client/jobs/autoUpload/AutoUploadWorker.kt

@@ -9,8 +9,6 @@ package com.nextcloud.client.jobs.autoUpload
 
 import android.app.Notification
 import android.content.Context
-import android.content.res.Resources
-import androidx.exifinterface.media.ExifInterface
 import androidx.localbroadcastmanager.content.LocalBroadcastManager
 import androidx.work.CoroutineWorker
 import androidx.work.ForegroundInfo
@@ -25,13 +23,11 @@ import com.nextcloud.client.jobs.upload.FileUploadBroadcastManager
 import com.nextcloud.client.jobs.upload.FileUploadWorker
 import com.nextcloud.client.jobs.utils.UploadErrorNotificationManager
 import com.nextcloud.client.network.ConnectivityService
-import com.nextcloud.client.preferences.SubFolderRule
 import com.nextcloud.utils.extensions.isNonRetryable
 import com.nextcloud.utils.extensions.updateStatus
 import com.owncloud.android.R
 import com.owncloud.android.datamodel.ArbitraryDataProviderImpl
 import com.owncloud.android.datamodel.FileDataStorageManager
-import com.owncloud.android.datamodel.MediaFolderType
 import com.owncloud.android.datamodel.SyncedFolder
 import com.owncloud.android.datamodel.SyncedFolderProvider
 import com.owncloud.android.datamodel.UploadsStorageManager
@@ -44,16 +40,10 @@ import com.owncloud.android.lib.common.operations.RemoteOperationResult
 import com.owncloud.android.lib.common.utils.Log_OC
 import com.owncloud.android.operations.UploadFileOperation
 import com.owncloud.android.ui.activity.SettingsActivity
-import com.owncloud.android.utils.FileStorageUtils
-import com.owncloud.android.utils.MimeType
 import com.owncloud.android.utils.theme.ViewThemeUtils
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
 import java.io.File
-import java.text.ParsePosition
-import java.text.SimpleDateFormat
-import java.util.Locale
-import java.util.TimeZone
 
 @Suppress("LongParameterList", "TooManyFunctions", "TooGenericExceptionCaught")
 class AutoUploadWorker(
@@ -79,6 +69,7 @@ class AutoUploadWorker(
     }
 
     private val helper = AutoUploadHelper()
+    private val syncFolderHelper = SyncFolderHelper(context)
     private val fileUploadBroadcastManager = FileUploadBroadcastManager(localBroadcastManager)
     private lateinit var syncedFolder: SyncedFolder
     private val notificationManager = AutoUploadNotificationManager(context, viewThemeUtils, NOTIFICATION_ID)
@@ -233,13 +224,6 @@ class AutoUploadWorker(
         Log_OC.d(TAG, "Exception collectFileChangesFromContentObserverWork: $e")
     }
 
-    private fun prepareDateFormat(): SimpleDateFormat {
-        val currentLocale = context.resources.configuration.locales[0]
-        return SimpleDateFormat("yyyy:MM:dd HH:mm:ss", currentLocale).apply {
-            timeZone = TimeZone.getTimeZone(TimeZone.getDefault().id)
-        }
-    }
-
     private fun getUserOrReturn(syncedFolder: SyncedFolder): User? {
         val optionalUser = userAccountManager.getUser(syncedFolder.account)
         if (!optionalUser.isPresent) {
@@ -274,13 +258,10 @@ class AutoUploadWorker(
 
     @Suppress("LongMethod", "DEPRECATION", "TooGenericExceptionCaught")
     private suspend fun uploadFiles(syncedFolder: SyncedFolder) = withContext(Dispatchers.IO) {
-        val dateFormat = prepareDateFormat()
         val user = getUserOrReturn(syncedFolder) ?: return@withContext
         val ocAccount = OwnCloudAccount(user.toPlatformAccount(), context)
         val client = OwnCloudClientManagerFactory.getDefaultSingleton()
             .getClientFor(ocAccount, context)
-        val lightVersion = context.resources.getBoolean(R.bool.syncedFolder_light)
-        val currentLocale = context.resources.configuration.locales[0]
 
         trySetForeground()
         updateNotification()
@@ -299,14 +280,7 @@ class AutoUploadWorker(
             filePathsWithIds.forEachIndexed { batchIndex, (path, id) ->
                 val file = File(path)
                 val localPath = file.absolutePath
-                val remotePath = getRemotePath(
-                    file,
-                    syncedFolder,
-                    dateFormat,
-                    lightVersion,
-                    context.resources,
-                    currentLocale
-                )
+                val remotePath = syncFolderHelper.getAutoUploadRemotePath(syncedFolder, file)
 
                 try {
                     val entityResult = getEntityResult(user, localPath, remotePath)
@@ -471,79 +445,6 @@ class AutoUploadWorker(
         FileDataStorageManager(user, context.contentResolver)
     )
 
-    private fun getRemotePath(
-        file: File,
-        syncedFolder: SyncedFolder,
-        sFormatter: SimpleDateFormat,
-        lightVersion: Boolean,
-        resources: Resources,
-        currentLocale: Locale
-    ): String {
-        val lastModificationTime = calculateLastModificationTime(file, syncedFolder, sFormatter)
-
-        val (remoteFolder, useSubfolders, subFolderRule) = if (lightVersion) {
-            Triple(
-                resources.getString(R.string.syncedFolder_remote_folder),
-                resources.getBoolean(R.bool.syncedFolder_light_use_subfolders),
-                SubFolderRule.YEAR_MONTH
-            )
-        } else {
-            Triple(
-                syncedFolder.remotePath,
-                syncedFolder.isSubfolderByDate,
-                syncedFolder.subfolderRule
-            )
-        }
-
-        return FileStorageUtils.getInstantUploadFilePath(
-            file,
-            currentLocale,
-            remoteFolder,
-            syncedFolder.localPath,
-            lastModificationTime,
-            useSubfolders,
-            subFolderRule
-        )
-    }
-
-    private fun hasExif(file: File): Boolean {
-        val mimeType = FileStorageUtils.getMimeTypeFromName(file.absolutePath)
-        return MimeType.JPEG.equals(mimeType, ignoreCase = true) || MimeType.TIFF.equals(mimeType, ignoreCase = true)
-    }
-
-    @Suppress("NestedBlockDepth")
-    private fun calculateLastModificationTime(
-        file: File,
-        syncedFolder: SyncedFolder,
-        formatter: SimpleDateFormat
-    ): Long {
-        var lastModificationTime = file.lastModified()
-        if (MediaFolderType.IMAGE == syncedFolder.type && hasExif(file)) {
-            Log_OC.d(TAG, "calculateLastModificationTime exif found")
-
-            @Suppress("TooGenericExceptionCaught")
-            try {
-                val exifInterface = ExifInterface(file.absolutePath)
-                val exifDate = exifInterface.getAttribute(ExifInterface.TAG_DATETIME)
-                if (!exifDate.isNullOrBlank()) {
-                    val pos = ParsePosition(0)
-                    val dateTime = formatter.parse(exifDate, pos)
-                    if (dateTime != null) {
-                        lastModificationTime = dateTime.time
-                        Log_OC.w(TAG, "calculateLastModificationTime calculatedTime is: $lastModificationTime")
-                    } else {
-                        Log_OC.w(TAG, "calculateLastModificationTime dateTime is empty")
-                    }
-                } else {
-                    Log_OC.w(TAG, "calculateLastModificationTime exifDate is empty")
-                }
-            } catch (e: Exception) {
-                Log_OC.d(TAG, "Failed to get the proper time " + e.localizedMessage)
-            }
-        }
-        return lastModificationTime
-    }
-
     private fun sendUploadFinishEvent(operation: UploadFileOperation, result: RemoteOperationResult<*>) {
         fileUploadBroadcastManager.sendFinished(
             operation,

app/src/main/java/com/nextcloud/client/jobs/autoUpload/FileSystemRepository.kt

@@ -15,19 +15,37 @@ import com.nextcloud.client.database.entity.FilesystemEntity
 import com.nextcloud.utils.extensions.shouldSkipFile
 import com.nextcloud.utils.extensions.toFile
 import com.owncloud.android.datamodel.SyncedFolder
+import com.owncloud.android.datamodel.UploadsStorageManager
 import com.owncloud.android.lib.common.utils.Log_OC
 import com.owncloud.android.utils.SyncedFolderUtils
 import java.io.File
 import java.util.zip.CRC32
 
 @Suppress("TooGenericExceptionCaught", "NestedBlockDepth", "MagicNumber", "ReturnCount")
-class FileSystemRepository(private val dao: FileSystemDao, private val context: Context) {
+class FileSystemRepository(
+    private val dao: FileSystemDao,
+    private val uploadsStorageManager: UploadsStorageManager,
+    private val context: Context
+) {
+    private val syncFolderHelper = SyncFolderHelper(context)
 
     companion object {
         private const val TAG = "FilesystemRepository"
         const val BATCH_SIZE = 50
     }
 
+    fun deleteAutoUploadAndUploadEntity(syncedFolder: SyncedFolder, localPath: String, entity: FilesystemEntity) {
+        Log_OC.d(TAG, "deleting auto upload entity and upload entity")
+
+        val file = File(localPath)
+        val remotePath = syncFolderHelper.getAutoUploadRemotePath(syncedFolder, file)
+        uploadsStorageManager.uploadDao.deleteByRemotePathAndAccountName(
+            remotePath = remotePath,
+            accountName = syncedFolder.account
+        )
+        dao.delete(entity)
+    }
+
     suspend fun deleteByLocalPathAndId(path: String, id: Int) {
         dao.deleteByLocalPathAndId(path, id)
     }
@@ -39,20 +57,23 @@ class FileSystemRepository(private val dao: FileSystemDao, private val context:
         val entities = dao.getAutoUploadFilesEntities(syncedFolderId, BATCH_SIZE, lastId)
         val filtered = mutableListOf<Pair<String, Int>>()
 
-        entities.forEach {
-            it.localPath?.let { path ->
+        entities.forEach { entity ->
+            entity.localPath?.let { path ->
                 val file = File(path)
                 if (!file.exists()) {
                     Log_OC.w(TAG, "Ignoring file for upload (doesn't exist): $path")
+                    deleteAutoUploadAndUploadEntity(syncedFolder, path, entity)
                 } else if (!SyncedFolderUtils.isQualifiedFolder(file.parent)) {
                     Log_OC.w(TAG, "Ignoring file for upload (unqualified folder): $path")
+                    deleteAutoUploadAndUploadEntity(syncedFolder, path, entity)
                 } else if (!SyncedFolderUtils.isFileNameQualifiedForAutoUpload(file.name)) {
                     Log_OC.w(TAG, "Ignoring file for upload (unqualified file): $path")
+                    deleteAutoUploadAndUploadEntity(syncedFolder, path, entity)
                 } else {
                     Log_OC.d(TAG, "Adding path to upload: $path")
 
-                    if (it.id != null) {
-                        filtered.add(path to it.id)
+                    if (entity.id != null) {
+                        filtered.add(path to entity.id)
                     } else {
                         Log_OC.w(TAG, "cant adding path to upload, id is null")
                     }
@@ -160,22 +181,29 @@ class FileSystemRepository(private val dao: FileSystemDao, private val context:
             }
 
             val entity = dao.getFileByPathAndFolder(localPath, syncedFolder.id.toString())
-            val fileSentForUpload = (entity != null && entity.fileSentForUpload == 1)
-            if (fileSentForUpload) {
-                Log_OC.d(TAG, "File was sent for upload, checking if it changed...")
-            }
 
             val fileModified = (lastModified ?: file.lastModified())
-            if (syncedFolder.shouldSkipFile(file, fileModified, creationTime, fileSentForUpload)) {
+            if (fileModified <= 0L) {
+                Log_OC.d(TAG, "file is deleted, skipping: $localPath")
+                entity?.let {
+                    deleteAutoUploadAndUploadEntity(syncedFolder, localPath, entity)
+                }
                 return
             }
 
-            if (fileSentForUpload) {
-                Log_OC.d(TAG, "File was sent for upload before but has changed, will re-upload: $localPath")
+            val hasNotChanged = entity?.fileModified == fileModified
+            val fileSentForUpload = entity?.fileSentForUpload == 1
+
+            if (hasNotChanged && fileSentForUpload) {
+                Log_OC.d(TAG, "File hasn't changed since last scan. skipping: $localPath")
+                return
             }
 
-            val crc = getFileChecksum(file)
+            if (syncedFolder.shouldSkipFile(file, fileModified, creationTime, fileSentForUpload)) {
+                return
+            }
 
+            val crc = getFileChecksum(file)
             val newEntity = FilesystemEntity(
                 id = entity?.id,
                 localPath = localPath,