diff --git a/changes/unreleased/489-sync-unreadable-local-items.md b/changes/unreleased/489-sync-unreadable-local-items.md new file mode 100644 index 000000000..a52638d6e --- /dev/null +++ b/changes/unreleased/489-sync-unreadable-local-items.md @@ -0,0 +1,7 @@ +category: fix +issue: 489 +pull: none +platforms: desktop +user-facing: yes + +Desktop folder sync, including Git working trees, now skips and lists a local item that vanishes, is locked, or is not a regular file instead of stopping the whole folder. diff --git a/tools/kotlin-file-size-baseline.txt b/tools/kotlin-file-size-baseline.txt index a875c8a79..11dcf9316 100644 --- a/tools/kotlin-file-size-baseline.txt +++ b/tools/kotlin-file-size-baseline.txt @@ -46,7 +46,7 @@ ui/src/commonTest/kotlin/dev/obiente/nextcloudnative/nativeui/runtime/NativeColl ui/src/commonTest/kotlin/dev/obiente/nextcloudnative/nativeui/runtime/NativeRecordActionsTest.kt|3161 ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopAppUpdates.kt|925 ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileReadCache.kt|1536 -ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncEngine.kt|884 +ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncEngine.kt|876 ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncRemoteTree.kt|857 ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopNextcloudServices.kt|6169 ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopVirtualRangeCache.kt|2759 diff --git a/ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncCoordinator.kt b/ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncCoordinator.kt index ff2c640e7..0048efac0 100644 --- a/ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncCoordinator.kt +++ b/ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncCoordinator.kt @@ -878,10 +878,10 @@ private fun requireValidFileSyncPair(pair: FileSyncPair) { "The persisted sync decision choices do not match the pair direction." } } - val expectedOperation = if (resolved != null) { - resolveDecisionOperation(pair, work, resolved.choice) - } else { - planFileSync( + val expectedOperation = when { + resolved != null -> resolveDecisionOperation(pair, work, resolved.choice) + work.isUnavailableLocalItemReport() -> work.operation + else -> planFileSync( localEntries = listOfNotNull(work.observedLocal), remoteEntries = listOfNotNull(work.observedRemote), baselines = listOfNotNull(work.observedBaseline), diff --git a/ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncLocalAvailability.kt b/ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncLocalAvailability.kt new file mode 100644 index 000000000..ef1b719fd --- /dev/null +++ b/ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncLocalAvailability.kt @@ -0,0 +1,159 @@ +package dev.obiente.nextcloudnative.app + +/** Why a platform scan could not observe one local item. */ +enum class FileSyncLocalUnavailableReason { + /** The item disappeared or was replaced between directory listing and reading. */ + Vanished, + + /** The item exists but could not be read, for example while another app holds it open. */ + Unreadable, + + /** The item is not a regular file or folder, for example a socket, device, or reparse point. */ + Unsupported, +} + +/** + * A local path whose current state is unknown for one scan. + * + * The item and every descendant are withheld from planning, so an unreadable subtree is never + * interpreted as a local deletion, a missing file to restore, or new content to upload. + */ +data class FileSyncUnavailableLocalItem( + val relativePath: String, + val reason: FileSyncLocalUnavailableReason, +) { + init { + requireValidSyncPath(relativePath) + require(relativePath.length <= MAX_FILE_SYNC_PATH_LENGTH) + } +} + +/** Local and remote observations that remain after unavailable local subtrees are withheld. */ +data class FileSyncAvailableObservations( + val localEntries: List, + val remoteEntries: List, + val unavailable: List, +) + +/** + * Withholds every local and remote observation at or below an unavailable local path. + * + * Persisted baselines are untouched, so a later complete scan resumes normal planning. Nested + * reports collapse into their outermost unavailable ancestor. + */ +fun withholdUnavailableFileSyncPaths( + localEntries: List, + remoteEntries: List, + unavailable: List, +): FileSyncAvailableObservations { + val withheldPaths = HashSet() + fun withheld(path: String): Boolean = path.syncPathAndAncestors().any { it in withheldPaths } + // Shallower paths first, so every nested report finds its already-withheld ancestor. + val outermost = unavailable + .sortedWith(compareBy({ it.relativePath.count { char -> char == '/' } }, { it.relativePath })) + .filter { item -> !withheld(item.relativePath) && withheldPaths.add(item.relativePath) } + .sortedBy(FileSyncUnavailableLocalItem::relativePath) + if (outermost.isEmpty()) return FileSyncAvailableObservations(localEntries, remoteEntries, emptyList()) + fun available(path: String) = !withheld(path) + return FileSyncAvailableObservations( + localEntries = localEntries.filter { available(it.relativePath) }, + remoteEntries = remoteEntries.filter { available(it.relativePath) }, + unavailable = outermost, + ) +} + +/** + * Narrows a scan predicate so remote traversal never descends into unavailable local subtrees. + * + * Their server content cannot take part in planning, so listing it would only add network work + * and listing failures. [withholdUnavailableFileSyncPaths] still applies as the second layer. + */ +fun ((relativePath: String, kind: SyncEntryKind) -> Boolean).excludingUnavailableFileSyncPaths( + unavailable: List, +): (relativePath: String, kind: SyncEntryKind) -> Boolean { + if (unavailable.isEmpty()) return this + val withheldPaths = unavailable.mapTo(HashSet(), FileSyncUnavailableLocalItem::relativePath) + return { path, kind -> path.syncPathAndAncestors().none { it in withheldPaths } && this(path, kind) } +} + +/** + * Records each unavailable local item as visible skipped work after planning. + * + * At most [maximumReports] items become work items; the run summary reports the complete count. + */ +fun FileSyncCoordinatorState.withUnavailableFileSyncReports( + pairId: String, + unavailable: List, + maximumReports: Int = MAX_FILE_SYNC_UNAVAILABLE_REPORTS, +): FileSyncCoordinatorState { + require(maximumReports in 0..MAX_FILE_SYNC_UNAVAILABLE_REPORTS) + if (unavailable.isEmpty() || maximumReports == 0) return this + return copy( + pairs = pairs.map { pair -> + if (pair.id != pairId) return@map pair + val plannedPaths = pair.workItems.mapTo(HashSet(), FileSyncWorkItem::relativePath) + var nextId = pair.nextWorkId + val reports = unavailable.asSequence() + .filter { it.relativePath !in plannedPaths } + .take(maximumReports) + .map { item -> + require(nextId < Long.MAX_VALUE) { "The sync work ID space is exhausted." } + FileSyncWorkItem( + id = nextId++, + relativePath = item.relativePath, + observedLocal = null, + observedRemote = null, + observedBaseline = null, + operation = FileSyncOperation.Skipped(item.relativePath, item.skippedReason()), + state = FileSyncExecutionState.Skipped, + ) + } + .toList() + pair.copy(workItems = pair.workItems + reports, nextWorkId = nextId) + }, + ) +} + +/** + * True for a skipped report produced by [withUnavailableFileSyncReports]. Such work has no observed + * generation to re-plan, so persisted validation instead requires an unexecuted item whose reason + * is exactly one generated for its path. + */ +internal fun FileSyncWorkItem.isUnavailableLocalItemReport(): Boolean { + val skipped = operation as? FileSyncOperation.Skipped ?: return false + return state == FileSyncExecutionState.Skipped && + observedLocal == null && observedRemote == null && observedBaseline == null && + decision == null && attemptCount == 0 && lastAttemptEpochMillis == null && + uploadCheckpoint == null && !contentMismatchVerified && + FileSyncLocalUnavailableReason.entries.any { reason -> + FileSyncUnavailableLocalItem(relativePath, reason).skippedReason() == skipped.reason + } +} + +/** One reviewable sentence per item; the path keeps each unavailable item individually visible. */ +internal fun FileSyncUnavailableLocalItem.skippedReason(): String { + val path = relativePath.boundedForSyncReason() + return when (reason) { + FileSyncLocalUnavailableReason.Vanished -> + "Not synced this time: $path disappeared or changed while the folder was checked. " + + "It will be checked again on the next sync." + FileSyncLocalUnavailableReason.Unreadable -> + "Not synced this time: $path could not be read. It may be open in another app or " + + "access may be denied. It will be checked again on the next sync." + FileSyncLocalUnavailableReason.Unsupported -> + "Not synced: $path is not a regular file or folder." + } +} + +private fun String.boundedForSyncReason(): String { + if (length <= MAX_FILE_SYNC_REASON_PATH_LENGTH) return this + val keep = (MAX_FILE_SYNC_REASON_PATH_LENGTH - 3) / 2 + return take(keep) + "..." + takeLast(keep) +} + +private fun String.syncPathAndAncestors(): Sequence = + generateSequence(this) { path -> path.substringBeforeLast('/', "").takeIf(String::isNotEmpty) } + +/** Visible skipped-work reports kept per run; every unavailable path is withheld regardless. */ +internal const val MAX_FILE_SYNC_UNAVAILABLE_REPORTS = 1_000 +private const val MAX_FILE_SYNC_REASON_PATH_LENGTH = 600 diff --git a/ui/src/commonTest/kotlin/dev/obiente/nextcloudnative/app/FileSyncLocalAvailabilityTest.kt b/ui/src/commonTest/kotlin/dev/obiente/nextcloudnative/app/FileSyncLocalAvailabilityTest.kt new file mode 100644 index 000000000..14c40cf50 --- /dev/null +++ b/ui/src/commonTest/kotlin/dev/obiente/nextcloudnative/app/FileSyncLocalAvailabilityTest.kt @@ -0,0 +1,191 @@ +package dev.obiente.nextcloudnative.app + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertIs +import kotlin.test.assertTrue + +class FileSyncLocalAvailabilityTest { + @Test + fun `unavailable local subtrees are withheld from both sides and nested reports collapse`() { + val observed = withholdUnavailableFileSyncPaths( + localEntries = listOf( + local(".git", SyncEntryKind.Directory), + local(".git/objects", SyncEntryKind.Directory), + local(".git/objects/ab", SyncEntryKind.Directory), + local(".git/HEAD"), + local(".git/objects pack"), + local("README.md"), + ), + remoteEntries = listOf( + remote(".git/objects", SyncEntryKind.Directory), + remote(".git/objects/ab/cdef"), + remote(".git/index.lock"), + remote(".git/HEAD"), + ), + unavailable = listOf( + FileSyncUnavailableLocalItem(".git/objects/ab", FileSyncLocalUnavailableReason.Vanished), + FileSyncUnavailableLocalItem(".git/objects", FileSyncLocalUnavailableReason.Unreadable), + FileSyncUnavailableLocalItem(".git/index.lock", FileSyncLocalUnavailableReason.Vanished), + ), + ) + + assertEquals( + listOf(".git", ".git/HEAD", ".git/objects pack", "README.md"), + observed.localEntries.map(LocalSyncEntry::relativePath), + ) + assertEquals(listOf(".git/HEAD"), observed.remoteEntries.map(RemoteSyncEntry::relativePath)) + assertEquals( + listOf( + FileSyncUnavailableLocalItem(".git/index.lock", FileSyncLocalUnavailableReason.Vanished), + FileSyncUnavailableLocalItem(".git/objects", FileSyncLocalUnavailableReason.Unreadable), + ), + observed.unavailable, + ) + } + + @Test + fun `an unreadable synced git file is never planned as a deletion and is reported`() { + val configuration = FileSyncConfiguration( + deviceLabel = "Workstation", + deletionPolicy = FileSyncDeletionPolicy.Propagate, + ) + val pair = FileSyncPair( + id = "pair", + accountId = "account", + localRootId = "root", + remoteRootPath = "Repo", + configuration = configuration, + baselines = listOf( + FileSyncBaseline(".git", SyncEntryKind.Directory, "local-dir", "remote-dir"), + FileSyncBaseline(".git/index", SyncEntryKind.File, "local-index", "remote-index"), + FileSyncBaseline(".git/objects", SyncEntryKind.Directory, "local-objects", "remote-objects"), + FileSyncBaseline(".git/objects/ab", SyncEntryKind.File, "local-object", "remote-object"), + ), + ) + val unavailable = listOf( + FileSyncUnavailableLocalItem(".git/index", FileSyncLocalUnavailableReason.Unreadable), + FileSyncUnavailableLocalItem(".git/objects", FileSyncLocalUnavailableReason.Unreadable), + ) + val observed = withholdUnavailableFileSyncPaths( + localEntries = listOf( + LocalSyncEntry(".git", SyncEntryKind.Directory, "local-dir"), + local(".git/COMMIT_EDITMSG"), + ), + remoteEntries = listOf( + RemoteSyncEntry(".git", SyncEntryKind.Directory, "remote-dir"), + RemoteSyncEntry(".git/index", SyncEntryKind.File, "remote-index"), + RemoteSyncEntry(".git/objects", SyncEntryKind.Directory, "remote-objects"), + RemoteSyncEntry(".git/objects/ab", SyncEntryKind.File, "remote-object"), + ), + unavailable = unavailable, + ) + + val scanned = scanFileSyncPair( + FileSyncCoordinatorState(listOf(pair)), + pair.id, + observed.localEntries, + observed.remoteEntries, + nowEpochMillis = 10L, + maximumWorkItems = MAX_FILE_SYNC_WORK_ITEMS - observed.unavailable.size, + ).withUnavailableFileSyncReports(pair.id, observed.unavailable).pairs.single() + + val operations = scanned.workItems.associate { it.relativePath to it.operation } + assertIs(operations.getValue(".git/COMMIT_EDITMSG")) + assertTrue(operations.values.none { it is FileSyncOperation.DeleteLocal || it is FileSyncOperation.DeleteRemote }) + val indexReport = assertIs(operations.getValue(".git/index")) + assertTrue(indexReport.reason.contains(".git/index could not be read")) + assertIs(operations.getValue(".git/objects")) + assertEquals(pair.baselines, scanned.baselines) + assertEquals( + setOf(".git/index", ".git/objects"), + scanned.workItems.filter { it.state == FileSyncExecutionState.Skipped }.map { it.relativePath }.toSet(), + ) + assertTrue(scanned.workItems.all { it.id < scanned.nextWorkId }) + + val recovered = scanFileSyncPair( + FileSyncCoordinatorState(listOf(scanned)), + pair.id, + localEntries = listOf( + LocalSyncEntry(".git", SyncEntryKind.Directory, "local-dir"), + local(".git/COMMIT_EDITMSG"), + LocalSyncEntry(".git/index", SyncEntryKind.File, "local-index"), + LocalSyncEntry(".git/objects", SyncEntryKind.Directory, "local-objects"), + LocalSyncEntry(".git/objects/ab", SyncEntryKind.File, "local-object"), + ), + remoteEntries = observed.remoteEntries + listOf( + RemoteSyncEntry(".git/index", SyncEntryKind.File, "remote-index"), + RemoteSyncEntry(".git/objects", SyncEntryKind.Directory, "remote-objects"), + RemoteSyncEntry(".git/objects/ab", SyncEntryKind.File, "remote-object"), + ), + nowEpochMillis = 20L, + ).pairs.single() + + assertEquals(listOf(".git/COMMIT_EDITMSG"), recovered.workItems.map(FileSyncWorkItem::relativePath)) + } + + @Test + fun `reports are bounded skip planned paths and keep reasons within persisted limits`() { + val pair = FileSyncPair( + id = "pair", + accountId = "account", + localRootId = "root", + remoteRootPath = "Repo", + configuration = FileSyncConfiguration(deviceLabel = "Workstation"), + ) + val planned = scanFileSyncPair( + FileSyncCoordinatorState(listOf(pair)), + pair.id, + localEntries = listOf(local("README.md")), + remoteEntries = emptyList(), + nowEpochMillis = 10L, + ) + val longName = "segment-".repeat(200).trimEnd('-') + val unavailable = listOf( + FileSyncUnavailableLocalItem("README.md", FileSyncLocalUnavailableReason.Vanished), + FileSyncUnavailableLocalItem(".git/$longName", FileSyncLocalUnavailableReason.Unsupported), + FileSyncUnavailableLocalItem(".git/index.lock", FileSyncLocalUnavailableReason.Vanished), + ) + + val reported = planned.withUnavailableFileSyncReports(pair.id, unavailable, maximumReports = 1) + .pairs.single() + + assertEquals(listOf("README.md", ".git/$longName"), reported.workItems.map(FileSyncWorkItem::relativePath)) + val reason = (reported.workItems.last().operation as FileSyncOperation.Skipped).reason + assertTrue(reason.length <= MAX_FILE_SYNC_FAILURE_LENGTH) + assertTrue(reason.contains("...")) + assertTrue(reason.endsWith("is not a regular file or folder.")) + assertEquals(planned, planned.withUnavailableFileSyncReports(pair.id, emptyList())) + } + + @Test + fun `persisted pairs accept generated reports and reject unobserved skips with other reasons`() { + val pair = FileSyncPair( + id = "pair", + accountId = "account", + localRootId = "root", + remoteRootPath = "Repo", + configuration = FileSyncConfiguration(deviceLabel = "Workstation"), + ) + val item = FileSyncUnavailableLocalItem(".git/index.lock", FileSyncLocalUnavailableReason.Vanished) + val reported = FileSyncCoordinatorState(listOf(pair)).withUnavailableFileSyncReports(pair.id, listOf(item)) + .pairs.single() + val report = reported.workItems.single() + + assertEquals(reported, reported.copy()) + assertFailsWith { + reported.copy( + workItems = listOf( + report.copy(operation = FileSyncOperation.Skipped(report.relativePath, "Forged skip.")), + ), + ) + } + } + + private fun local(path: String, kind: SyncEntryKind = SyncEntryKind.File) = + LocalSyncEntry(path, kind, "local-$path", size = if (kind == SyncEntryKind.File) 1L else null) + + private fun remote(path: String, kind: SyncEntryKind = SyncEntryKind.File) = + RemoteSyncEntry(path, kind, "remote-$path", size = if (kind == SyncEntryKind.File) 1L else null) +} diff --git a/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncContentSlices.kt b/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncContentSlices.kt index 2234f05be..02641d98e 100644 --- a/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncContentSlices.kt +++ b/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncContentSlices.kt @@ -1,26 +1,43 @@ package dev.obiente.nextcloudnative.app +import java.io.IOException +import kotlinx.coroutines.CancellationException + +/** + * Verifies one bounded slice, or returns null when the scanned local generation can no longer be + * read. The candidate then stays unverified, so planning skips it without writing either side. + */ internal fun verifyDesktopFileSyncContentSlice( slice: FileSyncContentVerificationSlice, local: DesktopFileSyncLocalTree, remote: DesktopFileSyncRemoteTree, localContentHash: String, shouldContinue: () -> Boolean, -): JvmFileSyncContentSliceOutcome { +): JvmFileSyncContentSliceOutcome? { val candidate = slice.candidate val expectedBytes = requireNotNull(candidate.expectedSizeBytes) val continueOrStop = { if (!shouldContinue()) throw DesktopFileSyncScanStoppedException() true } - val localHash = local.contentRangeHash( - candidate.relativePath, - candidate.localRevision, - expectedBytes, - slice.offset, - slice.length, - continueOrStop, - ) + val localHash = try { + local.contentRangeHash( + candidate.relativePath, + candidate.localRevision, + expectedBytes, + slice.offset, + slice.length, + continueOrStop, + ) + } catch (cancellation: CancellationException) { + throw cancellation + } catch (_: IOException) { + return null // Vanished or locked after the scan, for example a rewritten Git index. + } catch (_: DesktopFileSyncLocalRevisionChangedException) { + return null // The local revision changed after the scan; unsafe parents still propagate. + } catch (_: IllegalStateException) { + return null // The file was truncated while read, or is no longer a regular file or folder. + } val remoteHash = remote.contentRangeHash( candidate.relativePath, candidate.remoteEtag, diff --git a/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncDiagnostics.kt b/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncDiagnostics.kt index 93ded0263..2ed02c266 100644 --- a/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncDiagnostics.kt +++ b/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncDiagnostics.kt @@ -6,6 +6,10 @@ import java.nio.file.FileSystemException internal class DesktopFileSyncScanStoppedException : RuntimeException() +/** The scanned local file generation is gone; distinct from unsafe-path failures, which stop the folder. */ +internal class DesktopFileSyncLocalRevisionChangedException : + IllegalArgumentException("The local file changed during content verification.") + internal class DesktopFileSyncHttpStatusException( val statusCode: Int, operation: String, diff --git a/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncEngine.kt b/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncEngine.kt index f6db85e9e..3cdfa0eba 100644 --- a/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncEngine.kt +++ b/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncEngine.kt @@ -131,7 +131,7 @@ internal class DesktopFileSyncEngine( ?: return@transaction FileSyncCenterActionResult.Rejected("Choose the local folder again.") val canonical = selected.canonicalFile DesktopFileSyncLocalTree(canonical) - val normalizedRemote = normalizeRemoteRoot(remoteRootPath) + val normalizedRemote = normalizeDesktopFileSyncRemoteRoot(remoteRootPath) val accountId = desktopFileCacheAccountId(session) val current = store.load() if (current.coordinator.pairs.any { pair -> @@ -352,15 +352,23 @@ internal class DesktopFileSyncEngine( val cachedLocalRevisions = initialPair.baselines.mapNotNull { baseline -> baseline.localRevision?.let { revision -> baseline.relativePath to revision } }.toMap() - val scannedLocalEntries = try { - local.scan(cachedLocalRevisions, includes, shouldContinue).map(DesktopLocalSyncDocument::entry) + val localScan = try { + local.scan(cachedLocalRevisions, includes, shouldContinue) } catch (_: DesktopFileSyncScanStoppedException) { return FileSyncCenterActionResult.Stopped("The folder scan stopped before making changes.") } catch (failure: DesktopFileSyncScanLimitException) { onDiagnostic(failure.toDesktopFileSyncRunDiagnosticEvent(pairId, DesktopFileSyncScanStage.Local)) throw failure } - val scannedRemoteEntries = remote.scan(includes).map(DesktopRemoteSyncDocument::entry) + // Unreadable local subtrees are withheld from both sides and reported as skipped work. + val observed = withholdUnavailableFileSyncPaths( + localScan.documents.map(DesktopLocalSyncDocument::entry), + remote.scan(includes.excludingUnavailableFileSyncPaths(localScan.unavailable)).map { it.entry }, + localScan.unavailable, + ) + val scannedLocalEntries = observed.localEntries + val scannedRemoteEntries = observed.remoteEntries + val unavailableReports = observed.unavailable.take(MAX_FILE_SYNC_UNAVAILABLE_REPORTS) val cachedMismatchResults = currentFileSyncContentVerificationResults( scannedLocalEntries, scannedRemoteEntries, @@ -392,7 +400,7 @@ internal class DesktopFileSyncEngine( shouldContinue, ).also { outcome -> progressByPath.remove(slice.candidate.relativePath) - outcome.progress?.let { progress -> + outcome?.progress?.let { progress -> progressByPath[progress.candidate.relativePath] = progress } } @@ -400,7 +408,7 @@ internal class DesktopFileSyncEngine( } catch (_: DesktopFileSyncScanStoppedException) { return FileSyncCenterActionResult.Stopped("The folder scan stopped before making changes.") } - val verificationResults = cachedMismatchResults + completedSlices.mapNotNull { it.result } + val verificationResults = cachedMismatchResults + completedSlices.mapNotNull { it?.result } val verifiedPaths = verificationResults.mapTo(mutableSetOf()) { it.candidate.relativePath } val pendingCandidates = candidates.filterNot { it.relativePath in verifiedPaths } val verifiedMismatches = verificationResults.filter { it.matchingContentHash == null } @@ -426,11 +434,11 @@ internal class DesktopFileSyncEngine( localEntries, remoteEntries, System.currentTimeMillis(), - maximumWorkItems = MAX_FILE_SYNC_WORK_ITEMS, + maximumWorkItems = MAX_FILE_SYNC_WORK_ITEMS - unavailableReports.size, verifiedContentMismatches = verifiedMismatches, verifiedContentMismatchHashes = verifiedMismatchHashes, contentVerificationProgress = progressByPath.values.sortedBy { it.candidate.relativePath }, - ), + ).withUnavailableFileSyncReports(pairId, unavailableReports), ) val scannedPair = persisted.coordinator.pairs.single() if (!scannedPair.retainsResolvedFileSyncDecisions(expectedResolvedWorkIds)) { @@ -444,11 +452,11 @@ internal class DesktopFileSyncEngine( localEntries, remoteEntries, System.currentTimeMillis(), - maximumWorkItems = MAX_FILE_SYNC_WORK_ITEMS, + maximumWorkItems = MAX_FILE_SYNC_WORK_ITEMS - unavailableReports.size, verifiedContentMismatches = verifiedMismatches, verifiedContentMismatchHashes = verifiedMismatchHashes, contentVerificationProgress = progressByPath.values.sortedBy { it.candidate.relativePath }, - ), + ).withUnavailableFileSyncReports(pairId, unavailableReports), ) store.savePair(persisted, pairId) return FileSyncCenterActionResult.Rejected( @@ -588,13 +596,7 @@ internal class DesktopFileSyncEngine( if (execution.hasPendingDesktopUploadCleanup()) break } } - val message = buildString { - append(completed).append(" sync operation") - if (completed != 1) append('s') - append(" completed.") - if (conflicts > 0) append(' ').append(conflicts).append(" conflicts need review.") - if (failures > 0) append(' ').append(failures).append(" operations failed.") - } + val message = desktopFileSyncRunMessage(completed, conflicts, failures, observed.unavailable.size) return if (failures > 0) FileSyncCenterActionResult.Rejected(message) else FileSyncCenterActionResult.Completed(message) } @@ -867,16 +869,6 @@ internal class DesktopFileSyncEngine( stagingRoot, minimumFreeSpaceBytes, local, relativePath, downloadBytes, ) - private fun normalizeRemoteRoot(path: String): String { - val normalized = path.trim().trim('/') - if (normalized.isEmpty()) return "" - require(normalized.length <= MAX_FILE_SYNC_PATH_LENGTH) - normalized.split('/').forEach { segment -> - require(segment.isNotBlank() && segment !in setOf(".", "..") && segment.none(Char::isISOControl)) - } - return normalized - } - private fun safeFailureMessage(failure: Throwable, fallback: String): String = failure.message?.map { if (it.isISOControl()) ' ' else it }?.joinToString("") ?.trim()?.take(MAX_FILE_SYNC_FAILURE_LENGTH)?.takeIf(String::isNotBlank) ?: fallback diff --git a/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncEngineSupport.kt b/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncEngineSupport.kt index 501da382a..9e6ab1a2b 100644 --- a/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncEngineSupport.kt +++ b/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncEngineSupport.kt @@ -56,6 +56,35 @@ internal fun requireDesktopFileSyncBaselineCapacity( } } +internal fun normalizeDesktopFileSyncRemoteRoot(path: String): String { + val normalized = path.trim().trim('/') + if (normalized.isEmpty()) return "" + require(normalized.length <= MAX_FILE_SYNC_PATH_LENGTH) + normalized.split('/').forEach { segment -> + require(segment.isNotBlank() && segment !in setOf(".", "..") && segment.none(Char::isISOControl)) + } + return normalized +} + +/** Run summary; skipped local items are counted separately from failed operations. */ +internal fun desktopFileSyncRunMessage( + completed: Int, + conflicts: Int, + failures: Int, + unavailableLocalItems: Int, +): String = buildString { + append(completed).append(" sync operation") + if (completed != 1) append('s') + append(" completed.") + if (conflicts > 0) append(' ').append(conflicts).append(" conflicts need review.") + if (failures > 0) append(' ').append(failures).append(" operations failed.") + if (unavailableLocalItems > 0) { + append(' ').append(unavailableLocalItems) + append(if (unavailableLocalItems == 1) " local item was" else " local items were") + append(" skipped because they could not be read or are not regular files.") + } +} + internal fun desktopFileSyncRemoteMutationPath(remoteRootPath: String, relativePath: String): String { val relative = relativePath.trim('/') requireValidSyncPath(relative) diff --git a/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncLocalRecoveryNames.kt b/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncLocalRecoveryNames.kt new file mode 100644 index 000000000..83e62beff --- /dev/null +++ b/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncLocalRecoveryNames.kt @@ -0,0 +1,41 @@ +package dev.obiente.nextcloudnative.app + +import java.nio.file.Path +import java.util.UUID + +/** Local artifacts the desktop sync engine owns while it downloads or replaces a destination. */ +internal enum class DesktopLocalRecoveryKind(val marker: String) { + Download(".nextcloud-native-download-"), + Backup(".nextcloud-native-backup-"), +} + +/** A parsed `.` sibling of [destinationName]. */ +internal data class DesktopLocalRecoveryName( + val kind: DesktopLocalRecoveryKind, + val destinationName: String, + val token: String, +) + +/** The owned sibling name for [destinationName]; [token] must be a UUID so lookalikes are kept. */ +internal fun desktopLocalRecoveryName( + kind: DesktopLocalRecoveryKind, + destinationName: String, + token: String, +): String = ".$destinationName${kind.marker}$token" + +/** The owned sibling path of this destination. */ +internal fun Path.desktopLocalRecoverySibling(kind: DesktopLocalRecoveryKind, token: String): Path = + resolveSibling(desktopLocalRecoveryName(kind, fileName.toString(), token)) + +/** Parses an owned artifact name; user files that only resemble one return null. */ +internal fun parseDesktopLocalRecoveryName(name: String): DesktopLocalRecoveryName? { + if (!name.startsWith('.')) return null + return DesktopLocalRecoveryKind.entries.firstNotNullOfOrNull { kind -> + val markerIndex = name.lastIndexOf(kind.marker) + if (markerIndex <= 1) return@firstNotNullOfOrNull null + val token = name.substring(markerIndex + kind.marker.length) + if (runCatching { UUID.fromString(token) }.isFailure) return@firstNotNullOfOrNull null + val destinationName = name.substring(1, markerIndex) + destinationName.takeIf(String::isNotBlank)?.let { DesktopLocalRecoveryName(kind, it, token) } + } +} diff --git a/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncLocalTree.kt b/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncLocalTree.kt index e3c8f52a8..6cc718d72 100644 --- a/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncLocalTree.kt +++ b/ui/src/desktopMain/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncLocalTree.kt @@ -2,6 +2,7 @@ package dev.obiente.nextcloudnative.app import java.io.File import java.io.FileOutputStream +import java.io.IOException import java.io.InputStream import java.io.OutputStream import java.nio.channels.Channels @@ -11,6 +12,7 @@ import java.nio.file.AtomicMoveNotSupportedException import java.nio.file.FileVisitResult import java.nio.file.Files import java.nio.file.LinkOption +import java.nio.file.NoSuchFileException import java.nio.file.Path import java.nio.file.SimpleFileVisitor import java.nio.file.StandardCopyOption @@ -25,6 +27,12 @@ internal data class DesktopLocalSyncDocument( val path: Path, ) +/** Observed local documents plus items whose current state one scan could not establish. */ +internal data class DesktopLocalSyncScan( + val documents: List, + val unavailable: List, +) + /** Revision-guarded, symlink-rejecting local filesystem adapter for desktop folder sync. */ internal class DesktopFileSyncLocalTree( root: File, @@ -51,11 +59,12 @@ internal class DesktopFileSyncLocalTree( cachedLocalRevisions: Map = emptyMap(), includes: (relativePath: String, kind: SyncEntryKind) -> Boolean = { _, _ -> true }, shouldContinue: () -> Boolean = { true }, - ): List { + ): DesktopLocalSyncScan { requireSafeAncestors(root, includeLeaf = true, allowMissingTail = false) recoverOwnedStagingFiles() preflight(includes, shouldContinue) val result = ArrayList() + val unavailable = ArrayList() var observedFileCount = 0 var observedFileBytes = 0L Files.walkFileTree( @@ -84,34 +93,68 @@ internal class DesktopFileSyncLocalTree( require(!Files.isSymbolicLink(file)) { "Folder sync stopped because ${relative(file)} is a symbolic link." } - require(attrs.isRegularFile) { - "Folder sync stopped because ${relative(file)} is not a regular file." + requireWithinDepthLimit(file, attrs, includes) + if (attrs.isRegularFile) { + if (includes(relative(file), SyncEntryKind.File)) add(file, attrs, SyncEntryKind.File) + } else if (includesAnyKind(file, includes)) { + // A special item may replace a selected folder; withhold it instead of losing it. + report(file, FileSyncLocalUnavailableReason.Unsupported) } - if (includes(relative(file), SyncEntryKind.File)) add(file, attrs, SyncEntryKind.File) return FileVisitResult.CONTINUE } - private fun add(path: Path, attrs: BasicFileAttributes, kind: SyncEntryKind) { - if (kind == SyncEntryKind.File) { - observedFileCount += 1 - observedFileBytes = saturatingAdd(observedFileBytes, attrs.size()) - } - if (result.size >= maximumEntries) { + override fun visitFileFailed(file: Path, exc: IOException): FileVisitResult { + requireRecoverableWalkFailure(file, exc) + if (!shouldContinue()) throw DesktopFileSyncScanStoppedException() + if (includesAnyKind(file, includes)) report(file, exc.unavailableReason()) + return FileVisitResult.CONTINUE + } + + override fun postVisitDirectory(dir: Path, exc: IOException?): FileVisitResult { + if (exc == null) return FileVisitResult.CONTINUE + requireRecoverableWalkFailure(dir, exc) + // A partially listed folder must not make its unlisted children look deleted. + report(dir, FileSyncLocalUnavailableReason.Unreadable) + return FileVisitResult.CONTINUE + } + + private fun report(path: Path, reason: FileSyncLocalUnavailableReason) { + requireEntryCapacity() + unavailable += FileSyncUnavailableLocalItem(relative(path), reason) + } + + private fun requireEntryCapacity() { + val observedEntries = result.size + unavailable.size + if (observedEntries >= maximumEntries) { throw DesktopFileSyncScanLimitException( maximumEntries = maximumEntries, - observedEntries = result.size + 1, + observedEntries = observedEntries + 1, observedFiles = observedFileCount, observedFileBytes = observedFileBytes, ) } + } + + private fun add(path: Path, attrs: BasicFileAttributes, kind: SyncEntryKind) { + if (kind == SyncEntryKind.File) { + observedFileCount += 1 + observedFileBytes = saturatingAdd(observedFileBytes, attrs.size()) + } + requireEntryCapacity() val relative = relative(path) val metadata = metadataDigest(path, attrs) - val contentDigest = path.takeIf { kind == SyncEntryKind.File }?.let { - reusableContentDigest(cachedLocalRevisions[relative], metadata) - ?: run { - requireSafeAncestors(path, includeLeaf = true, allowMissingTail = false) - contentDigester(path, shouldContinue) + var contentDigest: String? = null + if (kind == SyncEntryKind.File) { + contentDigest = reusableContentDigest(cachedLocalRevisions[relative], metadata) + if (contentDigest == null) { + when (val scanned = scannedContentDigest(path, shouldContinue)) { + is ScannedContentDigest.Read -> contentDigest = scanned.value + is ScannedContentDigest.Unavailable -> { + report(path, scanned.reason) + return + } } + } } result += DesktopLocalSyncDocument( LocalSyncEntry( @@ -127,7 +170,50 @@ internal class DesktopFileSyncLocalTree( } }, ) - return result.sortedBy { it.entry.relativePath } + return DesktopLocalSyncScan( + documents = result.sortedBy { it.entry.relativePath }, + unavailable = unavailable.sortedBy(FileSyncUnavailableLocalItem::relativePath), + ) + } + + /** Hashes one listed file, isolating a file that vanishes or stays locked after listing. */ + private fun scannedContentDigest(path: Path, shouldContinue: () -> Boolean): ScannedContentDigest { + // Replaced or linked parent folders still stop the folder; only the leaf itself is isolated. + requireSafeAncestors(path, includeLeaf = false, allowMissingTail = false) + val leaf = try { + Files.readAttributes(path, BasicFileAttributes::class.java, LinkOption.NOFOLLOW_LINKS) + } catch (failure: IOException) { + return ScannedContentDigest.Unavailable(failure.unavailableReason()) + } + require(!leaf.isSymbolicLink) { "Folder sync stopped because ${relative(path)} is a symbolic link." } + if (!leaf.isRegularFile) return ScannedContentDigest.Unavailable(FileSyncLocalUnavailableReason.Vanished) + return try { + ScannedContentDigest.Read(contentDigester(path, shouldContinue)) + } catch (failure: IOException) { + ScannedContentDigest.Unavailable(failure.unavailableReason()) + } + } + + /** An unreadable root, or recovery artifact that may hold the only original, stops the folder. */ + private fun requireRecoverableWalkFailure(path: Path, failure: IOException) { + if (path == root) throw failure + val ancestors = generateSequence(path) { it.parent }.takeWhile { it != root && it.startsWith(root) } + if (ancestors.any(::isOwnedRecoveryPath)) throw IllegalStateException( + "Folder sync stopped because the interrupted-replacement item ${relative(path)} could not be read. " + + "Restore access to it, then sync again.", + failure, + ) + } + + /** True when [path] syncs as a file or as a folder, including a folder above a selected path. */ + private fun includesAnyKind(path: Path, includes: SyncPathFilter): Boolean = + relative(path).let { includes(it, SyncEntryKind.File) || includes(it, SyncEntryKind.Directory) } + + /** walkFileTree passes folders at its depth limit to visitFile; they stop the folder, never vanish. */ + private fun requireWithinDepthLimit(path: Path, attrs: BasicFileAttributes, includes: SyncPathFilter) { + require(!attrs.isDirectory || !includes(relative(path), SyncEntryKind.Directory)) { + "Folder sync stopped because ${relative(path)} is nested more than $MAX_DEPTH folders deep." + } } /** Counts the selected tree before any file content is hashed. */ @@ -162,10 +248,23 @@ internal class DesktopFileSyncLocalTree( require(!Files.isSymbolicLink(file)) { "Folder sync stopped because ${relative(file)} is a symbolic link." } - require(attrs.isRegularFile) { - "Folder sync stopped because ${relative(file)} is not a regular file." + requireWithinDepthLimit(file, attrs, includes) + // Unsupported items are reported per item by the scan; they count toward its bound. + if (attrs.isRegularFile && includes(relative(file), SyncEntryKind.File)) { + accept(fileBytes = attrs.size()) + } else if (!attrs.isRegularFile && includesAnyKind(file, includes)) { + accept(fileBytes = null) } - if (includes(relative(file), SyncEntryKind.File)) accept(attrs.size()) + return FileVisitResult.CONTINUE + } + + override fun visitFileFailed(file: Path, exc: IOException): FileVisitResult { + requireRecoverableWalkFailure(file, exc) + return FileVisitResult.CONTINUE + } + + override fun postVisitDirectory(dir: Path, exc: IOException?): FileVisitResult { + exc?.let { requireRecoverableWalkFailure(dir, it) } return FileVisitResult.CONTINUE } @@ -261,20 +360,25 @@ internal class DesktopFileSyncLocalTree( shouldContinue: () -> Boolean, ): String { require(offset >= 0L && length >= 0 && offset <= expectedBytes - length) - val before = requireNotNull(resolve(relativePath)) { "The local file no longer exists." } - require(before.entry.kind == SyncEntryKind.File && before.entry.revision == expectedLocalRevision) - require(before.entry.size == expectedBytes) + // Unsafe parents still throw IllegalArgumentException; only a changed generation is distinct. + val before = requireLocalRevision(resolve(relativePath), expectedLocalRevision, expectedBytes) val hash = FileChannel.open(before.path, StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS).use { channel -> channel.position(offset) hashExactJvmFileSyncSlice(Channels.newInputStream(channel), length, shouldContinue) } - val after = requireNotNull(resolve(relativePath)) { "The local file disappeared during verification." } - require(after.entry.revision == expectedLocalRevision && after.entry.size == expectedBytes) { - "The local file changed during content verification." - } + requireLocalRevision(resolve(relativePath), expectedLocalRevision, expectedBytes) return hash } + private fun requireLocalRevision( + current: DesktopLocalSyncDocument?, + expectedLocalRevision: String, + expectedBytes: Long, + ): DesktopLocalSyncDocument = current?.takeIf { + it.entry.kind == SyncEntryKind.File && it.entry.revision == expectedLocalRevision && + it.entry.size == expectedBytes + } ?: throw DesktopFileSyncLocalRevisionChangedException() + fun createDirectory(relativePath: String, expectedLocalRevision: String?) { val current = resolve(relativePath) if (expectedLocalRevision == null) { @@ -330,10 +434,10 @@ internal class DesktopFileSyncLocalTree( require(current.entry.kind == SyncEntryKind.File) { "The local item type changed after the sync scan." } - val parent = requireNotNull(destination.parent) + requireNotNull(destination.parent) requireSafeAncestors(destination, includeLeaf = false, allowMissingTail = false) val token = UUID.randomUUID().toString() - val backup = parent.resolve(".${destination.fileName}.nextcloud-native-backup-$token") + val backup = destination.desktopLocalRecoverySibling(DesktopLocalRecoveryKind.Backup, token) var protected = false try { requireSafeAncestors(destination, includeLeaf = true, allowMissingTail = false) @@ -358,12 +462,12 @@ internal class DesktopFileSyncLocalTree( current: DesktopLocalSyncDocument?, source: File, ): LocalSyncEntry { - val parent = requireNotNull(destination.parent) + requireNotNull(destination.parent) requireSafeAncestors(destination, includeLeaf = false, allowMissingTail = false) val expectedContentHash = "sha256:${contentDigester(source.toPath()) { true }}" val token = UUID.randomUUID().toString() - val staged = parent.resolve(".${destination.fileName}.nextcloud-native-download-$token") - val backup = parent.resolve(".${destination.fileName}.nextcloud-native-backup-$token") + val staged = destination.desktopLocalRecoverySibling(DesktopLocalRecoveryKind.Download, token) + val backup = destination.desktopLocalRecoverySibling(DesktopLocalRecoveryKind.Backup, token) Files.copy(source.toPath(), staged, StandardCopyOption.REPLACE_EXISTING) requireSafeAncestors(staged, includeLeaf = true, allowMissingTail = false) FileChannel.open(staged, setOf(StandardOpenOption.WRITE, LinkOption.NOFOLLOW_LINKS)).use { it.force(true) } @@ -415,7 +519,7 @@ internal class DesktopFileSyncLocalTree( object : SimpleFileVisitor() { override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult { val owned = ownedRecoveryPath(dir) - if (dir == root || owned?.kind != OwnedRecoveryKind.Backup) { + if (dir == root || owned?.kind != DesktopLocalRecoveryKind.Backup) { return FileVisitResult.CONTINUE } reconcileOwnedBackup(dir) @@ -428,23 +532,34 @@ internal class DesktopFileSyncLocalTree( override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult { when (ownedRecoveryPath(file)?.kind) { - OwnedRecoveryKind.Download -> Files.deleteIfExists(file) - OwnedRecoveryKind.Backup -> reconcileOwnedBackup(file) + DesktopLocalRecoveryKind.Download -> Files.deleteIfExists(file) + DesktopLocalRecoveryKind.Backup -> reconcileOwnedBackup(file) null -> Unit } return FileVisitResult.CONTINUE } + + // Ordinary unreadable items are reported by the scan; owned recovery items fail closed. + override fun visitFileFailed(file: Path, exc: IOException): FileVisitResult { + requireRecoverableWalkFailure(file, exc) + return FileVisitResult.CONTINUE + } + + override fun postVisitDirectory(dir: Path, exc: IOException?): FileVisitResult { + exc?.let { requireRecoverableWalkFailure(dir, it) } + return FileVisitResult.CONTINUE + } }, ) } private fun reconcileOwnedBackup(backup: Path) { - val owned = ownedRecoveryPath(backup)?.takeIf { it.kind == OwnedRecoveryKind.Backup } ?: return + val owned = ownedRecoveryPath(backup)?.takeIf { it.kind == DesktopLocalRecoveryKind.Backup } ?: return val finalPath = requireNotNull(backup.parent).resolve(owned.destinationName) if (Files.exists(finalPath, LinkOption.NOFOLLOW_LINKS)) { requireSafeAncestors(finalPath, includeLeaf = true, allowMissingTail = false) val incompleteDownload = backup.parent.resolve( - ".${owned.destinationName}$DOWNLOAD_MARKER${owned.token}", + desktopLocalRecoveryName(DesktopLocalRecoveryKind.Download, owned.destinationName, owned.token), ) if (!Files.exists(incompleteDownload, LinkOption.NOFOLLOW_LINKS)) deleteOwnedPath(backup) } else { @@ -452,27 +567,13 @@ internal class DesktopFileSyncLocalTree( } } - private fun ownedRecoveryPath(path: Path): OwnedRecoveryPath? { - val name = path.fileName.toString() - if (!name.startsWith('.')) return null - val candidates = listOf( - OwnedRecoveryKind.Download to DOWNLOAD_MARKER, - OwnedRecoveryKind.Backup to BACKUP_MARKER, - ) - return candidates.firstNotNullOfOrNull { (kind, marker) -> - val markerIndex = name.lastIndexOf(marker) - if (markerIndex <= 1) return@firstNotNullOfOrNull null - val token = name.substring(markerIndex + marker.length) - if (runCatching { UUID.fromString(token) }.isFailure) return@firstNotNullOfOrNull null - val destinationName = name.substring(1, markerIndex) - destinationName.takeIf(String::isNotBlank)?.let { OwnedRecoveryPath(kind, it, token) } - } - } + private fun ownedRecoveryPath(path: Path): DesktopLocalRecoveryName? = + parseDesktopLocalRecoveryName(path.fileName.toString()) private fun isOwnedRecoveryPath(path: Path): Boolean = ownedRecoveryPath(path) != null private fun deleteOwnedPath(path: Path) { - require(path.startsWith(root) && ownedRecoveryPath(path)?.kind == OwnedRecoveryKind.Backup) + require(path.startsWith(root) && ownedRecoveryPath(path)?.kind == DesktopLocalRecoveryKind.Backup) requireSafeAncestors(path, includeLeaf = true, allowMissingTail = false) Files.walkFileTree( path, @@ -640,13 +741,22 @@ internal class DesktopFileSyncLocalTree( const val MAX_ENTRIES = MAX_FILE_SYNC_ENTRIES const val MAX_DEPTH = 64 const val BUFFER_BYTES = 64 * 1024 - const val DOWNLOAD_MARKER = ".nextcloud-native-download-" - const val BACKUP_MARKER = ".nextcloud-native-backup-" } } private data class LocalMetadataDigest(val value: String, val reusable: Boolean) +private typealias SyncPathFilter = (relativePath: String, kind: SyncEntryKind) -> Boolean + +private sealed interface ScannedContentDigest { + data class Read(val value: String) : ScannedContentDigest + data class Unavailable(val reason: FileSyncLocalUnavailableReason) : ScannedContentDigest +} + +private fun IOException.unavailableReason(): FileSyncLocalUnavailableReason = + if (this is NoSuchFileException) FileSyncLocalUnavailableReason.Vanished + else FileSyncLocalUnavailableReason.Unreadable + private data class LocalDirectoryIdentity( val fileKey: String?, val creationTimeMillis: Long, @@ -671,11 +781,3 @@ private fun desktopSha256File(path: Path, shouldContinue: () -> Boolean): String } return digest.digest().joinToString("") { "%02x".format(it) } } - -private enum class OwnedRecoveryKind { Download, Backup } - -private data class OwnedRecoveryPath( - val kind: OwnedRecoveryKind, - val destinationName: String, - val token: String, -) diff --git a/ui/src/desktopTest/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncContentSliceSafetyTest.kt b/ui/src/desktopTest/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncContentSliceSafetyTest.kt new file mode 100644 index 000000000..03d61d9f7 --- /dev/null +++ b/ui/src/desktopTest/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncContentSliceSafetyTest.kt @@ -0,0 +1,69 @@ +package dev.obiente.nextcloudnative.app + +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.attribute.BasicFileAttributeView +import java.nio.file.attribute.FileTime +import kotlin.io.path.createDirectories +import kotlin.io.path.writeText +import kotlin.test.Test +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertNull + +/** Local content verification isolates changed generations but never unsafe folder hierarchies. */ +class DesktopFileSyncContentSliceSafetyTest { + @Test + fun `a git index rewritten after the scan leaves its candidate unverified`() = withScannedIndex { root, tree, slice, hash -> + root.resolve(".git/refs/heads/main").writeText("DIRC rewritten") // Same size, new generation. + + assertNull(verifyDesktopFileSyncContentSlice(slice, tree, unusedRemote(), hash) { true }) + } + + @Test + fun `a parent folder replaced after the scan still stops verification`() = withScannedIndex { root, tree, slice, hash -> + val heads = root.resolve(".git/refs/heads") + Files.move(heads, heads.resolveSibling("heads-original")) + heads.createDirectories() + Files.getFileAttributeView(heads, BasicFileAttributeView::class.java) + .setTimes(null, null, FileTime.fromMillis(1_000_000_000_000L)) + heads.resolve("main").writeText("DIRC synthetic") + + val failure = assertFailsWith { + verifyDesktopFileSyncContentSlice(slice, tree, unusedRemote(), hash) { true } + } + + assertFalse(failure is DesktopFileSyncLocalRevisionChangedException, failure.message) + } + + private fun withScannedIndex( + block: (Path, DesktopFileSyncLocalTree, FileSyncContentVerificationSlice, String) -> Unit, + ) { + val root = Files.createTempDirectory("desktop-sync-slice-safety-") + try { + root.resolve(".git/refs/heads").createDirectories() + root.resolve(".git/refs/heads/main").writeText("DIRC synthetic") + val tree = DesktopFileSyncLocalTree(root.toFile()) + val scanned = tree.scan().documents.single { it.entry.relativePath == ".git/refs/heads/main" }.entry + val size = requireNotNull(scanned.size) + val candidate = FileSyncContentVerificationCandidate(scanned.relativePath, scanned.revision, "etag", size) + val slice = FileSyncContentVerificationSlice( + candidate, + offset = 0L, + length = size.toInt(), + aggregateHash = EMPTY_FILE_SYNC_IDENTITY_AGGREGATE, + ) + block(root, tree, slice, requireNotNull(scanned.contentHash)) + } finally { + root.toFile().deleteRecursively() + } + } + + /** Both scenarios end before any server read; a request here would fail the test. */ + private fun unusedRemote() = DesktopFileSyncRemoteTree( + NextcloudSession("https://cloud.example.test", "alice", "secret"), + "alice", + "Repo", + okhttp3.OkHttpClient.Builder().addInterceptor { error("Unexpected server request") }.build(), + ) +} diff --git a/ui/src/desktopTest/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncFakeDav.kt b/ui/src/desktopTest/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncFakeDav.kt new file mode 100644 index 000000000..ebbb0b4d3 --- /dev/null +++ b/ui/src/desktopTest/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncFakeDav.kt @@ -0,0 +1,124 @@ +package dev.obiente.nextcloudnative.app + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicLong +import mockwebserver3.Dispatcher +import mockwebserver3.MockResponse +import mockwebserver3.RecordedRequest +import okio.Buffer + +/** + * Minimal in-memory WebDAV account used by desktop sync engine tests. + * + * It implements only the guarded verbs the desktop executor issues for creates and replacements: + * PROPFIND (depth 0 and 1), MKCOL, conditional PUT, and conditional or ranged GET. Every path is + * synthetic and relative to the account file root. + */ +internal class DesktopFileSyncFakeDav( + private val userId: String, + rootDirectories: List, + private val onPut: (path: String) -> Unit = {}, +) : Dispatcher() { + private data class Node(val directory: Boolean, val bytes: ByteArray, val etag: String) + + private val nodes = ConcurrentHashMap() + private val etags = AtomicLong() + private val prefix = listOf("remote.php", "dav", "files", userId) + private val methods = java.util.Collections.synchronizedList(ArrayList()) + + init { + rootDirectories.forEach { nodes[it] = Node(true, ByteArray(0), nextEtag()) } + } + + fun files(): Map = + nodes.filterValues { !it.directory }.mapValues { (_, node) -> node.bytes } + + fun directories(): Set = nodes.filterValues(Node::directory).keys + + /** Every received HTTP method, in arrival order; unsupported verbs such as DELETE answer 405. */ + fun requestMethods(): List = synchronized(methods) { methods.toList() } + + override fun dispatch(request: RecordedRequest): MockResponse { + methods += request.method.toString() + val segments = request.url.pathSegments.filter(String::isNotEmpty) + if (segments.size < prefix.size || segments.take(prefix.size) != prefix) return status(404) + val path = segments.drop(prefix.size).joinToString("/") + return synchronized(this) { + when (request.method) { + "PROPFIND" -> propfind(path, request.headers["Depth"] ?: "1") + "MKCOL" -> mkcol(path) + "PUT" -> put(path, request) + "GET" -> get(path, request) + else -> status(405) + } + } + } + + /** Paths whose PROPFIND answers HTTP 503, simulating a temporarily unlistable server folder. */ + val failingListings: MutableSet = ConcurrentHashMap.newKeySet() + + private fun propfind(path: String, depth: String): MockResponse { + if (path in failingListings) return status(503) + val node = nodes[path] ?: return status(404) + val children = if (node.directory && depth != "0") { + nodes.keys.filter { it.substringBeforeLast('/', "") == path && it != path }.sorted() + } else { + emptyList() + } + val body = buildString { + append("") + (listOf(path) + children).forEach { item -> append(davResponse(item, nodes.getValue(item))) } + append("") + } + return MockResponse.Builder().code(207).addHeader("Content-Type", "application/xml").body(body).build() + } + + private fun davResponse(path: String, node: Node): String { + // Synthetic test names are URL-safe, so hrefs need no percent-encoding. + val href = "/" + (prefix + path.split('/')).joinToString("/") + if (node.directory) "/" else "" + val type = if (node.directory) "" else "" + val length = if (node.directory) "" else "${node.bytes.size}" + val permissions = if (node.directory) "RGDNVCK" else "" + return "$href" + + "${node.etag}$length$type$permissions" + + "HTTP/1.1 200 OK" + } + + private fun mkcol(path: String): MockResponse { + if (nodes.containsKey(path)) return status(405) + if (nodes[path.substringBeforeLast('/', "")]?.directory != true) return status(409) + nodes[path] = Node(true, ByteArray(0), nextEtag()) + return status(201) + } + + private fun put(path: String, request: RecordedRequest): MockResponse { + val current = nodes[path] + if (request.headers["If-None-Match"] == "*" && current != null) return status(412) + request.headers["If-Match"]?.let { expected -> if (current?.etag != expected) return status(412) } + if (current?.directory == true) return status(409) + if (nodes[path.substringBeforeLast('/', "")]?.directory != true) return status(409) + val etag = nextEtag() + nodes[path] = Node(false, request.body?.toByteArray() ?: ByteArray(0), etag) + onPut(path) + return MockResponse.Builder().code(if (current == null) 201 else 204).addHeader("ETag", etag).build() + } + + private fun get(path: String, request: RecordedRequest): MockResponse { + val node = nodes[path]?.takeIf { !it.directory } ?: return status(404) + request.headers["If-Match"]?.let { expected -> if (node.etag != expected) return status(412) } + val range = request.headers["Range"]?.removePrefix("bytes=")?.split('-') + if (range != null) { + val start = range[0].toInt() + val end = range[1].toInt() + return MockResponse.Builder().code(206).addHeader("ETag", node.etag) + .addHeader("Content-Range", "bytes $start-$end/${node.bytes.size}") + .body(Buffer().write(node.bytes.copyOfRange(start, end + 1))).build() + } + return MockResponse.Builder().code(200).addHeader("ETag", node.etag) + .body(Buffer().write(node.bytes)).build() + } + + private fun status(code: Int): MockResponse = MockResponse.Builder().code(code).build() + + private fun nextEtag(): String = "\"etag-${etags.incrementAndGet()}\"" +} diff --git a/ui/src/desktopTest/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncGitRepositoryTest.kt b/ui/src/desktopTest/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncGitRepositoryTest.kt new file mode 100644 index 000000000..41a2a5dc1 --- /dev/null +++ b/ui/src/desktopTest/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncGitRepositoryTest.kt @@ -0,0 +1,272 @@ +package dev.obiente.nextcloudnative.app + +import java.io.File +import java.io.RandomAccessFile +import java.net.StandardProtocolFamily +import java.net.UnixDomainSocketAddress +import java.nio.channels.ServerSocketChannel +import java.nio.file.Files +import java.nio.file.Path +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertIs +import kotlin.test.assertTrue +import kotlinx.coroutines.runBlocking +import mockwebserver3.MockWebServer + +/** Synthetic Git working trees exercise the complete desktop scan, plan, upload, and verify path. */ +class DesktopFileSyncGitRepositoryTest { + @Test + fun `a folder with git metadata uploads every regular item and stays stable`() = withRepository { fixture -> + val result = fixture.run() + + assertIs(result, result.toString()) + assertTrue(fixture.dav.directories().containsAll(listOf("Repo/.git", "Repo/.git/refs/heads"))) + fixture.expectedFiles.forEach { relative -> + assertContentEquals( + fixture.local.resolve(relative).readBytes(), + fixture.dav.files()["Repo/$relative"], + relative, + ) + } + assertEquals(0, fixture.workItems().size) + + val second = fixture.run() + + assertIs(second) + assertEquals("0 sync operations completed.", second.message) + } + + @Test + fun `an unsupported item inside git metadata is reported without stopping the folder`() = + withRepository { fixture -> + val socketPath = fixture.local.resolve(".git/fsmonitor--daemon.ipc").toPath() + val socket = runCatching { + ServerSocketChannel.open(StandardProtocolFamily.UNIX).bind(UnixDomainSocketAddress.of(socketPath)) + }.getOrNull() ?: return@withRepository // The platform cannot create a socket file. + socket.use { + if (Files.isRegularFile(socketPath)) return@withRepository + + val result = fixture.run() + + assertIs(result, result.toString()) + assertTrue(result.message.contains("1 local item was skipped"), result.message) + val skipped = fixture.workItems().single() + assertEquals(".git/fsmonitor--daemon.ipc", skipped.relativePath) + assertEquals(FileSyncExecutionState.Skipped, skipped.state) + assertTrue( + (skipped.operation as FileSyncOperation.Skipped).reason + .contains(".git/fsmonitor--daemon.ipc is not a regular file or folder"), + ) + assertFalse("Repo/.git/fsmonitor--daemon.ipc" in fixture.dav.files()) + fixture.expectedFiles.forEach { relative -> assertTrue("Repo/$relative" in fixture.dav.files()) } + } + Files.deleteIfExists(socketPath) + } + + @Test + fun `a git file locked by another handle is skipped and synced once released`() = withRepository { fixture -> + // Windows byte-range locks are mandatory; other platforms only offer advisory locks. + if (!System.getProperty("os.name").startsWith("Windows")) return@withRepository + val lockFile = fixture.local.resolve(".git/index.lock").apply { writeText("pending index") } + + val result = RandomAccessFile(lockFile, "rw").use { handle -> + handle.channel.lock().use { fixture.run() } + } + + assertIs(result, result.toString()) + val skipped = fixture.workItems().single() + assertEquals(".git/index.lock", skipped.relativePath) + assertTrue((skipped.operation as FileSyncOperation.Skipped).reason.contains("could not be read")) + assertFalse("Repo/.git/index.lock" in fixture.dav.files()) + fixture.expectedFiles.forEach { relative -> assertTrue("Repo/$relative" in fixture.dav.files()) } + + val released = fixture.run() + + assertIs(released, released.toString()) + assertEquals(0, fixture.workItems().size) + assertEquals("pending index", fixture.dav.files()["Repo/.git/index.lock"]?.decodeToString()) + } + + @Test + fun `a git file rewritten during upload keeps the uploaded generation and syncs the rewrite next`() { + var rewrite: (() -> Unit)? = null + withRepository(onPut = { path -> if (path == "Repo/.git/COMMIT_EDITMSG") rewrite?.invoke() }) { fixture -> + val message = fixture.local.resolve(".git/COMMIT_EDITMSG") + rewrite = { + rewrite = null + message.writeText("fix: amended message\n") + } + + val first = fixture.run() + + assertIs(first, first.toString()) + assertEquals("fix: synthetic change\n", fixture.dav.files()["Repo/.git/COMMIT_EDITMSG"]?.decodeToString()) + + val second = fixture.run() + + assertIs(second, second.toString()) + assertEquals("1 sync operation completed.", second.message) + assertEquals("fix: amended message\n", fixture.dav.files()["Repo/.git/COMMIT_EDITMSG"]?.decodeToString()) + } + } + + @Test + fun `an unreadable interrupted replacement backup never propagates a deletion`() = withRepository( + configuration = FileSyncConfiguration( + deviceLabel = "Workstation", + deletionPolicy = FileSyncDeletionPolicy.Propagate, + ), + ) { fixture -> + assertIs(fixture.run()) + // A crash during a folder replacement leaves the only local original in an owned backup. + val refs = fixture.local.resolve(".git/refs").toPath() + val backup = refs.resolveSibling(".refs.nextcloud-native-backup-4d6f8828-7d52-4f2d-945b-f46aa4c97b41") + Files.move(refs, backup) + val restore = denyDesktopTestDirectoryListing(backup) ?: return@withRepository + try { + val blocked = runCatching { fixture.run() }.exceptionOrNull() + + assertIs(blocked, blocked.toString()) + assertFalse("DELETE" in fixture.dav.requestMethods()) + assertTrue("Repo/.git/refs/heads/main" in fixture.dav.files()) + } finally { + restore() + } + + val recovered = fixture.run() + + assertIs(recovered, recovered.toString()) + assertEquals("0 sync operations completed.", recovered.message) + assertTrue(fixture.local.resolve(".git/refs/heads/main").isFile) + assertFalse("DELETE" in fixture.dav.requestMethods()) + } + + @Test + fun `a special item replacing a selected folder never propagates a deletion`() = withRepository( + configuration = FileSyncConfiguration( + deviceLabel = "Workstation", + deletionPolicy = FileSyncDeletionPolicy.Propagate, + selectedPaths = listOf(".git/refs/heads/main"), + ), + ) { fixture -> + assertIs(fixture.run()) + assertTrue("Repo/.git/refs/heads/main" in fixture.dav.files()) + val refs = fixture.local.resolve(".git/refs") + refs.deleteRecursively() + val socket = runCatching { + ServerSocketChannel.open(StandardProtocolFamily.UNIX).bind(UnixDomainSocketAddress.of(refs.toPath())) + }.getOrNull() ?: return@withRepository // The platform cannot create a socket file. + socket.use { + if (Files.isRegularFile(refs.toPath())) return@withRepository + + val result = fixture.run() + + assertIs(result, result.toString()) + val skipped = fixture.workItems().single() + assertEquals(".git/refs", skipped.relativePath) + assertTrue((skipped.operation as FileSyncOperation.Skipped).reason.contains("not a regular file")) + assertFalse("DELETE" in fixture.dav.requestMethods()) + assertTrue("Repo/.git/refs/heads/main" in fixture.dav.files()) + } + Files.deleteIfExists(refs.toPath()) + } + + @Test + fun `an unlistable server folder below an unreadable local folder is not traversed`() = withRepository( + configuration = FileSyncConfiguration( + deviceLabel = "Workstation", + deletionPolicy = FileSyncDeletionPolicy.Propagate, + ), + ) { fixture -> + assertIs(fixture.run()) + val refs = fixture.local.resolve(".git/refs").toPath() + val restore = denyDesktopTestDirectoryListing(refs) ?: return@withRepository + fixture.dav.failingListings += "Repo/.git/refs" + try { + val result = fixture.run() + + assertIs(result, result.toString()) + assertEquals(".git/refs", fixture.workItems().single().relativePath) + assertFalse("DELETE" in fixture.dav.requestMethods()) + } finally { + restore() + } + } + + private class Fixture( + val dav: DesktopFileSyncFakeDav, + val local: File, + val expectedFiles: List, + private val store: DesktopFileSyncStore, + private val engine: DesktopFileSyncEngine, + private val session: NextcloudSession, + ) { + fun run(): FileSyncCenterActionResult = runBlocking { engine.runPair(session, "alice", PAIR_ID) } + + fun workItems(): List = store.loadPair(PAIR_ID).coordinator.pairs.single().workItems + } + + private fun withRepository( + onPut: (String) -> Unit = {}, + configuration: FileSyncConfiguration = FileSyncConfiguration(deviceLabel = "Workstation"), + block: (Fixture) -> Unit, + ) { + val directory = Files.createTempDirectory("desktop-sync-git-").toFile() + val dav = DesktopFileSyncFakeDav("alice", listOf("Repo"), onPut) + try { + MockWebServer().use { server -> + server.dispatcher = dav + server.start() + val local = directory.resolve("Repo").apply { mkdirs() } + val files = linkedMapOf( + "README.md" to "# Synthetic repository\n".encodeToByteArray(), + ".gitignore" to "build/\n".encodeToByteArray(), + ".git/HEAD" to "ref: refs/heads/main\n".encodeToByteArray(), + ".git/COMMIT_EDITMSG" to "fix: synthetic change\n".encodeToByteArray(), + ".git/config" to "[core]\n\trepositoryformatversion = 0\n".encodeToByteArray(), + ".git/refs/heads/main" to "0123456789abcdef0123456789abcdef01234567\n".encodeToByteArray(), + ".git/objects/ab/cdef0123" to ByteArray(512) { index -> (index * 31).toByte() }, + ) + files.forEach { (relative, bytes) -> + local.resolve(relative).apply { parentFile.mkdirs() }.writeBytes(bytes) + } + markHidden(local.resolve(".git").toPath()) + val session = NextcloudSession(server.url("/").toString(), "alice", "secret") + val store = DesktopFileSyncStore(directory.resolve("state.db"), legacyStateFile = null) + store.savePair( + DesktopFileSyncPersistedState( + coordinator = FileSyncCoordinatorState( + listOf( + FileSyncPair( + id = PAIR_ID, + accountId = desktopFileCacheAccountId(session), + localRootId = "root", + remoteRootPath = "Repo", + configuration = configuration, + ), + ), + ), + roots = listOf(DesktopFileSyncRootRecord("root", local.absolutePath, "Repo")), + ), + PAIR_ID, + ) + val engine = DesktopFileSyncEngine(store, directory.resolve("staging")) + block(Fixture(dav, local, files.keys.toList(), store, engine, session)) + } + } finally { + directory.deleteRecursively() + } + } + + /** Git for Windows marks `.git` hidden; the attribute must not change what syncs. */ + private fun markHidden(path: Path) { + runCatching { Files.setAttribute(path, "dos:hidden", true) } + } + + private companion object { + const val PAIR_ID = "pair" + } +} diff --git a/ui/src/desktopTest/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncListingDenial.kt b/ui/src/desktopTest/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncListingDenial.kt new file mode 100644 index 000000000..cd7341c08 --- /dev/null +++ b/ui/src/desktopTest/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncListingDenial.kt @@ -0,0 +1,45 @@ +package dev.obiente.nextcloudnative.app + +import java.nio.file.Files +import java.nio.file.LinkOption +import java.nio.file.Path +import java.nio.file.attribute.AclEntry +import java.nio.file.attribute.AclEntryPermission +import java.nio.file.attribute.AclEntryType +import java.nio.file.attribute.AclFileAttributeView +import java.nio.file.attribute.PosixFilePermissions + +/** + * Makes [directory] unlistable for the current user through an NTFS deny entry or POSIX mode. + * + * Returns the action that restores the original access, or null when this platform or user (for + * example a privileged account) still lists the directory, so callers can skip the scenario. + */ +internal fun denyDesktopTestDirectoryListing(directory: Path): (() -> Unit)? { + val restore = runCatching { + val acl = Files.getFileAttributeView(directory, AclFileAttributeView::class.java, LinkOption.NOFOLLOW_LINKS) + if (acl != null) denyAclListing(directory, acl) else denyPosixListing(directory) + }.getOrNull() ?: return null + val denied = runCatching { Files.newDirectoryStream(directory).use { it.iterator().hasNext() } }.isFailure + if (!denied) restore() + return restore.takeIf { denied } +} + +private fun denyAclListing(directory: Path, acl: AclFileAttributeView): () -> Unit { + val original = acl.acl + val user = directory.fileSystem.userPrincipalLookupService + .lookupPrincipalByName(System.getProperty("user.name")) + val deny = AclEntry.newBuilder() + .setType(AclEntryType.DENY) + .setPrincipal(user) + .setPermissions(AclEntryPermission.LIST_DIRECTORY) + .build() + acl.acl = listOf(deny) + original + return { acl.acl = original } +} + +private fun denyPosixListing(directory: Path): () -> Unit { + val original = Files.getPosixFilePermissions(directory, LinkOption.NOFOLLOW_LINKS) + Files.setPosixFilePermissions(directory, PosixFilePermissions.fromString("-wx------")) + return { Files.setPosixFilePermissions(directory, original) } +} diff --git a/ui/src/desktopTest/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncLocalAvailabilityTest.kt b/ui/src/desktopTest/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncLocalAvailabilityTest.kt new file mode 100644 index 000000000..2c6ab30f3 --- /dev/null +++ b/ui/src/desktopTest/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncLocalAvailabilityTest.kt @@ -0,0 +1,322 @@ +package dev.obiente.nextcloudnative.app + +import java.io.RandomAccessFile +import java.net.StandardProtocolFamily +import java.net.UnixDomainSocketAddress +import java.nio.channels.ServerSocketChannel +import java.nio.file.FileSystemException +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.attribute.BasicFileAttributeView +import java.nio.file.attribute.FileTime +import kotlin.io.path.createDirectories +import kotlin.io.path.writeText +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertIs +import kotlin.test.assertTrue + +private typealias SyncPathFilterForTest = (relativePath: String, kind: SyncEntryKind) -> Boolean + +/** Local scans keep folder sync running when individual Git working-tree items cannot be read. */ +class DesktopFileSyncLocalAvailabilityTest { + @Test + fun `git metadata with dot folders hidden attributes and extensionless names is scanned`() = withTree { root -> + writeGitTree(root) + runCatching { Files.setAttribute(root.resolve(".git"), "dos:hidden", true) } + + val scan = DesktopFileSyncLocalTree(root.toFile()).scan() + + assertEquals(emptyList(), scan.unavailable) + assertEquals( + listOf( + ".git", ".git/COMMIT_EDITMSG", ".git/HEAD", ".git/ORIG_HEAD", ".git/index", ".git/refs", + ".git/refs/heads", ".git/refs/heads/main", ".gitignore", "README.md", + ), + scan.documents.map { it.entry.relativePath }, + ) + } + + @Test + fun `a git lock file that disappears after listing is reported without stopping the scan`() = withTree { root -> + writeGitTree(root) + root.resolve(".git/index.lock").writeText("transient") + val tree = DesktopFileSyncLocalTree(root.toFile()) { path, _ -> + if (path.fileName.toString() == "index.lock") { + Files.delete(path) // Git renamed the lock into place after the directory was listed. + Files.newByteChannel(path).use { } + } + "a".repeat(64) + } + + val scan = tree.scan() + + assertEquals( + listOf(FileSyncUnavailableLocalItem(".git/index.lock", FileSyncLocalUnavailableReason.Vanished)), + scan.unavailable, + ) + assertTrue(scan.documents.any { it.entry.relativePath == ".git/COMMIT_EDITMSG" }) + assertTrue(scan.documents.none { it.entry.relativePath == ".git/index.lock" }) + } + + @Test + fun `a git file held open by another process is reported unreadable`() = withTree { root -> + writeGitTree(root) + val tree = DesktopFileSyncLocalTree(root.toFile()) { path, _ -> + if (path.fileName.toString() == "COMMIT_EDITMSG") { + throw FileSystemException(path.toString(), null, "The file is being used by another process.") + } + "b".repeat(64) + } + + val scan = tree.scan() + + assertEquals( + listOf(FileSyncUnavailableLocalItem(".git/COMMIT_EDITMSG", FileSyncLocalUnavailableReason.Unreadable)), + scan.unavailable, + ) + assertTrue(scan.documents.any { it.entry.relativePath == ".git/HEAD" }) + } + + @Test + fun `a mandatory byte range lock is reported unreadable on windows`() = withTree { root -> + if (!System.getProperty("os.name").startsWith("Windows")) return@withTree + writeGitTree(root) + val index = root.resolve(".git/index").toFile() + + val scan = RandomAccessFile(index, "rw").use { handle -> + handle.channel.lock().use { DesktopFileSyncLocalTree(root.toFile()).scan() } + } + + assertEquals( + listOf(FileSyncUnavailableLocalItem(".git/index", FileSyncLocalUnavailableReason.Unreadable)), + scan.unavailable, + ) + } + + @Test + fun `special files are reported unless an ignore rule excludes them`() = withTree { root -> + writeGitTree(root) + val socketPath = root.resolve(".git/fsmonitor--daemon.ipc") + val socket = runCatching { + ServerSocketChannel.open(StandardProtocolFamily.UNIX).bind(UnixDomainSocketAddress.of(socketPath)) + }.getOrNull() ?: return@withTree + socket.use { + if (Files.isRegularFile(socketPath)) return@withTree + val ignoring = FileSyncConfiguration(deviceLabel = "Desktop", ignoredPatterns = listOf("*.ipc")) + + val reported = DesktopFileSyncLocalTree(root.toFile()).scan() + val ignored = DesktopFileSyncLocalTree(root.toFile()).scan(includes = ignoring::includesSyncPath) + + assertEquals( + listOf( + FileSyncUnavailableLocalItem( + ".git/fsmonitor--daemon.ipc", + FileSyncLocalUnavailableReason.Unsupported, + ), + ), + reported.unavailable, + ) + assertEquals(emptyList(), ignored.unavailable) + } + Files.deleteIfExists(socketPath) + } + + @Test + fun `an unreadable git folder is reported once without listing its children`() = withTree { root -> + writeGitTree(root) + val objects = root.resolve(".git/objects").createDirectories() + objects.resolve("pack").createDirectories() + val restore = denyDesktopTestDirectoryListing(objects) ?: return@withTree + try { + val scan = DesktopFileSyncLocalTree(root.toFile()).scan() + + assertEquals( + listOf(FileSyncUnavailableLocalItem(".git/objects", FileSyncLocalUnavailableReason.Unreadable)), + scan.unavailable, + ) + assertTrue(scan.documents.none { it.entry.relativePath.startsWith(".git/objects") }) + } finally { + restore() + } + } + + @Test + fun `an unreadable sync root still stops the scan`() = withTree { root -> + writeGitTree(root) + val restore = denyDesktopTestDirectoryListing(root) ?: return@withTree + try { + assertFailsWith { DesktopFileSyncLocalTree(root.toFile()).scan() } + } finally { + restore() + } + } + + @Test + fun `an unreadable interrupted replacement backup stops the scan until it is recoverable`() = withTree { root -> + val backup = root.resolve("Notes/.today.nextcloud-native-backup-4d6f8828-7d52-4f2d-945b-f46aa4c97b41") + backup.createDirectories() + backup.resolve("page.md").writeText("only original") + val restore = denyDesktopTestDirectoryListing(backup) ?: return@withTree + try { + val failure = assertFailsWith { DesktopFileSyncLocalTree(root.toFile()).scan() } + assertTrue(failure.message.orEmpty().contains("interrupted-replacement item Notes/.today.")) + assertTrue(Files.isDirectory(backup)) + } finally { + restore() + } + + val recovered = DesktopFileSyncLocalTree(root.toFile()).scan() + + assertEquals(emptyList(), recovered.unavailable) + assertTrue(recovered.documents.any { it.entry.relativePath == "Notes/today/page.md" }) + assertEquals("only original", root.resolve("Notes/today/page.md").toFile().readText()) + } + + @Test + fun `a folder at the walk depth limit stops the scan unless it is ignored`() = withTree { root -> + val levels = List(64) { "d" } + val deepest = root.resolve(levels.joinToString("/")).createDirectories() + deepest.resolve("x.txt").writeText("deep") + root.resolve("keep.txt").writeText("keep") + val selected = FileSyncConfiguration( + deviceLabel = "Desktop", + selectedPaths = listOf((levels + "x.txt").joinToString("/")), + ) + val ignored = FileSyncConfiguration(deviceLabel = "Desktop", ignoredPatterns = listOf("d/d/d")) + + listOf({ _, _ -> true }, selected::includesSyncPath).forEach { includes -> + val failure = assertFailsWith { + DesktopFileSyncLocalTree(root.toFile()).scan(includes = includes) + } + assertTrue(failure.message.orEmpty().contains("nested more than 64 folders deep")) + } + val scan = DesktopFileSyncLocalTree(root.toFile()).scan(includes = ignored::includesSyncPath) + assertEquals(listOf("d", "d/d", "keep.txt"), scan.documents.map { it.entry.relativePath }) + assertEquals(emptyList(), scan.unavailable) + } + + @Test + fun `a leaf that vanishes during ancestor validation is reported as unavailable`() = withTree { root -> + writeGitTree(root) + val lock = root.resolve(".git/index.lock").apply { writeText("transient") } + val tree = DesktopFileSyncLocalTree(root.toFile(), changeTokenProvider = { path -> + if (path == lock) Files.deleteIfExists(path) // Runs after listing, before leaf validation. + null + }) + + val scan = tree.scan() + + assertEquals( + listOf(FileSyncUnavailableLocalItem(".git/index.lock", FileSyncLocalUnavailableReason.Vanished)), + scan.unavailable, + ) + assertTrue(scan.documents.any { it.entry.relativePath == ".git/HEAD" }) + } + + @Test + fun `a parent replaced during leaf validation still stops the scan`() = withTree { root -> + writeGitTree(root) + val heads = root.resolve(".git/refs/heads") + val leaf = heads.resolve("main") + val tree = DesktopFileSyncLocalTree(root.toFile(), changeTokenProvider = { path -> + if (path == leaf && Files.exists(heads.resolveSibling("heads-original")).not()) { + Files.move(heads, heads.resolveSibling("heads-original")) + heads.createDirectories() + Files.getFileAttributeView(heads, BasicFileAttributeView::class.java) + .setTimes(null, null, FileTime.fromMillis(1_000_000_000_000L)) + leaf.writeText("replacement") + } + null + }) + + val failure = assertFailsWith { tree.scan() } + + assertTrue(failure.message.orEmpty().contains("replaced after it was scanned")) + } + + @Test + fun `a leaf replaced by a symbolic link during validation still stops the scan`() = withTree { root -> + writeGitTree(root) + val outside = Files.createTempDirectory("desktop-sync-outside-") + try { + val target = outside.resolve("secret.txt").apply { writeText("outside") } + val leaf = root.resolve(".git/ORIG_HEAD") + var linked = false + val tree = DesktopFileSyncLocalTree(root.toFile(), changeTokenProvider = { path -> + if (path == leaf && !linked) { + Files.delete(leaf) + linked = runCatching { Files.createSymbolicLink(leaf, target) }.isSuccess + } + null + }) + + val result = runCatching { tree.scan() } + + if (!linked) return@withTree // Symbolic links need extra privileges on this platform. + val failure = assertIs(result.exceptionOrNull()) + assertTrue(failure.message.orEmpty().contains(".git/ORIG_HEAD is a symbolic link")) + } finally { + outside.toFile().deleteRecursively() + } + } + + @Test + fun `every vanished file is reported individually within the scan entry limit`() = withTree { root -> + root.resolve("one.txt").writeText("one") + root.resolve("two.txt").writeText("two") + val tree = DesktopFileSyncLocalTree(root.toFile(), maximumEntries = 2) { path, _ -> + Files.delete(path) + Files.newByteChannel(path).use { } + "c".repeat(64) + } + + val scan = tree.scan() + + assertEquals(emptyList(), scan.documents) + assertEquals(listOf("one.txt", "two.txt"), scan.unavailable.map(FileSyncUnavailableLocalItem::relativePath)) + } + + @Test + fun `a file rewritten while it is staged for upload is rejected for that item only`() = withTree { root -> + writeGitTree(root) + var changeToken = 0 + val tree = DesktopFileSyncLocalTree(root.toFile(), changeTokenProvider = { path -> + if (path.fileName.toString() == "index") "rewrite-${changeToken++}" else "stable" + }) + val staged = Files.createTempFile("desktop-sync-stage-", ".tmp").toFile() + try { + val failure = assertFailsWith { + tree.stageForUpload(".git/index", staged, maximumBytes = 1024L) + } + assertEquals("The local file changed while it was being prepared for upload.", failure.message) + assertEquals( + "fix: synthetic change\n".length.toLong(), + tree.stageForUpload(".git/COMMIT_EDITMSG", staged, maximumBytes = 1024L).size, + ) + } finally { + staged.delete() + } + } + + private fun writeGitTree(root: Path) { + root.resolve(".git/refs/heads").createDirectories() + root.resolve(".git/HEAD").writeText("ref: refs/heads/main\n") + root.resolve(".git/COMMIT_EDITMSG").writeText("fix: synthetic change\n") + root.resolve(".git/ORIG_HEAD").writeText("0123456789abcdef0123456789abcdef01234567\n") + root.resolve(".git/index").writeText("DIRC synthetic index") + root.resolve(".git/refs/heads/main").writeText("0123456789abcdef0123456789abcdef01234567\n") + root.resolve(".gitignore").writeText("build/\n") + root.resolve("README.md").writeText("# Synthetic repository\n") + } + + private fun withTree(block: (Path) -> Unit) { + val root = Files.createTempDirectory("desktop-sync-availability-") + try { + block(root) + } finally { + root.toFile().deleteRecursively() + } + } +} diff --git a/ui/src/desktopTest/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncLocalTreeTest.kt b/ui/src/desktopTest/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncLocalTreeTest.kt index cd849a0f9..5094076e1 100644 --- a/ui/src/desktopTest/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncLocalTreeTest.kt +++ b/ui/src/desktopTest/kotlin/dev/obiente/nextcloudnative/app/DesktopFileSyncLocalTreeTest.kt @@ -28,7 +28,7 @@ class DesktopFileSyncLocalTreeTest { val entries = DesktopFileSyncLocalTree(root.toFile()).scan(includes = { path, kind -> configuration.includesSyncPath(path, kind) - }).map { it.entry.relativePath } + }).documents.map { it.entry.relativePath } assertEquals(listOf("Photos", "Photos/Keep", "Photos/Keep/a.RAF", "Photos/Keep/a.jpg"), entries) } finally { @@ -88,7 +88,7 @@ class DesktopFileSyncLocalTreeTest { root.resolve(".notes.nextcloud-native-download-archive").writeText("keep") root.resolve(".notes.nextcloud-native-backup-personal").writeText("keep too") - val entries = DesktopFileSyncLocalTree(root.toFile()).scan().map { it.entry.relativePath } + val entries = DesktopFileSyncLocalTree(root.toFile()).scan().documents.map { it.entry.relativePath } assertTrue(".notes.nextcloud-native-download-archive" in entries) assertTrue(".notes.nextcloud-native-backup-personal" in entries) @@ -106,7 +106,7 @@ class DesktopFileSyncLocalTreeTest { val backup = ".notes.txt.nextcloud-native-backup-0b88c03f-55d1-4ccb-b92e-aa8ee32caf65" root.resolve(backup).writeText("protected original") - val entries = DesktopFileSyncLocalTree(root.toFile()).scan().map { it.entry.relativePath } + val entries = DesktopFileSyncLocalTree(root.toFile()).scan().documents.map { it.entry.relativePath } assertTrue("notes.txt" in entries) assertFalse(backup in entries) @@ -171,7 +171,7 @@ class DesktopFileSyncLocalTreeTest { Files.setLastModifiedTime(file, FileTime.fromMillis(-1_000L)) val tree = DesktopFileSyncLocalTree(root.toFile()) - assertEquals(null, tree.scan().single().entry.modifiedEpochMillis) + assertEquals(null, tree.scan().documents.single().entry.modifiedEpochMillis) assertEquals(null, requireNotNull(tree.resolve("archive.txt")).entry.modifiedEpochMillis) } finally { root.toFile().deleteRecursively() @@ -191,12 +191,12 @@ class DesktopFileSyncLocalTreeTest { digestCount += 1 "a".repeat(64) } - val first = tree.scan() + val first = tree.scan().documents val cachedRevisions = first.associate { document -> document.entry.relativePath to document.entry.revision } - val second = tree.scan(cachedRevisions) + val second = tree.scan(cachedRevisions).documents assertEquals(first.map { it.entry }, second.map { it.entry }) assertEquals(1, digestCount) @@ -218,7 +218,7 @@ class DesktopFileSyncLocalTreeTest { digestCount += 1 "b".repeat(64) } - val first = tree.scan() + val first = tree.scan().documents tree.scan(first.associate { it.entry.relativePath to it.entry.revision }) @@ -238,7 +238,7 @@ class DesktopFileSyncLocalTreeTest { root.resolve("Notes/today.md").writeText("inside") outside.resolve("today.md").writeText("outside") val tree = DesktopFileSyncLocalTree(root.toFile()) - val scanned = tree.scan().single { it.entry.relativePath == "Notes/today.md" } + val scanned = tree.scan().documents.single { it.entry.relativePath == "Notes/today.md" } Files.move(root.resolve("Notes"), root.resolve("Notes-original")) val linked = runCatching { Files.createSymbolicLink(root.resolve("Notes"), outside) }.isSuccess if (!linked) return diff --git a/website/content/guides/linux-folder-sync.md b/website/content/guides/linux-folder-sync.md index 3a9843734..6af60b3a2 100644 --- a/website/content/guides/linux-folder-sync.md +++ b/website/content/guides/linux-folder-sync.md @@ -8,15 +8,15 @@ device: Desktop platforms: Linux durationMinutes: 10 difficulty: Advanced -lastUpdated: 2026-08-30 +lastUpdated: 2026-10-08 captureScenarios: guide-linux-folder-sync-workspace, guide-linux-folder-sync-locations, guide-linux-folder-sync-rules prerequisites: A connected Linux account, A local directory you can safely test, Enough local and Nextcloud storage for the first scan --- # Sync a Linux folder with nati.ve -**Last reviewed: 2026-08-30.** The software and published packages may have -changed since this review. Check the [current releases](https://github.com/obiente/native/releases) +**Unreadable-item guidance last reviewed: 2026-10-08; other workflows: 2026-08-30.** +The software and published packages may have changed since these reviews. Check the [current releases](https://github.com/obiente/native/releases) and [compatibility notes](/compatibility/) before using this guide with important data. Linux folder sync keeps a normal local directory connected to one Nextcloud directory while the desktop process is running. It is intended for files used by ordinary desktop applications. It is separate from the Linux virtual filesystem mount, which represents remote placeholders and hydrates content on demand. @@ -57,3 +57,11 @@ A desktop sync pair supports no more than 100,000 selected entries or planned op Automatic checks wait longer after each repeated item failure. After five failed attempts, the item stays in the visible failed state until you use **Sync now** to request another attempt. Review the failure first. Repeatedly requesting a retry cannot repair a permissions error, an unsupported name, or a file that another application keeps changing. The **Review** setup step keeps direction, conflict policy and deletion policy visible before you start. In **Choose what syncs**, a partial checkbox means only some descendants are selected; opening that folder does not expand the sync scope. Removal is under the selected pair's **Settings** tab and still requires confirmation. + +### Items that cannot be read during a check + +**Last reviewed: 2026-10-08.** The following behavior is implemented in source; +availability in installed builds may change. Check the +[release notes](https://github.com/obiente/native/releases) for your version. + +A check continues when one local item cannot be read: for example a lock file that disappears while Git or another tool is working, a file held open by another application, a folder you cannot list, or a socket or other special file. That item and everything below it are skipped for that check, their Nextcloud copies are left unchanged, and the pair counts it as skipped and lists up to 20 skipped items with their reasons under **Current work**. The rest of the folder synchronizes normally, and the skipped item is checked again on the next sync. Use an ignore rule for items that should never sync. diff --git a/website/public/screenshots/capture-manifest.json b/website/public/screenshots/capture-manifest.json index 5976f9d17..0f3185e26 100644 --- a/website/public/screenshots/capture-manifest.json +++ b/website/public/screenshots/capture-manifest.json @@ -127,6 +127,7 @@ "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncCoordinator.kt", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncCoordinatorSnapshot.kt", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncExperience.kt", + "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncLocalAvailability.kt", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncPairConfiguration.kt", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncPairInspector.kt", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncPairList.kt", @@ -621,9 +622,10 @@ "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncConflictResolution.kt": "6af8d6d8e9eb4f2423d9f1074869ea2322c36dbc52a9149ce6455a0805c66400", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncConflictReview.kt": "b7a5aa4259795c547f4a212c09b9e0b1b968787303bc8d404e5f5867947a9aac", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncContentIdentity.kt": "e640d1241fdf423c0abaac0f61b1361131082eb19b8db2a7236d6d84ba38d115", - "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncCoordinator.kt": "0723dca3b19bab1937953097b5eecc3c2eb5b893e569c8f538155cd92d34689f", + "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncCoordinator.kt": "f63f08c678236f0f6f676bdd363165d06170483df28c95a8f4ae815693d02910", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncCoordinatorSnapshot.kt": "47b57f32edcb7c5c951c8fdd58c472d6446c841c7ca6550843777a6a1e016656", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncExperience.kt": "2e4330eda0ad3621f9e5d45a27ffba1fb72568c616113b48bd70d974ba20234a", + "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncLocalAvailability.kt": "2f799e8044d5e91b313cec9715b6b4751389093ae390816e3f72345e8942ce77", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncPairConfiguration.kt": "e7022629c32b99388a03675529886ac580a6def9cf5a4cbfea565ad6cc61fe01", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncPairInspector.kt": "32884a623b7331851d402434c6e5269d183f7a79b1cb89b31b3f8a1c833742ce", "ui/src/commonMain/kotlin/dev/obiente/nextcloudnative/app/FileSyncPairList.kt": "357d9749aa11419a6667850eb167530e30b0e721d40c4744c643e307834b7752",