From e48e1e36a51a37637ef00349143d419aea1f179b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D1=83=D1=80=D0=BD=D0=B0=D1=82=20=D0=90=D0=BD=D0=B4?= =?UTF-8?q?=D1=80=D0=B5=D0=B9?= Date: Wed, 15 Jul 2026 22:58:03 +0300 Subject: [PATCH] Implement background media ingestion and Pi deployment --- .github/dependabot.yml | 32 +++++++ .github/workflows/android.yml | 47 +++++++++ .github/workflows/server.yml | 62 ++++++++++++ Android/.gitignore | 4 + Android/qsferaApp/build.gradle | 24 ++++- .../main/java/eu/qsfera/android/MainApp.kt | 12 ++- .../SettingsPictureUploadsFragment.kt | 18 ---- .../SettingsPictureUploadsViewModel.kt | 26 +++-- .../SettingsVideoUploadsViewModel.kt | 13 ++- .../android/providers/FileContentProvider.kt | 9 +- .../android/providers/WorkManagerProvider.kt | 35 ++++++- .../UploadFileFromContentUriUseCase.kt | 5 + .../android/workers/AutomaticUploadsWorker.kt | 70 ++++++++++++-- .../android/workers/RemoveSourceFileWorker.kt | 17 +++- .../src/main/res/values-ru/strings.xml | 2 +- .../main/res/xml/settings_picture_uploads.xml | 11 +-- .../workers/AutomaticUploadsWorkerTest.kt | 53 ++++++++++ .../workers/RemoveSourceFileWorkerTest.kt | 47 +++++++++ .../model/AutomaticUploadsConfiguration.kt | 5 + .../model/FolderBackUpConfiguration.kt | 7 ++ .../model/FolderBackUpConfigurationTest.kt | 35 +++++++ README.md | 29 ++++++ Server/.woodpecker.star | 12 +-- Server/Dockerfile | 13 +++ Server/Makefile | 4 +- Server/deployments/raspberry-pi/.env.example | 22 +++++ Server/deployments/raspberry-pi/README.md | 96 +++++++++++++++++++ Server/deployments/raspberry-pi/backup.sh | 38 ++++++++ Server/deployments/raspberry-pi/compose.yaml | 47 +++++++++ Server/deployments/raspberry-pi/restore.sh | 51 ++++++++++ Server/deployments/raspberry-pi/smoke-test.sh | 53 ++++++++++ Server/qsfera/README.md | 2 +- 32 files changed, 825 insertions(+), 76 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/android.yml create mode 100644 .github/workflows/server.yml create mode 100644 Android/qsferaApp/src/test/java/eu/qsfera/android/workers/AutomaticUploadsWorkerTest.kt create mode 100644 Android/qsferaApp/src/test/java/eu/qsfera/android/workers/RemoveSourceFileWorkerTest.kt create mode 100644 README.md create mode 100644 Server/deployments/raspberry-pi/.env.example create mode 100644 Server/deployments/raspberry-pi/README.md create mode 100644 Server/deployments/raspberry-pi/backup.sh create mode 100644 Server/deployments/raspberry-pi/compose.yaml create mode 100644 Server/deployments/raspberry-pi/restore.sh create mode 100644 Server/deployments/raspberry-pi/smoke-test.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..c37bb64df --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,32 @@ +version: 2 +updates: + - package-ecosystem: gradle + directory: /Android + schedule: + interval: daily + labels: + - Dependencies + commit-message: + prefix: feat + + - package-ecosystem: gomod + directory: /Server + schedule: + interval: daily + open-pull-requests-limit: 2 + + - package-ecosystem: npm + directory: /Server/services/idp + schedule: + interval: weekly + open-pull-requests-limit: 2 + + - package-ecosystem: docker + directory: /Server + schedule: + interval: weekly + + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml new file mode 100644 index 000000000..77925540f --- /dev/null +++ b/.github/workflows/android.yml @@ -0,0 +1,47 @@ +name: Android + +on: + push: + branches: + - main + paths: + - Android/** + - .github/workflows/android.yml + pull_request: + paths: + - Android/** + - .github/workflows/android.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: android-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + working-directory: Android + +jobs: + test-and-build: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Set up JDK 17 + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: "17" + + - name: Set up Gradle + uses: gradle/actions/setup-gradle@v6 + + - name: Run unit tests + run: ./gradlew :qsferaComLibrary:testDebugUnitTest :qsferaDomain:testDebugUnitTest :qsferaData:testDebugUnitTest :qsferaApp:testOriginalDebugUnitTest --no-daemon + + - name: Build original debug APK + run: ./gradlew :qsferaApp:assembleOriginalDebug --no-daemon diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml new file mode 100644 index 000000000..f38cc8e6e --- /dev/null +++ b/.github/workflows/server.yml @@ -0,0 +1,62 @@ +name: Server + +on: + push: + branches: + - main + paths: + - Server/** + - .github/workflows/server.yml + pull_request: + paths: + - Server/** + - .github/workflows/server.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: server-${{ github.ref }} + cancel-in-progress: true + +jobs: + deployment-config: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Check deployment scripts + run: sh -n Server/deployments/raspberry-pi/backup.sh Server/deployments/raspberry-pi/restore.sh Server/deployments/raspberry-pi/smoke-test.sh + + - name: Validate Compose model + env: + QSFERA_IMAGE: example.invalid/qsfera:test + QSFERA_URL: https://cloud.example.invalid + QSFERA_ADMIN_PASSWORD: ci-only-placeholder + QSFERA_CONFIG_DIR: /tmp/qsfera-config + QSFERA_DATA_DIR: /tmp/qsfera-data + QSFERA_MEMORY_LIMIT: 1g + run: docker compose -f Server/deployments/raspberry-pi/compose.yaml config --quiet + + vulnerability-scan: + runs-on: ubuntu-latest + defaults: + run: + working-directory: Server + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Set up Go from go.mod + uses: actions/setup-go@v6 + with: + go-version-file: Server/go.mod + cache-dependency-path: Server/go.sum + + - name: Install native image dependency + run: sudo apt-get update && sudo apt-get install --yes libvips-dev + + - name: Run pinned govulncheck + run: make govulncheck diff --git a/Android/.gitignore b/Android/.gitignore index 91c8510f3..314e955de 100644 --- a/Android/.gitignore +++ b/Android/.gitignore @@ -12,6 +12,10 @@ # Local configuration files (sdk path, etc) local.properties +# Local release signing material +signing/*.jks +signing/*-signing.local.properties + # Mac .DS_Store files .DS_Store diff --git a/Android/qsferaApp/build.gradle b/Android/qsferaApp/build.gradle index 1782bed2e..0b7b26736 100644 --- a/Android/qsferaApp/build.gradle +++ b/Android/qsferaApp/build.gradle @@ -14,6 +14,20 @@ def envValue(String primary, String legacy = null) { return legacy ? System.getenv(legacy) : null } +def releaseSigningProperties = new Properties() +def releaseSigningPropertiesFile = rootProject.file('signing/qsfera-signing.local.properties') +if (releaseSigningPropertiesFile.exists()) { + releaseSigningPropertiesFile.withInputStream { stream -> + releaseSigningProperties.load(stream) + } +} + +def signingValue = { String primary, String legacy = null -> + envValue(primary, legacy) ?: + releaseSigningProperties.getProperty(primary) ?: + (legacy ? releaseSigningProperties.getProperty(legacy) : null) +} + dependencies { // Data and domain modules implementation project(':qsferaDomain') @@ -162,12 +176,12 @@ android { signingConfigs { release { - def releaseKeystore = envValue('QSFERA_RELEASE_KEYSTORE', 'OC_RELEASE_KEYSTORE') + def releaseKeystore = signingValue('QSFERA_RELEASE_KEYSTORE', 'OC_RELEASE_KEYSTORE') if (releaseKeystore) { storeFile file(releaseKeystore) // use an absolute path - storePassword envValue('QSFERA_RELEASE_KEYSTORE_PASSWORD', 'OC_RELEASE_KEYSTORE_PASSWORD') - keyAlias envValue('QSFERA_RELEASE_KEY_ALIAS', 'OC_RELEASE_KEY_ALIAS') - keyPassword envValue('QSFERA_RELEASE_KEY_PASSWORD', 'OC_RELEASE_KEY_PASSWORD') + storePassword signingValue('QSFERA_RELEASE_KEYSTORE_PASSWORD', 'OC_RELEASE_KEYSTORE_PASSWORD') + keyAlias signingValue('QSFERA_RELEASE_KEY_ALIAS', 'OC_RELEASE_KEY_ALIAS') + keyPassword signingValue('QSFERA_RELEASE_KEY_PASSWORD', 'OC_RELEASE_KEY_PASSWORD') } } } @@ -175,7 +189,7 @@ android { buildTypes { release { - if (envValue('QSFERA_RELEASE_KEYSTORE', 'OC_RELEASE_KEYSTORE')) { + if (signingValue('QSFERA_RELEASE_KEYSTORE', 'OC_RELEASE_KEYSTORE')) { signingConfig signingConfigs.release } } diff --git a/Android/qsferaApp/src/main/java/eu/qsfera/android/MainApp.kt b/Android/qsferaApp/src/main/java/eu/qsfera/android/MainApp.kt index 20d6f17f0..28628d61c 100644 --- a/Android/qsferaApp/src/main/java/eu/qsfera/android/MainApp.kt +++ b/Android/qsferaApp/src/main/java/eu/qsfera/android/MainApp.kt @@ -38,6 +38,7 @@ import android.view.WindowManager import android.widget.CheckBox import androidx.appcompat.app.AlertDialog import androidx.core.content.pm.PackageInfoCompat +import androidx.work.ExistingWorkPolicy import eu.qsfera.android.data.providers.implementation.OCSharedPreferencesProvider @@ -48,6 +49,7 @@ import eu.qsfera.android.dependecyinjection.remoteDataSourceModule import eu.qsfera.android.dependecyinjection.repositoryModule import eu.qsfera.android.dependecyinjection.useCaseModule import eu.qsfera.android.dependecyinjection.viewModelModule +import eu.qsfera.android.domain.automaticuploads.usecases.GetAutomaticUploadsConfigurationUseCase import eu.qsfera.android.domain.capabilities.usecases.GetStoredCapabilitiesUseCase import eu.qsfera.android.domain.spaces.model.OCSpace import eu.qsfera.android.domain.spaces.usecases.GetPersonalSpaceForAccountUseCase @@ -123,6 +125,7 @@ class MainApp : Application() { initDependencyInjection() val workManagerProvider: WorkManagerProvider by inject() + val getAutomaticUploadsConfigurationUseCase: GetAutomaticUploadsConfigurationUseCase by inject() var startedActivities = 0 // register global protection with pass code, pattern lock and biometric lock @@ -228,8 +231,15 @@ class MainApp : Application() { // (recovers if the chain was dropped) and trigger an immediate scan // so the user doesn't have to wait up to 15 min. CoroutineScope(Dispatchers.IO).launch { + val sourcePaths = getAutomaticUploadsConfigurationUseCase(Unit) + .getDataOrNull() + ?.sourcePaths + .orEmpty() workManagerProvider.enqueueAutomaticUploadsWorker() - workManagerProvider.enqueueMediaStoreAutomaticUploadsWorker() + workManagerProvider.enqueueMediaStoreAutomaticUploadsWorker( + existingWorkPolicy = ExistingWorkPolicy.REPLACE, + sourcePaths = sourcePaths, + ) workManagerProvider.enqueueImmediateAutomaticUploadsWorker() } } diff --git a/Android/qsferaApp/src/main/java/eu/qsfera/android/presentation/settings/automaticuploads/SettingsPictureUploadsFragment.kt b/Android/qsferaApp/src/main/java/eu/qsfera/android/presentation/settings/automaticuploads/SettingsPictureUploadsFragment.kt index 9f7eed147..d8a1cda7c 100644 --- a/Android/qsferaApp/src/main/java/eu/qsfera/android/presentation/settings/automaticuploads/SettingsPictureUploadsFragment.kt +++ b/Android/qsferaApp/src/main/java/eu/qsfera/android/presentation/settings/automaticuploads/SettingsPictureUploadsFragment.kt @@ -41,14 +41,12 @@ import androidx.preference.PreferenceFragmentCompat import androidx.preference.SwitchPreferenceCompat import eu.qsfera.android.R import eu.qsfera.android.db.PreferenceManager.PREF__CAMERA_PICTURE_UPLOADS_ACCOUNT_NAME -import eu.qsfera.android.db.PreferenceManager.PREF__CAMERA_PICTURE_UPLOADS_BEHAVIOUR import eu.qsfera.android.db.PreferenceManager.PREF__CAMERA_PICTURE_UPLOADS_CHARGING_ONLY import eu.qsfera.android.db.PreferenceManager.PREF__CAMERA_PICTURE_UPLOADS_ENABLED import eu.qsfera.android.db.PreferenceManager.PREF__CAMERA_PICTURE_UPLOADS_LAST_SYNC import eu.qsfera.android.db.PreferenceManager.PREF__CAMERA_PICTURE_UPLOADS_PATH import eu.qsfera.android.db.PreferenceManager.PREF__CAMERA_PICTURE_UPLOADS_SOURCE import eu.qsfera.android.db.PreferenceManager.PREF__CAMERA_PICTURE_UPLOADS_WIFI_ONLY -import eu.qsfera.android.domain.automaticuploads.model.UploadBehavior import eu.qsfera.android.extensions.collectLatestLifecycleFlow import eu.qsfera.android.extensions.showAlertDialog import eu.qsfera.android.extensions.showMessageInSnackbar @@ -71,7 +69,6 @@ class SettingsPictureUploadsFragment : PreferenceFragmentCompat() { private var prefPictureUploadsOnCharging: CheckBoxPreference? = null private var prefPictureUploadsSourcePath: Preference? = null private var prefPictureUploadsClearSourcePaths: Preference? = null - private var prefPictureUploadsBehaviour: ListPreference? = null private var prefPictureUploadsAccount: ListPreference? = null private var prefPictureUploadsLastSync: Preference? = null private var spaceId: String? = null @@ -104,13 +101,6 @@ class SettingsPictureUploadsFragment : PreferenceFragmentCompat() { prefPictureUploadsSourcePath = findPreference(PREF__CAMERA_PICTURE_UPLOADS_SOURCE) prefPictureUploadsClearSourcePaths = findPreference(PREF_PICTURE_UPLOADS_CLEAR_SOURCE_PATHS) prefPictureUploadsLastSync = findPreference(PREF__CAMERA_PICTURE_UPLOADS_LAST_SYNC) - prefPictureUploadsBehaviour = findPreference(PREF__CAMERA_PICTURE_UPLOADS_BEHAVIOUR)?.apply { - entries = listOf( - getString(R.string.pref_behaviour_entries_keep_file), - getString(R.string.pref_behaviour_entries_remove_original_file) - ).toTypedArray() - entryValues = listOf(UploadBehavior.COPY.name, UploadBehavior.MOVE.name).toTypedArray() - } prefPictureUploadsAccount = findPreference(PREF__CAMERA_PICTURE_UPLOADS_ACCOUNT_NAME) val comment = getString(R.string.prefs_camera_upload_source_path_title_required) @@ -158,7 +148,6 @@ class SettingsPictureUploadsFragment : PreferenceFragmentCompat() { prefPictureUploadsClearSourcePaths?.isEnabled = sourcePaths.isNotEmpty() prefPictureUploadsOnWifi?.isChecked = it.wifiOnly prefPictureUploadsOnCharging?.isChecked = it.chargingOnly - prefPictureUploadsBehaviour?.value = it.behavior.name prefPictureUploadsLastSync?.summary = DisplayUtils.unixTimeToHumanReadable(it.lastSyncTimestamp) spaceId = it.spaceId } ?: resetFields() @@ -258,11 +247,6 @@ class SettingsPictureUploadsFragment : PreferenceFragmentCompat() { true } - prefPictureUploadsBehaviour?.setOnPreferenceChangeListener { _, newValue -> - newValue as String - picturesViewModel.handleSelectBehaviour(newValue) - true - } } override fun onDestroy() { @@ -280,7 +264,6 @@ class SettingsPictureUploadsFragment : PreferenceFragmentCompat() { prefPictureUploadsOnCharging?.isEnabled = value prefPictureUploadsSourcePath?.isEnabled = value prefPictureUploadsClearSourcePaths?.isEnabled = value && picturesViewModel.getPictureUploadsSourcePaths().isNotEmpty() - prefPictureUploadsBehaviour?.isEnabled = value prefPictureUploadsAccount?.isEnabled = value prefPictureUploadsLastSync?.isEnabled = value } @@ -292,7 +275,6 @@ class SettingsPictureUploadsFragment : PreferenceFragmentCompat() { prefPictureUploadsClearSourcePaths?.isEnabled = false prefPictureUploadsOnWifi?.isChecked = false prefPictureUploadsOnCharging?.isChecked = false - prefPictureUploadsBehaviour?.value = UploadBehavior.COPY.name prefPictureUploadsLastSync?.summary = null } diff --git a/Android/qsferaApp/src/main/java/eu/qsfera/android/presentation/settings/automaticuploads/SettingsPictureUploadsViewModel.kt b/Android/qsferaApp/src/main/java/eu/qsfera/android/presentation/settings/automaticuploads/SettingsPictureUploadsViewModel.kt index e3d61d754..71cfbf541 100644 --- a/Android/qsferaApp/src/main/java/eu/qsfera/android/presentation/settings/automaticuploads/SettingsPictureUploadsViewModel.kt +++ b/Android/qsferaApp/src/main/java/eu/qsfera/android/presentation/settings/automaticuploads/SettingsPictureUploadsViewModel.kt @@ -26,12 +26,14 @@ import android.content.Intent import android.net.Uri import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope +import androidx.work.ExistingWorkPolicy import eu.qsfera.android.R import eu.qsfera.android.db.PreferenceManager.PREF__CAMERA_UPLOADS_DEFAULT_PATH import eu.qsfera.android.domain.automaticuploads.model.FolderBackUpConfiguration import eu.qsfera.android.domain.automaticuploads.model.FolderBackUpConfiguration.Companion.encodeSourcePaths import eu.qsfera.android.domain.automaticuploads.model.FolderBackUpConfiguration.Companion.pictureUploadsName import eu.qsfera.android.domain.automaticuploads.model.UploadBehavior +import eu.qsfera.android.domain.automaticuploads.usecases.GetAutomaticUploadsConfigurationUseCase import eu.qsfera.android.domain.automaticuploads.usecases.GetPictureUploadsConfigurationStreamUseCase import eu.qsfera.android.domain.automaticuploads.usecases.ResetPictureUploadsUseCase import eu.qsfera.android.domain.automaticuploads.usecases.SavePictureUploadsConfigurationUseCase @@ -53,6 +55,7 @@ import timber.log.Timber class SettingsPictureUploadsViewModel( private val accountProvider: AccountProvider, private val savePictureUploadsConfigurationUseCase: SavePictureUploadsConfigurationUseCase, + private val getAutomaticUploadsConfigurationUseCase: GetAutomaticUploadsConfigurationUseCase, private val getPictureUploadsConfigurationStreamUseCase: GetPictureUploadsConfigurationStreamUseCase, private val resetPictureUploadsUseCase: ResetPictureUploadsUseCase, private val getPersonalSpaceForAccountUseCase: GetPersonalSpaceForAccountUseCase, @@ -161,16 +164,6 @@ class SettingsPictureUploadsViewModel( } } - fun handleSelectBehaviour(behaviorString: String) { - val behavior = UploadBehavior.fromString(behaviorString) - - viewModelScope.launch(coroutinesDispatcherProvider.io) { - savePictureUploadsConfigurationUseCase( - SavePictureUploadsConfigurationUseCase.Params(composePictureUploadsConfiguration(behavior = behavior)) - ) - } - } - fun handleSelectPictureUploadsSourcePath(contentUriForTree: Uri) { val previousSourcePaths = getPictureUploadsSourcePaths() val newSourcePath = contentUriForTree.toString() @@ -202,8 +195,14 @@ class SettingsPictureUploadsViewModel( } fun schedulePictureUploads() { - workManagerProvider.enqueueAutomaticUploadsWorker() - workManagerProvider.enqueueMediaStoreAutomaticUploadsWorker() + viewModelScope.launch(coroutinesDispatcherProvider.io) { + val sourcePaths = getAutomaticUploadsConfigurationUseCase(Unit).getDataOrNull()?.sourcePaths.orEmpty() + workManagerProvider.enqueueAutomaticUploadsWorker() + workManagerProvider.enqueueMediaStoreAutomaticUploadsWorker( + existingWorkPolicy = ExistingWorkPolicy.REPLACE, + sourcePaths = sourcePaths, + ) + } } private fun composePictureUploadsConfiguration( @@ -212,12 +211,11 @@ class SettingsPictureUploadsViewModel( wifiOnly: Boolean? = _pictureUploads.value?.wifiOnly, chargingOnly: Boolean? = _pictureUploads.value?.chargingOnly, sourcePath: String? = _pictureUploads.value?.sourcePath, - behavior: UploadBehavior? = _pictureUploads.value?.behavior, timestamp: Long? = _pictureUploads.value?.lastSyncTimestamp, spaceId: String? = _pictureUploads.value?.spaceId, ): FolderBackUpConfiguration = FolderBackUpConfiguration( accountName = accountName ?: accountProvider.getCurrentQSferaAccount()!!.name, - behavior = behavior ?: UploadBehavior.COPY, + behavior = UploadBehavior.MOVE, sourcePath = sourcePath.orEmpty(), uploadPath = uploadPath ?: PREF__CAMERA_UPLOADS_DEFAULT_PATH, wifiOnly = wifiOnly ?: false, diff --git a/Android/qsferaApp/src/main/java/eu/qsfera/android/presentation/settings/automaticuploads/SettingsVideoUploadsViewModel.kt b/Android/qsferaApp/src/main/java/eu/qsfera/android/presentation/settings/automaticuploads/SettingsVideoUploadsViewModel.kt index 17488b3a2..548580b88 100644 --- a/Android/qsferaApp/src/main/java/eu/qsfera/android/presentation/settings/automaticuploads/SettingsVideoUploadsViewModel.kt +++ b/Android/qsferaApp/src/main/java/eu/qsfera/android/presentation/settings/automaticuploads/SettingsVideoUploadsViewModel.kt @@ -26,12 +26,14 @@ import android.content.Intent import android.net.Uri import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope +import androidx.work.ExistingWorkPolicy import eu.qsfera.android.R import eu.qsfera.android.db.PreferenceManager.PREF__CAMERA_UPLOADS_DEFAULT_PATH import eu.qsfera.android.domain.automaticuploads.model.FolderBackUpConfiguration import eu.qsfera.android.domain.automaticuploads.model.FolderBackUpConfiguration.Companion.encodeSourcePaths import eu.qsfera.android.domain.automaticuploads.model.FolderBackUpConfiguration.Companion.videoUploadsName import eu.qsfera.android.domain.automaticuploads.model.UploadBehavior +import eu.qsfera.android.domain.automaticuploads.usecases.GetAutomaticUploadsConfigurationUseCase import eu.qsfera.android.domain.automaticuploads.usecases.GetVideoUploadsConfigurationStreamUseCase import eu.qsfera.android.domain.automaticuploads.usecases.ResetVideoUploadsUseCase import eu.qsfera.android.domain.automaticuploads.usecases.SaveVideoUploadsConfigurationUseCase @@ -53,6 +55,7 @@ import timber.log.Timber class SettingsVideoUploadsViewModel( private val accountProvider: AccountProvider, private val saveVideoUploadsConfigurationUseCase: SaveVideoUploadsConfigurationUseCase, + private val getAutomaticUploadsConfigurationUseCase: GetAutomaticUploadsConfigurationUseCase, private val getVideoUploadsConfigurationStreamUseCase: GetVideoUploadsConfigurationStreamUseCase, private val resetVideoUploadsUseCase: ResetVideoUploadsUseCase, private val getPersonalSpaceForAccountUseCase: GetPersonalSpaceForAccountUseCase, @@ -202,8 +205,14 @@ class SettingsVideoUploadsViewModel( } fun scheduleVideoUploads() { - workManagerProvider.enqueueAutomaticUploadsWorker() - workManagerProvider.enqueueMediaStoreAutomaticUploadsWorker() + viewModelScope.launch(coroutinesDispatcherProvider.io) { + val sourcePaths = getAutomaticUploadsConfigurationUseCase(Unit).getDataOrNull()?.sourcePaths.orEmpty() + workManagerProvider.enqueueAutomaticUploadsWorker() + workManagerProvider.enqueueMediaStoreAutomaticUploadsWorker( + existingWorkPolicy = ExistingWorkPolicy.REPLACE, + sourcePaths = sourcePaths, + ) + } } private fun composeVideoUploadsConfiguration( diff --git a/Android/qsferaApp/src/main/java/eu/qsfera/android/providers/FileContentProvider.kt b/Android/qsferaApp/src/main/java/eu/qsfera/android/providers/FileContentProvider.kt index 5167e03e8..9306307d9 100644 --- a/Android/qsferaApp/src/main/java/eu/qsfera/android/providers/FileContentProvider.kt +++ b/Android/qsferaApp/src/main/java/eu/qsfera/android/providers/FileContentProvider.kt @@ -45,6 +45,7 @@ import android.net.Uri import android.os.CancellationSignal import android.os.ParcelFileDescriptor import android.text.TextUtils +import androidx.work.ExistingWorkPolicy import androidx.work.WorkManager import eu.qsfera.android.MainApp import eu.qsfera.android.R @@ -994,8 +995,14 @@ class FileContentProvider(val executors: Executors = Executors()) : ContentProvi videoUploadsConfiguration?.let { backupLocalDataSource.saveFolderBackupConfiguration(it) } if (pictureUploadsConfiguration != null || videoUploadsConfiguration != null) { val workManagerProvider = WorkManagerProvider(context!!) + val sourcePaths = listOfNotNull(pictureUploadsConfiguration, videoUploadsConfiguration) + .flatMap { it.sourcePaths } + .distinct() workManagerProvider.enqueueAutomaticUploadsWorker() - workManagerProvider.enqueueMediaStoreAutomaticUploadsWorker() + workManagerProvider.enqueueMediaStoreAutomaticUploadsWorker( + existingWorkPolicy = ExistingWorkPolicy.REPLACE, + sourcePaths = sourcePaths, + ) } } cursor.close() diff --git a/Android/qsferaApp/src/main/java/eu/qsfera/android/providers/WorkManagerProvider.kt b/Android/qsferaApp/src/main/java/eu/qsfera/android/providers/WorkManagerProvider.kt index 771db03fb..5b29be933 100644 --- a/Android/qsferaApp/src/main/java/eu/qsfera/android/providers/WorkManagerProvider.kt +++ b/Android/qsferaApp/src/main/java/eu/qsfera/android/providers/WorkManagerProvider.kt @@ -21,8 +21,12 @@ package eu.qsfera.android.providers +import android.content.ContentResolver import android.content.Context +import android.net.Uri +import android.provider.DocumentsContract import android.provider.MediaStore +import androidx.core.net.toUri import androidx.lifecycle.LiveData import androidx.work.Constraints import androidx.work.ExistingPeriodicWorkPolicy @@ -60,14 +64,21 @@ class WorkManagerProvider( } fun enqueueMediaStoreAutomaticUploadsWorker( - existingWorkPolicy: ExistingWorkPolicy = ExistingWorkPolicy.KEEP + existingWorkPolicy: ExistingWorkPolicy = ExistingWorkPolicy.KEEP, + sourcePaths: Collection = emptyList(), ) { - val mediaStoreTriggers = Constraints.Builder() + val constraintsBuilder = Constraints.Builder() .addContentUriTrigger(MediaStore.Images.Media.EXTERNAL_CONTENT_URI, true) .addContentUriTrigger(MediaStore.Video.Media.EXTERNAL_CONTENT_URI, true) .setTriggerContentUpdateDelay(AutomaticUploadsWorker.WRITE_SAFETY_BUFFER_MS, TimeUnit.MILLISECONDS) .setTriggerContentMaxDelay(AutomaticUploadsWorker.MEDIA_STORE_TRIGGER_MAX_DELAY_MS, TimeUnit.MILLISECONDS) - .build() + + sourcePaths + .flatMap(::sourcePathTriggerUris) + .distinct() + .forEach { constraintsBuilder.addContentUriTrigger(it, true) } + + val mediaStoreTriggers = constraintsBuilder.build() val mediaStoreWorker = OneTimeWorkRequestBuilder() .addTag(AutomaticUploadsWorker.MEDIA_STORE_UPLOADS_WORKER) @@ -84,6 +95,24 @@ class WorkManagerProvider( ) } + private fun sourcePathTriggerUris(sourcePath: String): List { + val treeUri = sourcePath.trim().takeIf { it.isNotEmpty() }?.toUri() ?: return emptyList() + if (treeUri.scheme != ContentResolver.SCHEME_CONTENT) return emptyList() + + val documentUri = runCatching { + if (DocumentsContract.isTreeUri(treeUri)) { + DocumentsContract.buildDocumentUriUsingTree( + treeUri, + DocumentsContract.getTreeDocumentId(treeUri), + ) + } else { + null + } + }.getOrNull() + + return listOfNotNull(treeUri, documentUri) + } + /** * Trigger an immediate one-time upload scan, e.g. when the app enters foreground. * Skips if either the periodic or immediate worker is already running to avoid diff --git a/Android/qsferaApp/src/main/java/eu/qsfera/android/usecases/transfers/uploads/UploadFileFromContentUriUseCase.kt b/Android/qsferaApp/src/main/java/eu/qsfera/android/usecases/transfers/uploads/UploadFileFromContentUriUseCase.kt index 1164324cd..e1f2f205e 100644 --- a/Android/qsferaApp/src/main/java/eu/qsfera/android/usecases/transfers/uploads/UploadFileFromContentUriUseCase.kt +++ b/Android/qsferaApp/src/main/java/eu/qsfera/android/usecases/transfers/uploads/UploadFileFromContentUriUseCase.kt @@ -83,6 +83,11 @@ class UploadFileFromContentUriUseCase( if (behavior == UploadBehavior.MOVE) { val removeSourceFileWorker = OneTimeWorkRequestBuilder() .setInputData(inputDataRemoveSourceFileWorker) + .setBackoffCriteria( + BackoffPolicy.EXPONENTIAL, + 10, + TimeUnit.SECONDS, + ) .build() workManager.beginUniqueWork( uniqueWorkName, diff --git a/Android/qsferaApp/src/main/java/eu/qsfera/android/workers/AutomaticUploadsWorker.kt b/Android/qsferaApp/src/main/java/eu/qsfera/android/workers/AutomaticUploadsWorker.kt index f5afeb82f..c07d966ff 100644 --- a/Android/qsferaApp/src/main/java/eu/qsfera/android/workers/AutomaticUploadsWorker.kt +++ b/Android/qsferaApp/src/main/java/eu/qsfera/android/workers/AutomaticUploadsWorker.kt @@ -79,6 +79,7 @@ class AutomaticUploadsWorker( override suspend fun doWork(): Result { Timber.i("Starting AutomaticUploadsWorker with UUID ${this.id}") var automaticUploadsEnabled = true + var configuredSourcePaths: List = emptyList() when (val useCaseResult = getAutomaticUploadsConfigurationUseCase(Unit)) { is UseCaseResult.Success -> { val cameraUploadsConfiguration = useCaseResult.data @@ -86,6 +87,7 @@ class AutomaticUploadsWorker( cancelWorker() automaticUploadsEnabled = false } else { + configuredSourcePaths = cameraUploadsConfiguration.sourcePaths cameraUploadsConfiguration.pictureUploadsConfiguration?.let { pictureUploadsConfiguration -> try { checkSourcePathsAreValidUrisOrThrowException(pictureUploadsConfiguration.sourcePaths) @@ -110,17 +112,21 @@ class AutomaticUploadsWorker( Timber.e(useCaseResult.throwable, "Worker ${useCaseResult.throwable}") } } - rescheduleMediaStoreTriggerIfNeeded(automaticUploadsEnabled) + rescheduleMediaStoreTriggerIfNeeded(automaticUploadsEnabled, configuredSourcePaths) Timber.i("Finishing CameraUploadsWorker with UUID ${this.id}") return Result.success() } - private fun rescheduleMediaStoreTriggerIfNeeded(automaticUploadsEnabled: Boolean) { + private fun rescheduleMediaStoreTriggerIfNeeded( + automaticUploadsEnabled: Boolean, + sourcePaths: List, + ) { val shouldReschedule = inputData.getBoolean(KEY_PARAM_RESCHEDULE_MEDIA_STORE_TRIGGER, false) if (!shouldReschedule || !automaticUploadsEnabled) return WorkManagerProvider(appContext).enqueueMediaStoreAutomaticUploadsWorker( - existingWorkPolicy = ExistingWorkPolicy.APPEND_OR_REPLACE + existingWorkPolicy = ExistingWorkPolicy.APPEND_OR_REPLACE, + sourcePaths = sourcePaths, ) } @@ -146,8 +152,30 @@ class AutomaticUploadsWorker( // Else should not happen for the moment. Maybe in upcoming features.. else -> SyncType.PICTURE_UPLOADS } + val effectiveBehavior = folderBackUpConfiguration.effectiveBehavior val currentTimestamp = System.currentTimeMillis() + val automaticUploadSourceType = when (syncType) { + SyncType.PICTURE_UPLOADS -> UploadEnqueuedBy.ENQUEUED_AS_AUTOMATIC_UPLOAD_PICTURE + SyncType.VIDEO_UPLOADS -> UploadEnqueuedBy.ENQUEUED_AS_AUTOMATIC_UPLOAD_VIDEO + } + val completedUploadTimesBySourceUri = if (effectiveBehavior == UploadBehavior.MOVE) { + transferRepository.getFinishedTransfers() + .asSequence() + .filter { + it.createdBy == automaticUploadSourceType && + it.accountName == folderBackUpConfiguration.accountName + } + .mapNotNull { transfer -> + val sourcePath = transfer.sourcePath + val completedAt = transfer.transferEndTimestamp + if (sourcePath != null && completedAt != null) sourcePath to completedAt else null + } + .groupBy(keySelector = { it.first }, valueTransform = { it.second }) + .mapValues { (_, completionTimes) -> completionTimes.maxOrNull()!! } + } else { + emptyMap() + } val localPicturesDocumentFiles: List = folderBackUpConfiguration.sourcePaths.flatMap { sourcePath -> getFilesReadyToUpload( @@ -155,6 +183,7 @@ class AutomaticUploadsWorker( sourcePath = sourcePath, lastSyncTimestamp = folderBackUpConfiguration.lastSyncTimestamp, currentTimestamp = currentTimestamp, + completedUploadTimesBySourceUri = completedUploadTimesBySourceUri, ) } @@ -175,7 +204,7 @@ class AutomaticUploadsWorker( documentFile = documentFile, uploadPath = folderBackUpConfiguration.uploadPath.plus(File.separator).plus(documentFile.name), accountName = folderBackUpConfiguration.accountName, - behavior = folderBackUpConfiguration.behavior, + behavior = effectiveBehavior, createdByWorker = when (syncType) { SyncType.PICTURE_UPLOADS -> UploadEnqueuedBy.ENQUEUED_AS_AUTOMATIC_UPLOAD_PICTURE SyncType.VIDEO_UPLOADS -> UploadEnqueuedBy.ENQUEUED_AS_AUTOMATIC_UPLOAD_VIDEO @@ -186,7 +215,7 @@ class AutomaticUploadsWorker( contentUri = documentFile.uri, uploadPath = folderBackUpConfiguration.uploadPath.plus(File.separator).plus(documentFile.name), lastModified = documentFile.lastModified(), - behavior = folderBackUpConfiguration.behavior.toString(), + behavior = effectiveBehavior.toString(), accountName = folderBackUpConfiguration.accountName, uploadId = uploadId, wifiOnly = folderBackUpConfiguration.wifiOnly, @@ -272,6 +301,7 @@ class AutomaticUploadsWorker( sourcePath: String, lastSyncTimestamp: Long, currentTimestamp: Long, + completedUploadTimesBySourceUri: Map, ): List { val sourceUri: Uri = sourcePath.toUri() val documentTree = DocumentFile.fromTreeUri(applicationContext, sourceUri) @@ -282,11 +312,28 @@ class AutomaticUploadsWorker( // can result in uploading a truncated or 0-byte JPEG. val safeTimestamp = currentTimestamp - WRITE_SAFETY_BUFFER_MS - val filteredList: List = arrayOfLocalFiles + val mediaFiles = arrayOfLocalFiles .sortedBy { it.lastModified() } + .filter { MimetypeIconUtil.getBestMimeTypeByFilename(it.name).startsWith(syncType.prefixForType) } + + val previouslyUploadedFiles = mediaFiles.filter { documentFile -> + shouldRemovePreviouslyUploadedSource( + sourceUri = documentFile.uri.toString(), + lastModified = documentFile.lastModified(), + completedUploadTimesBySourceUri = completedUploadTimesBySourceUri, + ) + } + previouslyUploadedFiles.forEach { documentFile -> + if (!removeSourceDocument(documentFile)) { + Timber.w("Uploaded source file could not be removed yet: %s", documentFile.uri) + } + } + val previouslyUploadedUris = previouslyUploadedFiles.mapTo(mutableSetOf()) { it.uri } + + val filteredList: List = mediaFiles + .filterNot { it.uri in previouslyUploadedUris } .filter { it.lastModified() >= lastSyncTimestamp } .filter { it.lastModified() < safeTimestamp } - .filter { MimetypeIconUtil.getBestMimeTypeByFilename(it.name).startsWith(syncType.prefixForType) } Timber.i("Last sync ${syncType.name}: ${Date(lastSyncTimestamp)}") Timber.i("CurrentTimestamp ${Date(currentTimestamp)}") @@ -359,3 +406,12 @@ class AutomaticUploadsWorker( const val MEDIA_STORE_TRIGGER_MAX_DELAY_MS = 60_000L } } + +internal fun shouldRemovePreviouslyUploadedSource( + sourceUri: String, + lastModified: Long, + completedUploadTimesBySourceUri: Map, +): Boolean { + val completedAt = completedUploadTimesBySourceUri[sourceUri] ?: return false + return lastModified > 0 && lastModified <= completedAt +} diff --git a/Android/qsferaApp/src/main/java/eu/qsfera/android/workers/RemoveSourceFileWorker.kt b/Android/qsferaApp/src/main/java/eu/qsfera/android/workers/RemoveSourceFileWorker.kt index f0fa80a4b..34477f29c 100644 --- a/Android/qsferaApp/src/main/java/eu/qsfera/android/workers/RemoveSourceFileWorker.kt +++ b/Android/qsferaApp/src/main/java/eu/qsfera/android/workers/RemoveSourceFileWorker.kt @@ -44,11 +44,15 @@ class RemoveSourceFileWorker( if (!areParametersValid()) return Result.failure() return try { val documentFile = DocumentFile.fromSingleUri(appContext, contentUri) - documentFile?.delete() - Result.success() + if (removeSourceDocument(documentFile)) { + Result.success() + } else { + Timber.w("Source file could not be removed yet: %s", contentUri) + Result.retry() + } } catch (throwable: Throwable) { Timber.e(throwable) - Result.failure() + Result.retry() } } @@ -60,3 +64,10 @@ class RemoveSourceFileWorker( return true } } + +internal fun removeSourceDocument(documentFile: DocumentFile?): Boolean { + if (documentFile == null) return false + if (!documentFile.exists()) return true + + return documentFile.delete() || !documentFile.exists() +} diff --git a/Android/qsferaApp/src/main/res/values-ru/strings.xml b/Android/qsferaApp/src/main/res/values-ru/strings.xml index af1cd66e9..bdfecc7a6 100644 --- a/Android/qsferaApp/src/main/res/values-ru/strings.xml +++ b/Android/qsferaApp/src/main/res/values-ru/strings.xml @@ -568,7 +568,7 @@ Скопировать файл Переместить файл остался в исходной папке - удалено из исходной папки + удалён из исходной папки Общий доступ Поделиться %1$s Пользователи и группы diff --git a/Android/qsferaApp/src/main/res/xml/settings_picture_uploads.xml b/Android/qsferaApp/src/main/res/xml/settings_picture_uploads.xml index 1ba11ab51..15120224b 100644 --- a/Android/qsferaApp/src/main/res/xml/settings_picture_uploads.xml +++ b/Android/qsferaApp/src/main/res/xml/settings_picture_uploads.xml @@ -44,14 +44,11 @@ app:iconSpaceReserved="false" app:key="picture_uploads_clear_source_paths" app:title="@string/prefs_camera_upload_source_paths_clear_title" /> - + app:selectable="false" + app:summary="@string/pref_behaviour_entries_remove_original_file" + app:title="@string/prefs_camera_upload_behaviour_title" /> () + every { documentFile.exists() } returns false + + assertTrue(removeSourceDocument(documentFile)) + } + + @Test + fun `successful delete removes source`() { + val documentFile = mockk() + every { documentFile.exists() } returns true + every { documentFile.delete() } returns true + + assertTrue(removeSourceDocument(documentFile)) + } + + @Test + fun `failed delete requests another attempt`() { + val documentFile = mockk() + every { documentFile.exists() } returnsMany listOf(true, true) + every { documentFile.delete() } returns false + + assertFalse(removeSourceDocument(documentFile)) + } +} diff --git a/Android/qsferaDomain/src/main/java/eu/qsfera/android/domain/automaticuploads/model/AutomaticUploadsConfiguration.kt b/Android/qsferaDomain/src/main/java/eu/qsfera/android/domain/automaticuploads/model/AutomaticUploadsConfiguration.kt index 081bdee0a..eeb555f66 100644 --- a/Android/qsferaDomain/src/main/java/eu/qsfera/android/domain/automaticuploads/model/AutomaticUploadsConfiguration.kt +++ b/Android/qsferaDomain/src/main/java/eu/qsfera/android/domain/automaticuploads/model/AutomaticUploadsConfiguration.kt @@ -23,4 +23,9 @@ data class AutomaticUploadsConfiguration( val videoUploadsConfiguration: FolderBackUpConfiguration? ) { fun areAutomaticUploadsDisabled() = pictureUploadsConfiguration == null && videoUploadsConfiguration == null + + val sourcePaths: List + get() = listOfNotNull(pictureUploadsConfiguration, videoUploadsConfiguration) + .flatMap { it.sourcePaths } + .distinct() } diff --git a/Android/qsferaDomain/src/main/java/eu/qsfera/android/domain/automaticuploads/model/FolderBackUpConfiguration.kt b/Android/qsferaDomain/src/main/java/eu/qsfera/android/domain/automaticuploads/model/FolderBackUpConfiguration.kt index 36d1aab5c..c36391b32 100644 --- a/Android/qsferaDomain/src/main/java/eu/qsfera/android/domain/automaticuploads/model/FolderBackUpConfiguration.kt +++ b/Android/qsferaDomain/src/main/java/eu/qsfera/android/domain/automaticuploads/model/FolderBackUpConfiguration.kt @@ -35,6 +35,13 @@ data class FolderBackUpConfiguration( val isVideoUploads get() = name == videoUploadsName val sourcePaths get() = parseSourcePaths(sourcePath) + /** + * Picture uploads are an ingest operation: the source is removed only after + * the upload worker has completed successfully. Video uploads keep their + * explicitly configured behavior. + */ + val effectiveBehavior get() = if (isPictureUploads) UploadBehavior.MOVE else behavior + companion object { const val pictureUploadsName = "Picture uploads" const val videoUploadsName = "Video uploads" diff --git a/Android/qsferaDomain/src/test/java/eu/qsfera/android/domain/automaticuploads/model/FolderBackUpConfigurationTest.kt b/Android/qsferaDomain/src/test/java/eu/qsfera/android/domain/automaticuploads/model/FolderBackUpConfigurationTest.kt index 3dc958718..5bbd9d6ee 100644 --- a/Android/qsferaDomain/src/test/java/eu/qsfera/android/domain/automaticuploads/model/FolderBackUpConfigurationTest.kt +++ b/Android/qsferaDomain/src/test/java/eu/qsfera/android/domain/automaticuploads/model/FolderBackUpConfigurationTest.kt @@ -59,4 +59,39 @@ class FolderBackUpConfigurationTest { assertEquals("$firstSourcePath\n$secondSourcePath", encodedSourcePaths) } + + @Test + fun `picture uploads always remove source after successful upload`() { + val configuration = folderBackUpConfiguration( + name = FolderBackUpConfiguration.pictureUploadsName, + behavior = UploadBehavior.COPY, + ) + + assertEquals(UploadBehavior.MOVE, configuration.effectiveBehavior) + } + + @Test + fun `video uploads preserve configured behavior`() { + val configuration = folderBackUpConfiguration( + name = FolderBackUpConfiguration.videoUploadsName, + behavior = UploadBehavior.COPY, + ) + + assertEquals(UploadBehavior.COPY, configuration.effectiveBehavior) + } + + private fun folderBackUpConfiguration( + name: String, + behavior: UploadBehavior, + ) = FolderBackUpConfiguration( + accountName = "account", + behavior = behavior, + sourcePath = "content://source", + uploadPath = "/CameraUpload", + wifiOnly = false, + chargingOnly = false, + lastSyncTimestamp = 0, + name = name, + spaceId = null, + ) } diff --git a/README.md b/README.md new file mode 100644 index 000000000..4ad6ad355 --- /dev/null +++ b/README.md @@ -0,0 +1,29 @@ +# QSfera + +This repository contains the QSfera clients and server as one workspace. + +| Component | Directory | Primary local check | +| --- | --- | --- | +| Android app | [`Android`](Android/) | `cd Android && ./gradlew :qsferaApp:assembleOriginalDebug` | +| Desktop client | [`Desktop`](Desktop/) | See [`Desktop/README.md`](Desktop/README.md) | +| Server | [`Server`](Server/) | `cd Server && make test` | + +## Android automatic photo uploads + +The Android app can monitor user-selected Storage Access Framework folders, +including the camera folder. New media is queued through WorkManager, uploaded, +and removed from the source folder only after a successful transfer. The +periodic worker remains as a fallback for document providers that do not emit a +content-change notification. + +## Raspberry Pi server + +The production-oriented ARM64 Compose profile, health check, SSD layout and +backup/restore scripts are documented in +[`Server/deployments/raspberry-pi`](Server/deployments/raspberry-pi/README.md). + +## CI and dependency updates + +Root GitHub workflows run Android checks and server security/deployment checks +from their actual monorepo paths. Root Dependabot configuration covers Gradle, +Go modules, the server IdP npm project, Docker and GitHub Actions. diff --git a/Server/.woodpecker.star b/Server/.woodpecker.star index 1468686dc..c1898ea2b 100644 --- a/Server/.woodpecker.star +++ b/Server/.woodpecker.star @@ -649,17 +649,7 @@ def buildWebCache(ctx): def testQsferaAndUploadResults(ctx): unit_pipeline = testQsfera(ctx) - - ###################################################################### - # The triggers have been disabled for now, since the govulncheck can # - # not silence single, acceptable vulnerabilities. # - # See https://github.com/owncloud/ocis/issues/9527 for more details. # - # FIXME: RE-ENABLE THIS ASAP!!! # - ###################################################################### - - #security_scan = scanQsfera(ctx) - #return [security_scan] + unit_pipeline + [scan_result_upload] - return unit_pipeline + return scanQsfera(ctx) + unit_pipeline def testPipelines(ctx): pipelines = [] diff --git a/Server/Dockerfile b/Server/Dockerfile index 88adcf29d..561b7a5b6 100644 --- a/Server/Dockerfile +++ b/Server/Dockerfile @@ -48,6 +48,19 @@ LABEL maintainer="QSfera" \ org.opencontainers.image.documentation="" \ org.opencontainers.image.source="" +RUN addgroup -g 1000 -S qsfera-group && \ + adduser -S --ingroup qsfera-group --uid 1000 qsfera-user --home /var/lib/qsfera && \ + mkdir -p /var/lib/qsfera/web/assets/apps /etc/qsfera && \ + chown -R qsfera-user:qsfera-group /var/lib/qsfera /etc/qsfera && \ + chmod -R 751 /var/lib/qsfera /etc/qsfera + +VOLUME ["/var/lib/qsfera", "/etc/qsfera"] +WORKDIR /var/lib/qsfera + +USER 1000 + +EXPOSE 9200/tcp + ENTRYPOINT ["/usr/bin/qsfera"] CMD ["server"] diff --git a/Server/Makefile b/Server/Makefile index 99f6b18ab..86040f00c 100644 --- a/Server/Makefile +++ b/Server/Makefile @@ -210,11 +210,11 @@ protobuf: .PHONY: golangci-lint golangci-lint: $(GOLANGCI_LINT) - $(GOLANGCI_LINT) run --modules-download-mode vendor --timeout 15m0s --issues-exit-code 0 --out-format checkstyle > checkstyle.xml + $(GOLANGCI_LINT) run --modules-download-mode vendor --timeout 15m0s --issues-exit-code 1 --out-format checkstyle > checkstyle.xml .PHONY: ci-golangci-lint ci-golangci-lint: - $(GOLANGCI_LINT) run --modules-download-mode vendor --timeout 15m0s --issues-exit-code 0 --out-format checkstyle > checkstyle.xml + $(GOLANGCI_LINT) run --modules-download-mode vendor --timeout 15m0s --issues-exit-code 1 --out-format checkstyle > checkstyle.xml .PHONY: golangci-lint-fix golangci-lint-fix: $(GOLANGCI_LINT) diff --git a/Server/deployments/raspberry-pi/.env.example b/Server/deployments/raspberry-pi/.env.example new file mode 100644 index 000000000..3f43c0da4 --- /dev/null +++ b/Server/deployments/raspberry-pi/.env.example @@ -0,0 +1,22 @@ +# Use an immutable release tag or digest that contains linux/arm64. +QSFERA_IMAGE= + +# Public URL used by the Android, desktop and web clients. +QSFERA_URL= + +# Bind to localhost when a reverse proxy runs on the Pi. +QSFERA_BIND_IP=127.0.0.1 +QSFERA_PROXY_TLS=true +QSFERA_INSECURE=false +QSFERA_ENABLE_BASIC_AUTH=false +QSFERA_LOG_LEVEL=info + +# Do not reuse the example or demo password. +QSFERA_ADMIN_PASSWORD= + +# Absolute directories on persistent storage. An external SSD is recommended. +QSFERA_CONFIG_DIR= +QSFERA_DATA_DIR= + +# Set after measuring RAM used by the OS and other services, for example 2g. +QSFERA_MEMORY_LIMIT= diff --git a/Server/deployments/raspberry-pi/README.md b/Server/deployments/raspberry-pi/README.md new file mode 100644 index 000000000..d2401f6a8 --- /dev/null +++ b/Server/deployments/raspberry-pi/README.md @@ -0,0 +1,96 @@ +# Raspberry Pi deployment + +This profile runs the single-container QSfera server on a 64-bit Raspberry Pi +OS and persists configuration and user data outside the container. It expects a +separate reverse proxy for a browser-trusted TLS certificate; by default port +9200 is therefore published only on `127.0.0.1`. + +## Requirements + +- `uname -m` must report `aarch64` or `arm64`. +- Docker Engine with the Compose plugin must be installed. +- The selected QSfera image tag or digest must contain `linux/arm64`. +- The configuration and data directories must be on persistent storage and + writable by UID/GID `1000:1000`, which is the non-root user in the image. + +For an external disk mounted at `/mnt/qsfera`, create the directories with: + +```sh +sudo install -d -o 1000 -g 1000 -m 0750 /mnt/qsfera/config /mnt/qsfera/data +findmnt /mnt/qsfera +``` + +`findmnt` must show the expected external filesystem before the service starts. +This prevents an unavailable disk from silently placing data on the Pi's root +filesystem. + +## Configuration + +```sh +cd Server/deployments/raspberry-pi +cp .env.example .env +chmod 600 .env +``` + +Fill every empty value in `.env`. The Compose file deliberately has no default +for the image, public URL, admin password, persistent paths or memory limit. +`docker compose config` fails when any of these values is empty. + +Choose `QSFERA_MEMORY_LIMIT` from measurements on the target Pi: + +```text +QSFERA_MEMORY_LIMIT = total RAM - OS reserve - other services reserve +``` + +Use `free -h` for total/current host memory and `docker stats` for the other +containers. Leave enough headroom for the kernel, filesystem cache and reverse +proxy. QSfera's Go runtime then derives its own default `GOMEMLIMIT` from the +container limit with a 0.9 ratio. + +Keep `QSFERA_BIND_IP=127.0.0.1` when the reverse proxy runs on the same Pi. Set +it to a reachable interface only when access to port 9200 is protected by an +equivalent network and TLS design. + +## Start and verify + +```sh +sh ./smoke-test.sh +docker compose ps +``` + +The smoke test validates the Compose file, starts the service and waits for the +container health check. Its default 180-second startup budget can be changed +with `SMOKE_TIMEOUT_SECONDS`. + +## Backup + +The backup script stops QSfera, archives both persistent directories, writes a +SHA-256 checksum and starts the service again: + +```sh +sh ./backup.sh /mnt/backups/qsfera +``` + +Copy the resulting `.tar.gz` and `.sha256` files to storage that is independent +of the Pi and its data disk. + +## Restore + +Restore replaces the current configuration and data. Check the selected archive +and its checksum first, then run: + +```sh +RESTORE_CONFIRM=restore sh ./restore.sh /mnt/backups/qsfera/qsfera-TIMESTAMP.tar.gz +sh ./smoke-test.sh +``` + +The restore script verifies the checksum when the matching `.sha256` file is +present and stops the service during replacement. It starts QSfera only after a +successful extraction; after an error the service remains stopped so a partial +data set is never served automatically. + +## References + +- [Raspberry Pi external storage documentation](https://www.raspberrypi.com/documentation/computers/raspberry-pi.html#external-storage) +- [Docker Compose service configuration](https://docs.docker.com/reference/compose-file/services/) +- [Go garbage collector guide](https://go.dev/doc/gc-guide) diff --git a/Server/deployments/raspberry-pi/backup.sh b/Server/deployments/raspberry-pi/backup.sh new file mode 100644 index 000000000..c61db791f --- /dev/null +++ b/Server/deployments/raspberry-pi/backup.sh @@ -0,0 +1,38 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")" + +backup_root=${1:-./backups} +timestamp=$(date -u +%Y%m%dT%H%M%SZ) +archive="$backup_root/qsfera-$timestamp.tar.gz" +temporary="$archive.incomplete" +archive_directory=$(dirname "$archive") +archive_name=$(basename "$archive") + +umask 077 +mkdir -p "$backup_root" + +compose() { + docker compose "$@" +} + +cleanup() { + rm -f "$temporary" + compose start qsfera >/dev/null +} + +compose config --quiet +compose stop qsfera +trap cleanup EXIT +trap 'exit 1' HUP INT TERM + +compose run --rm --no-deps -T --entrypoint /bin/sh qsfera -ec \ + 'tar -C / -czf - etc/qsfera var/lib/qsfera' >"$temporary" +mv "$temporary" "$archive" +(cd "$archive_directory" && sha256sum "$archive_name" >"$archive_name.sha256") + +compose start qsfera >/dev/null +trap - EXIT HUP INT TERM + +printf 'Backup: %s\nChecksum: %s\n' "$archive" "$archive.sha256" diff --git a/Server/deployments/raspberry-pi/compose.yaml b/Server/deployments/raspberry-pi/compose.yaml new file mode 100644 index 000000000..53353f7e0 --- /dev/null +++ b/Server/deployments/raspberry-pi/compose.yaml @@ -0,0 +1,47 @@ +name: qsfera-pi + +services: + qsfera: + image: ${QSFERA_IMAGE:?Set QSFERA_IMAGE to a pinned ARM64 image tag} + platform: linux/arm64 + init: true + entrypoint: + - /bin/sh + command: + - -ec + - | + if [ ! -f /etc/qsfera/qsfera.yaml ]; then + qsfera init + fi + exec qsfera server + environment: + OC_BASE_DATA_PATH: /var/lib/qsfera + OC_CONFIG_DIR: /etc/qsfera + OC_URL: ${QSFERA_URL:?Set QSFERA_URL to the public server URL} + OC_LOG_LEVEL: ${QSFERA_LOG_LEVEL:-info} + OC_LOG_COLOR: "false" + OC_LOG_PRETTY: "false" + OC_INSECURE: "${QSFERA_INSECURE:-false}" + PROXY_TLS: "${QSFERA_PROXY_TLS:-true}" + PROXY_ENABLE_BASIC_AUTH: "${QSFERA_ENABLE_BASIC_AUTH:-false}" + IDM_ADMIN_PASSWORD: ${QSFERA_ADMIN_PASSWORD:?Set a strong QSFERA_ADMIN_PASSWORD} + IDM_CREATE_DEMO_USERS: "false" + ports: + - "${QSFERA_BIND_IP:-127.0.0.1}:9200:9200" + volumes: + - type: bind + source: ${QSFERA_CONFIG_DIR:?Set QSFERA_CONFIG_DIR to a persistent directory} + target: /etc/qsfera + - type: bind + source: ${QSFERA_DATA_DIR:?Set QSFERA_DATA_DIR to a persistent directory} + target: /var/lib/qsfera + mem_limit: "${QSFERA_MEMORY_LIMIT:?Set QSFERA_MEMORY_LIMIT after measuring available RAM}" + healthcheck: + test: + - CMD-SHELL + - 'scheme=https; [ "$${PROXY_TLS}" = "false" ] && scheme=http; curl --fail --silent --show-error --insecure "$${scheme}://127.0.0.1:9200/status.php" >/dev/null' + security_opt: + - no-new-privileges:true + logging: + driver: local + restart: unless-stopped diff --git a/Server/deployments/raspberry-pi/restore.sh b/Server/deployments/raspberry-pi/restore.sh new file mode 100644 index 000000000..68c98fec2 --- /dev/null +++ b/Server/deployments/raspberry-pi/restore.sh @@ -0,0 +1,51 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")" + +archive=${1:-} +if [ -z "$archive" ] || [ ! -f "$archive" ]; then + echo "Usage: RESTORE_CONFIRM=restore $0 /path/to/qsfera-TIMESTAMP.tar.gz" >&2 + exit 2 +fi + +if [ "${RESTORE_CONFIRM:-}" != "restore" ]; then + echo "Restore replaces the current QSfera configuration and data." >&2 + echo "Re-run with RESTORE_CONFIRM=restore after checking the archive path." >&2 + exit 2 +fi + +if [ -f "$archive.sha256" ]; then + archive_directory=$(dirname "$archive") + archive_name=$(basename "$archive") + (cd "$archive_directory" && sha256sum --check "$archive_name.sha256") +fi + +compose() { + docker compose "$@" +} + +restore_completed=false + +finish_restore() { + if [ "$restore_completed" = "true" ]; then + compose start qsfera >/dev/null + else + echo "Restore failed; QSfera remains stopped to protect the data set." >&2 + fi +} + +compose config --quiet +compose stop qsfera +trap finish_restore EXIT +trap 'exit 1' HUP INT TERM + +compose run --rm --no-deps -T --entrypoint /bin/sh qsfera -ec \ + 'find /etc/qsfera -mindepth 1 -maxdepth 1 -exec rm -rf {} +; find /var/lib/qsfera -mindepth 1 -maxdepth 1 -exec rm -rf {} +; tar -C / -xzf -' \ + <"$archive" + +restore_completed=true +compose start qsfera >/dev/null +trap - EXIT HUP INT TERM + +echo "Restore completed. Run sh ./smoke-test.sh to verify startup and health." diff --git a/Server/deployments/raspberry-pi/smoke-test.sh b/Server/deployments/raspberry-pi/smoke-test.sh new file mode 100644 index 000000000..373093e35 --- /dev/null +++ b/Server/deployments/raspberry-pi/smoke-test.sh @@ -0,0 +1,53 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")" + +case "$(uname -m)" in + aarch64|arm64) ;; + *) + echo "This profile requires a 64-bit ARM OS; uname -m returned $(uname -m)." >&2 + exit 1 + ;; +esac + +timeout_seconds=${SMOKE_TIMEOUT_SECONDS:-180} +started_at=$(date +%s) + +docker compose config --quiet +docker compose up -d +container_id=$(docker compose ps -q qsfera) + +if [ -z "$container_id" ]; then + echo "QSfera container was not created." >&2 + exit 1 +fi + +while :; do + state=$(docker inspect --format '{{.State.Status}}' "$container_id") + if [ "$state" != "running" ]; then + docker compose logs --tail=100 qsfera >&2 + echo "QSfera container state is $state." >&2 + exit 1 + fi + + health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}missing{{end}}' "$container_id") + case "$health" in + healthy) + echo "QSfera container is healthy." + exit 0 + ;; + unhealthy) + docker compose logs --tail=100 qsfera >&2 + exit 1 + ;; + esac + + now=$(date +%s) + if [ $((now - started_at)) -ge "$timeout_seconds" ]; then + docker compose logs --tail=100 qsfera >&2 + echo "QSfera did not become healthy within ${timeout_seconds}s." >&2 + exit 1 + fi + sleep 2 +done diff --git a/Server/qsfera/README.md b/Server/qsfera/README.md index dd2b75063..f857626a3 100644 --- a/Server/qsfera/README.md +++ b/Server/qsfera/README.md @@ -15,7 +15,7 @@ To configure which registry to use, you have to set the environment variable `MI ## Memory limits -КуСфера will automatically set the go native `GOMEMLIMIT` to `0.9`. To disable the limit set `AUTOMEMEMLIMIT=off`. For more information take a look at the official [Guide to the Go Garbage Collector](https://go.dev/doc/gc-guide). +КуСфера will automatically set the go native `GOMEMLIMIT` to `0.9`. To disable the limit set `AUTOMEMLIMIT=off`. For more information take a look at the official [Guide to the Go Garbage Collector](https://go.dev/doc/gc-guide). ## CLI Commands