diff --git a/Dockerfile b/Dockerfile index 91be63c..80a6b72 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,5 +17,4 @@ RUN pipenv install --system --deploy WORKDIR /mergin-media-sync COPY version.py config.py drivers.py media_sync.py media_sync_daemon.py ./ -# create deafult config file (can be overridden with env variables) -COPY config.yaml.default ./config.yaml \ No newline at end of file +ENTRYPOINT ["python3", "media_sync_daemon.py"] diff --git a/README.md b/README.md index bcd1b39..84334b0 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,9 @@ # Mergin Maps Media Sync -Sync media files from Mergin Maps projects to other storage backends. Currently, supported backend are MinIO (S3-like) backend, Google Drive and local drive (mostly used for testing). +Sync media files from Mergin Maps projects to other storage backends. Currently, supported backends are MinIO (S3-like), Google Drive and local drive (mostly used for testing). -Sync works in two modes, in COPY mode, where media files are only copied to external drive and MOVE mode, where files are -subsequently removed from Mergin Maps project (on cloud). +Sync works in two modes: in **COPY** mode media files are only copied to the external storage, in **MOVE** mode files are additionally removed from the Mergin Maps project (cloud). The user can choose whether references to media files in the project should be updated to point to the new location (e.g. S3 URL) or left unchanged. -Also user can choose whether references to media files should be updated. - -**IMPORTANT**: structure of the config file was changed in the latest version. Therefore old .ini config files should be migrated and enviromnent values should be updated. +**IMPORTANT**: The config file format was updated. There is now a single `config.yaml` file and project settings are defined in a `projects` list. Old single-project configs must be migrated (see the Config file structure section below). ### Quick start @@ -17,57 +14,87 @@ Not sure where to start? Check out our [quick start](docs/quick_start.md) guide Basic principle is that media-sync deamon COPY or MOVE your pictures from Mergin Maps server to other storage backend. Here is an example of MOVE operation: ![Overview](docs/images/overview.png) -#### Running with Docker -To run the container, use a command like the following one: -```shell - docker run -it \ - -e MERGIN__USERNAME=john \ - -e MERGIN__PASSWORD=myStrongPassword \ - -e MERGIN__PROJECT_NAME=john/my_project \ - lutraconsulting/mergin-media-sync python3 media_sync_daemon.py +### Config file structure + +Multiple Mergin Maps projects can be configured in a single `config.yaml`. The Mergin user credentials and the storage driver are shared across all projects; each project specifies only its full project name, its own driver destination path and its reference table configuration. + +For a quick start, copy `config.yaml.default` to `config.yaml` and edit it. The key sections are: + +```yaml +project_working_dir: /tmp/mediasync # base dir; each project gets its own sub-folder +allowed_extensions: [jpg, png] +operation_mode: copy # copy | move +driver: minio # local | minio | google_drive + +mergin: + url: https://app.merginmaps.com # optional, defaults to public Mergin Maps instance + username: myuser + password: mypassword + +# Driver connection settings – global, shared by all projects +minio: + endpoint: localhost:9000 + access_key: ACCESS + secret_key: SECRET + bucket: mybucket + secure: false + region: + public_url: # optional: public base URL for DB links (e.g. https://cdn.example.com) + +# Per-project settings +projects: + - project_name: myworkspace/project1 # full Mergin project name + bucket_subpath: project1 # minio: optional sub-path inside the bucket + references: + - file: survey.gpkg + table: notes + local_path_column: photo + driver_path_column: ext_url + + - project_name: myworkspace/project2 + bucket_subpath: project2 + references: [] + +daemon: + sleep_time: 10 ``` -The sync process will start, regularly checking Mergin Maps service copy/move media files from a Mergin Maps project to an external storage. -Local drive is a default backend, you need to mount volume from host machine for data to persist. -#### Update reference table in geopackage -If you'd like to update references to media files (probably useful with MOVE mode), you can run: -```shell -docker run -it \ - -v /tmp/mediasync:/data \ - --name mergin-media-sync \ - -e MERGIN__USERNAME=john \ - -e MERGIN__PASSWORD=myStrongPassword \ - -e MERGIN__PROJECT_NAME=john/my_project \ - -e LOCAL__DEST=/data \ - -e OPERATION_MODE=move \ - -e REFERENCES = "[{file='my_survey.gpkg', table='my_table', local_path_column='col_with_path', driver_path_column='col_with_ext_url'}]" \ - lutraconsulting/mergin-media-sync python3 media_sync_daemon.py +Per-project path field per driver: + +| Driver | Field | Description | +|----------------|------------------|-------------------------------------------| +| `local` | `dest` | Destination directory | +| `minio` | `bucket_subpath` | Optional sub-path inside the bucket | +| `google_drive` | `folder` | Google Drive folder name | + +#### MinIO: separate public URL for database links + +By default, the URL stored in the reference table is constructed from `minio.endpoint` and `minio.bucket`. +If the public-facing domain differs from the upload endpoint (e.g. a CDN, reverse proxy, or load-balanced hostname), +set `minio.public_url` to the desired base URL: + +```yaml +minio: + endpoint: internal-minio.corp:9000 # used for file uploads + bucket: media + public_url: https://cdn.example.com # used for URLs stored in the database ``` -Make sure you have correct structure of you .gpkg file. Otherwise leave all `REFERENCE__` variables empty. -#### Using MinIO backend -Last, in case you want to switch to different driver, you can run: +The stored link will then be `https://cdn.example.com/` instead of +`http://internal-minio.corp:9000/media/`. +`bucket_subpath` is still applied on top of `public_url` when set. + +#### Running with Docker + +Prepare a `config.yaml` (use `config.yaml.default` as a template) and mount it into the container: + ```shell docker run -it \ --name mergin-media-sync \ - -e MERGIN__USERNAME=john \ - -e MERGIN__PASSWORD=myStrongPassword \ - -e MERGIN__PROJECT_NAME=john/my_project \ - -e MERGIN__PROJECT_NAME=ttest/mediasync_test \ - -e DRIVER=minio \ - -e MINIO__ENDPOINT="minio-server-url" \ - -e MINIO__ACCESS_KEY=access-key \ - -e MINIO__SECRET_KEY=secret-key \ - -e MINIO__BUCKET=destination-bucket \ - -e MINIO__SECURE=1 \ - -e MINIO__BUCKET_SUBPATH=SubFolder \ - lutraconsulting/mergin-media-sync python3 media_sync_daemon.py + -v ${PWD}:/settings \ + lutraconsulting/mergin-media-sync /settings/config.yaml ``` -**Please note double underscore `__` is used to separate [config](config.yaml.default) group and item.** - -The specification of `MINIO__BUCKET_SUBPATH` is optional and can be skipped if the files should be stored directly in `MINIO__BUCKET`. - #### Using Google Drive backend For setup instructions and more details, please refer to our [Google Drive guide](./docs/google-drive-setup.md). @@ -85,36 +112,36 @@ And then building the image: ``` docker build -t mergin_media_sync . ``` + #### Manual installation If you would like to avoid the manual installation steps, please follow the guide on using sync with Docker above. We use pipenv for managing python virtual environment. - ```shell - pipenv install --three +pipenv install --three ``` If you get `ModuleNotFoundError: No module named 'skbuild'` error, try to update pip with command `python -m pip install --upgrade pip` - ### How to use If you want to modify references to media files in some geopackage in your project, please make sure you have two columns there, one with reference to local file and another for external URL where file can be downloaded from. - Initialization: -1. set up configuration in config.yaml (see config.yaml.default for a sample) -2. all settings can be overridden with env variables (see docker example above) -3. run media-sync +1. Set up configuration in config.yaml (see config.yaml.default for a sample). +2. All settings can be overridden with env variables. +3. Run media-sync. + ```shell - pipenv run python3 media_sync.py +pipenv run python3 media_sync.py ``` ### Running Tests + You need to install also dev packages: ```shell - pipenv install --three --dev +pipenv install --three --dev ``` and run local minio server: @@ -150,3 +177,4 @@ To run automatic tests: docker tag lutraconsulting/mergin-media-sync lutraconsulting/mergin-media-sync:0.1.0 docker push lutraconsulting/mergin-media-sync:0.1.0 docker push lutraconsulting/mergin-media-sync:latest + ``` diff --git a/config.py b/config.py index ca379ab..ef05eba 100644 --- a/config.py +++ b/config.py @@ -21,11 +21,57 @@ class ConfigError(Exception): pass +def _validate_project(project, driver, index): + """Validate a single project entry inside the projects list.""" + if not hasattr(project, "project_name") or not project.project_name: + raise ConfigError( + f"Config error: Project #{index} is missing 'project_name'" + ) + + if driver == DriverType.LOCAL and not ( + hasattr(project, "dest") and project.dest + ): + raise ConfigError( + f"Config error: Project '{project.project_name}' is missing 'dest' for local driver" + ) + + if driver == DriverType.MINIO: + # bucket_subpath is optional for minio – no mandatory check needed + pass + + if driver == DriverType.GOOGLE_DRIVE and not ( + hasattr(project, "folder") and project.folder + ): + raise ConfigError( + f"Config error: Project '{project.project_name}' is missing 'folder' for google_drive driver" + ) + + if not hasattr(project, "base_path") or project.base_path is None: + project.update({"base_path": ""}) + + if not hasattr(project, "references") or project.references is None: + project.update({"references": []}) + + if not isinstance(project.references, list): + raise ConfigError( + f"Config error: Project '{project.project_name}': 'references' must be a list" + ) + + for ref in project.references: + if not all( + hasattr(ref, attr) + for attr in ["file", "table", "local_path_column", "driver_path_column"] + ): + raise ConfigError( + f"Config error: Project '{project.project_name}': incorrect media reference settings" + ) + + def validate_config(config): """Validate config - make sure values are consistent""" if not ( - config.mergin.username and config.mergin.password and config.mergin.project_name + config.mergin.username and config.mergin.password and config.mergin.url ): raise ConfigError("Config error: Incorrect mergin settings") @@ -39,9 +85,6 @@ def validate_config(config): if config.operation_mode not in ["move", "copy"]: raise ConfigError("Config error: Unsupported operation mode") - if config.driver == "local" and not config.local.dest: - raise ConfigError("Config error: Incorrect Local driver settings") - if config.driver == DriverType.MINIO and not ( config.minio.endpoint and config.minio.access_key @@ -53,31 +96,26 @@ def validate_config(config): if not (config.allowed_extensions and len(config.allowed_extensions)): raise ConfigError("Config error: Allowed extensions can not be empty") - if "references" not in config: - config.update({"references": []}) - - if config.references is None: - config.update({"references": []}) - - if not isinstance(config.references, list): - raise ConfigError( - "Config error: Incorrect reference settings. Needs to be list of references." - ) - - for ref in config.references: - if not all( - hasattr(ref, attr) - for attr in ["file", "table", "local_path_column", "driver_path_column"] - ): - raise ConfigError("Config error: Incorrect media reference settings") - if config.driver == DriverType.GOOGLE_DRIVE and not ( hasattr(config.google_drive, "service_account_file") - and hasattr(config.google_drive, "folder") and hasattr(config.google_drive, "share_with") ): raise ConfigError("Config error: Incorrect GoogleDrive driver settings") + # Validate projects list + if "projects" not in config or not config.projects: + raise ConfigError("Config error: 'projects' list is missing or empty") + + if not isinstance(config.projects, list): + raise ConfigError("Config error: 'projects' must be a list") + + projects = config.projects + + for i, project in enumerate(projects): + _validate_project(project, config.driver, i) + + config.update({"projects": projects}) + def update_config_path( path_param: str, diff --git a/config.yaml.default b/config.yaml.default index d72d66f..6c7bb0e 100644 --- a/config.yaml.default +++ b/config.yaml.default @@ -3,17 +3,12 @@ allowed_extensions: - jpg - png operation_mode: copy -base_path: driver: local mergin: url: https://app.merginmaps.com username: media-sync password: super-secret - project_name: media-sync/project - -local: - dest: /tmp/mediasync_copy minio: endpoint: localhost:9000 @@ -22,18 +17,26 @@ minio: bucket: test secure: false region: - bucket_subpath: + public_url: google_drive: - service_account_file: - folder: + service_account_file: share_with: -references: - - file: survey.gpkg - table: notes - local_path_column: photo - driver_path_column: ext_url +# List of projects to sync. Each project specifies its full name, the +# driver-specific destination path and its own reference table configuration. +projects: + - project_name: workspace/project1 + base_path: + # Path settings - only the field matching the active driver is required: + dest: # local driver: destination directory + bucket_subpath: # minio driver: subpath inside the bucket + folder: # google_drive driver: target folder name + references: + - file: survey.gpkg + table: notes + local_path_column: photo + driver_path_column: ext_url daemon: sleep_time: 10 diff --git a/docs/quick_start.md b/docs/quick_start.md index 9f2ee77..1f591aa 100644 --- a/docs/quick_start.md +++ b/docs/quick_start.md @@ -42,27 +42,49 @@ Upload your project to Mergin Maps, either via web browser or [Mergin plugin](ht You have now your project ready in Mergin Maps. -## 4. Start syncing -Download and run media-sync docker image with configuration based on above (you will need to tweak that): +## 4. Create a config file +Copy `config.yaml.default` to `config.yaml` and edit it. The important parts for this quick start are: + +```yaml +project_working_dir: /tmp/mediasync +allowed_extensions: [jpg, png] +operation_mode: copy +driver: minio + +mergin: + url: https://app.merginmaps.com + username: test000 + password: myStrongPassword + +minio: + endpoint: minio-server-url + access_key: access-key + secret_key: secret-key + bucket: destination-bucket + secure: true + +projects: + - project_name: test000/media-sync + bucket_subpath: media-sync + references: + - file: survey.gpkg + table: notes + local_path_column: photo + driver_path_column: external_url +``` + +You can add more entries under `projects:` to sync multiple Mergin Maps projects with the same driver. -Note that you may need to escape some characters in your password or username +## 5. Start syncing +Run media-sync docker image with the config file from above: ``` -$ sudo docker run -it \ +docker run -it \ --name mergin-media-sync \ - -e MERGIN__USERNAME=test000 \ - -e MERGIN__PASSWORD=myStrongPassword \ - -e MERGIN__PROJECT_NAME=test000/media-sync \ - -e DRIVER=minio \ - -e MINIO__ENDPOINT="minio-server-url" \ - -e MINIO__ACCESS_KEY=access-key \ - -e MINIO__SECRET_KEY=secret-key \ - -e MINIO__BUCKET=destination-bucket \ - -e MINIO__SECRET=1 \ - -e OPERATION_MODE=copy \ - -e REFERENCES="[{file='survey.gpkg', table='notes', local_path_column='photo', driver_path_column='external_url'}]" \ - mergin-media-sync python3 media_sync_daemon.py + -v ${PWD}:/settings \ + lutraconsulting/mergin-media-sync /settings/config.yaml ``` + and you should see photos copied from your Mergin Maps project to the bucket: ![bucket](images/bucket.png) @@ -71,7 +93,4 @@ and your references in QGIS project updated: ![bucket](images/qgis_proj2.png) -If you wish to specify folder within the bucket to store the files you can specify `MINIO__BUCKET_SUBPATH` variable -ie. `MINIO__BUCKET_SUBPATH=MyFolder` to store files in `MyFolder` instead of root of the bucket. - In order to stop syncing simply stop docker container. diff --git a/drivers.py b/drivers.py index 0ec20bf..c25bd6e 100644 --- a/drivers.py +++ b/drivers.py @@ -54,9 +54,9 @@ def upload_file(self, src, obj_path): class LocalDriver(Driver): """Driver to work with local drive, for testing purpose mainly""" - def __init__(self, config): + def __init__(self, config, project): super(LocalDriver, self).__init__(config) - self.dest = config.local.dest + self.dest = project.dest try: if not os.path.exists(self.dest): @@ -79,7 +79,7 @@ def upload_file(self, src, obj_path): class MinioDriver(Driver): """Driver to handle connection to minio-like server""" - def __init__(self, config): + def __init__(self, config, project): super(MinioDriver, self).__init__(config) try: @@ -96,19 +96,21 @@ def __init__(self, config): self.client.make_bucket(self.bucket) self.bucket_subpath = None - if hasattr(config.minio, "bucket_subpath"): - if config.minio.bucket_subpath: - self.bucket_subpath = config.minio.bucket_subpath + if hasattr(project, "bucket_subpath") and project.bucket_subpath: + self.bucket_subpath = project.bucket_subpath - # construct base url for bucket + # construct base url from endpoint and bucket scheme = "https://" if config.as_bool("minio.secure") else "http://" - if config.minio.region and "amazonaws" in config.minio.endpoint.lower(): + if hasattr(config.minio, "public_url") and config.minio.public_url: + self.base_url = config.minio.public_url.rstrip("/") + elif config.minio.region and "amazonaws" in config.minio.endpoint.lower(): self.base_url = ( f"{scheme}{self.bucket}.s3.{config.minio.region}.amazonaws.com" ) else: self.base_url = scheme + config.minio.endpoint + "/" + self.bucket + except S3Error as e: raise DriverError("MinIO driver init error: " + str(e)) @@ -126,7 +128,7 @@ def upload_file(self, src, obj_path): class GoogleDriveDriver(Driver): """Driver to handle connection to Google Drive""" - def __init__(self, config): + def __init__(self, config, project): super(GoogleDriveDriver, self).__init__(config) try: @@ -139,7 +141,7 @@ def __init__(self, config): "drive", "v3", credentials=self._credentials ) - self._folder = config.google_drive.folder + self._folder = project.folder self._folder_id = self._folder_exists(self._folder) if not self._folder_id: @@ -282,13 +284,13 @@ def _get_share_with(self, config_google_drive) -> typing.List[str]: return emails_to_share_with -def create_driver(config): - """Create driver object based on type defined in config""" +def create_driver(config, project): + """Create driver object based on type defined in config and project-specific settings""" driver = None if config.driver == DriverType.LOCAL: - driver = LocalDriver(config) + driver = LocalDriver(config, project) elif config.driver == DriverType.MINIO: - driver = MinioDriver(config) + driver = MinioDriver(config, project) elif config.driver == DriverType.GOOGLE_DRIVE: - driver = GoogleDriveDriver(config) + driver = GoogleDriveDriver(config, project) return driver diff --git a/media_sync.py b/media_sync.py index f3a6ec8..c1c08a1 100644 --- a/media_sync.py +++ b/media_sync.py @@ -24,29 +24,39 @@ def _quote_identifier(identifier): return '"' + identifier + '"' -def _get_project_version(): +def _project_working_dir(project): + """Return the working directory for a specific project. + + The base directory from config is combined with the project name so that + each project gets its own isolated sub-directory, e.g.: + /tmp/mediasync/workspace/my_project + """ + return os.path.join(config.project_working_dir, project.project_name) + + +def _get_project_version(project): """Returns the current version of the project""" - mp = MerginProject(config.project_working_dir) + mp = MerginProject(_project_working_dir(project)) return mp.version() -def _check_has_working_dir(): - if not os.path.exists(config.project_working_dir): +def _check_has_working_dir(project): + working_dir = _project_working_dir(project) + if not os.path.exists(working_dir): raise MediaSyncError( - "The project working directory does not exist: " - + config.project_working_dir + "The project working directory does not exist: " + working_dir ) - if not os.path.exists(os.path.join(config.project_working_dir, ".mergin")): + if not os.path.exists(os.path.join(working_dir, ".mergin")): raise MediaSyncError( "The project working directory does not seem to contain Mergin project: " - + config.project_working_dir + + working_dir ) -def _check_pending_changes(): +def _check_pending_changes(project): """Check working directory was not modified manually - this is probably uncommitted change from last attempt""" - mp = MerginProject(config.project_working_dir) + mp = MerginProject(_project_working_dir(project)) status_push = mp.get_push_changes() if status_push["added"] or status_push["updated"] or status_push["removed"]: raise MediaSyncError( @@ -55,7 +65,7 @@ def _check_pending_changes(): ) -def _get_media_sync_files(files): +def _get_media_sync_files(files, project): """Return files relevant to media sync from project files""" allowed_extensions = config.allowed_extensions files_to_upload = [ @@ -64,11 +74,10 @@ def _get_media_sync_files(files): if os.path.splitext(f["path"])[1].lstrip(".") in allowed_extensions ] # filter out files which are not under particular directory in mergin project - if "base_path" in config and config.base_path: - filtered_files = [ - f for f in files_to_upload if f["path"].startswith(config.base_path) + if project.base_path: + files_to_upload = [ + f for f in files_to_upload if f["path"].startswith(project.base_path) ] - files_to_upload = filtered_files return files_to_upload @@ -92,32 +101,35 @@ def create_mergin_client(): raise MediaSyncError("Mergin client error: " + str(e)) -def mc_download(mc): +def mc_download(mc, project): """Clone mergin project to local dir :param mc: mergin client instance + :param project: project config object :return: list(dict) list of project files metadata """ - print("Downloading project from Mergin server ...") + working_dir = _project_working_dir(project) + print(f"Downloading project '{project.project_name}' from Mergin server ...") try: - mc.download_project(config.mergin.project_name, config.project_working_dir) + mc.download_project(project.project_name, working_dir) except ClientError as e: - # this could be e.g. DNS error raise MediaSyncError("Mergin client error on download: " + str(e)) - mp = MerginProject(config.project_working_dir) - print(f"Downloaded {_get_project_version()} from Mergin") - files_to_upload = _get_media_sync_files(mp.inspect_files()) + mp = MerginProject(working_dir) + print(f"Downloaded {_get_project_version(project)} from Mergin") + files_to_upload = _get_media_sync_files(mp.inspect_files(), project) return files_to_upload -def mc_pull(mc): +def mc_pull(mc, project): """Pull latest version to synchronize with local dir :param mc: mergin client instance + :param project: project config object :return: list(dict) list of project files metadata """ - print("Pulling from mergin server ...") - _check_pending_changes() + working_dir = _project_working_dir(project) + print(f"Pulling project '{project.project_name}' from mergin server ...") + _check_pending_changes(project) - mp = MerginProject(config.project_working_dir) + mp = MerginProject(working_dir) local_version = mp.version() try: @@ -125,31 +137,31 @@ def mc_pull(mc): projects = mc.get_projects_by_names([mp.project_full_name()]) server_version = projects[mp.project_full_name()]["version"] except ClientError as e: - # this could be e.g. DNS error raise MediaSyncError("Mergin client error: " + str(e)) - _check_pending_changes() + _check_pending_changes(project) if server_version == local_version: - print("No changes on Mergin.") + print(f"No changes on Mergin for '{project.project_name}'.") return try: status_pull = mp.get_pull_changes(project_info["files"]) - mc.pull_project(config.project_working_dir) + mc.pull_project(working_dir) except ClientError as e: raise MediaSyncError("Mergin client error on pull: " + str(e)) - print("Pulled new version from Mergin: " + _get_project_version()) + print("Pulled new version from Mergin: " + _get_project_version(project)) files_to_upload = _get_media_sync_files( - status_pull["added"] + status_pull["updated"] + status_pull["added"] + status_pull["updated"], project ) return files_to_upload -def _update_references(files): +def _update_references(project, files): """Update references to media files in reference table""" - for ref in config.references: + working_dir = _project_working_dir(project) + for ref in project.references: reference_config = [ ref.file, ref.table, @@ -162,7 +174,7 @@ def _update_references(files): print("Updating references ...") try: gpkg_conn = sqlite3.connect( - os.path.join(config.project_working_dir, ref.file) + os.path.join(working_dir, ref.file) ) gpkg_conn.enable_load_extension(True) gpkg_cur = gpkg_conn.cursor() @@ -188,16 +200,17 @@ def _update_references(files): raise MediaSyncError("SQLITE error: " + str(e)) -def media_sync_push(mc, driver, files): +def media_sync_push(mc, driver, project, files): if not files: return - print("Synchronizing files with external drive...") - _check_has_working_dir() + working_dir = _project_working_dir(project) + print(f"Synchronizing files for project '{project.project_name}' with external drive...") + _check_has_working_dir(project) migrated_files = {} # TODO make async and parallel for better performance for file in files: - src = os.path.join(config.project_working_dir, file["path"]) + src = os.path.join(working_dir, file["path"]) if not os.path.exists(src): print("Missing local file: " + str(file["path"])) continue @@ -212,25 +225,25 @@ def media_sync_push(mc, driver, files): continue # update reference table (if applicable) - _update_references(migrated_files) + _update_references(project, migrated_files) # remove from local dir if move mode if config.operation_mode == "move": for file in migrated_files.keys(): - src = os.path.join(config.project_working_dir, file) + src = os.path.join(working_dir, file) os.remove(src) # push changes to mergin back (with changed references and removed files) if applicable try: - mp = MerginProject(config.project_working_dir) + mp = MerginProject(working_dir) status_push = mp.get_push_changes() if status_push["added"]: raise MediaSyncError( "There are changes to be added - it should never happen" ) if status_push["updated"] or status_push["removed"]: - mc.push_project(config.project_working_dir) - version = _get_project_version() + mc.push_project(working_dir) + version = _get_project_version(project) print("Pushed new version to Mergin: " + version) except (ClientError, MediaSyncError) as e: # this could be either because of some temporal error (network, server lock) @@ -240,6 +253,30 @@ def media_sync_push(mc, driver, files): print("Sync finished") +def _sync_project(mc, project): + """Run the full download-or-pull + push cycle for a single project.""" + working_dir = _project_working_dir(project) + try: + driver = create_driver(config, project) + except DriverError as e: + print(f"Error initialising driver for '{project.project_name}': " + str(e)) + return + + try: + if os.path.exists(working_dir): + files_to_sync = mc_pull(mc, project) + else: + files_to_sync = mc_download(mc, project) + + if not files_to_sync: + print(f"No files to sync for '{project.project_name}'") + return + + media_sync_push(mc, driver, project, files_to_sync) + except MediaSyncError as err: + print(f"Error syncing '{project.project_name}': " + str(err)) + + def main(): print(f"== Starting Mergin Media Sync version {__version__} ==") try: @@ -248,30 +285,17 @@ def main(): print("Error: " + str(e)) return - try: - driver = create_driver(config) - except DriverError as e: - print("Error: " + str(e)) - return - try: print("Logging in to Mergin...") mc = create_mergin_client() - # initialize or pull changes to sync with latest project version - if os.path.exists(config.project_working_dir): - files_to_sync = mc_pull(mc) - else: - files_to_sync = mc_download(mc) - - if not files_to_sync: - print("No files to sync") - return - - # sync media files with external driver - media_sync_push(mc, driver, files_to_sync) - print("== Media sync done! ==") except MediaSyncError as err: print("Error: " + str(err)) + return + + for project in config.projects: + _sync_project(mc, project) + + print("== Media sync done! ==") if __name__ == "__main__": diff --git a/media_sync_daemon.py b/media_sync_daemon.py index 816cbfb..cddb77a 100644 --- a/media_sync_daemon.py +++ b/media_sync_daemon.py @@ -16,7 +16,8 @@ create_mergin_client, mc_download, media_sync_push, - mc_pull, + _sync_project, + _project_working_dir, MediaSyncError, ) from config import config, validate_config, ConfigError, update_config_path @@ -55,33 +56,33 @@ def main(): print("Error: " + str(e)) return - try: - driver = create_driver(config) - except DriverError as e: - print("Error: " + str(e)) - return - print("Logging in to Mergin...") try: mc = create_mergin_client() - # initialize or pull changes to sync with latest project version - if not os.path.exists(config.project_working_dir): - files_to_sync = mc_download(mc) - media_sync_push(mc, driver, files_to_sync) + # Initial download for projects that have not been downloaded yet + for project in config.projects: + working_dir = _project_working_dir(project) + if not os.path.exists(working_dir): + try: + driver = create_driver(config, project) + files_to_sync = mc_download(mc, project) + media_sync_push(mc, driver, project, files_to_sync) + except (DriverError, MediaSyncError) as e: + print(f"Error initialising project '{project.project_name}': " + str(e)) + except MediaSyncError as e: print("Error: " + str(e)) return # keep running until killed by ctrl+c: # - sleep N seconds - # - pull - # - push + # - pull + push for every project while True: print(datetime.datetime.now()) try: - files_to_sync = mc_pull(mc) - media_sync_push(mc, driver, files_to_sync) + for project in config.projects: + _sync_project(mc, project) # check mergin client token expiration delta = mc._auth_session["expire"] - datetime.datetime.now( diff --git a/test/conftest.py b/test/conftest.py index 8f9af2a..1824c74 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -36,12 +36,10 @@ def setup_config(): "MERGIN__USERNAME": API_USER, "MERGIN__PASSWORD": USER_PWD, "MERGIN__URL": SERVER_URL, - "MERGIN__PROJECT_NAME": "", - "PROJECT_WORKING_DIR": "", + "PROJECT_WORKING_DIR": TMP_DIR, "OPERATION_MODE": "copy", "DRIVER": "", - "REFERENCES": [], - "BASE_PATH": "", + "PROJECTS": [], "MINIO__ENDPOINT": "", "MINIO__ACCESS_KEY": "", "MINIO__SECRET_KEY": "", @@ -60,6 +58,8 @@ def cleanup(mc, project, dirs): mc.delete_project_now(project) except ClientError as e: print("Deleting project error: " + str(e)) + except Exception as e: + print("Deleting project error: " + str(e)) for d in dirs: if os.path.exists(d): shutil.rmtree(d) diff --git a/test/test_config.py b/test/test_config.py index c1bb340..e821d37 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -28,7 +28,6 @@ def _reset_config(): "MERGIN__USERNAME": API_USER, "MERGIN__PASSWORD": USER_PWD, "MERGIN__URL": SERVER_URL, - "MERGIN__PROJECT_NAME": "test/mediasync", "PROJECT_WORKING_DIR": "/tmp/working_project", "OPERATION_MODE": "copy", "DRIVER": "minio", @@ -36,7 +35,13 @@ def _reset_config(): "MINIO__ACCESS_KEY": MINIO_ACCESS_KEY, "MINIO__SECRET_KEY": MINIO_SECRET_KEY, "MINIO__BUCKET": "test", - "BASE_PATH": "", + "PROJECTS": [ + { + "project_name": "test/mediasync", + "bucket_subpath": "", + "references": [], + } + ], } ) @@ -46,19 +51,37 @@ def test_config(): _reset_config() validate_config(config) + # references None is normalised to [] _reset_config() - config.update({"REFERENCES": None}) + config.update( + { + "PROJECTS": [ + { + "project_name": "test/mediasync", + "bucket_subpath": "", + "references": None, + } + ] + } + ) validate_config(config) + # valid references list _reset_config() config.update( { - "REFERENCES": [ + "PROJECTS": [ { - "file": "survey.gpkg", - "table": "table", - "local_path_column": "local_path_column", - "driver_path_column": "driver_path_column", + "project_name": "test/mediasync", + "bucket_subpath": "", + "references": [ + { + "file": "survey.gpkg", + "table": "table", + "local_path_column": "local_path_column", + "driver_path_column": "driver_path_column", + } + ], } ] } @@ -76,9 +99,21 @@ def test_config(): _reset_config() with pytest.raises( - ConfigError, match="Config error: Incorrect Local driver settings" + ConfigError, + match="Config error: Project 'test/mediasync' is missing 'dest' for local driver", ): - config.update({"DRIVER": "local", "LOCAL__DEST": None}) + config.update( + { + "DRIVER": "local", + "PROJECTS": [ + { + "project_name": "test/mediasync", + "dest": None, + "references": [], + } + ], + } + ) validate_config(config) _reset_config() @@ -97,9 +132,19 @@ def test_config(): _reset_config() with pytest.raises( - ConfigError, match="Config error: Incorrect media reference settings" + ConfigError, match="incorrect media reference settings" ): - config.update({"REFERENCES": [{"file": "survey.gpkg"}]}) + config.update( + { + "PROJECTS": [ + { + "project_name": "test/mediasync", + "bucket_subpath": "", + "references": [{"file": "survey.gpkg"}], + } + ] + } + ) validate_config(config) _reset_config() @@ -108,6 +153,6 @@ def test_config(): validate_config(config) _reset_config() - with pytest.raises(ConfigError, match="Config error: Incorrect reference settings"): - config.update({"REFERENCES": "text"}) + with pytest.raises(ConfigError, match="'projects' list is missing or empty"): + config.update({"PROJECTS": []}) validate_config(config) diff --git a/test/test_sync.py b/test/test_sync.py index a9cfe00..7ac6f4f 100644 --- a/test/test_sync.py +++ b/test/test_sync.py @@ -14,13 +14,12 @@ from drivers import MinioDriver, LocalDriver, GoogleDriveDriver from media_sync import ( main, - config, mc_pull, media_sync_push, mc_download, MediaSyncError, ) -from config import validate_config, ConfigError +from config import config, validate_config, ConfigError from .conftest import ( API_USER, @@ -51,12 +50,10 @@ def test_sync(mc): """ project_name = "mediasync_test" full_project_name = WORKSPACE + "/" + project_name - work_project_dir = os.path.join( - TMP_DIR, project_name + "_work" - ) # working dir for mediasync - driver_dir = os.path.join( - TMP_DIR, project_name + "_driver" - ) # destination dir for 'local' driver + driver_dir = os.path.join(TMP_DIR, project_name + "_driver") + + # working dir is derived automatically: TMP_DIR / full_project_name + work_project_dir = os.path.join(TMP_DIR, full_project_name) cleanup(mc, full_project_name, [work_project_dir, driver_dir]) prepare_mergin_project(mc, full_project_name) @@ -68,17 +65,21 @@ def test_sync(mc): "MERGIN__USERNAME": API_USER, "MERGIN__PASSWORD": USER_PWD, "MERGIN__URL": SERVER_URL, - "MERGIN__PROJECT_NAME": full_project_name, - "PROJECT_WORKING_DIR": work_project_dir, + "PROJECT_WORKING_DIR": TMP_DIR, "DRIVER": "local", - "LOCAL__DEST": driver_dir, "OPERATION_MODE": "copy", - "REFERENCES": [ + "PROJECTS": [ { - "file": None, - "table": None, - "local_path_column": None, - "driver_path_column": None, + "project_name": full_project_name, + "dest": driver_dir, + "references": [ + { + "file": None, + "table": None, + "local_path_column": None, + "driver_path_column": None, + } + ], } ], } @@ -112,12 +113,18 @@ def test_sync(mc): # enable references updates config.update( { - "REFERENCES": [ + "PROJECTS": [ { - "file": "survey.gpkg", - "table": "notes", - "local_path_column": "photo", - "driver_path_column": "ext_url", + "project_name": full_project_name, + "dest": driver_dir, + "references": [ + { + "file": "survey.gpkg", + "table": "notes", + "local_path_column": "photo", + "driver_path_column": "ext_url", + } + ], } ] } @@ -131,11 +138,10 @@ def test_sync(mc): # check references have been changed and pushed project_info = mc.project_info(full_project_name) assert project_info["version"] == "v2" - gpkg_conn = sqlite3.connect( - os.path.join(work_project_dir, config.references[0].file) - ) + ref = config.projects[0].references[0] + gpkg_conn = sqlite3.connect(os.path.join(work_project_dir, ref.file)) gpkg_cur = gpkg_conn.cursor() - sql = f"SELECT count(*) FROM {config.references[0].table} WHERE {config.references[0].driver_path_column}='{copied_file}'" + sql = f"SELECT count(*) FROM {ref.table} WHERE {ref.driver_path_column}='{copied_file}'" gpkg_cur.execute(sql) assert gpkg_cur.fetchone()[0] == 1 shutil.rmtree(work_project_dir) @@ -156,20 +162,37 @@ def test_sync(mc): (f for f in project_info["files"] if f["path"] == "images/img2.jpg"), None ) assert not moved_file - gpkg_conn = sqlite3.connect( - os.path.join(work_project_dir, config.references[0].file) - ) + ref = config.projects[0].references[0] + gpkg_conn = sqlite3.connect(os.path.join(work_project_dir, ref.file)) gpkg_cur = gpkg_conn.cursor() - sql = f"SELECT count(*) FROM {config.references[0].table} WHERE {config.references[0].local_path_column} is Null AND {config.references[0].driver_path_column}='{copied_file}'" + sql = f"SELECT count(*) FROM {ref.table} WHERE {ref.local_path_column} is Null AND {ref.driver_path_column}='{copied_file}'" gpkg_cur.execute(sql) assert gpkg_cur.fetchone()[0] == 1 shutil.rmtree(work_project_dir) shutil.rmtree(driver_dir) # change mode to .png and also base project path to 'images' - nothing should be done - config.update({"allowed_extensions": ["png"], "base_path": "images"}) - main() - # check synced images + config.update( + { + "ALLOWED_EXTENSIONS": ["png"], + "PROJECTS": [ + { + "project_name": full_project_name, + "base_path": "images", + "dest": driver_dir, + "references": [ + { + "file": None, + "table": None, + "local_path_column": None, + "driver_path_column": None, + } + ], + } + ] + } + ) + main() # check synced images copied_file = os.path.join(driver_dir, "images", "img1.png") assert not os.path.exists(copied_file) project_info = mc.project_info(full_project_name) @@ -180,7 +203,7 @@ def test_pull_and_sync(mc): """Test media sync if mergin project is ahead of locally downloaded one""" project_name = "mediasync_test_pull" full_project_name = WORKSPACE + "/" + project_name - work_project_dir = os.path.join(TMP_DIR, project_name + "_work") + work_project_dir = os.path.join(TMP_DIR, full_project_name) driver_dir = os.path.join(TMP_DIR, project_name + "_driver") cleanup(mc, full_project_name, [work_project_dir, driver_dir]) @@ -191,19 +214,22 @@ def test_pull_and_sync(mc): "MERGIN__USERNAME": API_USER, "MERGIN__PASSWORD": USER_PWD, "MERGIN__URL": SERVER_URL, - "MERGIN__PROJECT_NAME": full_project_name, - "PROJECT_WORKING_DIR": work_project_dir, + "PROJECT_WORKING_DIR": TMP_DIR, "DRIVER": "local", - "LOCAL__DEST": driver_dir, "OPERATION_MODE": "copy", "ALLOWED_EXTENSIONS": ["png", "jpg"], - "BASE_PATH": None, - "REFERENCES": [ + "PROJECTS": [ { - "file": None, - "table": None, - "local_path_column": None, - "driver_path_column": None, + "project_name": full_project_name, + "dest": driver_dir, + "references": [ + { + "file": None, + "table": None, + "local_path_column": None, + "driver_path_column": None, + } + ], } ], } @@ -229,9 +255,10 @@ def test_pull_and_sync(mc): project_info = mc.project_info(full_project_name) assert project_info["version"] == "v2" - files_to_sync = mc_pull(mc) - driver = LocalDriver(config) - media_sync_push(mc, driver, files_to_sync) + project = config.projects[0] + files_to_sync = mc_pull(mc, project) + driver = LocalDriver(config, project) + media_sync_push(mc, driver, project, files_to_sync) # check synced image assert os.path.exists(os.path.join(driver_dir, "img_new.png")) @@ -240,7 +267,7 @@ def test_minio_backend(mc): """Test media sync connected to minio backend (needs local service running)""" project_name = "mediasync_test_minio" full_project_name = WORKSPACE + "/" + project_name - work_project_dir = os.path.join(TMP_DIR, project_name + "_work") + work_project_dir = os.path.join(TMP_DIR, full_project_name) cleanup(mc, full_project_name, [work_project_dir]) prepare_mergin_project(mc, full_project_name) @@ -251,28 +278,33 @@ def test_minio_backend(mc): "MERGIN__USERNAME": API_USER, "MERGIN__PASSWORD": USER_PWD, "MERGIN__URL": SERVER_URL, - "MERGIN__PROJECT_NAME": full_project_name, - "PROJECT_WORKING_DIR": work_project_dir, + "PROJECT_WORKING_DIR": TMP_DIR, "OPERATION_MODE": "copy", - "REFERENCES": [ - { - "file": None, - "table": None, - "local_path_column": None, - "driver_path_column": None, - } - ], "DRIVER": "minio", "MINIO__ENDPOINT": MINIO_URL, "MINIO__ACCESS_KEY": MINIO_ACCESS_KEY, "MINIO__SECRET_KEY": MINIO_SECRET_KEY, "MINIO__BUCKET": "test", + "PROJECTS": [ + { + "project_name": full_project_name, + "references": [ + { + "file": None, + "table": None, + "local_path_column": None, + "driver_path_column": None, + } + ], + } + ], } ) main() # check synced images - driver = MinioDriver(config) + project = config.projects[0] + driver = MinioDriver(config, project) minio_objects = [ o.object_name for o in driver.client.list_objects("test", recursive=True) ] @@ -291,29 +323,34 @@ def test_minio_backend(mc): "MERGIN__USERNAME": API_USER, "MERGIN__PASSWORD": USER_PWD, "MERGIN__URL": SERVER_URL, - "MERGIN__PROJECT_NAME": full_project_name, - "PROJECT_WORKING_DIR": work_project_dir, + "PROJECT_WORKING_DIR": TMP_DIR, "OPERATION_MODE": "copy", - "REFERENCES": [ - { - "file": None, - "table": None, - "local_path_column": None, - "driver_path_column": None, - } - ], "DRIVER": "minio", "MINIO__ENDPOINT": MINIO_URL, "MINIO__ACCESS_KEY": MINIO_ACCESS_KEY, "MINIO__SECRET_KEY": MINIO_SECRET_KEY, "MINIO__BUCKET": "test1", - "MINIO__BUCKET_SUBPATH": "subPath", + "PROJECTS": [ + { + "project_name": full_project_name, + "bucket_subpath": "subPath", + "references": [ + { + "file": None, + "table": None, + "local_path_column": None, + "driver_path_column": None, + } + ], + } + ], } ) main() # check synced images - driver = MinioDriver(config) + project = config.projects[0] + driver = MinioDriver(config, project) minio_objects = [ o.object_name for o in driver.client.list_objects("test1", recursive=True) ] @@ -325,12 +362,8 @@ def test_sync_failures(mc): """Test common sync failures""" project_name = "mediasync_fail" full_project_name = WORKSPACE + "/" + project_name - work_project_dir = os.path.join( - TMP_DIR, project_name + "_work" - ) # working dir for mediasync - driver_dir = os.path.join( - TMP_DIR, project_name + "_driver" - ) # destination dir for 'local' driver + work_project_dir = os.path.join(TMP_DIR, full_project_name) + driver_dir = os.path.join(TMP_DIR, project_name + "_driver") cleanup(mc, full_project_name, [work_project_dir, driver_dir]) prepare_mergin_project(mc, full_project_name) @@ -341,23 +374,29 @@ def test_sync_failures(mc): "MERGIN__USERNAME": API_USER, "MERGIN__PASSWORD": USER_PWD, "MERGIN__URL": SERVER_URL, - "MERGIN__PROJECT_NAME": full_project_name, - "PROJECT_WORKING_DIR": work_project_dir, + "PROJECT_WORKING_DIR": TMP_DIR, "DRIVER": "local", - "LOCAL__DEST": driver_dir, "OPERATION_MODE": "copy", - "REFERENCES": [ + "PROJECTS": [ { - "file": None, - "table": None, - "local_path_column": None, - "driver_path_column": None, + "project_name": full_project_name, + "dest": driver_dir, + "base_path": "", + "references": [ + { + "file": None, + "table": None, + "local_path_column": None, + "driver_path_column": None, + } + ], } ], } ) - driver = LocalDriver(config) - files_to_sync = mc_download(mc) + project = config.projects[0] + driver = LocalDriver(config, project) + files_to_sync = mc_download(mc, project) # "remove" .mergin hidden dir to mimic broken working directory os.rename( os.path.join(work_project_dir, ".mergin"), @@ -365,7 +404,7 @@ def test_sync_failures(mc): ) with pytest.raises(MediaSyncError) as exc: - media_sync_push(mc, driver, files_to_sync) + media_sync_push(mc, driver, project, files_to_sync) assert ( "The project working directory does not seem to contain Mergin project" in exc.value.args[0] @@ -378,38 +417,53 @@ def test_sync_failures(mc): # "remove" working project os.rename(work_project_dir, work_project_dir + "_renamed") with pytest.raises(MediaSyncError) as exc: - media_sync_push(mc, driver, files_to_sync) + media_sync_push(mc, driver, project, files_to_sync) assert "The project working directory does not exist" in exc.value.args[0] os.rename(work_project_dir + "_renamed", work_project_dir) # incorrect gpkg details for reference table config.update( { - "REFERENCES": [ + "PROJECTS": [ { - "file": "survey.gpkg", - "table": "notes_error", - "local_path_column": "photo", - "driver_path_column": "ext_url", + "project_name": full_project_name, + "dest": driver_dir, + "references": [ + { + "file": "survey.gpkg", + "table": "notes_error", + "local_path_column": "photo", + "driver_path_column": "ext_url", + } + ], } ], } ) + project = config.projects[0] with pytest.raises(MediaSyncError) as exc: - media_sync_push(mc, driver, files_to_sync) + media_sync_push(mc, driver, project, files_to_sync) assert "SQLITE error" in exc.value.args[0] + config.update( { - "REFERENCES": [ + "PROJECTS": [ { - "file": "survey.gpkg", - "table": "notes", - "local_path_column": "photo", - "driver_path_column": "ext_url", + "project_name": full_project_name, + "dest": driver_dir, + "references": [ + { + "file": "survey.gpkg", + "table": "notes", + "local_path_column": "photo", + "driver_path_column": "ext_url", + } + ], } ], } ) + project = config.projects[0] # introduce some unknown local file shutil.copyfile( @@ -417,14 +471,14 @@ def test_sync_failures(mc): os.path.join(work_project_dir, "new.gpkg"), ) with pytest.raises(MediaSyncError) as exc: - media_sync_push(mc, driver, files_to_sync) + media_sync_push(mc, driver, project, files_to_sync) assert "There are changes to be added - it should never happen" in exc.value.args[0] os.remove(os.path.join(work_project_dir, "new.gpkg")) # remove local file which should have been synced -> just skipped os.remove(os.path.join(work_project_dir, "img1.png")) shutil.rmtree(driver_dir) - media_sync_push(mc, driver, files_to_sync) + media_sync_push(mc, driver, project, files_to_sync) # check synced images assert not os.path.exists(os.path.join(driver_dir, "img1.png")) assert os.path.exists(os.path.join(driver_dir, "images", "img2.jpg")) @@ -433,12 +487,8 @@ def test_sync_failures(mc): def test_multiple_tables(mc): project_name = "mediasync_test_multiple" full_project_name = WORKSPACE + "/" + project_name - work_project_dir = os.path.join( - TMP_DIR, project_name + "_work" - ) # working dir for mediasync - driver_dir = os.path.join( - TMP_DIR, project_name + "_driver" - ) # destination dir for 'local' driver + work_project_dir = os.path.join(TMP_DIR, full_project_name) + driver_dir = os.path.join(TMP_DIR, project_name + "_driver") cleanup(mc, full_project_name, [work_project_dir, driver_dir]) prepare_mergin_project(mc, full_project_name) @@ -450,24 +500,28 @@ def test_multiple_tables(mc): "MERGIN__USERNAME": API_USER, "MERGIN__PASSWORD": USER_PWD, "MERGIN__URL": SERVER_URL, - "MERGIN__PROJECT_NAME": full_project_name, - "PROJECT_WORKING_DIR": work_project_dir, + "PROJECT_WORKING_DIR": TMP_DIR, "DRIVER": "local", - "LOCAL__DEST": driver_dir, "OPERATION_MODE": "copy", - "REFERENCES": [ - { - "file": "survey.gpkg", - "table": "notes", - "local_path_column": "photo", - "driver_path_column": "ext_url", - }, + "PROJECTS": [ { - "file": "survey.gpkg", - "table": "photos", - "local_path_column": "123_photo", - "driver_path_column": "ext_url", - }, + "project_name": full_project_name, + "dest": driver_dir, + "references": [ + { + "file": "survey.gpkg", + "table": "notes", + "local_path_column": "photo", + "driver_path_column": "ext_url", + }, + { + "file": "survey.gpkg", + "table": "photos", + "local_path_column": "123_photo", + "driver_path_column": "ext_url", + }, + ], + } ], } ) @@ -481,26 +535,23 @@ def test_multiple_tables(mc): assert project_info["version"] == "v2" copied_file = os.path.join(driver_dir, "img1.png") + refs = config.projects[0].references # check that both tables were updated - gpkg_conn = sqlite3.connect( - os.path.join(work_project_dir, config.references[0].file) - ) + gpkg_conn = sqlite3.connect(os.path.join(work_project_dir, refs[0].file)) gpkg_cur = gpkg_conn.cursor() - sql = f"SELECT count(*) FROM {config.references[0].table} WHERE {config.references[0].driver_path_column}='{copied_file}'" + sql = f"SELECT count(*) FROM {refs[0].table} WHERE {refs[0].driver_path_column}='{copied_file}'" gpkg_cur.execute(sql) assert gpkg_cur.fetchone()[0] == 1 - # check that both tables were updated - gpkg_conn = sqlite3.connect( - os.path.join(work_project_dir, config.references[1].file) - ) + + gpkg_conn = sqlite3.connect(os.path.join(work_project_dir, refs[1].file)) gpkg_cur = gpkg_conn.cursor() - sql = f"SELECT count(*) FROM {config.references[1].table} WHERE {config.references[1].driver_path_column}='{copied_file}'" + sql = f"SELECT count(*) FROM {refs[1].table} WHERE {refs[1].driver_path_column}='{copied_file}'" gpkg_cur.execute(sql) assert gpkg_cur.fetchone()[0] == 1 @pytest.mark.parametrize( - "project_name,config_update", + "project_name,project_overrides", [ ( "mediasync_test_without_references", @@ -508,45 +559,44 @@ def test_multiple_tables(mc): ), ( "mediasync_test_with_references_none", - {"REFERENCES": None}, + {"references": None}, ), ( "mediasync_test_with_references_empty_list", - {"REFERENCES": []}, + {"references": []}, ), ], ) -def test_sync_without_references(mc, project_name: str, config_update: dict): +def test_sync_without_references(mc, project_name: str, project_overrides: dict): """ Test media sync running sync without references. It should not fail and just copy files. The test just checks that main() runs without errors. """ full_project_name = WORKSPACE + "/" + project_name - work_project_dir = os.path.join( - TMP_DIR, project_name + "_work" - ) # working dir for mediasync - driver_dir = os.path.join( - TMP_DIR, project_name + "_driver" - ) # destination dir for 'local' driver + work_project_dir = os.path.join(TMP_DIR, full_project_name) + driver_dir = os.path.join(TMP_DIR, project_name + "_driver") cleanup(mc, full_project_name, [work_project_dir, driver_dir]) prepare_mergin_project(mc, full_project_name) + project_config = { + "project_name": full_project_name, + "dest": driver_dir, + } + project_config.update(project_overrides) + config.update( { "ALLOWED_EXTENSIONS": ["png", "jpg"], "MERGIN__USERNAME": API_USER, "MERGIN__PASSWORD": USER_PWD, "MERGIN__URL": SERVER_URL, - "MERGIN__PROJECT_NAME": full_project_name, - "PROJECT_WORKING_DIR": work_project_dir, + "PROJECT_WORKING_DIR": TMP_DIR, "DRIVER": "local", - "LOCAL__DEST": driver_dir, "OPERATION_MODE": "copy", - "BASE_PATH": None, + "PROJECTS": [project_config], } ) - config.update(config_update) main() @@ -565,31 +615,35 @@ def test_google_drive_backend(mc): """Test media sync connected to google drive backend""" project_name = "mediasync_test_googledrive" full_project_name = WORKSPACE + "/" + project_name - work_project_dir = os.path.join(TMP_DIR, project_name + "_work") + work_project_dir = os.path.join(TMP_DIR, full_project_name) cleanup(mc, full_project_name, [work_project_dir]) prepare_mergin_project(mc, full_project_name) - # invalid config + # invalid config – missing service_account_file config.update( { "MERGIN__USERNAME": API_USER, "MERGIN__PASSWORD": USER_PWD, "MERGIN__URL": SERVER_URL, - "MERGIN__PROJECT_NAME": full_project_name, - "PROJECT_WORKING_DIR": work_project_dir, + "PROJECT_WORKING_DIR": TMP_DIR, "OPERATION_MODE": "copy", - "REFERENCES": [ + "DRIVER": "google_drive", + "GOOGLE_DRIVE__SHARE_WITH": "", + "PROJECTS": [ { - "file": None, - "table": None, - "local_path_column": None, - "driver_path_column": None, + "project_name": full_project_name, + "folder": GOOGLE_DRIVE_FOLDER, + "references": [ + { + "file": None, + "table": None, + "local_path_column": None, + "driver_path_column": None, + } + ], } ], - "DRIVER": "google_drive", - "GOOGLE_DRIVE__FOLDER": GOOGLE_DRIVE_FOLDER, - "GOOGLE_DRIVE__SHARE_WITH": "", } ) @@ -602,30 +656,35 @@ def test_google_drive_backend(mc): "MERGIN__USERNAME": API_USER, "MERGIN__PASSWORD": USER_PWD, "MERGIN__URL": SERVER_URL, - "MERGIN__PROJECT_NAME": full_project_name, - "PROJECT_WORKING_DIR": work_project_dir, + "PROJECT_WORKING_DIR": TMP_DIR, "OPERATION_MODE": "copy", - "REFERENCES": [ - { - "file": None, - "table": None, - "local_path_column": None, - "driver_path_column": None, - } - ], "DRIVER": "google_drive", "GOOGLE_DRIVE__SERVICE_ACCOUNT_FILE": GOOGLE_DRIVE_SERVICE_ACCOUNT_FILE, - "GOOGLE_DRIVE__FOLDER": GOOGLE_DRIVE_FOLDER, "GOOGLE_DRIVE__SHARE_WITH": "", + "PROJECTS": [ + { + "project_name": full_project_name, + "folder": GOOGLE_DRIVE_FOLDER, + "references": [ + { + "file": None, + "table": None, + "local_path_column": None, + "driver_path_column": None, + } + ], + } + ], } ) - google_drive_delete_folder(GoogleDriveDriver(config), GOOGLE_DRIVE_FOLDER) + project = config.projects[0] + google_drive_delete_folder(GoogleDriveDriver(config, project), GOOGLE_DRIVE_FOLDER) main() google_drive_files = google_drive_list_files_in_folder( - GoogleDriveDriver(config), GOOGLE_DRIVE_FOLDER + GoogleDriveDriver(config, project), GOOGLE_DRIVE_FOLDER ) assert "img1.png" in google_drive_files