diff --git a/.gitignore b/.gitignore index ed6eebde88..b0fac3d19e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ +# for rn +package/doc/sphinx/build_docs.sh +testsuite/MDAnalysisTests/fetch/run_tests.sh +package/MDAnalysis/TEST.py + # Ignore python bytecoded files *.py[cod] *.[oa] diff --git a/package/MDAnalysis/fetch/__init__.py b/package/MDAnalysis/fetch/__init__.py index 1ebe1c8bd1..5f73ef4fdf 100644 --- a/package/MDAnalysis/fetch/__init__.py +++ b/package/MDAnalysis/fetch/__init__.py @@ -37,3 +37,4 @@ __all__ = ["from_PDB"] from .pdb import from_PDB +from .fetchers import StaticFetcher diff --git a/package/MDAnalysis/fetch/fetchers.py b/package/MDAnalysis/fetch/fetchers.py new file mode 100644 index 0000000000..dc027e718c --- /dev/null +++ b/package/MDAnalysis/fetch/fetchers.py @@ -0,0 +1,678 @@ +# -*- Mode: python; tab-width: 4; indent-tabs-mode:nil; coding: utf-8 -*- +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 +# +# MDAnalysis --- https://www.mdanalysis.org +# Copyright (c) 2006-2017 The MDAnalysis Development Team and contributors +# (see the file AUTHORS for the full list of names) +# +# Released under the Lesser GNU Public Licence, v2.1 or any higher version +# +# Please cite your use of MDAnalysis in published work: +# +# R. J. Gowers, M. Linke, J. Barnoud, T. J. E. Reddy, M. N. Melo, S. L. Seyler, +# D. L. Dotson, J. Domanski, S. Buchoux, I. M. Kenney, and O. Beckstein. +# MDAnalysis: A Python package for the rapid analysis of molecular dynamics +# simulations. In S. Benthall and S. Rostrup editors, Proceedings of the 15th +# Python in Science Conference, pages 102-109, Austin, TX, 2016. SciPy. +# doi: 10.25080/majora-629e541a-00e +# +# N. Michaud-Agrawal, E. J. Denning, T. B. Woolf, and O. Beckstein. +# MDAnalysis: A Toolkit for the Analysis of Molecular Dynamics Simulations. +# J. Comput. Chem. 32 (2011), 2319--2327, doi:10.1002/jcc.21787 +# + +""" +Fetchers --- :mod:`MDAnalysis.fetch.fetchers` +============================================= + +This module contains Fetcher classes which are able to retrieve files from +remote servers.These classes use the third-party library :mod:`pooch` as +a dependency. + +Classes +------- + +.. autoclass:: StaticFetcher + :members: + :inherited-members: + +Variables +--------- + +These module-level variables affect the runtime behavior across all Fetcher +classes. Changing these values affects all initialized Fetchers. + + +.. autodata:: DEFAULT_CACHE_NAME_DOWNLOADER +.. autodata:: DEFAULT_TIMEOUT +.. autodata:: DEFAULT_RETRIES + +""" + +import hashlib +import re +import os + +from pathlib import Path +from abc import ABC, abstractmethod + +try: + import pooch +except ImportError: + HAS_POOCH = False +else: + HAS_POOCH = True + + +#: Name of the :mod:`pooch` cache directory +#: ``pooch.os_cache(DEFAULT_CACHE_NAME_DOWNLOADER)``; +#: +#: See :func:`pooch.os_cache` for further details. +#: +#: .. versionadded:: 2.11.0 +DEFAULT_CACHE_NAME_DOWNLOADER = "MDAnalysis_pdbs" + +#: Default time in seconds to wait for a response from the server before +#: timing out. +#: +#: .. versionadded:: 2.11.0 +DEFAULT_TIMEOUT = 10 + +#: Default number of attempts to retry a download if it fails. +#: +#: .. versionadded:: 2.11.0 +DEFAULT_RETRIES = 2 + + +class _BaseFetcher(ABC): + """Base class for all fetchers. + + This class should not be initialized directly; fetcher implementations + should inherit from it. + + .. versionadded:: 2.11.0 + + """ + + def __init__( + self, + ): + pass + + @abstractmethod + def fetch(self, base_url, verbose, timeout, retries): + """Retrieve files from a remote server.""" + # Starts file retrieval workflow + # All fetchers should call _check_pooch() + # + # These arguments should be implemented by all child Fetchers. + + def _check_pooch( + self, + ): + """Raise an error if :mod:`pooch` is not installed.""" + if not HAS_POOCH: + raise ModuleNotFoundError( + "pooch is needed as a dependency for Fetchers" + ) + + def _validate_fetch_args(self, args): + """Add the default global timeout and retry variables to fetch + arguments.""" + + args.setdefault("timeout", DEFAULT_TIMEOUT) + args.setdefault("retries", DEFAULT_RETRIES) + + return args + + +class StaticFetcher(_BaseFetcher): + """ + Downloads files from a URL to disk and caches them to a local directory. + + Parameters + ---------- + cache_path : str or pathlib.Path + Path to the cache directory. If set to ``None``, the default cache + directory will be used as specified by + :data:`DEFAULT_CACHE_NAME_DOWNLOADER`. + If the directory does not exist, it will attempt to be created. + + hash : str + Hash algorithm to use for verifying the integrity of downloaded files. + The default is ``sha256``. Valid options are any hash algorithm + available in the :mod:`hashlib` module. + + + Attributes + ---------- + cache_path : pathlib.Path + Path to the cache directory. + + hash : str + Hash algorithm used for verifying the integrity of downloaded files. + + Notes + ----- + The download directory can be overridden by setting the environment + variable ``MDANALYSIS_FETCHER_DATA`` to a valid path. This class uses + :mod:`pooch` as a backend for downloading and caching files. + + .. versionadded:: 2.11.0 + + """ + + def __init__(self, cache_path=None, hash="sha256"): + + self._check_pooch() + + self.cache_path = self._check_cache_path_input(cache_path) + self.hash = self._check_hash_input(hash) + + def fetch( + self, + base_url, + file_name, + verbose=False, + db_name="hashes.txt", + append_db=False, + downloader="auto", + **kwargs, + ): + """ + Download one or more files from a static base URL and cache them + locally. + + Primarily designed to be working with `FAIR`_ + databases, this method works by sending a request to a web server and + caching them to a registry.The registry is in the format of a + `pooch registry file`_, and it will be created or read relative to + :attr:`cache_path`. + + .. _FAIR: https://www.nature.com/articles/s41592-025-02635-0 + .. _`pooch registry file`: https://www.fatiando.org/pooch/latest/registry-files.html#registry-file-format + + Parameters + ---------- + base_url : str + Base URL from which to download the file(s). This should be a valid + URL pointing to the directory containing the files to be + downloaded. + file_name : str or sequence of str + Name of the file or files to download. + The requested URL has the form ``{base_url}/{file_name}``. + verbose : bool, optional + If ``True``, show download progress. The default is ``False``. + db_name : str or None, optional + Name of the local hash database file used to verify cached + downloads. The default is ``"hashes.txt"``. If ``None``, + no registry database is read or written. + append_db : bool, optional + If ``True``, add downloaded files that are missing from an existing + registry to that registry. If ``False``, missing registry entries + raise a :class:`ValueError`. The default is ``False``. + timeout : float, optional + Time in seconds to wait for a response from the server before + timing out. The default is :data:`DEFAULT_TIMEOUT`. + retries : int, optional + Number of times to retry a failed download. The default is + :data:`DEFAULT_RETRIES`. + downloader : str, optional + Downloader backend to use. Supported values are ``"auto"``, + ``"http"``, ``"ftp"``, ``"sftp"``, and ``"doi"``. + The default is ``"auto"``. + + Returns + ------- + pathlib.Path or list of pathlib.Path + The downloaded file path for a single file, or a list of paths for + multiple files. + + Examples + -------- + + .. code-block:: pycon + + Download a single CIF file from the RCSB Protein Data Bank. + + >>> StaticFetcher().fetch(file_name="1AKE.cif", + base_url="https://files.wwpdb.org/download/") + './MDAnalysis_pdbs/1AKE.cif' + + Download multiple CIF files from the RCSB Protein Data Bank. + + >>> StaticFetcher().fetch(file_name=["1AKE.cif", "4AKE.cif"], + base_url="https://files.wwpdb.org/download/") + ['./MDAnalysis_pdbs/1AKE.cif', './MDAnalysis_pdbs/4AKE.cif'] + + Notes + ----- + The download directory can be overridden by setting the environment + variable ``MDANALYSIS_FETCHER_DATA`` to a valid path. This class uses + :mod:`pooch` as a backend for downloading and caching files. The + cache database is created on demand when ``db_name`` does not + exist relative to :attr:`cache_path`. + + .. versionadded:: 2.11.0 + + """ + # Keywords arguments that are reserved for common + # _BaseFetcher.fetch() arguments. + kwargs = self._validate_fetch_args(kwargs) + + LOAD_FROM_CACHE = False + CREATE_DATABASE = False + MISSING_FILES = False + APPEND_DATABASE = append_db + + registry_dictionary = {} + + # Process file names + requested_files = ( + (file_name,) if isinstance(file_name, str) else tuple(file_name) + ) + + # Reading from Registry + if db_name is not None: + db_path = self.cache_path / Path(db_name) + + if db_path.exists(): + LOAD_FROM_CACHE = True + else: + CREATE_DATABASE = True + + if LOAD_FROM_CACHE: + registry_dictionary = self.read_registry(db_path) + missing_files_list = self.check_registry( + db_path, files=list(requested_files) + ) + + if len(missing_files_list) != 0: + MISSING_FILES = True + + if MISSING_FILES and not APPEND_DATABASE: + raise ValueError( + "fetch() is requesting files not found in the registry. " + + f"The missing files are {missing_files_list}. " + + "To fix this, please set append_db=True to append the " + + "registry." + ) + + for name in requested_files: + registry_dictionary.setdefault(name, None) + + ## + + # Download code using pooch + main_downloader = pooch.create( + path=self.cache_path, + base_url=base_url, + registry=registry_dictionary, + retry_if_failed=kwargs["retries"], + ) + + download_kwargs = kwargs.copy() + download_kwargs.pop("retries") + fetch_downloader = self._set_downloader( + base_url, downloader, **download_kwargs + ) + + paths = [ + Path( + main_downloader.fetch( + fname=file_name, + progressbar=verbose, + downloader=fetch_downloader, + ) + ) + for file_name in requested_files + ] + + ## + + # Registry write code + if CREATE_DATABASE: + self.write_registry(db_path, paths) + + if APPEND_DATABASE and LOAD_FROM_CACHE: + self.append_registry(db_path, requested_files) + + ## + + return paths[0] if len(paths) == 1 else paths + + def append_registry(self, db_path, files, write_duplicate=False): + """ + Append cached files to an existing Pooch registry. + + Each entry in ``files`` is resolved relative to :attr:`cache_path`. The + file hash is computed using the fetcher's configured hash algorithm + :attr:`hash` and a new registry line is appended to ``db_path``. + + Parameters + ---------- + db_path : str or path-like + Path to the registry file to update. + files : iterable of str or path-like + File names or paths for cached files to append to the registry. + Relative paths are interpreted relative to :attr:`cache_path`. + write_duplicate : bool + If set to True, append_registry will write the file and its hash + to the registry regardless of the existing presence of a entry + in the registry. Default behavior is False. + + Returns + ------- + None + This method updates the registry file in place and does not + return a value. + + Example + ------- + >>> from MDAnalysis.fetch.fetchers import StaticFetcher + >>> fetcher = StaticFetcher() + >>> file1 = fetcher.fetch( + ... file_name="1AKE.cif", + ... base_url="https://files.wwpdb.org/download/", + ... db_name="db_hash1.txt", + ... ) + >>> file2 = fetcher.fetch( + ... file_name="4AKE.cif", + ... base_url="https://files.wwpdb.org/download/", + ... db_name="db_hash2.txt", + ... ) + >>> registry = file1.parent / "db_hash1.txt" + >>> fetcher.append_registry(registry, ["4AKE.cif"]) + >>> registry.read_text() + 1AKE.cif sha256:01f41b1b42318a1a5df7f650dbab881677aa0e8d825f7c42dd26ae16a94c0948 + 4AKE.cif sha256:fcb2ff49a3e255797fee277ce28e0acace67f6e6ddf432841f8451f00cbde9e9 + + Notes + ----- + Existing registry entries are preserved. This method does not check + for or remove duplicate file entries. + + Each appended registry line has the format:: + + : + + .. versionadded:: 2.11.0 + + """ + new_files = [self.cache_path / file_name for file_name in files] + + if not write_duplicate: + files_dict = self.read_registry(db_path) + + _new_files = [ + file for file in new_files if file.name not in files_dict + ] + + else: + _new_files = new_files + + self.write_registry(Path(db_path), _new_files, mode="a") + + def check_registry(self, db_path, files=None, ignore=None): + """ + Return paths relative to :attr:`cache_path` for cache files that are + missing from the registry. + + This method compares filenames within the registry against files found + recursively under :attr:`cache_path`. A cache file is considered + missing when it is on disk, but it is not recorded in the registry. + + Parameters + ---------- + db_path : str or path-like + Path to the registry file to read. + files : list of pathlib.Path + Paths to additional files to check. Each path must be relative to + :attr:`cache_path`. + ignore : list of pathlib.Path + Files to be ignored. Each path must be relative to + :attr:`cache_path`. + + Returns + ------- + missing_files : list of pathlib.Path + Cache file paths whose filenames are not present in the registry. + The registry database file itself is excluded from the result. + + Example + ------- + .. code-block:: python + + >>> files = { + ... "file1.txt": "Molecular \\n", + ... "file2.txt": "Dynamics. \\n", + ... "file3.txt": "Analysis. \\n" + ... } + >>> for filename, content in files.items(): + ... with open(filename, "w") as f: + ... f.write(content) + ... + >>> fetcher = StaticFetcher() + >>> fetcher.write_registry( + ... "file_1_2_and_3_hash.txt", + ... files=["file1.txt"], + ... ) + >>> fetcher.check_registry("file_1_2_and_3_hash.txt") + [Path('./MDAnalysis_pdbs/file3.txt'), Path('./MDAnalysis_pdbs/file2.txt')] + >>> fetcher.check_registry("file_1_2_and_3_hash.txt", ignore=["file2.txt"]) + [Path('./MDAnalysis_pdbs/file3.txt')] + + + Notes + ----- + Each line in the registry file is expected to have the format:: + + : + + + """ + files = [] if files is None else files + ignore = [] if ignore is None else ignore + + registry_dictionary = self.read_registry(db_path) + database_files = set(registry_dictionary.keys()) + + cache_files = [ + path + for path in self.cache_path.rglob("*") + if path != db_path and path.is_file() + ] + [self.cache_path / file_name for file_name in files] + + return [ + path + for path in cache_files + if (path.name not in database_files) and (path not in ignore) + ] + + def read_registry(self, db_path): + """ + Read a Pooch registry file into a dictionary. + + This method returns filenames within the registry against files found + recursively under :attr:`cache_path`. Each key in the returned + dictionary corresponds to a filename in the registry relative to + :attr:`cache_path`. + + Parameters + ---------- + db_path : str or path-like + Path to the registry file to read. + + Returns + ------- + hash_dict : dict + Dictionary mapping each filename in the registry to its stored hash + value. Hash values are expected to include the hash + algorithm prefix. + + Example + ------- + .. code-block:: python + + >>> files = { + ... "file1.txt": "Molecular \\n", + ... "file2.txt": "Dynamics. \\n", + ... } + >>> for filename, content in files.items(): + ... with open(filename, "w") as f: + ... f.write(content) + ... + >>> fetcher = StaticFetcher() + >>> fetcher.write_registry( + ... "file_1_and_2_hash.txt", + ... ["file1.txt", "file2.txt"], + ... ) + >>> fetcher.read_registry("file_1_and_2_hash.txt") + {'file1.txt': 'sha256:2da169c5aae36a823c202da49fb11935b76277efcb5cd42a4cf238ddda2a9b20', + 'file2.txt': 'sha256:3a0dbd9e2abc4a7bbae6adfe92e2858218135926dacd4a7d3fb4ca2dbdbe457a'} + + Notes + ----- + Each line in the registry file is expected to have the format:: + + : + + .. versionadded:: 2.11.0 + + """ + hash_dict = {} + + with open(db_path, mode="r") as f: + for line in f: + key, value = line.strip().split() + hash_dict[key] = value + + return hash_dict + + def write_registry(self, db_path, files, mode="w"): + """ + Write a Pooch registry file with hashes for the given files. + + This method computes the hash for each file and writes it to the + registry file.The registry file maps each filename to its + corresponding hash value. The hash algorithm used is determined + by the :attr:`hash` of the fetcher. + + Parameters + ---------- + db_path : str or path-like + Path to the registry file to write. + files : iterable of str or path-like + Files to be include in the registry. Each file must be relative + to :attr:`cache_path`. + mode : str, optional + File opening mode used when writing the registry. + Default is ``"w"``. + + Returns + ------- + None + This method writes the registry to disk and does not return + a value. + + Example + ------- + .. code-block:: python + + >>> files = { + ... "file1.txt": "Molecular \\n", + ... "file2.txt": "Dynamics. \\n", + ... } + >>> for filename, content in files.items(): + ... with open(filename, "w") as f: + ... f.write(content) + ... + >>> fetcher = StaticFetcher() + >>> fetcher.write_registry( + ... "file_1_and_2_hash.txt", + ... ["file1.txt", "file2.txt"], + ... ) + >>> Path("file_1_and_2_hash.txt").read_text() + file1.txt sha256:2da169c5aae36a823c202da49fb11935b76277efcb5cd42a4cf238ddda2a9b20 + file2.txt sha256:3a0dbd9e2abc4a7bbae6adfe92e2858218135926dacd4a7d3fb4ca2dbdbe457a + + Notes + ----- + Each registry line is written in the format:: + + : + + .. versionadded:: 2.11.0 + + """ + with open(db_path, mode=mode) as f: + for file in files: + file = Path(file) + digest = pooch.file_hash(file, alg=self.hash) + f.write(f"{file.name} {self.hash}:{digest}\n") + + # Argument validation methods + def _check_cache_path_input(self, cache_path): + + if cache_path is None: + path = Path(pooch.os_cache(DEFAULT_CACHE_NAME_DOWNLOADER)) + else: + path = Path(cache_path) + + # Environment variable master override + if not os.environ.get("MDANALYSIS_FETCHER_DATA") is None: + path = Path(os.environ.get("MDANALYSIS_FETCHER_DATA")) + + Path(path).mkdir(parents=True, exist_ok=True) + return path + + def _check_hash_input(self, hash): + if hash in hashlib.algorithms_available: + return hash + else: + raise ValueError( + f'Invalid hash "{hash}". Valid hashes algorithms' + + f" are {hashlib.algorithms_available}." + ) + + def _set_downloader(self, base_url, downloader, **kwargs): + """Sets Downloader in fetch() by matching a regex against the + download link""" + + SUPPORTED_DOWNLOADERS = ("auto", "http", "https", "ftp", "sftp", "doi") + + if downloader not in SUPPORTED_DOWNLOADERS: + raise ValueError( + f"Invalid downloader '{downloader}'. Valid options " + + f"are {SUPPORTED_DOWNLOADERS}" + ) + + # Regex matching if downloader is set to auto + if downloader == "auto": + regex = r"^([^:]+):" + match = re.match(regex, base_url) + + if match: + _downloader = match.group(1) + else: + raise ValueError( + f"Unable to determine downloader for URL '{base_url}'." + ) + + else: + _downloader = downloader + + match _downloader: + case "http" | "https": + return pooch.HTTPDownloader(**kwargs) + case "ftp": + return pooch.FTPDownloader(**kwargs) + case "sftp": + return pooch.SFTPDownloader(**kwargs) + case "doi": + return pooch.DOIDownloader(**kwargs) + case _: + raise ValueError( + f"Invalid downloader '{_downloader}'. Valid options " + + f"are {SUPPORTED_DOWNLOADERS}" + ) diff --git a/package/MDAnalysis/fetch/pdb.py b/package/MDAnalysis/fetch/pdb.py index 5820035d30..b9ba3166f4 100644 --- a/package/MDAnalysis/fetch/pdb.py +++ b/package/MDAnalysis/fetch/pdb.py @@ -25,8 +25,9 @@ PDB Fetchers --- :mod:`MDAnalysis.fetch.pdb` ============================================ -This suite of functions download structure files from the Research Collaboratory for -Structural Bioinformatics (RCSB) `Protein Data Batabank`_ (PDB). +This suite of functions download structure files from the Research +Collaboratory for Structural Bioinformatics (RCSB) +`Protein Data Batabank`_ (PDB). .. _Protein Data Batabank: https://www.rcsb.org/ @@ -35,30 +36,23 @@ .. autodata:: DEFAULT_CACHE_NAME_DOWNLOADER - Functions --------- .. autofunction:: from_PDB """ -from pathlib import Path -try: - import pooch -except ImportError: - HAS_POOCH = False -else: - HAS_POOCH = True +from .fetchers import StaticFetcher -#: Name of the :mod:`pooch` cache directory ``pooch.os_cache(DEFAULT_CACHE_NAME_DOWNLOADER)``; -#: see :func:`pooch.os_cache` for further details. +#: Alias to fetchers/DEFAULT_CACHE_NAME_DOWNLOADER +#: +#: Maintained for backwards compatiblity #: -#: .. versionadded:: 2.11.0 -DEFAULT_CACHE_NAME_DOWNLOADER = "MDAnalysis_pdbs" +from .fetchers import DEFAULT_CACHE_NAME_DOWNLOADER # These file formats are here https://www.rcsb.org/docs/programmatic-access/file-download-services#pdb-entry-files" -SUPPORTED_FILE_FORMATS_DOWNLOADER = ( +_SUPPORTED_FILE_FORMATS_PDB = ( "cif", "cif.gz", "bcif", @@ -100,7 +94,8 @@ def from_PDB( The file extension/format to download (e.g., "cif", "pdb"). See the Notes section below for a list of all supported file formats. progressbar : bool - If True, display a progress bar during file downloads. Default is False. + If True, display a progress bar during file downloads. Default + is False. Returns ------- @@ -119,8 +114,8 @@ def from_PDB( Notes ----- - This function uses the `RCSB File Download Services`_ for directly downloading - structure files via https. + This function uses the `RCSB File Download Services`_ for directly + downloading structure files via https. .. _`RCSB File Download Services`: https://www.rcsb.org/docs/programmatic-access/file-download-services @@ -132,9 +127,9 @@ def from_PDB( Caching, controlled by the `cache_path` parameter, is handled internally by :mod:`pooch`. The default cache name is taken from - :data:`DEFAULT_CACHE_NAME_DOWNLOADER`. To clear cache (and subsequently force - re-fetching), it is required to delete the cache folder as specified by - `cache_path`. + :data:`DEFAULT_CACHE_NAME_DOWNLOADER`. To clear cache (and subsequently + force re-fetching), it is required to delete the cache folder + as specified by `cache_path`. Examples -------- @@ -162,39 +157,21 @@ def from_PDB( .. versionadded:: 2.11.0 """ - if not HAS_POOCH: - raise ModuleNotFoundError( - "pooch is needed as a dependency for from_PDB()" - ) - elif file_format not in SUPPORTED_FILE_FORMATS_DOWNLOADER: + if file_format not in _SUPPORTED_FILE_FORMATS_PDB: raise ValueError( "Invalid file format. Supported file formats " - f"are {SUPPORTED_FILE_FORMATS_DOWNLOADER}" + f"are {_SUPPORTED_FILE_FORMATS_PDB}" ) if isinstance(pdb_ids, str): - _pdb_ids = (pdb_ids,) + _pdb_ids = (pdb_ids + "." + file_format,) else: - _pdb_ids = pdb_ids - - if cache_path is None: - cache_path = pooch.os_cache(DEFAULT_CACHE_NAME_DOWNLOADER) + _pdb_ids = [pdb + "." + file_format for pdb in pdb_ids] - # Have to do this dictionary approach instead of using pooch.retrieve in order - # to prevent the hardcoded known_hash warning from showing up. - registry_dictionary = { - f"{pdb_id}.{file_format}": None for pdb_id in _pdb_ids - } - - downloader = pooch.create( - path=cache_path, + fetcher = StaticFetcher(cache_path=cache_path) + return fetcher.fetch( + file_name=_pdb_ids, base_url="https://files.wwpdb.org/download/", - registry=registry_dictionary, + progressbar=progressbar, + append_db=True, ) - - paths = [ - Path(downloader.fetch(fname=file_name, progressbar=progressbar)) - for file_name in registry_dictionary.keys() - ] - - return paths if not isinstance(pdb_ids, str) else paths[0] diff --git a/package/doc/sphinx/source/documentation_pages/fetchers/fetchers.rst b/package/doc/sphinx/source/documentation_pages/fetchers/fetchers.rst new file mode 100644 index 0000000000..58961f4806 --- /dev/null +++ b/package/doc/sphinx/source/documentation_pages/fetchers/fetchers.rst @@ -0,0 +1 @@ +.. automodule:: MDAnalysis.fetch.fetchers diff --git a/package/doc/sphinx/source/documentation_pages/fetchers_modules.rst b/package/doc/sphinx/source/documentation_pages/fetchers_modules.rst index bdcff49164..e04988b2b7 100644 --- a/package/doc/sphinx/source/documentation_pages/fetchers_modules.rst +++ b/package/doc/sphinx/source/documentation_pages/fetchers_modules.rst @@ -15,4 +15,5 @@ module. :maxdepth: 1 fetchers/init + fetchers/fetchers fetchers/PDB diff --git a/testsuite/MDAnalysisTests/fetch/servers.py b/testsuite/MDAnalysisTests/fetch/servers.py new file mode 100644 index 0000000000..4bc983b85c --- /dev/null +++ b/testsuite/MDAnalysisTests/fetch/servers.py @@ -0,0 +1,73 @@ +# -*- Mode: python; tab-width: 4; indent-tabs-mode:nil; coding:utf-8 -*- +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 fileencoding=utf-8 +# +# MDAnalysis --- https://www.mdanalysis.org +# Copyright (c) 2006-2017 The MDAnalysis Development Team and contributors +# (see the file AUTHORS for the full list of names) +# +# Released under the Lesser GNU Public Licence, v2.1 or any higher version +# +# Please cite your use of MDAnalysis in published work: +# +# R. J. Gowers, M. Linke, J. Barnoud, T. J. E. Reddy, N. M. Melo, S. L. Seyler, +# D. L. Dotson, J. Domanski, S. Buchoux, I. M. Kenney, and O. Beckstein. +# MDAnalysis: A Python package for the rapid analysis of molecular dynamics +# simulations. In S. Benthall and S. Rostrup editors, Proceedings of the 15th +# Python in Science Conference, pages 102-109, Austin, TX, 2016. SciPy. +# doi: 10.25080/majora-629e541a-00e +# +# N. Michaud-Agrawal, E. J. Denning, T. B. Woolf, and O. Beckstein. +# MDAnalysis: A Toolkit for the Analysis of Molecular Dynamics Simulations. +# J. Comput. Chem. 32 (2011), 2319--2327, doi:10.1002/jcc.21787 +# + +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from functools import partial + +import tempfile +import threading + +from contextlib import contextmanager + + +@contextmanager +def temporary_http_server(): + with tempfile.TemporaryDirectory() as temp_dir: + temp_folder = Path(temp_dir) + + (temp_folder / "TEST_FILE1.txt").write_text( + "Sally sells seashells by the seashore" + ) + + (temp_folder / "TEST_FILE2.txt").write_text( + "Life, Liberty and the pursuit of Happiness" + ) + + # This is a beautiful quote from the GROMACS source code + (temp_folder / "TEST_FILE3.txt").write_text( + "Unlike teachers or doctors, our efforts improve the lives of " + + "people we'll never meet. \n" + + "- Katie Busch-Sorensen" + ) + + http_handler = partial( + SimpleHTTPRequestHandler, + directory=str(temp_folder), + ) + + server = ThreadingHTTPServer(("127.0.0.1", 0), http_handler) + host, port = server.server_address + + thread = threading.Thread( + target=server.serve_forever, + daemon=True, + ) + thread.start() + + try: + yield host, port, temp_folder + finally: + server.shutdown() + server.server_close() + thread.join() diff --git a/testsuite/MDAnalysisTests/fetch/test_from_PDB.py b/testsuite/MDAnalysisTests/fetch/test_from_PDB.py index 881490680b..4ec983076e 100644 --- a/testsuite/MDAnalysisTests/fetch/test_from_PDB.py +++ b/testsuite/MDAnalysisTests/fetch/test_from_PDB.py @@ -22,23 +22,15 @@ # import pytest - import MDAnalysis as mda -from MDAnalysis.fetch.pdb import ( - DEFAULT_CACHE_NAME_DOWNLOADER, - HAS_POOCH, - SUPPORTED_FILE_FORMATS_DOWNLOADER, -) import re + +from MDAnalysis.fetch.fetchers import HAS_POOCH +from MDAnalysis.fetch.pdb import _SUPPORTED_FILE_FORMATS_PDB from urllib import request -from shutil import rmtree from pathlib import Path -if HAS_POOCH: - from requests.exceptions import HTTPError - import pooch - try: request.urlopen("https://files.wwpdb.org/", timeout=2) HAS_ACCESS_TO_WWPDB = True @@ -46,16 +38,16 @@ HAS_ACCESS_TO_WWPDB = False +@pytest.mark.skipif(not HAS_POOCH, reason="Pooch is not installed.") @pytest.mark.skipif( - HAS_POOCH, - reason="Pooch is installed.", + not HAS_ACCESS_TO_WWPDB, + reason="Can not connect to https://files.wwpdb.org/", ) -def test_pooch_installation(tmp_path): - with pytest.raises( - ModuleNotFoundError, - match="pooch is needed as a dependency for from_PDB()", - ): - mda.fetch.from_PDB("1AKE", cache_path=tmp_path, file_format="cif") +def test_download_one_file_str(tmp_path): + + path = mda.fetch.from_PDB("1AKE", cache_path=tmp_path) + assert path.exists() + assert path.name == "1AKE.cif.gz" @pytest.mark.skipif(not HAS_POOCH, reason="Pooch is not installed.") @@ -63,51 +55,25 @@ def test_pooch_installation(tmp_path): not HAS_ACCESS_TO_WWPDB, reason="Can not connect to https://files.wwpdb.org/", ) -class TestDocstringExamples: - """This class tests all the examples found in from_PDB's docstring""" - - @pytest.mark.parametrize("pdb_id", ["1AKE", "4BWZ"]) - def test_one_file_download(self, tmp_path, pdb_id): - path = mda.fetch.from_PDB( - pdb_id, cache_path=tmp_path, file_format="cif" - ) - assert isinstance(path, Path) - assert Path(path).name == f"{pdb_id}.cif" +def test_download_one_file_list(tmp_path): - def test_multiple_files_download(self, tmp_path): - list_of_path_strings = mda.fetch.from_PDB( - ["1AKE", "4BWZ"], cache_path=tmp_path, progressbar=True - ) - assert all(isinstance(pdb_id, Path) for pdb_id in list_of_path_strings) - assert all( - [ - Path(path).name == f"{name}.cif.gz" - for path, name in zip( - list_of_path_strings, ["1AKE", "4BWZ"], strict=True - ) - ] - ) + path = mda.fetch.from_PDB(["1AKE"], cache_path=tmp_path) + assert path.exists() + assert path.name == "1AKE.cif.gz" - @pytest.mark.parametrize( - "pdb_id, n_atoms", [("1AKE", 3816), ("4BWZ", 2824)] - ) - def test_files_to_universe(self, tmp_path, pdb_id, n_atoms): - u = mda.Universe( - mda.fetch.from_PDB( - pdb_id, - file_format="pdb.gz", - cache_path=tmp_path, - progressbar=True, - ) - ) - assert isinstance(u, mda.Universe) and (len(u.atoms) == n_atoms) +@pytest.mark.skipif(not HAS_POOCH, reason="Pooch is not installed.") +@pytest.mark.skipif( + not HAS_ACCESS_TO_WWPDB, + reason="Can not connect to https://files.wwpdb.org/", +) +def test_download_multiple_files(tmp_path): -@pytest.fixture() -def clean_up_default_cache(): - rmtree(pooch.os_cache(DEFAULT_CACHE_NAME_DOWNLOADER), ignore_errors=True) - yield - rmtree(pooch.os_cache(DEFAULT_CACHE_NAME_DOWNLOADER)) + paths = mda.fetch.from_PDB(["1AKE", "4AKE"], cache_path=tmp_path) + assert all(isinstance(path, Path) for path in paths) + assert [path.name for path in paths] == list( + ["1AKE.cif.gz", "4AKE.cif.gz"] + ) @pytest.mark.skipif(not HAS_POOCH, reason="Pooch is not installed.") @@ -115,22 +81,28 @@ def clean_up_default_cache(): not HAS_ACCESS_TO_WWPDB, reason="Can not connect to https://files.wwpdb.org/", ) -class TestExpectedBehaviors: +def test_download_file_format(tmp_path): - def test_no_cache_path(self, clean_up_default_cache): - assert isinstance(mda.fetch.from_PDB("1AKE", cache_path=None), Path) + path = mda.fetch.from_PDB(["1AKE"], cache_path=tmp_path, file_format="pdb") + assert path.exists() + assert path.name == "1AKE.pdb" - def test_str_input_gives_path_output(self, tmp_path): - assert isinstance( - mda.fetch.from_PDB( - pdb_ids="1AKE", cache_path=tmp_path, file_format="cif" - ), - Path, - ) - def test_list_input_gives_list_output(self, tmp_path): - assert isinstance( - mda.fetch.from_PDB(pdb_ids=["1AKE"], cache_path=tmp_path), list +@pytest.mark.skipif(not HAS_POOCH, reason="Pooch is not installed.") +@pytest.mark.skipif( + not HAS_ACCESS_TO_WWPDB, + reason="Can not connect to https://files.wwpdb.org/", +) +def test_invalid_file_format(tmp_path): + with pytest.raises( + ValueError, + match=re.escape( + "Invalid file format. Supported file formats " + f"are {_SUPPORTED_FILE_FORMATS_PDB}" + ), + ): + mda.fetch.from_PDB( + pdb_ids="1AKE", cache_path=tmp_path, file_format="barfoo" ) @@ -139,20 +111,10 @@ def test_list_input_gives_list_output(self, tmp_path): not HAS_ACCESS_TO_WWPDB, reason="Can not connect to https://files.wwpdb.org/", ) -class TestExpectedErrors: - - def test_invalid_pdb(self, tmp_path): - with pytest.raises(HTTPError): - mda.fetch.from_PDB(pdb_ids="foobar", cache_path=tmp_path) - - def test_invalid_file_format(self, tmp_path): - with pytest.raises( - ValueError, - match=re.escape( - "Invalid file format. Supported file formats " - f"are {SUPPORTED_FILE_FORMATS_DOWNLOADER}" - ), - ): - mda.fetch.from_PDB( - pdb_ids="1AKE", cache_path=tmp_path, file_format="barfoo" - ) +def test_download_multiple_calls(tmp_path): + + p1 = mda.fetch.from_PDB(["9BUY"], cache_path=tmp_path) + p2 = mda.fetch.from_PDB(["3SN6"], cache_path=tmp_path) + + assert p1.exists() + assert p2.exists() diff --git a/testsuite/MDAnalysisTests/fetch/test_static_fetcher.py b/testsuite/MDAnalysisTests/fetch/test_static_fetcher.py new file mode 100644 index 0000000000..5b8dea84b0 --- /dev/null +++ b/testsuite/MDAnalysisTests/fetch/test_static_fetcher.py @@ -0,0 +1,557 @@ +# -*- Mode: python; tab-width: 4; indent-tabs-mode:nil; coding:utf-8 -*- +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 fileencoding=utf-8 +# +# MDAnalysis --- https://www.mdanalysis.org +# Copyright (c) 2006-2017 The MDAnalysis Development Team and contributors +# (see the file AUTHORS for the full list of names) +# +# Released under the Lesser GNU Public Licence, v2.1 or any higher version +# +# Please cite your use of MDAnalysis in published work: +# +# R. J. Gowers, M. Linke, J. Barnoud, T. J. E. Reddy, N. M. Melo, S. L. Seyler, +# D. L. Dotson, J. Domanski, S. Buchoux, I. M. Kenney, and O. Beckstein. +# MDAnalysis: A Python package for the rapid analysis of molecular dynamics +# simulations. In S. Benthall and S. Rostrup editors, Proceedings of the 15th +# Python in Science Conference, pages 102-109, Austin, TX, 2016. SciPy. +# doi: 10.25080/majora-629e541a-00e +# +# N. Michaud-Agrawal, E. J. Denning, T. B. Woolf, and O. Beckstein. +# MDAnalysis: A Toolkit for the Analysis of Molecular Dynamics Simulations. +# J. Comput. Chem. 32 (2011), 2319--2327, doi:10.1002/jcc.21787 +# + +import re +from pathlib import Path +from shutil import rmtree +from unittest.mock import Mock + +from servers import temporary_http_server + +import hashlib +import pytest + +from MDAnalysis.fetch.fetchers import ( + DEFAULT_CACHE_NAME_DOWNLOADER, + HAS_POOCH, + StaticFetcher, +) + +if HAS_POOCH: + import pooch + +REGISTRY_NAME = "hashes.txt" + + +@pytest.fixture() +def clean_up_default_cache(): + rmtree(pooch.os_cache(DEFAULT_CACHE_NAME_DOWNLOADER), ignore_errors=True) + yield + rmtree(pooch.os_cache(DEFAULT_CACHE_NAME_DOWNLOADER), ignore_errors=True) + + +@pytest.mark.skipif(HAS_POOCH, reason="Pooch is installed.") +def test_pooch_installation(): + with pytest.raises( + ModuleNotFoundError, + match="pooch is needed as a dependency for Fetchers", + ): + StaticFetcher() + + +@pytest.mark.skipif(not HAS_POOCH, reason="Pooch is not installed.") +class TestExpectedErrors: + + def test_invalid_downloader(self, tmp_path): + with temporary_http_server() as (host, port, temp_folder): + base_url = f"http://{host}:{port}/" + downloader = StaticFetcher(cache_path=tmp_path) + + with pytest.raises( + ValueError, + match=re.escape( + "Invalid downloader 'barfoo'. Valid options " + + "are ('auto', 'http', 'https', 'ftp', 'sftp', 'doi')" + ), + ): + downloader.fetch( + base_url=base_url, + file_name="TEST_FILE1.txt", + downloader="barfoo", + ) + + def test_invalid_hash(self, tmp_path): + hash = "foo" + + with temporary_http_server() as (host, port, temp_folder): + + with pytest.raises( + ValueError, + match=re.escape( + f'Invalid hash "{hash}". Valid hashes algorithms ' + + f"are {hashlib.algorithms_available}." + ), + ): + StaticFetcher(cache_path=tmp_path, hash=hash) + + def test_append_error(self, tmp_path): + + with temporary_http_server() as (host, port, temp_folder): + base_url = f"http://{host}:{port}/" + fetcher = StaticFetcher(cache_path=tmp_path) + + file1 = fetcher.fetch( + base_url=base_url, + file_name="TEST_FILE1.txt", + db_name=REGISTRY_NAME, + ) + + # match later + with pytest.raises(ValueError): + file2 = fetcher.fetch( + base_url=base_url, + file_name="TEST_FILE2.txt", + db_name=REGISTRY_NAME, + ) + + def test_invalid_auto_downloader(self, tmp_path): + + base_url = "foo" + with pytest.raises(ValueError): + + fetcher = StaticFetcher( + cache_path=tmp_path, + ) + fetcher.fetch( + base_url=base_url, file_name="bar", downloader="auto" + ) + + def test_invalid_manual_downloader(self, tmp_path): + + base_url = "foo" + with pytest.raises(ValueError): + + fetcher = StaticFetcher( + cache_path=tmp_path, + ) + fetcher.fetch(base_url=base_url, file_name="bar", downloader="FOO") + + +@pytest.mark.skipif(not HAS_POOCH, reason="Pooch is not installed.") +class TestExpectedBehaviors: + + def test_default_cache_path(self, clean_up_default_cache): + + with temporary_http_server() as (host, port, temp_folder): + base_url = f"http://{host}:{port}/" + downloader = StaticFetcher() + path = downloader.fetch( + base_url=base_url, file_name="TEST_FILE1.txt" + ) + + assert isinstance(path, Path) + assert path.name == "TEST_FILE1.txt" + assert path.read_text() == "Sally sells seashells by the seashore" + assert path.exists() + + def test_create_database(self, tmp_path): + with temporary_http_server() as (host, port, temp_folder): + base_url = f"http://{host}:{port}/" + downloader = StaticFetcher(cache_path=tmp_path) + path = downloader.fetch( + base_url=base_url, + file_name="TEST_FILE1.txt", + db_name=REGISTRY_NAME, + ) + + assert isinstance(path, Path) + assert path.name == "TEST_FILE1.txt" + assert path.read_text() == "Sally sells seashells by the seashore" + assert path.exists() + + assert Path(downloader.cache_path / REGISTRY_NAME).exists() + assert ( + downloader.cache_path / REGISTRY_NAME + ).read_text() == "TEST_FILE1.txt sha256:a625aaf4ca5e2d358b216165cee3247a93a40e699bb864193499d230ab7aad7e\n" + + def test_append_database(self, tmp_path): + with temporary_http_server() as (host, port, temp_folder): + base_url = f"http://{host}:{port}/" + downloader = StaticFetcher(cache_path=tmp_path) + path = downloader.fetch( + base_url=base_url, + file_name="TEST_FILE1.txt", + db_name=REGISTRY_NAME, + ) + + path = downloader.fetch( + base_url=base_url, + file_name="TEST_FILE2.txt", + db_name=REGISTRY_NAME, + append_db=True, + ) + + assert Path(downloader.cache_path / REGISTRY_NAME).read_text() == ( + "TEST_FILE1.txt sha256:a625aaf4ca5e2d358b216165cee3247a93a40e699bb864193499d230ab7aad7e\n" + "TEST_FILE2.txt sha256:eff7c015c379263afdf464bd1baf266909d0e4d4af7cccb722dd4994ff4e998c\n" + ) + + # Add paramertization + def test_different_hashes(self, tmp_path): + with temporary_http_server() as (host, port, temp_folder): + base_url = f"http://{host}:{port}/" + downloader = StaticFetcher(cache_path=tmp_path, hash="md5") + path = downloader.fetch( + base_url=base_url, + file_name="TEST_FILE1.txt", + db_name=REGISTRY_NAME, + ) + + assert ( + Path(downloader.cache_path / REGISTRY_NAME).read_text() + == "TEST_FILE1.txt md5:adf1020ce2ffe073600990e1c9c72ce8\n" + ) + + def test_existing_database(self, tmp_path): + with temporary_http_server() as (host, port, temp_folder): + base_url = f"http://{host}:{port}/" + downloader = StaticFetcher(cache_path=tmp_path) + + p1 = downloader.fetch( + base_url=base_url, + file_name="TEST_FILE1.txt", + db_name=REGISTRY_NAME, + ) + + downloader = StaticFetcher(cache_path=tmp_path) + p2 = downloader.fetch( + base_url=base_url, + file_name="TEST_FILE1.txt", + db_name=REGISTRY_NAME, + ) + + assert p1.stat().st_mtime == p2.stat().st_mtime + + def test_no_database(self, tmp_path): + with temporary_http_server() as (host, port, temp_folder): + base_url = f"http://{host}:{port}/" + downloader = StaticFetcher(cache_path=tmp_path) + path = downloader.fetch( + base_url=base_url, file_name="TEST_FILE1.txt", db_name=None + ) + + assert isinstance(path, Path) + assert path.exists() + assert path.name == "TEST_FILE1.txt" + + assert not Path(downloader.cache_path / REGISTRY_NAME).exists() + assert not (tmp_path / REGISTRY_NAME).exists() + + def test_multiple_downloads_no_database(self, tmp_path): + + with temporary_http_server() as (host, port, temp_folder): + base_url = f"http://{host}:{port}/" + downloader = StaticFetcher(cache_path=tmp_path) + paths = downloader.fetch( + base_url=base_url, + file_name=("TEST_FILE1.txt", "TEST_FILE2.txt"), + db_name=None, + ) + + assert all(isinstance(path, Path) for path in paths) + assert all(path.exists() for path in paths) + assert [path.name for path in paths] == list( + ("TEST_FILE1.txt", "TEST_FILE2.txt") + ) + + assert not Path(downloader.cache_path / REGISTRY_NAME).exists() + assert not (tmp_path / REGISTRY_NAME).exists() + + def test_multiple_downloads_create_database(self, tmp_path): + + with temporary_http_server() as (host, port, temp_folder): + base_url = f"http://{host}:{port}/" + downloader = StaticFetcher(cache_path=tmp_path) + paths = downloader.fetch( + base_url=base_url, + file_name=("TEST_FILE1.txt", "TEST_FILE2.txt"), + db_name=REGISTRY_NAME, + ) + + assert Path(downloader.cache_path / REGISTRY_NAME).read_text() == ( + "TEST_FILE1.txt sha256:a625aaf4ca5e2d358b216165cee3247a93a40e699bb864193499d230ab7aad7e\n" + "TEST_FILE2.txt sha256:eff7c015c379263afdf464bd1baf266909d0e4d4af7cccb722dd4994ff4e998c\n" + ) + + def test_multiple_downloads_existing_database(self, tmp_path): + with temporary_http_server() as (host, port, temp_folder): + base_url = f"http://{host}:{port}/" + downloader = StaticFetcher(cache_path=tmp_path) + paths1 = downloader.fetch( + base_url=base_url, + file_name=("TEST_FILE1.txt", "TEST_FILE2.txt"), + db_name=REGISTRY_NAME, + ) + mtime1 = [path.stat().st_mtime for path in paths1] + + downloader = StaticFetcher(cache_path=tmp_path) + paths2 = downloader.fetch( + base_url=base_url, + file_name=("TEST_FILE1.txt", "TEST_FILE2.txt"), + db_name=REGISTRY_NAME, + ) + + mtime2 = [path.stat().st_mtime for path in paths2] + + assert mtime1 == mtime2 + + def test_environment_variable_override(self, tmp_path, monkeypatch): + monkeypatch.setenv("MDANALYSIS_FETCHER_DATA", str(tmp_path)) + + with temporary_http_server() as (host, port, temp_folder): + base_url = f"http://{host}:{port}/" + downloader = StaticFetcher() + + path = downloader.fetch( + base_url=base_url, + file_name="TEST_FILE1.txt", + ) + + assert Path(tmp_path / "TEST_FILE1.txt").exists() + + +@pytest.mark.skipif(not HAS_POOCH, reason="Pooch is not installed.") +class TestRegistry: + def test_append_registry(self, tmp_path): + + with temporary_http_server() as (host, port, temp_folder): + base_url = f"http://{host}:{port}/" + fetcher = StaticFetcher(cache_path=tmp_path) + + file1 = fetcher.fetch( + base_url=base_url, + file_name="TEST_FILE1.txt", + db_name="db_hash1.txt", + ) + + file2 = fetcher.fetch( + base_url=base_url, + file_name="TEST_FILE2.txt", + db_name="db_hash2.txt", + ) + + registry = file1.parent / "db_hash1.txt" + fetcher.append_registry(registry, ["TEST_FILE2.txt"]) + + assert Path(registry).read_text() == ( + "TEST_FILE1.txt sha256:a625aaf4ca5e2d358b216165cee3247a93a40e699bb864193499d230ab7aad7e\n" + "TEST_FILE2.txt sha256:eff7c015c379263afdf464bd1baf266909d0e4d4af7cccb722dd4994ff4e998c\n" + ) + + def test_append_registry_duplicate(self, tmp_path): + + with temporary_http_server() as (host, port, temp_folder): + base_url = f"http://{host}:{port}/" + fetcher = StaticFetcher(cache_path=tmp_path) + + file1 = fetcher.fetch( + base_url=base_url, + file_name="TEST_FILE1.txt", + db_name="db_hash1.txt", + ) + + registry = file1.parent / "db_hash1.txt" + fetcher.append_registry( + registry, ["TEST_FILE1.txt"], write_duplicate=True + ) + + assert Path(registry).read_text() == ( + "TEST_FILE1.txt sha256:a625aaf4ca5e2d358b216165cee3247a93a40e699bb864193499d230ab7aad7e\n" + "TEST_FILE1.txt sha256:a625aaf4ca5e2d358b216165cee3247a93a40e699bb864193499d230ab7aad7e\n" + ) + + def test_append_registry_no_duplicate(self, tmp_path): + + with temporary_http_server() as (host, port, temp_folder): + base_url = f"http://{host}:{port}/" + fetcher = StaticFetcher(cache_path=tmp_path) + + file1 = fetcher.fetch( + base_url=base_url, + file_name="TEST_FILE1.txt", + db_name="db_hash1.txt", + ) + + registry = file1.parent / "db_hash1.txt" + fetcher.append_registry( + registry, ["TEST_FILE1.txt"], write_duplicate=False + ) + + assert Path(registry).read_text() == ( + "TEST_FILE1.txt sha256:a625aaf4ca5e2d358b216165cee3247a93a40e699bb864193499d230ab7aad7e\n" + ) + + def test_write_registry(self, tmp_path): + # This can't call StaticFetcher directly for an effective test + # Maybe refactor the file creation into a function handle or fixture + files = { + "file1.txt": "Molecular \n", + "file2.txt": "Dynamics. \n", + } + for filename, content in files.items(): + with open(tmp_path / filename, "w") as f: + f.write(content) + + fetcher = StaticFetcher(cache_path=tmp_path) + + fetcher.write_registry( + tmp_path / "file_1_hash.txt", [tmp_path / "file1.txt"] + ) + assert ( + tmp_path / "file_1_hash.txt" + ).read_text() == "file1.txt sha256:2da169c5aae36a823c202da49fb11935b76277efcb5cd42a4cf238ddda2a9b20\n" + + fetcher.write_registry( + tmp_path / "file_2_hash.txt", [tmp_path / "file2.txt"] + ) + assert ( + tmp_path / "file_2_hash.txt" + ).read_text() == "file2.txt sha256:3a0dbd9e2abc4a7bbae6adfe92e2858218135926dacd4a7d3fb4ca2dbdbe457a\n" + + fetcher.write_registry( + tmp_path / "file_1_and_2_hash.txt", + [tmp_path / "file1.txt", tmp_path / "file2.txt"], + ) + assert (tmp_path / "file_1_and_2_hash.txt").read_text() == ( + "file1.txt sha256:2da169c5aae36a823c202da49fb11935b76277efcb5cd42a4cf238ddda2a9b20\n" + "file2.txt sha256:3a0dbd9e2abc4a7bbae6adfe92e2858218135926dacd4a7d3fb4ca2dbdbe457a\n" + ) + + ## Test works, but doesn't work on github action. Idk why and need to find out at some point. + def test_check_registry(self, tmp_path): + + files = { + "file1.txt": "Molecular \\n", + "file2.txt": "Dynamics. \\n", + "file3.txt": "Analysis. \\n", + } + + for filename, content in files.items(): + with open(tmp_path / filename, "w") as f: + f.write(content) + + fetcher = StaticFetcher(cache_path=tmp_path) + + # Write only file1 + fetcher.write_registry( + tmp_path / "file_1_2_and_3_hash.txt", + [tmp_path / "file1.txt"], + ) + + # Show that file2 and file3 are missing + assert sorted( + fetcher.check_registry(tmp_path / "file_1_2_and_3_hash.txt") + ) == sorted( + [ + tmp_path / "file3.txt", + tmp_path / "file2.txt", + ] + ) + + def test_check_registry_ignore(self, tmp_path): + + files = { + "file1.txt": "Molecular \\n", + "file2.txt": "Dynamics. \\n", + "file3.txt": "Analysis. \\n", + } + + for filename, content in files.items(): + with open(tmp_path / filename, "w") as f: + f.write(content) + + fetcher = StaticFetcher(cache_path=tmp_path) + + # Write only file1 + fetcher.write_registry( + tmp_path / "file_1_2_and_3_hash.txt", + [tmp_path / "file1.txt"], + ) + + assert fetcher.check_registry( + tmp_path / "file_1_2_and_3_hash.txt", + ignore=[tmp_path / "file2.txt"], + ) == [ + tmp_path / "file3.txt", + ] + + +@pytest.mark.skipif(not HAS_POOCH, reason="Pooch is not installed.") +class TestDownloaders: + + @pytest.mark.parametrize( + ("downloader", "constructor_name"), + [ + ("ftp", "FTPDownloader"), + ("sftp", "SFTPDownloader"), + ("doi", "DOIDownloader"), + ], + ) + def test_monkeypatch_other_downloaders( + self, + monkeypatch, + tmp_path, + downloader, + constructor_name, + ): + downloaded_path = tmp_path / "example.dat" + + # Monkeypatches the pooch.create to return a fake path + pooch_fetch = Mock(return_value=str(downloaded_path)) + pooch_instance = Mock(fetch=pooch_fetch) + + monkeypatch.setattr( + pooch, + "create", + Mock(return_value=pooch_instance), + ) + + # Monkekpatch a generic pooch Downloader + # This will be used to mock the behavior of a generic pooch Downloader + downloader_instance = object() + downloader_constructor = Mock(return_value=downloader_instance) + + monkeypatch.setattr( + pooch, + constructor_name, + downloader_constructor, + ) + + # Since pooch.create() is monkeypatched to return a fake path. This won't raise an exception + # And the fake downloader will be passed into the fetch method to mock its behavior. + result = StaticFetcher(cache_path=tmp_path).fetch( + base_url="https://example.com/", + file_name="example.dat", + db_name=None, + downloader=downloader, + ) + + assert result == downloaded_path + + def test_downloader_error(self, tmp_path): + with pytest.raises( + ValueError, + ): + + with temporary_http_server() as (host, port, temp_folder): + base_url = f"http://{host}:{port}/" + downloader = StaticFetcher() + + downloader = StaticFetcher( + cache_path=tmp_path, + ) + path = downloader.fetch( + base_url=base_url, + file_name="TEST_FILE1.txt", + downloader="FOO", + )