SciSpacy UmlsLinkerPaths Linker does not cache!

[ddofer]

The nlp/linker does not cache the downloaded files. This means it redownloads the (large) file each time and won't work offline.

Example :

`MaxRetryError: HTTPSConnectionPool(host='s3-us-west-2.amazonaws.com', port=443): Max retries exceeded with url: /ai2-s2-scispacy/data/linkers/2023-04-23/umls/tfidf_vectors_sparse.npz (Caused by NameResolutionError("<urllib3.connection.HTTPSConnection object at 0x6ccd66190>: Failed to resolve 's3-us-west-2.amazonaws.com' ([Errno 8] nodename nor servname provided, or not known)"))

During handling of the above exception, another exception occurred:

ConnectionError Traceback (most recent call last)
Cell In[22], line 4
1 nlp = spacy.load("en_core_sci_lg")
2 # nlp.add_pipe("abbreviation_detector")
----> 4 nlp.add_pipe("scispacy_linker",
5 config={"resolve_abbreviations": True,
6 "linker_name": "umls",
7 "max_entities_per_mention": 3 # 3, #6, #4, #5
8 # , "threshold": 0.85 ## default is 0.8, paper mentions 0.99 as thresh
9 })

File /opt/anaconda3/envs/Medrag/lib/python3.11/site-packages/spacy/language.py:821, in Language.add_pipe(self, factory_name, name, before, after, first, last, source, config, raw_config, validate)
817 pipe_component, factory_name = self.create_pipe_from_source(
818 factory_name, source, name=name
819 )
820 else:
--> 821 pipe_component = self.create_pipe(
822 factory_name,
823 name=name,
824 config=config,
825 raw_config=raw_config,
826 validate=validate,
827 )
828 pipe_index = self._get_pipe_index(before, after, first, last)
829 self._pipe_meta[name] = self.get_factory_meta(factory_name)

File /opt/anaconda3/envs/Medrag/lib/python3.11/site-packages/spacy/language.py:709, in Language.create_pipe(self, factory_name, name, config, raw_config, validate)
706 cfg = {factory_name: config}
707 # We're calling the internal _fill here to avoid constructing the
708 # registered functions twice
--> 709 resolved = registry.resolve(cfg, validate=validate)
710 filled = registry.fill({"cfg": cfg[factory_name]}, validate=validate)["cfg"]
711 filled = Config(filled)

File /opt/anaconda3/envs/Medrag/lib/python3.11/site-packages/confection/init.py:759, in registry.resolve(cls, config, schema, overrides, validate)
750 @classmethod
751 def resolve(
752 cls,
(...)
757 validate: bool = True,
758 ) -> Dict[str, Any]:
--> 759 resolved, _ = cls._make(
760 config, schema=schema, overrides=overrides, validate=validate, resolve=True
761 )
762 return resolved

File /opt/anaconda3/envs/Medrag/lib/python3.11/site-packages/confection/init.py:808, in registry._make(cls, config, schema, overrides, resolve, validate)
806 if not is_interpolated:
807 config = Config(orig_config).interpolate()
--> 808 filled, _, resolved = cls._fill(
809 config, schema, validate=validate, overrides=overrides, resolve=resolve
810 )
811 filled = Config(filled, section_order=section_order)
812 # Check that overrides didn't include invalid properties not in config

File /opt/anaconda3/envs/Medrag/lib/python3.11/site-packages/confection/init.py:880, in registry._fill(cls, config, schema, validate, resolve, parent, overrides)
877 getter = cls.get(reg_name, func_name)
878 # We don't want to try/except this and raise our own error
879 # here, because we want the traceback if the function fails.
--> 880 getter_result = getter(*args, **kwargs)
881 else:
882 # We're not resolving and calling the function, so replace
883 # the getter_result with a Promise class
884 getter_result = Promise(
885 registry=reg_name, name=func_name, args=args, kwargs=kwargs
886 )

File /opt/anaconda3/envs/Medrag/lib/python3.11/site-packages/scispacy/linking.py:85, in EntityLinker.init(self, nlp, name, candidate_generator, resolve_abbreviations, k, threshold, no_definition_threshold, filter_for_definitions, max_entities_per_mention, linker_name)
70 def init(
71 self,
72 nlp: Optional[Language] = None,
(...)
81 linker_name: Optional[str] = None,
82 ):
83 Span.set_extension("kb_ents", default=[], force=True)
---> 85 self.candidate_generator = candidate_generator or CandidateGenerator(
86 name=linker_name
87 )
88 self.resolve_abbreviations = resolve_abbreviations
89 self.k = k

File /opt/anaconda3/envs/Medrag/lib/python3.11/site-packages/scispacy/candidate_generation.py:222, in CandidateGenerator.init(self, ann_index, tfidf_vectorizer, ann_concept_aliases_list, kb, verbose, ef_search, name)
218 name = "umls"
220 linker_paths = DEFAULT_PATHS.get(name, UmlsLinkerPaths)
--> 222 self.ann_index = ann_index or load_approximate_nearest_neighbours_index(
223 linker_paths=linker_paths, ef_search=ef_search
224 )
225 self.vectorizer = tfidf_vectorizer or joblib.load(
226 cached_path(linker_paths.tfidf_vectorizer)
227 )
228 self.ann_concept_aliases_list = ann_concept_aliases_list or json.load(
229 open(cached_path(linker_paths.concept_aliases_list))
230 )

File /opt/anaconda3/envs/Medrag/lib/python3.11/site-packages/scispacy/candidate_generation.py:134, in load_approximate_nearest_neighbours_index(linker_paths, ef_search)
117 def load_approximate_nearest_neighbours_index(
118 linker_paths: LinkerPaths,
119 ef_search: int = 200,
120 ) -> FloatIndex:
121 """
122 Load an approximate nearest neighbours index from disk.
123
(...)
131 of magnitude for a small performance hit.
132 """
133 concept_alias_tfidfs = scipy.sparse.load_npz(
--> 134 cached_path(linker_paths.tfidf_vectors)
135 ).astype(numpy.float32)
136 ann_index = nmslib.init(
137 method="hnsw",
138 space="cosinesimil_sparse",
139 data_type=nmslib.DataType.SPARSE_VECTOR,
140 )
141 ann_index.addDataPointBatch(concept_alias_tfidfs)

File /opt/anaconda3/envs/Medrag/lib/python3.11/site-packages/scispacy/file_cache.py:39, in cached_path(url_or_filename, cache_dir)
35 parsed = urlparse(url_or_filename)
37 if parsed.scheme in ("http", "https"):
38 # URL, so get it from the cache (downloading if necessary)
---> 39 return get_from_cache(url_or_filename, cache_dir)
40 elif os.path.exists(url_or_filename):
41 # File, and it exists.
42 return url_or_filename

File /opt/anaconda3/envs/Medrag/lib/python3.11/site-packages/scispacy/file_cache.py:119, in get_from_cache(url, cache_dir)
115 cache_dir = DATASET_CACHE
117 os.makedirs(cache_dir, exist_ok=True)
--> 119 response = requests.head(url, allow_redirects=True)
120 if response.status_code != 200:
121 raise IOError(
122 "HEAD request failed for url {} with status code {}".format(
123 url, response.status_code
124 )
125 )

File /opt/anaconda3/envs/Medrag/lib/python3.11/site-packages/requests/api.py:100, in head(url, **kwargs)
89 r"""Sends a HEAD request.
90
91 :param url: URL for the new :class:Request object.
(...)
96 :rtype: requests.Response
97 """
99 kwargs.setdefault("allow_redirects", False)
--> 100 return request("head", url, **kwargs)

File /opt/anaconda3/envs/Medrag/lib/python3.11/site-packages/requests/api.py:59, in request(method, url, **kwargs)
55 # By using the 'with' statement we are sure the session is closed, thus we
56 # avoid leaving sockets open which can trigger a ResourceWarning in some
57 # cases, and look like a memory leak in others.
58 with sessions.Session() as session:
---> 59 return session.request(method=method, url=url, **kwargs)

File /opt/anaconda3/envs/Medrag/lib/python3.11/site-packages/requests/sessions.py:589, in Session.request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
584 send_kwargs = {
585 "timeout": timeout,
586 "allow_redirects": allow_redirects,
587 }
588 send_kwargs.update(settings)
--> 589 resp = self.send(prep, **send_kwargs)
591 return resp

File /opt/anaconda3/envs/Medrag/lib/python3.11/site-packages/requests/sessions.py:703, in Session.send(self, request, **kwargs)
700 start = preferred_clock()
702 # Send the request
--> 703 r = adapter.send(request, **kwargs)
705 # Total elapsed time of the request (approximately)
706 elapsed = preferred_clock() - start

File /opt/anaconda3/envs/Medrag/lib/python3.11/site-packages/requests/adapters.py:700, in HTTPAdapter.send(self, request, stream, timeout, verify, cert, proxies)
696 if isinstance(e.reason, _SSLError):
697 # This branch is for urllib3 v1.22 and later.
698 raise SSLError(e, request=request)
--> 700 raise ConnectionError(e, request=request)
702 except ClosedPoolError as e:
703 raise ConnectionError(e, request=request)

ConnectionError: HTTPSConnectionPool(host='s3-us-west-2.amazonaws.com', port=443): Max retries exceeded with url: /ai2-s2-scispacy/data/linkers/2023-04-23/umls/tfidf_vectors_sparse.npz (Caused by NameResolutionError("<urllib3.connection.HTTPSConnection object at 0x6ccd66190>: Failed to resolve 's3-us-west-2.amazonaws.com' ([Errno 8] nodename nor servname provided, or not known)"))`

This does work if i;m connected to the internet, but nothing seems to be saved, i.e even rerunning in the same notebook and runtime , after turning the internet off results in the error.

I am using the latest package version. Mac M3 env. Python 3.11, conda.
This is a pretty annoying bug since it breaks stuff. The same issue also occurs for me when running in WSL2 so it doesn;t seem to be mac or admin rights specific?

[dakinggg]

Hi, the caching does rely on an internet call to determine if it needs to redownload files. You should be able to load things from local paths via something like #343, but I'd also be happy to accept a PR making this easier!

[dakinggg]

Another issue related to this as well #469

[ddofer]

The thing is, this means that it doesn't run if there's no internet connection

[Mohammadaminrmz]

This is because in scispacy.file_cache.get_from_cache, It send request to Urls hard coded in scispacy.candidate_generation.LinkerPaths to retrieve ETag to reconstruct filename from url and Etag.

You can get url and ETag from cached files at: ~/.scispacy/datasets/<filename.json>
and pass them to file_cache.get_from_cache function somehow.
I have hardcoded ETags in a lookup dictionary in function body as a simple approach.
To resolve it, you can change file_cache.py:

"""
Utilities for working with the local dataset cache.
"""

import os
import shutil
import tempfile
import json
from urllib.parse import urlparse
from pathlib import Path
from typing import Optional, Tuple, Union, IO
from hashlib import sha256

import requests
from tqdm import tqdm

CACHE_ROOT = Path(os.getenv("SCISPACY_CACHE", str(Path.home() / ".scispacy")))
DATASET_CACHE = str(CACHE_ROOT / "datasets")


def cached_path(
    url_or_filename: Union[str, Path], cache_dir: Optional[str] = None
) -> str:
    """
    Given something that might be a URL (or might be a local path),
    determine which. If it's a URL, download the file and cache it, and
    return the path to the cached file. If it's already a local path,
    make sure the file exists and then return the path.
    """
    if cache_dir is None:
        cache_dir = DATASET_CACHE
    if isinstance(url_or_filename, Path):
        url_or_filename = str(url_or_filename)

    parsed = urlparse(url_or_filename)

    if parsed.scheme in ("http", "https"):
        # URL, so get it from the cache (downloading if necessary)
        return get_from_cache(url_or_filename, cache_dir)
    elif os.path.exists(url_or_filename):
        # File, and it exists.
        return url_or_filename
    elif parsed.scheme == "":
        # File, but it doesn't exist.
        raise FileNotFoundError("file {} not found".format(url_or_filename))
    else:
        # Something unknown
        raise ValueError(
            "unable to parse {} as a URL or as a local path".format(url_or_filename)
        )


def url_to_filename(url: str, etag: Optional[str] = None) -> str:
    """
    Convert `url` into a hashed filename in a repeatable way.
    If `etag` is specified, append its hash to the url's, delimited
    by a period.
    """

    last_part = url.split("/")[-1]
    url_bytes = url.encode("utf-8")
    url_hash = sha256(url_bytes)
    filename = url_hash.hexdigest()

    if etag:
        etag_bytes = etag.encode("utf-8")
        etag_hash = sha256(etag_bytes)
        filename += "." + etag_hash.hexdigest()

    filename += "." + last_part
    return filename


def filename_to_url(filename: str, cache_dir: Optional[str] = None) -> Tuple[str, str]:
    """
    Return the url and etag (which may be ``None``) stored for `filename`.
    Raise ``FileNotFoundError`` if `filename` or its stored metadata do not exist.
    """
    if cache_dir is None:
        cache_dir = DATASET_CACHE

    cache_path = os.path.join(cache_dir, filename)
    if not os.path.exists(cache_path):
        raise FileNotFoundError("file {} not found".format(cache_path))

    meta_path = cache_path + ".json"
    if not os.path.exists(meta_path):
        raise FileNotFoundError("file {} not found".format(meta_path))

    with open(meta_path) as meta_file:
        metadata = json.load(meta_file)
    url = metadata["url"]
    etag = metadata["etag"]

    return url, etag


def http_get(url: str, temp_file: IO) -> None:
    req = requests.get(url, stream=True)
    total = int(req.headers.get("content-length", 0))
    pbar = tqdm(total=total, unit="iB", unit_scale=True, unit_divisor=1024)
    for chunk in req.iter_content(chunk_size=1024):
        if chunk:  # filter out keep-alive new chunks
            size = temp_file.write(chunk)
            pbar.update(size)
    pbar.close()


def get_from_cache(url: str, cache_dir: Optional[str] = None) -> str:
    """
    Given a URL, look for the corresponding dataset in the local cache.
    If it's not there, download it. Then return the path to the cached file.
    """
    if cache_dir is None:
        cache_dir = DATASET_CACHE

    os.makedirs(cache_dir, exist_ok=True)

    # response = requests.head(url, allow_redirects=True)
    # if response.status_code != 200:
    #     raise IOError(
    #         "HEAD request failed for url {} with status code {}".format(
    #             url, response.status_code
    #         )
    #     )
    # etag = response.headers.get("ETag")
    import json
    fns = [fn for fn in os.listdir(cache_dir) if fn.endswith('json')] 
    lookup = {}
    for fpath in fns:
        try:
            with open('/root/.scispacy/datasets/' +fpath) as f:
                j = json.load(f)
                lookup[j['url']] = j['etag']
    except: pass
    etag = lookup[url]
    filename = url_to_filename(url ,etag) #etag

    # get cache path to put the file
    cache_path = os.path.join(cache_dir, filename)

    if not os.path.exists(cache_path):
        # Download to temporary file, then copy to cache dir once finished.
        # Otherwise you get corrupt cache entries if the download gets interrupted.
        with tempfile.NamedTemporaryFile() as temp_file:  # type: IO
            print(f"{url} not found in cache, downloading to {temp_file.name}")

            # GET file object
            http_get(url, temp_file)

            # we are copying the file before closing it, so flush to avoid truncation
            temp_file.flush()
            # shutil.copyfileobj() starts at the current position, so go to the start
            temp_file.seek(0)

            print(
                f"Finished download, copying {temp_file.name} to cache at {cache_path}"
            )
            with open(cache_path, "wb") as cache_file:
                shutil.copyfileobj(temp_file, cache_file)

            meta = {"url": url, "etag": etag}
            meta_path = cache_path + ".json"
            with open(meta_path, "w") as meta_file:
                json.dump(meta, meta_file)

    return cache_path

I am willing to accept PR for fixing this.