|
|
import copy |
|
|
from dataclasses import dataclass, field |
|
|
from pathlib import Path |
|
|
from typing import Any, Optional, Union |
|
|
|
|
|
from .. import config |
|
|
|
|
|
|
|
|
@dataclass |
|
|
class DownloadConfig: |
|
|
"""Configuration for our cached path manager. |
|
|
|
|
|
Attributes: |
|
|
cache_dir (`str` or `Path`, *optional*): |
|
|
Specify a cache directory to save the file to (overwrite the |
|
|
default cache dir). |
|
|
force_download (`bool`, defaults to `False`): |
|
|
If `True`, re-download the file even if it's already cached in |
|
|
the cache dir. |
|
|
resume_download (`bool`, defaults to `False`): |
|
|
If `True`, resume the download if an incompletely received file is |
|
|
found. |
|
|
proxies (`dict`, *optional*): |
|
|
user_agent (`str`, *optional*): |
|
|
Optional string or dict that will be appended to the user-agent on remote |
|
|
requests. |
|
|
extract_compressed_file (`bool`, defaults to `False`): |
|
|
If `True` and the path point to a zip or tar file, |
|
|
extract the compressed file in a folder along the archive. |
|
|
force_extract (`bool`, defaults to `False`): |
|
|
If `True` when `extract_compressed_file` is `True` and the archive |
|
|
was already extracted, re-extract the archive and override the folder where it was extracted. |
|
|
delete_extracted (`bool`, defaults to `False`): |
|
|
Whether to delete (or keep) the extracted files. |
|
|
extract_on_the_fly (`bool`, defaults to `False`): |
|
|
If `True`, extract compressed files while they are being read. |
|
|
use_etag (`bool`, defaults to `True`): |
|
|
Whether to use the ETag HTTP response header to validate the cached files. |
|
|
num_proc (`int`, *optional*): |
|
|
The number of processes to launch to download the files in parallel. |
|
|
max_retries (`int`, default to `1`): |
|
|
The number of times to retry an HTTP request if it fails. |
|
|
token (`str` or `bool`, *optional*): |
|
|
Optional string or boolean to use as Bearer token |
|
|
for remote files on the Datasets Hub. If `True`, or not specified, will get token from `~/.huggingface`. |
|
|
storage_options (`dict`, *optional*): |
|
|
Key/value pairs to be passed on to the dataset file-system backend, if any. |
|
|
download_desc (`str`, *optional*): |
|
|
A description to be displayed alongside with the progress bar while downloading the files. |
|
|
disable_tqdm (`bool`, defaults to `False`): |
|
|
Whether to disable the individual files download progress bar |
|
|
""" |
|
|
|
|
|
cache_dir: Optional[Union[str, Path]] = None |
|
|
force_download: bool = False |
|
|
resume_download: bool = False |
|
|
local_files_only: bool = False |
|
|
proxies: Optional[dict] = None |
|
|
user_agent: Optional[str] = None |
|
|
extract_compressed_file: bool = False |
|
|
force_extract: bool = False |
|
|
delete_extracted: bool = False |
|
|
extract_on_the_fly: bool = False |
|
|
use_etag: bool = True |
|
|
num_proc: Optional[int] = None |
|
|
max_retries: int = 1 |
|
|
token: Optional[Union[str, bool]] = None |
|
|
storage_options: dict[str, Any] = field(default_factory=dict) |
|
|
download_desc: Optional[str] = None |
|
|
disable_tqdm: bool = False |
|
|
|
|
|
def copy(self) -> "DownloadConfig": |
|
|
return self.__class__(**{k: copy.deepcopy(v) for k, v in self.__dict__.items()}) |
|
|
|
|
|
def __setattr__(self, name, value): |
|
|
if name == "token" and getattr(self, "storage_options", None) is not None: |
|
|
if "hf" not in self.storage_options: |
|
|
self.storage_options["hf"] = {"token": value, "endpoint": config.HF_ENDPOINT} |
|
|
elif getattr(self.storage_options["hf"], "token", None) is None: |
|
|
self.storage_options["hf"]["token"] = value |
|
|
super().__setattr__(name, value) |
|
|
|
