 nothing.
                pass
            else:
                # Very unlikely to happen. Means a file `dst` has been created exactly between `os.remove` and
                # `os.symlink` and is not a symlink to the `abs_src` blob file. Raise exception.
                raise
    elif new_blob:
        logger.info(f"Symlink not supported. Moving file from {abs_src} to {abs_dst}")
        shutil.move(src, dst)
    else:
        logger.info(f"Symlink not supported. Copying file from {abs_src} to {abs_dst}")
        shutil.copyfile(src, dst)


def _cache_commit_hash_for_specific_revision(storage_folder: str, revision: str, commit_hash: str) -> None:
    """Cache reference between a revision (tag, branch or truncated commit hash) and the corresponding commit hash.

    Does nothing if `revision` is already a proper `commit_hash` or reference is already cached.
    """
    if revision != commit_hash:
        ref_path = Path(storage_folder) / "refs" / revision
        ref_path.parent.mkdir(parents=True, exist_ok=True)
        if not ref_path.exists() or commit_hash != ref_path.read_text():
            # Update ref only if has been updated. Could cause useless error in case
            # repo is already cached and user doesn't have write access to cache folder.
            # See https://github.com/huggingface/huggingface_hub/issues/1216.
            ref_path.write_text(commit_hash)


@validate_hf_hub_args
def repo_folder_name(*, repo_id: str, repo_type: str) -> str:
    """Return a serialized version of a hf.co repo name and type, safe for disk storage
    as a single non-nested folder.

    Example: models--julien-c--EsperBERTo-small
    """
    # remove all `/` occurrences to correctly convert repo to directory name
    parts = [f"{repo_type}s", *repo_id.split("/")]
    return REPO_ID_SEPARATOR.join(parts)


@validate_hf_hub_args
def hf_hub_download(
    repo_id: str,
    filename: str,
    *,
    subfolder: Optional[str] = None,
    repo_type: Optional[str] = None,
    revision: Optional[str] = None,
    library_name: Optional[str] = None,
    library_version: Optional[str] = None,
    cache_dir: Union[str, Path, None] = None,
    local_dir: Union[str, Path, None] = None,
    local_dir_use_symlinks: Union[bool, Literal["auto"]] = "auto",
    user_agent: Union[Dict, str, None] = None,
    force_download: bool = False,
    force_filename: Optional[str] = None,
    proxies: Optional[Dict] = None,
    etag_timeout: float = 10,
    resume_download: bool = False,
    token: Union[bool, str, None] = None,
    local_files_only: bool = False,
    legacy_cache_layout: bool = False,
) -> str:
    """Download a given file if it's not already present in the local cache.

    The new cache file layout looks like this:
    - The cache directory contains one subfolder per repo_id (namespaced by repo type)
    - inside each repo folder:
        - refs is a list of the latest known revision => commit_hash pairs
        - blobs contains the actual file blobs (identified by their git-sha or sha256, depending on
          whether they're LFS files or not)
        - snapshots contains one subfolder per commit, each "commit" contains the subset of the files
          that have been resolved at that particular commit. Each filename is a symlink to the blob
          at that particular commit.

    If `local_dir` is provided, the file structure from the repo will be replicated in this location. You can configure
    how you want to move those files:
      - If `local_dir_use_symlinks="auto"` (default), files are downloaded and stored in the cache directory as blob
        files. Small files (<5MB) are duplicated in `local_dir` while a symlink is created for bigger files. The goal
        is to be able to manually edit and save small files without corrupting the cache while saving disk space for
        binary files. The 5MB threshold can be configured with the `HF_HUB_LOCAL_DIR_AUTO_SYMLINK_THRESHOLD`
        environment variable.
      - If `local_dir_use_symlinks=True`, files are downloaded, stored in the cache directory and symlinked in `local_dir`.
        This is optimal in term of disk usage but files must not be manually edited.
      - If `local_dir_use_symlinks=False` and the blob files exist in the cache directory, they are duplicated in the
        local dir. This means disk usage is not optimized.
      - Finally, if `local_dir_use_symlinks=False` and the blob files do not exist in the cache directory, then the
        files are downloaded and directly placed under `local_dir`. This means if you need to download them again later,
        they will be re-downloaded entirely.

    ```
    [  96]  .
    └── [ 160]  models--julien-c--EsperBERTo-small
        ├── [ 160]  blobs
        │   ├── [321M]  403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd
        │   ├── [ 398]  7cb18dc9bafbfcf74629a4b760af1b160957a83e
        │   └── [1.4K]  d7edf6bd2a681fb0175f7735299831ee1b22b812
        ├── [  96]  refs
        │   └── [  40]  main
        └── [ 128]  snapshots
            ├── [ 128]  2439f60ef33a0d46d85da5001d52aeda5b00ce9f
            │   ├── [  52]  README.md -> ../../blobs/d7edf6bd2a681fb0175f7735299831ee1b22b812
            │   └── [  76]  pytorch_model.bin -> ../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd
            └── [ 128]  bbc77c8132af1cc5cf678da3f1ddf2de43606d48
                ├── [  52]  README.md -> ../../blobs/7cb18dc9bafbfcf74629a4b760af1b160957a83e
                └── [  76]  pytorch_model.bin -> ../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd
    ```

    Args:
        repo_id (`str`):
            A user or an organization name and a repo name separated by a `/`.
        filename (`str`):
            The name of the file in the repo.
        subfolder (`str`, *optional*):
            An optional value corresponding to a folder inside the model repo.
        repo_type (`str`, *optional*):
            Set to `"dataset"` or `"space"` if downloading from a dataset or space,
            `None` or `"model"` if downloading from a model. Default is `None`.
        revision (`str`, *optional*):
            An optional Git revision id which can be a branch name, a tag, or a
            commit hash.
        library_name (`str`, *optional*):
            The name of the library to which the object corresponds.
        library_version (`str`, *optional*):
            The version of the library.
        cache_dir (`str`, `Path`, *optional*):
            Path to the folder where cached files are stored.
        local_dir (`str` or `Path`, *optional*):
            If provided, the downloaded file will be placed under this directory, either as a symlink (default) or
            a regular file (see description for more details).
        local_dir_use_symlinks (`"auto"` or `bool`, defaults to `"auto"`):
            To be used with `local_dir`. If set to "auto", the cache directory will be used and the file will be either
            duplicated or symlinked to the local directory depending on its size. It set to `True`, a symlink will be
            created, no matter the file size. If set to `False`, the file will either be duplicated from cache (if
            already exists) or downloaded from the Hub and not cached. See description for more details.
        user_agent (`dict`, `str`, *optional*):
            The user-agent info in the form of a dictionary or a string.
        force_download (`bool`, *optional*, defaults to `False`):
            Whether the file should be downloaded even if it already exists in
            the local cache.
        proxies (`dict`, *optional*):
            Dictionary mapping protocol to the URL of the proxy passed to
            `requests.request`.
        etag_timeout (`float`, *optional*, defaults to `10`):
            When fetching ETag, how many seconds to wait for the server to send
            data before giving up which is passed to `requests.request`.
        resume_download (`bool`, *optional*, defaults to `False`):
            If `True`, resume a previously interrupted download.
        token (`str`, `bool`, *optional*):
            A token to be used for the download.
                - If `True`, the token is read from the HuggingFace config
                  folder.
                - If a string, it's used as the authentication token.
        local_files_only (`bool`, *optional*, defaults to `False`):
            If `True`, avoid downloading the file and return the path to the
            local cached file if it exists.
        legacy_cache_layout (`bool`, *optional*, defaults to `False`):
            If `True`, uses the legacy file cache layout i.e. just call [`hf_hub_url`]
            then `cached_download`. This is deprecated as the new cache layout is
            more powerful.

    Returns:
        Local path (string) of file or if networking is off, last version of
        file cached on disk.

    <Tip>

    Raises the following errors:

        - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
          if `token=True` and the token cannot be found.
        - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError)
          if ETag cannot be determined.
        - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
          if some parameter value is invalid
        - [`~utils.RepositoryNotFoundError`]
          If the repository to download from cannot be found. This may be because it doesn't exist,
          or because it is set to `private` and you do not have access.
        - [`~utils.RevisionNotFoundError`]
          If the revision to download from cannot be found.
        - [`~utils.EntryNotFoundError`]
          If the file to download cannot be found.
        - [`~utils.LocalEntryNotFoundError`]
          If network is disabled or unavailable and file is not found in cache.

    </Tip>
    """
    if force_filename is not None:
        warnings.warn(
            (
                "The `force_filename` parameter is deprecated as a new caching system, "
                "which keeps the filenames as they are on the Hub, is now in place."
            ),
            FutureWarning,
        )
        legacy_cache_layout = True

    if legacy_cache_layout:
        url = hf_hub_url(
            repo_id,
            filename,
            subfolder=subfolder,
            repo_type=repo_type,
            revision=revision,
        )

        return cached_download(
            url,
            library_name=library_name,
            library_version=library_version,
            cache_dir=cache_dir,
            user_agent=user_agent,
            force_download=force_download,
            force_filename=force_filename,
            proxies=proxies,
            etag_timeout=etag_timeout,
            resume_download=resume_download,
            token=token,
            local_files_only=local_files_only,
            legacy_cache_layout=legacy_cache_layout,
        )

    if cache_dir is None:
        cache_dir = HUGGINGFACE_HUB_CACHE
    if revision is None:
        revision = DEFAULT_REVISION
    if isinstance(cache_dir, Path):
        cache_dir = str(cache_dir)
    if isinstance(local_dir, Path):
        local_dir = str(local_dir)

    if subfolder == "":
        subfolder = None
    if subfolder is not None:
        # This is used to create a URL, and not a local path, hence the forward slash.
        filename = f"{subfolder}/{filename}"

    if repo_type is None:
        repo_type = "model"
    if repo_type not in REPO_TYPES:
        raise ValueError(f"Invalid repo type: {repo_type}. Accepted repo types are: {str(REPO_TYPES)}")

    storage_folder = os.path.join(cache_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type))
    os.makedirs(storage_folder, exist_ok=True)

    # cross platform transcription of filename, to be used as a local file path.
    relative_filename = os.path.join(*filename.split("/"))
    if os.name == "nt":
        if relative_filename.startswith("..\\") or "\\..\\" in relative_filename:
            raise ValueError(
                f"Invalid filename: cannot handle filename '{relative_filename}' on Windows. Please ask the repository"
                " owner to rename this file."
            )

    # if user provides a commit_hash and they already have the file on disk,
    # shortcut everything.
    if REGEX_COMMIT_HASH.match(revision):
        pointer_path = _get_pointer_path(storage_folder, revision, relative_filename)
        if os.path.exists(pointer_path):
            if local_dir is not None:
                return _to_local_dir(pointer_path, local_dir, relative_filename, use_symlinks=local_dir_use_symlinks)
            return pointer_path

    url = hf_hub_url(repo_id, filename, repo_type=repo_type, revision=revision)

    headers = build_hf_headers(
        token=token,
        library_name=library_name,
        library_version=library_version,
        user_agent=user_agent,
    )

    url_to_download = url
    etag = None
    commit_hash = None
    expected_size = None
    if not local_files_only:
        try:
            try:
                metadata = get_hf_file_metadata(
                    url=url,
                    token=token,
                    proxies=proxies,
                    timeout=etag_timeout,
                )
            except EntryNotFoundError as http_error:
                # Cache the non-existence of the file and raise
                commit_hash = http_error.response.headers.get(HUGGINGFACE_HEADER_X_REPO_COMMIT)
                if commit_hash is not None and not legacy_cache_layout:
                    no_exist_file_path = Path(storage_folder) / ".no_exist" / commit_hash / relative_filename
                    no_exist_file_path.parent.mkdir(parents=True, exist_ok=True)
                    no_exist_file_path.touch()
                    _cache_commit_hash_for_specific_revision(storage_folder, revision, commit_hash)
                raise

            # Commit hash must exist
            commit_hash = metadata.commit_hash
            if commit_hash is None:
                raise OSError("Distant resource does not seem to be on huggingface.co (missing commit header).")

            # Etag must exist
            etag = metadata.etag
            # We favor a custom header indicating the etag of the linked resource, and
            # we fallback to the regular etag header.
            # If we don't have any of those, raise an error.
            if etag is None:
                raise OSError(
                    "Distant resource does not have an ETag, we won't be able to reliably ensure reproducibility."
                )

            # Expected (uncompressed) size
            expected_size = metadata.size

            # In case of a redirect, save an extra redirect on the request.get call,
            # and ensure we download the exact atomic version even if it changed
            # between the HEAD and the GET (unlikely, but hey).
            # Useful for lfs blobs that are stored on a CDN.
            if metadata.location != url:
                url_to_download = metadata.location
                # Remove authorization header when downloading a LFS blob
                headers.pop("authorization", None)
        except (requests.exceptions.SSLError, requests.exceptions.ProxyError):
            # Actually raise for those subclasses of ConnectionError
            raise
        except (
            requests.exceptions.ConnectionError,
            requests.exceptions.Timeout,
            OfflineModeIsEnabled,
        ):
            # Otherwise, our Internet connection is down.
            # etag is None
            pass

    # etag is None == we don't have a connection or we passed local_files_only.
    # try to get the last downloaded one from the specified revision.
    # If the specified revision is a commit hash, look inside "snapshots".
    # If the specified revision is a branch or tag, look inside "refs".
    if etag is None:
        # In those cases, we cannot force download.
        if force_download:
            raise ValueError(
                "We have no connection or you passed local_files_only, so force_download is not an accepted option."
            )

        # Try to get "commit_hash" from "revision"
        commit_hash = None
        if REGEX_COMMIT_HASH.match(revision):
            commit_hash = revision
        else:
            ref_path = os.path.join(storage_folder, "refs", revision)
            if os.path.isfile(ref_path):
                with open(ref_path) as f:
                    commit_hash = f.read()

        # Return pointer file if exists
        if commit_hash is not None:
            pointer_path = _get_pointer_path(storage_folder, commit_hash, relative_filename)
            if os.path.exists(pointer_path):
                if local_dir is not None:
                    return _to_local_dir(
                        pointer_path, local_dir, relative_filename, use_symlinks=local_dir_use_symlinks
                    )
                return pointer_path

        # If we couldn't find an appropriate file on disk, raise an error.
        # If files cannot be found and local_files_only=True,
        # the models might've been found if local_files_only=False
        # Notify the user about that
        if local_files_only:
            raise LocalEntryNotFoundError(
                "Cannot find the requested files in the disk cache and"
                " outgoing traffic has been disabled. To enable hf.co look-ups"
                " and downloads online, set 'local_files_only' to False."
            )
        else:
            raise LocalEntryNotFoundError(
                "Connection error, and we cannot find the requested files in"
                " the disk cache. Please try again or make sure your Internet"
                " connection is on."
            )

    # From now on, etag and commit_hash are not None.
    assert etag is not None, "etag must have been retrieved from server"
    assert commit_hash is not None, "commit_hash must have been retrieved from server"
    blob_path = os.path.join(storage_folder, "blobs", etag)
    pointer_path = _get_pointer_path(storage_folder, commit_hash, relative_filename)

    os.makedirs(os.path.dirname(blob_path), exist_ok=True)
    os.makedirs(os.path.dirname(pointer_path), exist_ok=True)
    # if passed revision is not identical to commit_hash
    # then revision has to be a branch name or tag name.
    # In that case store a ref.
    _cache_commit_hash_for_specific_revision(storage_folder, revision, commit_hash)

    if os.path.exists(pointer_path) and not force_download:
        if local_dir is not None:
            return _to_local_dir(pointer_path, local_dir, relative_filename, use_symlinks=local_dir_use_symlinks)
        return pointer_path

    if os.path.exists(blob_path) and not force_download:
        # we have the blob already, but not the pointer
        if local_dir is not None:  # to local dir
            return _to_local_dir(blob_path, local_dir, relative_filename, use_symlinks=local_dir_use_symlinks)
        else:  # or in snapshot cache
            _create_symlink(blob_path, pointer_path, new_blob=False)
            return pointer_path

    # Prevent parallel downloads of the same file with a lock.
    lock_path = blob_path + ".lock"

    # Some Windows versions do not allow for paths longer than 255 characters.
    # In this case, we must specify it is an extended path by using the "\\?\" prefix.
    if os.name == "nt" and len(os.path.abspath(lock_path)) > 255:
        lock_path = "\\\\?\\" + os.path.abspath(lock_path)

    if os.name == "nt" and len(os.path.abspath(blob_path)) > 255:
        blob_path = "\\\\?\\" + os.path.abspath(blob_path)

    with FileLock(lock_path):
        # If the download just completed while the lock was activated.
        if os.path.exists(pointer_path) and not force_download:
            # Even if returning early like here, the lock will be released.
            return pointer_path

        if resume_download:
            incomplete_path = blob_path + ".incomplete"

            @contextmanager
            def _resumable_file_manager() -> Generator[io.BufferedWriter, None, None]:
                with open(incomplete_path, "ab") as f:
                    yield f

            temp_file_manager = _resumable_file_manager
            if os.path.exists(incomplete_path):
                resume_size = os.stat(incomplete_path).st_size
            else:
                resume_size = 0
        else:
            temp_file_manager = partial(  # type: ignore
                tempfile.NamedTemporaryFile, mode="wb", dir=cache_dir, delete=False
            )
            resume_size = 0

        # Download to temporary file, then copy to cache dir once finished.
        # Otherwise you get corrupt cache entries if the download gets interrupted.
        with temp_file_manager() as temp_file:
            logger.info("downloading %s to %s", url, temp_file.name)

            http_get(
                url_to_download,
                temp_file,
                proxies=proxies,
                resume_size=resume_size,
                headers=headers,
                expected_size=expected_size,
            )

        if local_dir is None:
            logger.info(f"Storing {url} in cache at {blob_path}")
            _chmod_and_replace(temp_file.name, blob_path)
            _create_symlink(blob_path, pointer_path, new_blob=True)
        else:
            local_dir_filepath = os.path.join(local_dir, relative_filename)
            os.makedirs(os.path.dirname(local_dir_filepath), exist_ok=True)

            # If "auto" (default) copy-paste small files to ease manual editing but symlink big files to save disk
            # In both cases, blob file is cached.
            is_big_file = os.stat(temp_file.name).st_size > constants.HF_HUB_LOCAL_DIR_AUTO_SYMLINK_THRESHOLD
            if local_dir_use_symlinks is True or (local_dir_use_symlinks == "auto" and is_big_file):
                logger.info(f"Storing {url} in cache at {blob_path}")
                _chmod_and_replace(temp_file.name, blob_path)
                logger.info("Create symlink to local dir")
                _create_symlink(blob_path, local_dir_filepath, new_blob=False)
            elif local_dir_use_symlinks == "auto" and not is_big_file:
                logger.info(f"Storing {url} in cache at {blob_path}")
                _chmod_and_replace(temp_file.name, blob_path)
                logger.info("Duplicate in local dir (small file and use_symlink set to 'auto')")
                shutil.copyfile(blob_path, local_dir_filepath)
            else:
                logger.info(f"Storing {url} in local_dir at {local_dir_filepath} (not cached).")
                _chmod_and_replace(temp_file.name, local_dir_filepath)
            pointer_path = local_dir_filepath  # for return value

    try:
        os.remove(lock_path)
    except OSError:
        pass

    return pointer_path


@validate_hf_hub_args
def try_to_load_from_cache(
    repo_id: str,
    filename: str,
    cache_dir: Union[str, Path, None] = None,
    revision: Optional[str] = None,
    repo_type: Optional[str] = None,
) -> Union[str, _CACHED_NO_EXIST_T, None]:
    """
    Explores the cache to return the latest cached file for a given revision if found.

    This function will not raise any exception if the file in not cached.

    Args:
        cache_dir (`str` or `os.PathLike`):
            The folder where the cached files lie.
        repo_id (`str`):
            The ID of the repo on huggingface.co.
        filename (`str`):
            The filename to look for inside `repo_id`.
        revision (`str`, *optional*):
            The specific model version to use. Will default to `"main"` if it's not provided and no `commit_hash` is
            provided either.
        repo_type (`str`, *optional*):
            The type of the repository. Will default to `"model"`.

    Returns:
        `Optional[str]` or `_CACHED_NO_EXIST`:
            Will return `None` if the file was not cached. Otherwise:
            - The exact path to the cached file if it's found in the cache
            - A special value `_CACHED_NO_EXIST` if the file does not exist at the given commit hash and this fact was
              cached.

    Example:

    ```python
    from huggingface_hub import try_to_load_from_cache, _CACHED_NO_EXIST

    filepath = try_to_load_from_cache()
    if isinstance(filepath, str):
        # file exists and is cached
        ...
    elif filepath is _CACHED_NO_EXIST:
        # non-existence of file is cached
        ...
    else:
        # file is not cached
        ...
    ```
    """
    if revision is None:
        revision = "main"
    if repo_type is None:
        repo_type = "model"
    if repo_type not in REPO_TYPES:
        raise ValueError(f"Invalid repo type: {repo_type}. Accepted repo types are: {str(REPO_TYPES)}")
    if cache_dir is None:
        cache_dir = HUGGINGFACE_HUB_CACHE

    object_id = repo_id.replace("/", "--")
    repo_cache = os.path.join(cache_dir, f"{repo_type}s--{object_id}")
    if not os.path.isdir(repo_cache):
        # No cache for this model
        return None

    refs_dir = os.path.join(repo_cache, "refs")
    snapshots_dir = os.path.join(repo_cache, "snapshots")
    no_exist_dir = os.path.join(repo_cache, ".no_exist")

    # Resolve refs (for instance to convert main to the associated commit sha)
    if os.path.isdir(refs_dir):
        revision_file = os.path.join(refs_dir, revision)
        if os.path.isfile(revision_file):
            with open(revision_file) as f:
                revision = f.read()

    # Check if file is cached as "no_exist"
    if os.path.isfile(os.path.join(no_exist_dir, revision, filename)):
        return _CACHED_NO_EXIST

    # Check if revision folder exists
    if not os.path.exists(snapshots_dir):
        return None
    cached_shas = os.listdir(snapshots_dir)
    if revision not in cached_shas:
        # No cache for this revision and we won't try to return a random revision
        return None

    # Check if file exists in cache
    cached_file = os.path.join(snapshots_dir, revision, filename)
    return cached_file if os.path.isfile(cached_file) else None


@validate_hf_hub_args
def get_hf_file_metadata(
    url: str,
    token: Union[bool, str, None] = None,
    proxies: Optional[Dict] = None,
    timeout: Optional[float] = 10.0,
) -> HfFileMetadata:
    """Fetch metadata of a file versioned on the Hub for a given url.

    Args:
        url (`str`):
            File url, for example returned by [`hf_hub_url`].
        token (`str` or `bool`, *optional*):
            A token to be used for the download.
                - If `True`, the token is read from the HuggingFace config
                  folder.
                - If `False` or `None`, no token is provided.
                - If a string, it's used as the authentication token.
        proxies (`dict`, *optional*):
            Dictionary mapping protocol to the URL of the proxy passed to
            `requests.request`.
        timeout (`float`, *optional*, defaults to 10):
            How many seconds to wait for the server to send metadata before giving up.

    Returns:
        A [`HfFileMetadata`] object containing metadata such as location, etag, size and
        commit_hash.
    """
    headers = build_hf_headers(token=token)
    headers["Accept-Encoding"] = "identity"  # prevent any compression => we want to know the real size of the file

    # Retrieve metadata
    r = _request_wrapper(
        method="HEAD",
        url=url,
        headers=headers,
        allow_redirects=False,
        follow_relative_redirects=True,
        proxies=proxies,
        timeout=timeout,
    )
    hf_raise_for_status(r)

    # Return
    return HfFileMetadata(
        commit_hash=r.headers.get(HUGGINGFACE_HEADER_X_REPO_COMMIT),
        etag=_normalize_etag(
            # We favor a custom header indicating the etag of the linked resource, and
            # we fallback to the regular etag header.
            r.headers.get(HUGGINGFACE_HEADER_X_LINKED_ETAG)
            or r.headers.get("ETag")
        ),
        # Either from response headers (if redirected) or defaults to request url
        # Do not use directly `url`, as `_request_wrapper` might have followed relative
        # redirects.
        location=r.headers.get("Location") or r.request.url,  # type: ignore
        size=_int_or_none(r.headers.get(HUGGINGFACE_HEADER_X_LINKED_SIZE) or r.headers.get("Content-Length")),
    )


def _int_or_none(value: Optional[str]) -> Optional[int]:
    try:
        return int(value)  # type: ignore
    except (TypeError, ValueError):
        return None


def _chmod_and_replace(src: str, dst: str) -> None:
    """Set correct permission before moving a blob from tmp directory to cache dir.

    Do not take into account the `umask` from the process as there is no convenient way
    to get it that is thread-safe.

    See:
    - About umask: https://docs.python.org/3/library/os.html#os.umask
    - Thread-safety: https://stackoverflow.com/a/70343066
    - About solution: https://github.com/huggingface/huggingface_hub/pull/1220#issuecomment-1326211591
    - Fix issue: https://github.com/huggingface/huggingface_hub/issues/1141
    - Fix issue: https://github.com/huggingface/huggingface_hub/issues/1215
    """
    # Get umask by creating a temporary file in the cached repo folder.
    tmp_file = Path(dst).parent.parent / f"tmp_{uuid.uuid4()}"
    try:
        tmp_file.touch()
        cache_dir_mode = Path(tmp_file).stat().st_mode
        os.chmod(src, stat.S_IMODE(cache_dir_mode))
    finally:
        tmp_file.unlink()

    shutil.move(src, dst)


def _get_pointer_path(storage_folder: str, revision: str, relative_filename: str) -> str:
    # Using `os.path.abspath` instead of `Path.resolve()` to avoid resolving symlinks
    snapshot_path = os.path.join(storage_folder, "snapshots")
    pointer_path = os.path.join(snapshot_path, revision, relative_filename)
    if Path(os.path.abspath(snapshot_path)) not in Path(os.path.abspath(pointer_path)).parents:
        raise ValueError(
            "Invalid pointer path: cannot create pointer path in snapshot folder if"
            f" `storage_folder='{storage_folder}'`, `revision='{revision}'` and"
            f" `relative_filename='{relative_filename}'`."
        )
    return pointer_path


def _to_local_dir(
    path: str, local_dir: str, relative_filename: str, use_symlinks: Union[bool, Literal["auto"]]
) -> str:
    """Place a file in a local dir (different than cache_dir).

    Either symlink to blob file in cache or duplicate file depending on `use_symlinks` and file size.
    """
    # Using `os.path.abspath` instead of `Path.resolve()` to avoid resolving symlinks
    local_dir_filepath = os.path.join(local_dir, relative_filename)
    if Path(os.path.abspath(local_dir)) not in Path(os.path.abspath(local_dir_filepath)).parents:
        raise ValueError(
            f"Cannot copy file '{relative_filename}' to local dir '{local_dir}': file would not be in the local"
            " directory."
        )

    os.makedirs(os.path.dirname(local_dir_filepath), exist_ok=True)
    real_blob_path = os.path.realpath(path)

    # If "auto" (default) copy-paste small files to ease manual editing but symlink big files to save disk
    if use_symlinks == "auto":
        use_symlinks = os.stat(real_blob_path).st_size > constants.HF_HUB_LOCAL_DIR_AUTO_SYMLINK_THRESHOLD

    if use_symlinks:
        _create_symlink(real_blob_path, local_dir_filepath, new_blob=False)
    else:
        shutil.copyfile(real_blob_path, local_dir_filepath)
    return local_dir_filepath


# src/huggingface_hub/hf_api.py

# coding=utf-8
# Copyright 2019-present, the HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations

import inspect
import json
import pprint
import re
import textwrap
import warnings
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass, field
from datetime import datetime
from functools import wraps
from itertools import islice
from pathlib import Path
from typing import Any, BinaryIO, Callable, Dict, Iterable, Iterator, List, Optional, Tuple, TypeVar, Union, overload
from urllib.parse import quote

import requests
from requests.exceptions import HTTPError

from huggingface_hub.utils import (
    IGNORE_GIT_FOLDER_PATTERNS,
    EntryNotFoundError,
    LocalTokenNotFoundError,
    RepositoryNotFoundError,
    experimental,
    get_session,
)

from ._commit_api import (
    CommitOperation,
    CommitOperationAdd,
    CommitOperationCopy,
    CommitOperationDelete,
    fetch_lfs_files_to_copy,
    fetch_upload_modes,
    prepare_commit_payload,
    upload_lfs_files,
    warn_on_overwriting_operations,
)
from ._multi_commits import (
    MULTI_COMMIT_PR_CLOSE_COMMENT_FAILURE_BAD_REQUEST_TEMPLATE,
    MULTI_COMMIT_PR_CLOSE_COMMENT_FAILURE_NO_CHANGES_TEMPLATE,
    MULTI_COMMIT_PR_CLOSING_COMMENT_TEMPLATE,
    MULTI_COMMIT_PR_COMPLETION_COMMENT_TEMPLATE,
    MultiCommitException,
    MultiCommitStep,
    MultiCommitStrategy,
    multi_commit_create_pull_request,
    multi_commit_generate_comment,
    multi_commit_parse_pr_description,
    plan_multi_commits,
)
from ._space_api import SpaceHardware, SpaceRuntime
from .community import (
    Discussion,
    DiscussionComment,
    DiscussionStatusChange,
    DiscussionTitleChange,
    DiscussionWithDetails,
    deserialize_event,
)
from .constants import (
    DEFAULT_REVISION,
    ENDPOINT,
    REGEX_COMMIT_OID,
    REPO_TYPE_MODEL,
    REPO_TYPES,
    REPO_TYPES_MAPPING,
    REPO_TYPES_URL_PREFIXES,
    SPACES_SDK_TYPES,
)
from .utils import (  # noqa: F401 # imported for backward compatibility
    BadRequestError,
    HfFolder,
    HfHubHTTPError,
    build_hf_headers,
    filter_repo_objects,
    hf_raise_for_status,
    logging,
    paginate,
    parse_datetime,
    validate_hf_hub_args,
)
from .utils._deprecation import (
    _deprecate_arguments,
)
from .utils._typing import CallableT, Literal, TypedDict
from .utils.endpoint_helpers import (
    AttributeDictionary,
    DatasetFilter,
    DatasetTags,
    ModelFilter,
    ModelTags,
    _filter_emissions,
)


R = TypeVar("R")  # Return type

USERNAME_PLACEHOLDER = "hf_user"
_REGEX_DISCUSSION_URL = re.compile(r".*/discussions/(\d+)$")


logger = logging.get_logger(__name__)


class ReprMixin:
    """Mixin to create the __repr__ for a class"""

    def __repr__(self):
        formatted_value = pprint.pformat(self.__dict__, width=119, compact=True)
        if "\n" in formatted_value:
            return f"{self.__class__.__name__}: {{ \n{textwrap.indent(formatted_value, '  ')}\n}}"
        else:
            return f"{self.__class__.__name__}: {formatted_value}"


def repo_type_and_id_from_hf_id(hf_id: str, hub_url: Optional[str] = None) -> Tuple[Optional[str], Optional[str], str]:
    """
    Returns the repo type and ID from a huggingface.co URL linking to a
    repository

    Args:
        hf_id (`str`):
            An URL or ID of a repository on the HF hub. Accepted values are:

            - https://huggingface.co/<repo_type>/<namespace>/<repo_id>
            - https://huggingface.co/<namespace>/<repo_id>
            - hf://<repo_type>/<namespace>/<repo_id>
            - hf://<namespace>/<repo_id>
            - <repo_type>/<namespace>/<repo_id>
            - <namespace>/<repo_id>
            - <repo_id>
        hub_url (`str`, *optional*):
            The URL of the HuggingFace Hub, defaults to https://huggingface.co

    Returns:
        A tuple with three items: repo_type (`str` or `None`), namespace (`str` or
        `None`) and repo_id (`str`).

    Raises:
        - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
            If URL cannot be parsed.
        - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
            If `repo_type` is unknown.
    """
    input_hf_id = hf_id
    hub_url = re.sub(r"https?://", "", hub_url if hub_url is not None else ENDPOINT)
    is_hf_url = hub_url in hf_id and "@" not in hf_id

    HFFS_PREFIX = "hf://"
    if hf_id.startswith(HFFS_PREFIX):  # Remove "hf://" prefix if exists
        hf_id = hf_id[len(HFFS_PREFIX) :]

    url_segments = hf_id.split("/")
    is_hf_id = len(url_segments) <= 3

    namespace: Optional[str]
    if is_hf_url:
        namespace, repo_id = url_segments[-2:]
        if namespace == hub_url:
            namespace = None
        if len(url_segments) > 2 and hub_url not in url_segments[-3]:
            repo_type = url_segments[-3]
        elif namespace in REPO_TYPES_MAPPING:
            # Mean canonical dataset or model
            repo_type = REPO_TYPES_MAPPING[namespace]
            namespace = None
        else:
            repo_type = None
    elif is_hf_id:
        if len(url_segments) == 3:
            # Passed <repo_type>/<user>/<model_id> or <repo_type>/<org>/<model_id>
            repo_type, namespace, repo_id = url_segments[-3:]
        elif len(url_segments) == 2:
            if url_segments[0] in REPO_TYPES_MAPPING:
                # Passed '<model_id>' or 'datasets/<dataset_id>' for a canonical model or dataset
                repo_type = REPO_TYPES_MAPPING[url_segments[0]]
                namespace = None
                repo_id = hf_id.split("/")[-1]
            else:
                # Passed <user>/<model_id> or <org>/<model_id>
                namespace, repo_id = hf_id.split("/")[-2:]
                repo_type = None
        else:
            # Passed <model_id>
            repo_id = url_segments[0]
            namespace, repo_type = None, None
    else:
        raise ValueError(f"Unable to retrieve user and repo ID from the passed HF ID: {hf_id}")

    # Check if repo type is known (mapping "spaces" => "space" + empty value => `None`)
    if repo_type in REPO_TYPES_MAPPING:
        repo_type = REPO_TYPES_MAPPING[repo_type]
    if repo_type == "":
        repo_type = None
    if repo_type not in REPO_TYPES:
        raise ValueError(f"Unknown `repo_type`: '{repo_type}' ('{input_hf_id}')")

    return repo_type, namespace, repo_id


class BlobLfsInfo(TypedDict, total=False):
    size: int
    sha256: str
    pointer_size: int


@dataclass
class CommitInfo:
    """Data structure containing information about a newly created commit.

    Returned by [`create_commit`].

    Args:
        commit_url (`str`):
            Url where to find the commit.

        commit_message (`str`):
            The summary (first line) of the commit that has been created.

        commit_description (`str`):
            Description of the commit that has been created. Can be empty.

        oid (`str`):
            Commit hash id. Example: `"91c54ad1727ee830252e457677f467be0bfd8a57"`.

        pr_url (`str`, *optional*):
            Url to the PR that has been created, if any. Populated when `create_pr=True`
            is passed.

        pr_revision (`str`, *optional*):
            Revision of the PR that has been created, if any. Populated when
            `create_pr=True` is passed. Example: `"refs/pr/1"`.

        pr_num (`int`, *optional*):
            Number of the PR discussion that has been created, if any. Populated when
            `create_pr=True` is passed. Can be passed as `discussion_num` in
            [`get_discussion_details`]. Example: `1`.
    """

    commit_url: str
    commit_message: str
    commit_description: str
    oid: str
    pr_url: Optional[str] = None

    # Computed from `pr_url` in `__post_init__`
    pr_revision: Optional[str] = field(init=False)
    pr_num: Optional[str] = field(init=False)

    def __post_init__(self):
        """Populate pr-related fields after initialization.

        See https://docs.python.org/3.10/library/dataclasses.html#post-init-processing.
        """
        if self.pr_url is not None:
            self.pr_revision = _parse_revision_from_pr_url(self.pr_url)
            self.pr_num = int(self.pr_revision.split("/")[-1])
        else:
            self.pr_revision = None
            self.pr_num = None


class RepoUrl(str):
    """Subclass of `str` describing a repo URL on the Hub.

    `RepoUrl` is returned by `HfApi.create_repo`. It inherits from `str` for backward
    compatibility. At initialization, the URL is parsed to populate properties:
    - endpoint (`str`)
    - namespace (`Optional[str]`)
    - repo_name (`str`)
    - repo_id (`str`)
    - repo_type (`Literal["model", "dataset", "space"]`)
    - url (`str`)

    Args:
        url (`Any`):
            String value of the repo url.
        endpoint (`str`, *optional*):
            Endpoint of the Hub. Defaults to <https://huggingface.co>.

    Example:
    ```py
    >>> RepoUrl('https://huggingface.co/gpt2')
    RepoUrl('https://huggingface.co/gpt2', endpoint='https://huggingface.co', repo_type='model', repo_id='gpt2')

    >>> RepoUrl('https://hub-ci.huggingface.co/datasets/dummy_user/dummy_dataset', endpoint='https://hub-ci.huggingface.co')
    RepoUrl('https://hub-ci.huggingface.co/datasets/dummy_user/dummy_dataset', endpoint='https://hub-ci.huggingface.co', repo_type='dataset', repo_id='dummy_user/dummy_dataset')

    >>> RepoUrl('hf://datasets/my-user/my-dataset')
    RepoUrl('hf://datasets/my-user/my-dataset', endpoint='https://huggingface.co', repo_type='dataset', repo_id='user/dataset')

    >>> HfApi.create_repo("dummy_model")
    RepoUrl('https://huggingface.co/Wauplin/dummy_model', endpoint='https://huggingface.co', repo_type='model', repo_id='Wauplin/dummy_model')
    ```

    Raises:
        - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
            If URL cannot be parsed.
        - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
            If `repo_type` is unknown.
    """

    def __new__(cls, url: Any, endpoint: Optional[str] = None):
        return super(RepoUrl, cls).__new__(cls, url)

    def __init__(self, url: Any, endpoint: Optional[str] = None) -> None:
        super().__init__()
        # Parse URL
        self.endpoint = endpoint or ENDPOINT
        repo_type, namespace, repo_name = repo_type_and_id_from_hf_id(self, hub_url=self.endpoint)

        # Populate fields
        self.namespace = namespace
        self.repo_name = repo_name
        self.repo_id = repo_name if namespace is None else f"{namespace}/{repo_name}"
        self.repo_type = repo_type or REPO_TYPE_MODEL
        self.url = str(self)  # just in case it's needed

    def __repr__(self) -> str:
        return f"RepoUrl('{self}', endpoint='{self.endpoint}', repo_type='{self.repo_type}', repo_id='{self.repo_id}')"


class RepoFile(ReprMixin):
    """
    Data structure that represents a public file inside a repo, accessible from huggingface.co

    Args:
        rfilename (str):
            file name, relative to the repo root. This is the only attribute that's guaranteed to be here, but under
            certain conditions there can certain other stuff.
        size (`int`, *optional*):
            The file's size, in bytes. This attribute is present when `files_metadata` argument of [`repo_info`] is set
            to `True`. It's `None` otherwise.
        blob_id (`str`, *optional*):
            The file's git OID. This attribute is present when `files_metadata` argument of [`repo_info`] is set to
            `True`. It's `None` otherwise.
        lfs (`BlobLfsInfo`, *optional*):
            The file's LFS metadata. This attribute is present when`files_metadata` argument of [`repo_info`] is set to
            `True` and the file is stored with Git LFS. It's `None` otherwise.
    """

    def __init__(
        self,
        rfilename: str,
        size: Optional[int] = None,
        blobId: Optional[str] = None,
        lfs: Optional[BlobLfsInfo] = None,
        **kwargs,
    ):
        self.rfilename = rfilename  # filename relative to the repo root

        # Optional file metadata
        self.size = size
        self.blob_id = blobId
        self.lfs = lfs

        # Hack to ensure backward compatibility with future versions of the API.
        # See discussion in https://github.com/huggingface/huggingface_hub/pull/951#discussion_r926460408
        for k, v in kwargs.items():
            setattr(self, k, v)


class ModelInfo(ReprMixin):
    """
    Info about a model accessible from huggingface.co

    Attributes:
        modelId (`str`, *optional*):
            ID of model repository.
        sha (`str`, *optional*):
            repo sha at this particular revision
        lastModified (`str`, *optional*):
            date of last commit to repo
        tags (`List[str]`, *optional*):
            List of tags.
        pipeline_tag (`str`, *optional*):
            Pipeline tag to identify the correct widget.
        siblings (`List[RepoFile]`, *optional*):
            list of ([`huggingface_hub.hf_api.RepoFile`]) objects that constitute the model.
        private (`bool`, *optional*, defaults to `False`):
            is the repo private
        author (`str`, *optional*):
            repo author
        config (`Dict`, *optional*):
            Model configuration information
        securityStatus (`Dict`, *optional*):
            Security status of the model.
            Example: `{"containsInfected": False}`
        kwargs (`Dict`, *optional*):
            Kwargs that will be become attributes of the class.
    """

    def __init__(
        self,
        *,
        modelId: Optional[str] = None,
        sha: Optional[str] = None,
        lastModified: Optional[str] = None,
        tags: Optional[List[str]] = None,
        pipeline_tag: Optional[str] = None,
        siblings: Optional[List[Dict]] = None,
        private: bool = False,
        author: Optional[str] = None,
        config: Optional[Dict] = None,
        securityStatus: Optional[Dict] = None,
        **kwargs,
    ):
        self.modelId = modelId
        self.sha = sha
        self.lastModified = lastModified
        self.tags = tags
        self.pipeline_tag = pipeline_tag
        self.siblings = [RepoFile(**x) for x in siblings] if siblings is not None else []
        self.private = private
        self.author = author
        self.config = config
        self.securityStatus = securityStatus
        for k, v in kwargs.items():
            setattr(self, k, v)

    def __str__(self):
        r = f"Model Name: {self.modelId}, Tags: {self.tags}"
        if self.pipeline_tag:
            r += f", Task: {self.pipeline_tag}"
        return r


class DatasetInfo(ReprMixin):
    """
    Info about a dataset accessible from huggingface.co

    Attributes:
        id (`str`, *optional*):
            ID of dataset repository.
        sha (`str`, *optional*):
            repo sha at this particular revision
        lastModified (`str`, *optional*):
            date of last commit to repo
        tags (`List[str]`, *optional*):
            List of tags.
        siblings (`List[RepoFile]`, *optional*):
            list of [`huggingface_hub.hf_api.RepoFile`] objects that constitute the dataset.
        private (`bool`, *optional*, defaults to `False`):
            is the repo private
        author (`str`, *optional*):
            repo author
        description (`str`, *optional*):
            Description of the dataset
        citation (`str`, *optional*):
            Dataset citation
        cardData (`Dict`, *optional*):
            Metadata of the model card as a dictionary.
        kwargs (`Dict`, *optional*):
            Kwargs that will be become attributes of the class.
    """

    def __init__(
        self,
        *,
        id: Optional[str] = None,
        sha: Optional[str] = None,
        lastModified: Optional[str] = None,
        tags: Optional[List[str]] = None,
        siblings: Optional[List[Dict]] = None,
        private: bool = False,
        author: Optional[str] = None,
        description: Optional[str] = None,
        citation: Optional[str] = None,
        cardData: Optional[dict] = None,
        **kwargs,
    ):
        self.id = id
        self.sha = sha
        self.lastModified = lastModified
        self.tags = tags
        self.private = private
        self.author = author
        self.description = description
        self.citation = citation
        self.cardData = cardData
        self.siblings = [RepoFile(**x) for x in siblings] if siblings is not None else []
        # Legacy stuff, "key" is always returned with an empty string
        # because of old versions of the datasets lib that need this field
        kwargs.pop("key", None)
        # Store all the other fields returned by the API
        for k, v in kwargs.items():
            setattr(self, k, v)

    def __str__(self):
        r = f"Dataset Name: {self.id}, Tags: {self.tags}"
        return r


class SpaceInfo(ReprMixin):
    """
    Info about a Space accessible from huggingface.co

    This is a "dataclass" like container that just sets on itself any attribute
    passed by the server.

    Attributes:
        id (`str`, *optional*):
            id of space
        sha (`str`, *optional*):
            repo sha at this particular revision
        lastModified (`str`, *optional*):
            date of last commit to repo
        siblings (`List[RepoFile]`, *optional*):
            list of [`huggingface_hub.hf_api.RepoFIle`] objects that constitute the Space
        private (`bool`, *optional*, defaults to `False`):
            is the repo private
        author (`str`, *optional*):
            repo author
        kwargs (`Dict`, *optional*):
            Kwargs that will be become attributes of the class.
    """

    def __init__(
        self,
        *,
        id: Optional[str] = None,
        sha: Optional[str] = None,
        lastModified: Optional[str] = None,
        siblings: Optional[List[Dict]] = None,
        private: bool = False,
        author: Optional[str] = None,
        **kwargs,
    ):
        self.id = id
        self.sha = sha
        self.lastModified = lastModified
        self.siblings = [RepoFile(**x) for x in siblings] if siblings is not None else []
        self.private = private
        self.author = author
        for k, v in kwargs.items():
            setattr(self, k, v)


class MetricInfo(ReprMixin):
    """
    Info about a public metric accessible from huggingface.co
    """

    def __init__(
        self,
        *,
        id: Optional[str] = None,  # id of metric
        description: Optional[str] = None,
        citation: Optional[str] = None,
        **kwargs,
    ):
        self.id = id
        self.description = description
        self.citation = citation
        # Legacy stuff, "key" is always returned with an empty string
        # because of old versions of the datasets lib that need this field
        kwargs.pop("key", None)
        # Store all the other fields returned by the API
        for k, v in kwargs.items():
            setattr(self, k, v)

    def __str__(self):
        r = f"Metric Name: {self.id}"
        return r


class ModelSearchArguments(AttributeDictionary):
    """
    A nested namespace object holding all possible values for properties of
    models currently hosted in the Hub with tab-completion. If a value starts
    with a number, it will only exist in the dictionary

    Example:

    ```python
    >>> args = ModelSearchArguments()

    >>> args.author.huggingface
    'huggingface'

    >>> args.language.en
    'en'
    ```

    <Tip warning={true}>

    `ModelSearchArguments` is a legacy class meant for exploratory purposes only. Its
    initialization requires listing all models on the Hub which makes it increasingly
    slower as the number of repos on the Hub increases.

    </Tip>
    """

    def __init__(self, api: Optional["HfApi"] = None):
        self._api = api if api is not None else HfApi()
        tags = self._api.get_model_tags()
        super().__init__(tags)
        self._process_models()

    def _process_models(self):
        def clean(s: str) -> str:
            return s.replace(" ", "").replace("-", "_").replace(".", "_")

        models = self._api.list_models()
        author_dict, model_name_dict = AttributeDictionary(), AttributeDictionary()
        for model in models:
            if "/" in model.modelId:
                author, name = model.modelId.split("/")
                author_dict[author] = clean(author)
            else:
                name = model.modelId
            model_name_dict[name] = clean(name)
        self["model_name"] = model_name_dict
        self["author"] = author_dict


class DatasetSearchArguments(AttributeDictionary):
    """
    A nested namespace object holding all possible values for properties of
    datasets currently hosted in the Hub with tab-completion. If a value starts
    with a number, it will only exist in the dictionary

    Example:

    ```python
    >>> args = DatasetSearchArguments()

    >>> args.author.huggingface
    'huggingface'

    >>> args.language.en
    'language:en'
    ```

    <Tip warning={true}>

    `DatasetSearchArguments` is a legacy class meant for exploratory purposes only. Its
    initialization requires listing all datasets on the Hub which makes it increasingly
    slower as the number of repos on the Hub increases.

    </Tip>
    """

    def __init__(self, api: Optional["HfApi"] = None):
        self._api = api if api is not None else HfApi()
        tags = self._api.get_dataset_tags()
        super().__init__(tags)
        self._process_models()

    def _process_models(self):
        def clean(s: str):
            return s.replace(" ", "").replace("-", "_").replace(".", "_")

        datasets = self._api.list_datasets()
        author_dict, dataset_name_dict = AttributeDictionary(), AttributeDictionary()
        for dataset in datasets:
            if "/" in dataset.id:
                author, name = dataset.id.split("/")
                author_dict[author] = clean(author)
            else:
                name = dataset.id
            dataset_name_dict[name] = clean(name)
        self["dataset_name"] = dataset_name_dict
        self["author"] = author_dict


@dataclass
class GitRefInfo:
    """
    Contains information about a git reference for a repo on the Hub.

    Args:
        name (`str`):
            Name of the reference (e.g. tag name or branch name).
        ref (`str`):
            Full git ref on the Hub (e.g. `"refs/heads/main"` or `"refs/tags/v1.0"`).
        target_commit (`str`):
            OID of the target commit for the ref (e.g. `"e7da7f221d5bf496a48136c0cd264e630fe9fcc8"`)
    """

    name: str
    ref: str
    target_commit: str

    def __init__(self, data: Dict) -> None:
        self.name = data["name"]
        self.ref = data["ref"]
        self.target_commit = data["targetCommit"]


@dataclass
class GitRefs:
    """
    Contains information about all git references for a repo on the Hub.

    Object is returned by [`list_repo_refs`].

    Args:
        branches (`List[GitRefInfo]`):
            A list of [`GitRefInfo`] containing information about branches on the repo.
        converts (`List[GitRefInfo]`):
            A list of [`GitRefInfo`] containing information about "convert" refs on the repo.
            Converts are refs used (internally) to push preprocessed data in Dataset repos.
        tags (`List[GitRefInfo]`):
            A list of [`GitRefInfo`] containing information about tags on the repo.
    """

    branches: List[GitRefInfo]
    converts: List[GitRefInfo]
    tags: List[GitRefInfo]


@dataclass
class GitCommitInfo:
    """
    Contains information about a git commit for a repo on the Hub. Check out [`list_repo_commits`] for more details.

    Args:
        commit_id (`str`):
            OID of the commit (e.g. `"e7da7f221d5bf496a48136c0cd264e630fe9fcc8"`)
        authors (`List[str]`):
            List of authors of the commit.
        created_at (`datetime`):
            Datetime when the commit was created.
        title (`str`):
            Title of the commit. This is a free-text value entered by the authors.
        message (`str`):
            Description of the commit. This is a free-text value entered by the authors.
        formatted_title (`str`):
            Title of the commit formatted as HTML. Only returned if `formatted=True` is set.
        formatted_message (`str`):
            Description of the commit formatted as HTML. Only returned if `formatted=True` is set.
    """

    commit_id: str

    authors: List[str]
    created_at: datetime
    title: str
    message: str

    formatted_title: Optional[str]
    formatted_message: Optional[str]

    def __init__(self, data: Dict) -> None:
        self.commit_id = data["id"]
        self.authors = [author["user"] for author in data["authors"]]
        self.created_at = parse_datetime(data["date"])
        self.title = data["title"]
        self.message = data["message"]

        self.formatted_title = data.get("formatted", {}).get("title")
        self.formatted_message = data.get("formatted", {}).get("message")


@dataclass
class UserLikes:
    """
    Contains information about a user likes on the Hub.

    Args:
        user (`str`):
            Name of the user for which we fetched the likes.
        total (`int`):
            Total number of likes.
        datasets (`List[str]`):
            List of datasets liked by the user (as repo_ids).
        models (`List[str]`):
            List of models liked by the user (as repo_ids).
        spaces (`List[str]`):
            List of spaces liked by the user (as repo_ids).
    """

    # Metadata
    user: str
    total: int

    # User likes
    datasets: List[str]
    models: List[str]
    spaces: List[str]


def future_compatible(fn: CallableT) -> CallableT:
    """Wrap a method of `HfApi` to handle `run_as_future=True`.

    A method flagged as "future_compatible" will be called in a thread if `run_as_future=True` and return a
    `concurrent.futures.Future` instance. Otherwise, it will be called normally and return the result.
    """
    sig = inspect.signature(fn)
    args_params = list(sig.parameters)[1:]  # remove "self" from list

    @wraps(fn)
    def _inner(self, *args, **kwargs):
        # Get `run_as_future` value if provided (default to False)
        if "run_as_future" in kwargs:
            run_as_future = kwargs["run_as_future"]
            kwargs["run_as_future"] = False  # avoid recursion error
        else:
            run_as_future = False
            for param, value in zip(args_params, args):
                if param == "run_as_future":
                    run_as_future = value
                    break

        # Call the function in a thread if `run_as_future=True`
        if run_as_future:
            return self.run_as_future(fn, self, *args, **kwargs)

        # Otherwise, call the function normally
        return fn(self, *args, **kwargs)

    _inner.is_future_compatible = True  # type: ignore
    return _inner  # type: ignore


class HfApi:
    def __init__(
        self,
        endpoint: Optional[str] = None,
        token: Optional[str] = None,
        library_name: Optional[str] = None,
        library_version: Optional[str] = None,
        user_agent: Union[Dict, str, None] = None,
    ) -> None:
        """Create a HF client to interact with the Hub via HTTP.

        The client is initialized with some high-level settings used in all requests
        made to the Hub (HF endpoint, authentication, user agents...). Using the `HfApi`
        client is preferred but not mandatory as all of its public methods are exposed
        directly at the root of `huggingface_hub`.

        Args:
            endpoint (`str`, *optional*):
                Hugging Face Hub base url. Will default to https://huggingface.co/. To
                be set if you are using a private hub. Otherwise, one can set the
                `HF_ENDPOINT` environment variable.
            token (`str`, *optional*):
                Hugging Face token. Will default to the locally saved token if
                not provided.
            library_name (`str`, *optional*):
                The name of the library that is making the HTTP request. Will be added to
                the user-agent header. Example: `"transformers"`.
            library_version (`str`, *optional*):
                The version of the library that is making the HTTP request. Will be added
                to the user-agent header. Example: `"4.24.0"`.
            user_agent (`str`, `dict`, *optional*):
                The user agent info in the form of a dictionary or a single string. It will
                be completed with information about the installed packages.
        """
        self.endpoint = endpoint if endpoint is not None else ENDPOINT
        self.token = token
        self.library_name = library_name
        self.library_version = library_version
        self.user_agent = user_agent
        self._thread_pool: Optional[ThreadPoolExecutor] = None

    def run_as_future(self, fn: Callable[..., R], *args, **kwargs) -> Future[R]:
        """
        Run a method in the background and return a Future instance.

        The main goal is to run methods without blocking the main thread (e.g. to push data during a training).
        Background jobs are queued to preserve order but are not ran in parallel. If you need to speed-up your scripts
        by parallelizing lots of call to the API, you must setup and use your own [ThreadPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#threadpoolexecutor).

        Note: Most-used methods like [`upload_file`], [`upload_folder`] and [`create_commit`] have a `run_as_future: bool`
        argument to directly call them in the background. This is equivalent to calling `api.run_as_future(...)` on them
        but less verbose.

        Args:
            fn (`Callable`):
                The method to run in the background.
            *args, **kwargs:
                Arguments with which the method will be called.

        Return:
            `Future`: a [Future](https://docs.python.org/3/library/concurrent.futures.html#future-objects) instance to
            get the result of the task.

        Example:
            ```py
            >>> from huggingface_hub import HfApi
            >>> api = HfApi()
            >>> future = api.run_as_future(api.whoami) # instant
            >>> future.done()
            False
            >>> future.result() # wait until complete and return result
            (...)
            >>> future.done()
            True
            ```
        """
        if self._thread_pool is None:
            self._thread_pool = ThreadPoolExecutor(max_workers=1)
        self._thread_pool
        return self._thread_pool.submit(fn, *args, **kwargs)

    @validate_hf_hub_args
    def whoami(self, token: Optional[str] = None) -> Dict:
        """
        Call HF API to know "whoami".

        Args:
            token (`str`, *optional*):
                Hugging Face token. Will default to the locally saved token if
                not provided.
        """
        r = get_session().get(
            f"{self.endpoint}/api/whoami-v2",
            headers=self._build_hf_headers(
                # If `token` is provided and not `None`, it will be used by default.
                # Otherwise, the token must be retrieved from cache or env variable.
                token=(token or self.token or True),
            ),
        )
        try:
            hf_raise_for_status(r)
        except HTTPError as e:
            raise HTTPError(
                "Invalid user token. If you didn't pass a user token, make sure you "
                "are properly logged in by executing `huggingface-cli login`, and "
                "if you did pass a user token, double-check it's correct."
            ) from e
        return r.json()

    def get_token_permission(self, token: Optional[str] = None) -> Literal["read", "write", None]:
        """
        Check if a given `token` is valid and return its permissions.

        For more details about tokens, please refer to https://huggingface.co/docs/hub/security-tokens#what-are-user-access-tokens.

        Args:
            token (`str`, *optional*):
                The token to check for validity. Defaults to the one saved locally.

        Returns:
            `Literal["read", "write", None]`: Permission granted by the token ("read" or "write"). Returns `None` if no
            token passed or token is invalid.
        """
        try:
            return self.whoami(token=token)["auth"]["accessToken"]["role"]
        except (LocalTokenNotFoundError, HTTPError):
            return None

    def get_model_tags(self) -> ModelTags:
        """
        List all valid model tags as a nested namespace object
        """
        path = f"{self.endpoint}/api/models-tags-by-type"
        r = get_session().get(path)
        hf_raise_for_status(r)
        d = r.json()
        return ModelTags(d)

    def get_dataset_tags(self) -> DatasetTags:
        """
        List all valid dataset tags as a nested namespace object.
        """
        path = f"{self.endpoint}/api/datasets-tags-by-type"
        r = get_session().get(path)
        hf_raise_for_status(r)
        d = r.json()
        return DatasetTags(d)

    @validate_hf_hub_args
    def list_models(
        self,
        *,
        filter: Union[ModelFilter, str, Iterable[str], None] = None,
        author: Optional[str] = None,
        search: Optional[str] = None,
        emissions_thresholds: Optional[Tuple[float, float]] = None,
        sort: Union[Literal["lastModified"], str, None] = None,
        direction: Optional[Literal[-1]] = None,
        limit: Optional[int] = None,
        full: Optional[bool] = None,
        cardData: bool = False,
        fetch_config: bool = False,
        token: Optional[Union[bool, str]] = None,
    ) -> Iterable[ModelInfo]:
        """
        List models hosted on the Huggingface Hub, given some filters.

        Args:
            filter ([`ModelFilter`] or `str` or `Iterable`, *optional*):
                A string or [`ModelFilter`] which can be used to identify models
                on the Hub.
            author (`str`, *optional*):
                A string which identify the author (user or organization) of the
                returned models
            search (`str`, *optional*):
                A string that will be contained in the returned model ids.
            emissions_thresholds (`Tuple`, *optional*):
                A tuple of two ints or floats representing a minimum and maximum
                carbon footprint to filter the resulting models with in grams.
            sort (`Literal["lastModified"]` or `str`, *optional*):
                The key with which to sort the resulting models. Possible values
                are the properties of the [`huggingface_hub.hf_api.ModelInfo`] class.
            direction (`Literal[-1]` or `int`, *optional*):
                Direction in which to sort. The value `-1` sorts by descending
                order while all other values sort by ascending order.
            limit (`int`, *optional*):
                The limit on the number of models fetched. Leaving this option
                to `None` fetches all models.
            full (`bool`, *optional*):
                Whether to fetch all model data, including the `lastModified`,
                the `sha`, the files and the `tags`. This is set to `True` by
                default when using a filter.
            cardData (`bool`, *optional*):
                Whether to grab the metadata for the model as well. Can contain
                useful information such as carbon emissions, metrics, and
                datasets trained on.
            fetch_config (`bool`, *optional*):
                Whether to fetch the model configs as well. This is not included
                in `full` due to its size.
            token (`bool` or `str`, *optional*):
                A valid authentication token (see https://huggingface.co/settings/token).
                If `None` or `True` and machine is logged in (through `huggingface-cli login`
                or [`~huggingface_hub.login`]), token will be retrieved from the cache.
                If `False`, token is not sent in the request header.

        Returns:
            `Iterable[ModelInfo]`: an iterable of [`huggingface_hub.hf_api.ModelInfo`] objects.

        Example usage with the `filter` argument:

        ```python
        >>> from huggingface_hub import HfApi

        >>> api = HfApi()

        >>> # List all models
        >>> api.list_models()

        >>> # Get all valid search arguments
        >>> args = ModelSearchArguments()

        >>> # List only the text classification models
        >>> api.list_models(filter="text-classification")
        >>> # Using the `ModelFilter`
        >>> filt = ModelFilter(task="text-classification")
        >>> # With `ModelSearchArguments`
        >>> filt = ModelFilter(task=args.pipeline_tags.TextClassification)
        >>> api.list_models(filter=filt)

        >>> # Using `ModelFilter` and `ModelSearchArguments` to find text classification in both PyTorch and TensorFlow
        >>> filt = ModelFilter(
        ...     task=args.pipeline_tags.TextClassification,
        ...     library=[args.library.PyTorch, args.library.TensorFlow],
        ... )
        >>> api.list_models(filter=filt)

        >>> # List only models from the AllenNLP library
        >>> api.list_models(filter="allennlp")
        >>> # Using `ModelFilter` and `ModelSearchArguments`
        >>> filt = ModelFilter(library=args.library.allennlp)
        ```

        Example usage with the `search` argument:

        ```python
        >>> from huggingface_hub import HfApi

        >>> api = HfApi()

        >>> # List all models with "bert" in their name
        >>> api.list_models(search="bert")

        >>> # List all models with "bert" in their name made by google
        >>> api.list_models(search="bert", author="google")
        ```
        """
        if emissions_thresholds is not None and cardData is None:
            raise ValueError("`emissions_thresholds` were passed without setting `cardData=True`.")

        path = f"{self.endpoint}/api/models"
        headers = self._build_hf_headers(token=token)
        params = {}
        if filter is not None:
            if isinstance(filter, ModelFilter):
                params = self._unpack_model_filter(filter)
            else:
                params.update({"filter": filter})
            params.update({"full": True})
        if author is not None:
            params.update({"author": author})
        if search is not None:
            params.update({"search": search})
        if sort is not None:
            params.update({"sort": sort})
        if direction is not None:
            params.update({"direction": direction})
        if limit is not None:
            params.update({"limit": limit})
        if full is not None:
            if full:
                params.update({"full": True})
            elif "full" in params:
                del params["full"]
        if fetch_config:
            params.update({"config": True})
        if cardData:
            params.update({"cardData": True})

        # `items` is a generator
        items = paginate(path, params=params, headers=headers)
        if limit is not None:
            items = islice(items, limit)  # Do not iterate over all pages
        if emissions_thresholds is not None:
            items = _filter_emissions(items, *emissions_thresholds)
        for item in items:
            yield ModelInfo(**item)

    def _unpack_model_filter(self, model_filter: ModelFilter):
        """
        Unpacks a [`ModelFilter`] into something readable for `list_models`
        """
        model_str = ""
        tags = []

        # Handling author
        if model_filter.author is not None:
            model_str = f"{model_filter.author}/"

        # Handling model_name
        if model_filter.model_name is not None:
            model_str += model_filter.model_name

        filter_list: List[str] = []

        # Handling tasks
        if model_filter.task is not None:
            filter_list.extend([model_filter.task] if isinstance(model_filter.task, str) else model_filter.task)

        # Handling dataset
        if model_filter.trained_dataset is not None:
            if not isinstance(model_filter.trained_dataset, (list, tuple)):
                model_filter.trained_dataset = [model_filter.trained_dataset]
            for dataset in model_filter.trained_dataset:
                if "dataset:" not in dataset:
                    dataset = f"dataset:{dataset}"
                filter_list.append(dataset)

        # Handling library
        if model_filter.library:
            filter_list.extend(
                [model_filter.library] if isinstance(model_filter.library, str) else model_filter.library
            )

        # Handling tags
        if model_filter.tags:
            tags.extend([model_filter.tags] if isinstance(model_filter.tags, str) else model_filter.tags)

        query_dict: Dict[str, Any] = {}
        if model_str is not None:
            query_dict["search"] = model_str
        if len(tags) > 0:
            query_dict["tags"] = tags
        if isinstance(model_filter.language, list):
            filter_list.extend(model_filter.language)
        elif isinstance(model_filter.language, str):
            filter_list.append(model_filter.language)
        query_dict["filter"] = tuple(filter_list)
        return query_dict

    @validate_hf_hub_args
    def list_datasets(
        self,
        *,
        filter: Union[DatasetFilter, str, Iterable[str], None] = None,
        author: Optional[str] = None,
        search: Optional[str] = None,
        sort: Union[Literal["lastModified"], str, None] = None,
        direction: Optional[Literal[-1]] = None,
        limit: Optional[int] = None,
        full: Optional[bool] = None,
        token: Optional[str] = None,
    ) -> Iterable[DatasetInfo]:
        """
        List datasets hosted on the Huggingface Hub, given some filters.

        Args:
            filter ([`DatasetFilter`] or `str` or `Iterable`, *optional*):
                A string or [`DatasetFilter`] which can be used to identify
                datasets on the hub.
            author (`str`, *optional*):
                A string which identify the author of the returned datasets.
            search (`str`, *optional*):
                A string that will be contained in the returned datasets.
            sort (`Literal["lastModified"]` or `str`, *optional*):
                The key with which to sort the resulting datasets. Possible
                values are the properties of the [`huggingface_hub.hf_api.DatasetInfo`] class.
            direction (`Literal[-1]` or `int`, *optional*):
                Direction in which to sort. The value `-1` sorts by descending
                order while all other values sort by ascending order.
            limit (`int`, *optional*):
                The limit on the number of datasets fetched. Leaving this option
                to `None` fetches all datasets.
            full (`bool`, *optional*):
                Whether to fetch all dataset data, including the `lastModified`
                and the `cardData`. Can contain useful information such as the
                PapersWithCode ID.
            token (`bool` or `str`, *optional*):
                A valid authentication token (see https://huggingface.co/settings/token).
                If `None` or `True` and machine is logged in (through `huggingface-cli login`
                or [`~huggingface_hub.login`]), token will be retrieved from the cache.
                If `False`, token is not sent in the request header.

        Returns:
            `Iterable[DatasetInfo]`: an iterable of [`huggingface_hub.hf_api.DatasetInfo`] objects.

        Example usage with the `filter` argument:

        ```python
        >>> from huggingface_hub import HfApi

        >>> api = HfApi()

        >>> # List all datasets
        >>> api.list_datasets()

        >>> # Get all valid search arguments
        >>> args = DatasetSearchArguments()

        >>> # List only the text classification datasets
        >>> api.list_datasets(filter="task_categories:text-classification")
        >>> # Using the `DatasetFilter`
        >>> filt = DatasetFilter(task_categories="text-classification")
        >>> # With `DatasetSearchArguments`
        >>> filt = DatasetFilter(task=args.task_categories.text_classification)
        >>> api.list_models(filter=filt)

        >>> # List only the datasets in russian for language modeling
        >>> api.list_datasets(
        ...     filter=("language:ru", "task_ids:language-modeling")
        ... )
        >>> # Using the `DatasetFilter`
        >>> filt = DatasetFilter(language="ru", task_ids="language-modeling")
        >>> # With `DatasetSearchArguments`
        >>> filt = DatasetFilter(
        ...     language=args.language.ru,
        ...     task_ids=args.task_ids.language_modeling,
        ... )
        >>> api.list_datasets(filter=filt)
        ```

        Example usage with the `search` argument:

        ```python
        >>> from huggingface_hub import HfApi

        >>> api = HfApi()

        >>> # List all datasets with "text" in their name
        >>> api.list_datasets(search="text")

        >>> # List all datasets with "text" in their name made by google
        >>> api.list_datasets(search="text", author="google")
        ```
        """
        path = f"{self.endpoint}/api/datasets"
        headers = self._build_hf_headers(token=token)
        params = {}
        if filter is not None:
            if isinstance(filter, DatasetFilter):
                params = self._unpack_dataset_filter(filter)
            else:
                params.update({"filter": filter})
        if author is not None:
            params.update({"author": author})
        if search is not None:
            params.update({"search": search})
        if sort is not None:
            params.update({"sort": sort})
        if direction is not None:
            params.update({"direction": direction})
        if limit is not None:
            params.update({"limit": limit})
        if full:
            params.update({"full": True})

        items = paginate(path, params=params, headers=headers)
        if limit is not None:
            items = islice(items, limit)  # Do not iterate over all pages
        for item in items:
            yield DatasetInfo(**item)

    def _unpack_dataset_filter(self, dataset_filter: DatasetFilter):
        """
        Unpacks a [`DatasetFilter`] into something readable for `list_datasets`
        """
        dataset_str = ""

        # Handling author
        if dataset_filter.author is not None:
            dataset_str = f"{dataset_filter.author}/"

        # Handling dataset_name
        if dataset_filter.dataset_name is not None:
            dataset_str += dataset_filter.dataset_name

        filter_list = []
        data_attributes = [
            "benchmark",
            "language_creators",
            "language",
            "multilinguality",
            "size_categories",
            "task_categories",
            "task_ids",
        ]

        for attr in data_attributes:
            curr_attr = getattr(dataset_filter, attr)
            if curr_attr is not None:
                if not isinstance(curr_attr, (list, tuple)):
                    curr_attr = [curr_attr]
                for data in curr_attr:
                    if f"{attr}:" not in data:
                        data = f"{attr}:{data}"
                    filter_list.append(data)

        query_dict: Dict[str, Any] = {}
        if dataset_str is not None:
            query_dict["search"] = dataset_str
        query_dict["filter"] = tuple(filter_list)
        return query_dict

    def list_metrics(self) -> List[MetricInfo]:
        """
        Get the public list of all the metrics on huggingface.co

        Returns:
            `List[MetricInfo]`: a list of [`MetricInfo`] objects which.
        """
        path = f"{self.endpoint}/api/metrics"
        r = get_session().get(path)
        hf_raise_for_status(r)
        d = r.json()
        return [MetricInfo(**x) for x in d]

    @validate_hf_hub_args
    def list_spaces(
        self,
        *,
        filter: Union[str, Iterable[str], None] = None,
        author: Optional[str] = None,
        search: Optional[str] = None,
        sort: Union[Literal["lastModified"], str, None] = None,
        direction: Optional[Literal[-1]] = None,
        limit: Optional[int] = None,
        datasets: Union[str, Iterable[str], None] = None,
        models: Union[str, Iterable[str], None] = None,
        linked: bool = False,
        full: Optional[bool] = None,
        token: Optional[str] = None,
    ) -> Iterable[SpaceInfo]:
        """
        List spaces hosted on the Huggingface Hub, given some filters.

        Args:
            filter (`str` or `Iterable`, *optional*):
                A string tag or list of tags that can be used to identify Spaces on the Hub.
            author (`str`, *optional*):
                A string which identify the author of the returned Spaces.
            search (`str`, *optional*):
                A string that will be contained in the returned Spaces.
            sort (`Literal["lastModified"]` or `str`, *optional*):
                The key with which to sort the resulting Spaces. Possible
                values are the properties of the [`huggingface_hub.hf_api.SpaceInfo`]` class.
            direction (`Literal[-1]` or `int`, *optional*):
                Direction in which to sort. The value `-1` sorts by descending
                order while all other values sort by ascending order.
            limit (`int`, *optional*):
                The limit on the number of Spaces fetched. Leaving this option
                to `None` fetches all Spaces.
            datasets (`str` or `Iterable`, *optional*):
                Whether to return Spaces that make use of a dataset.
                The name of a specific dataset can be passed as a string.
            models (`str` or `Iterable`, *optional*):
                Whether to return Spaces that make use of a model.
                The name of a specific model can be passed as a string.
            linked (`bool`, *optional*):
                Whether to return Spaces that make use of either a model or a dataset.
            full (`bool`, *optional*):
                Whether to fetch all Spaces data, including the `lastModified`
                and the `cardData`.
            token (`bool` or `str`, *optional*):
                A valid authentication token (see https://huggingface.co/settings/token).
                If `None` or `True` and machine is logged in (through `huggingface-cli login`
                or [`~huggingface_hub.login`]), token will be retrieved from the cache.
                If `False`, token is not sent in the request header.

        Returns:
            `Iterable[SpaceInfo]`: an iterable of [`huggingface_hub.hf_api.SpaceInfo`] objects.
        """
        path = f"{self.endpoint}/api/spaces"
        headers = self._build_hf_headers(token=token)
        params: Dict[str, Any] = {}
        if filter is not None:
            params.update({"filter": filter})
        if author is not None:
            params.update({"author": author})
        if search is not None:
            params.update({"search": search})
        if sort is not None:
            params.update({"sort": sort})
        if direction is not None:
            params.update({"direction": direction})
        if limit is not None:
            params.update({"limit": limit})
        if full:
            params.update({"full": True})
        if linked:
            params.update({"linked": True})
        if datasets is not None:
            params.update({"datasets": datasets})
        if models is not None:
            params.update({"models": models})

        items = paginate(path, params=params, headers=headers)
        if limit is not None:
            items = islice(items, limit)  # Do not iterate over all pages
        for item in items:
            yield SpaceInfo(**item)

    @validate_hf_hub_args
    def like(
        self,
        repo_id: str,
        *,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
    ) -> None:
        """
        Like a given repo on the Hub (e.g. set as favorite).

        See also [`unlike`] and [`list_liked_repos`].

        Args:
            repo_id (`str`):
                The repository to like. Example: `"user/my-cool-model"`.

            token (`str`, *optional*):
                Authentication token. Will default to the stored token.

            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if liking a dataset or space, `None` or
                `"model"` if liking a model. Default is `None`.

        Raises:
            [`~utils.RepositoryNotFoundError`]:
                If repository is not found (error 404): wrong repo_id/repo_type, private
                but not authenticated or repo does not exist.

        Example:
        ```python
        >>> from huggingface_hub import like, list_liked_repos, unlike
        >>> like("gpt2")
        >>> "gpt2" in list_liked_repos().models
        True
        >>> unlike("gpt2")
        >>> "gpt2" in list_liked_repos().models
        False
        ```
        """
        if repo_type is None:
            repo_type = REPO_TYPE_MODEL
        response = get_session().post(
            url=f"{self.endpoint}/api/{repo_type}s/{repo_id}/like",
            headers=self._build_hf_headers(token=token),
        )
        hf_raise_for_status(response)

    @validate_hf_hub_args
    def unlike(
        self,
        repo_id: str,
        *,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
    ) -> None:
        """
        Unlike a given repo on the Hub (e.g. remove from favorite list).

        See also [`like`] and [`list_liked_repos`].

        Args:
            repo_id (`str`):
                The repository to unlike. Example: `"user/my-cool-model"`.

            token (`str`, *optional*):
                Authentication token. Will default to the stored token.

            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if unliking a dataset or space, `None` or
                `"model"` if unliking a model. Default is `None`.

        Raises:
            [`~utils.RepositoryNotFoundError`]:
                If repository is not found (error 404): wrong repo_id/repo_type, private
                but not authenticated or repo does not exist.

        Example:
        ```python
        >>> from huggingface_hub import like, list_liked_repos, unlike
        >>> like("gpt2")
        >>> "gpt2" in list_liked_repos().models
        True
        >>> unlike("gpt2")
        >>> "gpt2" in list_liked_repos().models
        False
        ```
        """
        if repo_type is None:
            repo_type = REPO_TYPE_MODEL
        response = get_session().delete(
            url=f"{self.endpoint}/api/{repo_type}s/{repo_id}/like", headers=self._build_hf_headers(token=token)
        )
        hf_raise_for_status(response)

    @validate_hf_hub_args
    def list_liked_repos(
        self,
        user: Optional[str] = None,
        *,
        token: Optional[str] = None,
    ) -> UserLikes:
        """
        List all public repos liked by a user on huggingface.co.

        This list is public so token is optional. If `user` is not passed, it defaults to
        the logged in user.

        See also [`like`] and [`unlike`].

        Args:
            user (`str`, *optional*):
                Name of the user for which you want to fetch the likes.
            token (`str`, *optional*):
                A valid authentication token (see https://huggingface.co/settings/token).
                Used only if `user` is not passed to implicitly determine the current
                user name.

        Returns:
            [`UserLikes`]: object containing the user name and 3 lists of repo ids (1 for
            models, 1 for datasets and 1 for Spaces).

        Raises:
            [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
                If `user` is not passed and no token found (either from argument or from machine).

        Example:
        ```python
        >>> from huggingface_hub import list_liked_repos

        >>> likes = list_liked_repos("julien-c")

        >>> likes.user
        "julien-c"

        >>> likes.models
        ["osanseviero/streamlit_1.15", "Xhaheen/ChatGPT_HF", ...]
        ```
        """
        # User is either provided explicitly or retrieved from current token.
        if user is None:
            me = self.whoami(token=token)
            if me["type"] == "user":
                user = me["name"]
            else:
                raise ValueError(
                    "Cannot list liked repos. You must provide a 'user' as input or be logged in as a user."
                )

        path = f"{self.endpoint}/api/users/{user}/likes"
        headers = self._build_hf_headers(token=token)

        likes = list(paginate(path, params={}, headers=headers))
        # Looping over a list of items similar to:
        #   {
        #       'createdAt': '2021-09-09T21:53:27.000Z',
        #       'repo': {
        #           'name': 'PaddlePaddle/PaddleOCR',
        #           'type': 'space'
        #        }
        #   }
        # Let's loop 3 times over the received list. Less efficient but more straightforward to read.
        return UserLikes(
            user=user,
            total=len(likes),
            models=[like["repo"]["name"] for like in likes if like["repo"]["type"] == "model"],
            datasets=[like["repo"]["name"] for like in likes if like["repo"]["type"] == "dataset"],
            spaces=[like["repo"]["name"] for like in likes if like["repo"]["type"] == "space"],
        )

    @validate_hf_hub_args
    def model_info(
        self,
        repo_id: str,
        *,
        revision: Optional[str] = None,
        timeout: Optional[float] = None,
        securityStatus: Optional[bool] = None,
        files_metadata: bool = False,
        token: Optional[Union[bool, str]] = None,
    ) -> ModelInfo:
        """
        Get info on one specific model on huggingface.co

        Model can be private if you pass an acceptable token or are logged in.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            revision (`str`, *optional*):
                The revision of the model repository from which to get the
                information.
            timeout (`float`, *optional*):
                Whether to set a timeout for the request to the Hub.
            securityStatus (`bool`, *optional*):
                Whether to retrieve the security status from the model
                repository as well.
            files_metadata (`bool`, *optional*):
                Whether or not to retrieve metadata for files in the repository
                (size, LFS metadata, etc). Defaults to `False`.
            token (`bool` or `str`, *optional*):
                A valid authentication token (see https://huggingface.co/settings/token).
                If `None` or `True` and machine is logged in (through `huggingface-cli login`
                or [`~huggingface_hub.login`]), token will be retrieved from the cache.
                If `False`, token is not sent in the request header.

        Returns:
            [`huggingface_hub.hf_api.ModelInfo`]: The model repository information.

        <Tip>

        Raises the following errors:

            - [`~utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.
            - [`~utils.RevisionNotFoundError`]
              If the revision to download from cannot be found.

        </Tip>
        """
        headers = self._build_hf_headers(token=token)
        path = (
            f"{self.endpoint}/api/models/{repo_id}"
            if revision is None
            else (f"{self.endpoint}/api/models/{repo_id}/revision/{quote(revision, safe='')}")
        )
        params = {}
        if securityStatus:
            params["securityStatus"] = True
        if files_metadata:
            params["blobs"] = True
        r = get_session().get(path, headers=headers, timeout=timeout, params=params)
        hf_raise_for_status(r)
        d = r.json()
        return ModelInfo(**d)

    @validate_hf_hub_args
    def dataset_info(
        self,
        repo_id: str,
        *,
        revision: Optional[str] = None,
        timeout: Optional[float] = None,
        files_metadata: bool = False,
        token: Optional[Union[bool, str]] = None,
    ) -> DatasetInfo:
        """
        Get info on one specific dataset on huggingface.co.

        Dataset can be private if you pass an acceptable token.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            revision (`str`, *optional*):
                The revision of the dataset repository from which to get the
                information.
            timeout (`float`, *optional*):
                Whether to set a timeout for the request to the Hub.
            files_metadata (`bool`, *optional*):
                Whether or not to retrieve metadata for files in the repository
                (size, LFS metadata, etc). Defaults to `False`.
            token (`bool` or `str`, *optional*):
                A valid authentication token (see https://huggingface.co/settings/token).
                If `None` or `True` and machine is logged in (through `huggingface-cli login`
                or [`~huggingface_hub.login`]), token will be retrieved from the cache.
                If `False`, token is not sent in the request header.

        Returns:
            [`hf_api.DatasetInfo`]: The dataset repository information.

        <Tip>

        Raises the following errors:

            - [`~utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.
            - [`~utils.RevisionNotFoundError`]
              If the revision to download from cannot be found.

        </Tip>
        """
        headers = self._build_hf_headers(token=token)
        path = (
            f"{self.endpoint}/api/datasets/{repo_id}"
            if revision is None
            else (f"{self.endpoint}/api/datasets/{repo_id}/revision/{quote(revision, safe='')}")
        )
        params = {}
        if files_metadata:
            params["blobs"] = True

        r = get_session().get(path, headers=headers, timeout=timeout, params=params)
        hf_raise_for_status(r)
        d = r.json()
        return DatasetInfo(**d)

    @validate_hf_hub_args
    def space_info(
        self,
        repo_id: str,
        *,
        revision: Optional[str] = None,
        timeout: Optional[float] = None,
        files_metadata: bool = False,
        token: Optional[Union[bool, str]] = None,
    ) -> SpaceInfo:
        """
        Get info on one specific Space on huggingface.co.

        Space can be private if you pass an acceptable token.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            revision (`str`, *optional*):
                The revision of the space repository from which to get the
                information.
            timeout (`float`, *optional*):
                Whether to set a timeout for the request to the Hub.
            files_metadata (`bool`, *optional*):
                Whether or not to retrieve metadata for files in the repository
                (size, LFS metadata, etc). Defaults to `False`.
            token (`bool` or `str`, *optional*):
                A valid authentication token (see https://huggingface.co/settings/token).
                If `None` or `True` and machine is logged in (through `huggingface-cli login`
                or [`~huggingface_hub.login`]), token will be retrieved from the cache.
                If `False`, token is not sent in the request header.

        Returns:
            [`~hf_api.SpaceInfo`]: The space repository information.

        <Tip>

        Raises the following errors:

            - [`~utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.
            - [`~utils.RevisionNotFoundError`]
              If the revision to download from cannot be found.

        </Tip>
        """
        headers = self._build_hf_headers(token=token)
        path = (
            f"{self.endpoint}/api/spaces/{repo_id}"
            if revision is None
            else (f"{self.endpoint}/api/spaces/{repo_id}/revision/{quote(revision, safe='')}")
        )
        params = {}
        if files_metadata:
            params["blobs"] = True

        r = get_session().get(path, headers=headers, timeout=timeout, params=params)
        hf_raise_for_status(r)
        d = r.json()
        return SpaceInfo(**d)

    @validate_hf_hub_args
    def repo_info(
        self,
        repo_id: str,
        *,
        revision: Optional[str] = None,
        repo_type: Optional[str] = None,
        timeout: Optional[float] = None,
        files_metadata: bool = False,
        token: Optional[Union[bool, str]] = None,
    ) -> Union[ModelInfo, DatasetInfo, SpaceInfo]:
        """
        Get the info object for a given repo of a given type.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            revision (`str`, *optional*):
                The revision of the repository from which to get the
                information.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if getting repository info from a dataset or a space,
                `None` or `"model"` if getting repository info from a model. Default is `None`.
            timeout (`float`, *optional*):
                Whether to set a timeout for the request to the Hub.
            files_metadata (`bool`, *optional*):
                Whether or not to retrieve metadata for files in the repository
                (size, LFS metadata, etc). Defaults to `False`.
            token (`bool` or `str`, *optional*):
                A valid authentication token (see https://huggingface.co/settings/token).
                If `None` or `True` and machine is logged in (through `huggingface-cli login`
                or [`~huggingface_hub.login`]), token will be retrieved from the cache.
                If `False`, token is not sent in the request header.

        Returns:
            `Union[SpaceInfo, DatasetInfo, ModelInfo]`: The repository information, as a
            [`huggingface_hub.hf_api.DatasetInfo`], [`huggingface_hub.hf_api.ModelInfo`]
            or [`huggingface_hub.hf_api.SpaceInfo`] object.

        <Tip>

        Raises the following errors:

            - [`~utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.
            - [`~utils.RevisionNotFoundError`]
              If the revision to download from cannot be found.

        </Tip>
        """
        if repo_type is None or repo_type == "model":
            method = self.model_info
        elif repo_type == "dataset":
            method = self.dataset_info  # type: ignore
        elif repo_type == "space":
            method = self.space_info  # type: ignore
        else:
            raise ValueError("Unsupported repo type.")
        return method(
            repo_id,
            revision=revision,
            token=token,
            timeout=timeout,
            files_metadata=files_metadata,
        )

    @validate_hf_hub_args
    def list_files_info(
        self,
        repo_id: str,
        paths: Union[List[str], str, None] = None,
        *,
        expand: bool = False,
        revision: Optional[str] = None,
        repo_type: Optional[str] = None,
        token: Optional[Union[bool, str]] = None,
    ) -> Iterable[RepoFile]:
        """
        List files on a repo and get information about them.

        Takes as input a list of paths. Those paths can be either files or folders. Two server endpoints are called:
        1. POST "/paths-info" to get information about the provided paths. Called once.
        2. GET  "/tree?recursive=True" to paginate over the input folders. Called only if a folder path is provided as
           input. Will be called multiple times to follow pagination.
        If no path is provided as input, step 1. is ignored and all files from the repo are listed.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated by a `/`.
            paths (`Union[List[str], str, None]`, *optional*):
                The paths to get information about. Paths to files are directly resolved. Paths to folders are resolved
                recursively which means that information is returned about all files in the folder and its subfolders.
                If `None`, all files are returned (the default). If a path do not exist, it is ignored without raising
                an exception.
            expand (`bool`, *optional*, defaults to `False`):
                Whether to fetch more information about the files (e.g. last commit and security scan results). This
                operation is more expensive for the server so only 50 results are returned per page (instead of 1000).
                As pagination is implemented in `huggingface_hub`, this is transparent for you except for the time it
                takes to get the results.
            revision (`str`, *optional*):
                The revision of the repository from which to get the information. Defaults to `"main"` branch.
            repo_type (`str`, *optional*):
                The type of the repository from which to get the information (`"model"`, `"dataset"` or `"space"`.
                Defaults to `"model"`.
            token (`bool` or `str`, *optional*):
                A valid authentication token (see https://huggingface.co/settings/token). If `None` or `True` and
                machine is logged in (through `huggingface-cli login` or [`~huggingface_hub.login`]), token will be
                retrieved from the cache. If `False`, token is not sent in the request header.

        Returns:
            `Iterable[RepoFile]`:
                The information about the files, as an iterable of [`RepoFile`] objects. The order of the files is
                not guaranteed.

        Raises:
            [`~utils.RepositoryNotFoundError`]:
                If repository is not found (error 404): wrong repo_id/repo_type, private but not authenticated or repo
                does not exist.
            [`~utils.RevisionNotFoundError`]:
                If revision is not found (error 404) on the repo.

        Examples:

            Get information about files on a repo.
            ```py
            >>> from huggingface_hub import list_files_info
            >>> files_info = list_files_info("lysandre/arxiv-nlp", ["README.md", "config.json"])
            >>> files_info
            <generator object HfApi.list_files_info at 0x7f93b848e730>
            >>> list(files_info)
            [
                RepoFile: {"blob_id": "43bd404b159de6fba7c2f4d3264347668d43af25", "lfs": None, "rfilename": "README.md", "size": 391},
                RepoFile: {"blob_id": "2f9618c3a19b9a61add74f70bfb121335aeef666", "lfs": None, "rfilename": "config.json", "size": 554},
            ]
            ```

            Get even more information about files on a repo (last commit and security scan results)
            ```py
            >>> from huggingface_hub import list_files_info
            >>> files_info = list_files_info("prompthero/openjourney-v4", expand=True)
            >>> list(files_info)
            [
                RepoFile: {
                {'blob_id': '815004af1a321eaed1d93f850b2e94b0c0678e42',
                'lastCommit': {'date': '2023-03-21T09:05:27.000Z',
                                'id': '47b62b20b20e06b9de610e840282b7e6c3d51190',
                                'title': 'Upload diffusers weights (#48)'},
                'lfs': None,
                'rfilename': 'model_index.json',
                'security': {'avScan': {'virusFound': False, 'virusNames': None},
                                'blobId': '815004af1a321eaed1d93f850b2e94b0c0678e42',
                                'name': 'model_index.json',
                                'pickleImportScan': None,
                                'repositoryId': 'models/prompthero/openjourney-v4',
                                'safe': True},
                'size': 584}
                },
                RepoFile: {
                {'blob_id': 'd2343d78b33ac03dade1d525538b02b130d0a3a0',
                'lastCommit': {'date': '2023-03-21T09:05:27.000Z',
                                'id': '47b62b20b20e06b9de610e840282b7e6c3d51190',
                                'title': 'Upload diffusers weights (#48)'},
                'lfs': {'pointer_size': 134,
                        'sha256': 'dcf4507d99b88db73f3916e2a20169fe74ada6b5582e9af56cfa80f5f3141765',
                        'size': 334711857},
                'rfilename': 'vae/diffusion_pytorch_model.bin',
                'security': {'avScan': {'virusFound': False, 'virusNames': None},
                                'blobId': 'd2343d78b33ac03dade1d525538b02b130d0a3a0',
                                'name': 'vae/diffusion_pytorch_model.bin',
                                'pickleImportScan': {'highestSafetyLevel': 'innocuous',
                                                    'imports': [{'module': 'torch._utils',
                                                                'name': '_rebuild_tensor_v2',
                                                                'safety': 'innocuous'},
                                                                {'module': 'collections', 'name': 'OrderedDict', 'safety': 'innocuous'},
                                                                {'module': 'torch', 'name': 'FloatStorage', 'safety': 'innocuous'}]},
                                'repositoryId': 'models/prompthero/openjourney-v4',
                                'safe': True},
                'size': 334711857}
                },
                (...)
            ]
            ```

            List LFS files from the "vae/" folder in "stabilityai/stable-diffusion-2" repository.

            ```py
            >>> from huggingface_hub import list_files_info
            >>> [info.rfilename for info in list_files_info("stabilityai/stable-diffusion-2", "vae") if info.lfs is not None]
            ['vae/diffusion_pytorch_model.bin', 'vae/diffusion_pytorch_model.safetensors']
            ```

            List all files on a repo.
            ```py
            >>> from huggingface_hub import list_files_info
            >>> [info.rfilename for info in list_files_info("glue", repo_type="dataset")]
            ['.gitattributes', 'README.md', 'dataset_infos.json', 'glue.py']
            ```
        """
        repo_type = repo_type or REPO_TYPE_MODEL
        revision = quote(revision, safe="") if revision is not None else DEFAULT_REVISION
        headers = self._build_hf_headers(token=token)

        def _format_as_repo_file(info: Dict) -> RepoFile:
            # Quick alias very specific to the server return type of /paths-info and /tree endpoints. Let's keep this
            # logic here.
            rfilename = info.pop("path")
            size = info.pop("size")
            blobId = info.pop("oid")
            lfs = info.pop("lfs", None)
            info.pop("type", None)  # "file" or "folder" -> not needed in practice since we know it's a file
            if lfs is not None:
                lfs = BlobLfsInfo(size=lfs["size"], sha256=lfs["oid"], pointer_size=lfs["pointerSize"])
            return RepoFile(rfilename=rfilename, size=size, blobId=blobId, lfs=lfs, **info)

        folder_paths = []
        if paths is None:
            # `paths` is not provided => list all files from the repo
            folder_paths.append("")
        elif paths == []:
            # corner case: server would return a 400 error if `paths` is an empty list. Let's return early.
            return
        else:
            # `paths` is provided => get info about those
            response = get_session().post(
                f"{self.endpoint}/api/{repo_type}s/{repo_id}/paths-info/{revision}",
                data={
                    "paths": paths if isinstance(paths, list) else [paths],
                    "expand": True,
                },
                headers=headers,
            )
            hf_raise_for_status(response)
            paths_info = response.json()

            # List top-level files first
            for path_info in paths_info:
                if path_info["type"] == "file":
                    yield _format_as_repo_file(path_info)
                else:
                    folder_paths.append(path_info["path"])

        # List files in subdirectories
        for path in folder_paths:
            encoded_path = "/" + quote(path, safe="") if path else ""
            tree_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/tree/{revision}{encoded_path}"
            for subpath_info in paginate(path=tree_url, headers=headers, params={"recursive": True, "expand": expand}):
                if subpath_info["type"] == "file":
                    yield _format_as_repo_file(subpath_info)

    @_deprecate_arguments(version="0.17", deprecated_args=["timeout"], custom_message="timeout is not used anymore.")
    @validate_hf_hub_args
    def list_repo_files(
        self,
        repo_id: str,
        *,
        revision: Optional[str] = None,
        repo_type: Optional[str] = None,
        timeout: Optional[float] = None,
        token: Optional[Union[bool, str]] = None,
    ) -> List[str]:
        """
        Get the list of files in a given repo.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated by a `/`.
            revision (`str`, *optional*):
                The revision of the model repository from which to get the information.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or space, `None` or `"model"` if uploading to
                a model. Default is `None`.
            token (`bool` or `str`, *optional*):
                A valid authentication token (see https://huggingface.co/settings/token). If `None` or `True` and
                machine is logged in (through `huggingface-cli login` or [`~huggingface_hub.login`]), token will be
                retrieved from the cache. If `False`, token is not sent in the request header.

        Returns:
            `List[str]`: the list of files in a given repository.
        """
        return [
            f.rfilename
            for f in self.list_files_info(
                repo_id=repo_id, paths=None, revision=revision, repo_type=repo_type, token=token
            )
        ]

    @validate_hf_hub_args
    def list_repo_refs(
        self,
        repo_id: str,
        *,
        repo_type: Optional[str] = None,
        token: Optional[Union[bool, str]] = None,
    ) -> GitRefs:
        """
        Get the list of refs of a given repo (both tags and branches).

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if listing refs from a dataset or a Space,
                `None` or `"model"` if listing from a model. Default is `None`.
            token (`bool` or `str`, *optional*):
                A valid authentication token (see https://huggingface.co/settings/token).
                If `None` or `True` and machine is logged in (through `huggingface-cli login`
                or [`~huggingface_hub.login`]), token will be retrieved from the cache.
                If `False`, token is not sent in the request header.

        Example:
        ```py
        >>> from huggingface_hub import HfApi
        >>> api = HfApi()
        >>> api.list_repo_refs("gpt2")
        GitRefs(branches=[GitRefInfo(name='main', ref='refs/heads/main', target_commit='e7da7f221d5bf496a48136c0cd264e630fe9fcc8')], converts=[], tags=[])

        >>> api.list_repo_refs("bigcode/the-stack", repo_type='dataset')
        GitRefs(
            branches=[
                GitRefInfo(name='main', ref='refs/heads/main', target_commit='18edc1591d9ce72aa82f56c4431b3c969b210ae3'),
                GitRefInfo(name='v1.1.a1', ref='refs/heads/v1.1.a1', target_commit='f9826b862d1567f3822d3d25649b0d6d22ace714')
            ],
            converts=[],
            tags=[
                GitRefInfo(name='v1.0', ref='refs/tags/v1.0', target_commit='c37a8cd1e382064d8aced5e05543c5f7753834da')
            ]
        )
        ```

        Returns:
            [`GitRefs`]: object containing all information about branches and tags for a
            repo on the Hub.
        """
        repo_type = repo_type or REPO_TYPE_MODEL
        response = get_session().get(
            f"{self.endpoint}/api/{repo_type}s/{repo_id}/refs", headers=self._build_hf_headers(token=token)
        )
        hf_raise_for_status(response)
        data = response.json()
        return GitRefs(
            branches=[GitRefInfo(item) for item in data["branches"]],
            converts=[GitRefInfo(item) for item in data["converts"]],
            tags=[GitRefInfo(item) for item in data["tags"]],
        )

    @validate_hf_hub_args
    def list_repo_commits(
        self,
        repo_id: str,
        *,
        repo_type: Optional[str] = None,
        token: Optional[Union[bool, str]] = None,
        revision: Optional[str] = None,
        formatted: bool = False,
    ) -> List[GitCommitInfo]:
        """
        Get the list of commits of a given revision for a repo on the Hub.

        Commits are sorted by date (last commit first).

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated by a `/`.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if listing commits from a dataset or a Space, `None` or `"model"` if
                listing from a model. Default is `None`.
            token (`bool` or `str`, *optional*):
                A valid authentication token (see https://huggingface.co/settings/token).
                If `None` or `True` and machine is logged in (through `huggingface-cli login`
                or [`~huggingface_hub.login`]), token will be retrieved from the cache.
                If `False`, token is not sent in the request header.
            revision (`str`, *optional*):
                The git revision to commit from. Defaults to the head of the `"main"` branch.
            formatted (`bool`):
                Whether to return the HTML-formatted title and description of the commits. Defaults to False.

        Example:
        ```py
        >>> from huggingface_hub import HfApi
        >>> api = HfApi()

        # Commits are sorted by date (last commit first)
        >>> initial_commit = api.list_repo_commits("gpt2")[-1]

        # Initial commit is always a system commit containing the `.gitattributes` file.
        >>> initial_commit
        GitCommitInfo(
            commit_id='9b865efde13a30c13e0a33e536cf3e4a5a9d71d8',
            authors=['system'],
            created_at=datetime.datetime(2019, 2, 18, 10, 36, 15, tzinfo=datetime.timezone.utc),
            title='initial commit',
            message='',
            formatted_title=None,
            formatted_message=None
        )

        # Create an empty branch by deriving from initial commit
        >>> api.create_branch("gpt2", "new_empty_branch", revision=initial_commit.commit_id)
        ```

        Returns:
            List[[`GitCommitInfo`]]: list of objects containing information about the commits for a repo on the Hub.

        Raises:
            [`~utils.RepositoryNotFoundError`]:
                If repository is not found (error 404): wrong repo_id/repo_type, private but not authenticated or repo
                does not exist.
            [`~utils.RevisionNotFoundError`]:
                If revision is not found (error 404) on the repo.
        """
        repo_type = repo_type or REPO_TYPE_MODEL
        revision = quote(revision, safe="") if revision is not None else DEFAULT_REVISION

        # Paginate over results and return the list of commits.
        return [
            GitCommitInfo(item)
            for item in paginate(
                f"{self.endpoint}/api/{repo_type}s/{repo_id}/commits/{revision}",
                headers=self._build_hf_headers(token=token),
                params={"expand[]": "formatted"} if formatted else {},
            )
        ]

    @validate_hf_hub_args
    def create_repo(
        self,
        repo_id: str,
        *,
        token: Optional[str] = None,
        private: bool = False,
        repo_type: Optional[str] = None,
        exist_ok: bool = False,
        space_sdk: Optional[str] = None,
        space_hardware: Optional[str] = None,
    ) -> RepoUrl:
        """Create an empty repo on the HuggingFace Hub.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)
            private (`bool`, *optional*, defaults to `False`):
                Whether the model repo should be private.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model. Default is
                `None`.
            exist_ok (`bool`, *optional*, defaults to `False`):
                If `True`, do not raise an error if repo already exists.
            space_sdk (`str`, *optional*):
                Choice of SDK to use if repo_type is "space". Can be "streamlit", "gradio", "docker", or "static".
            space_hardware (`SpaceHardware` or `str`, *optional*):
                Choice of Hardware if repo_type is "space". See [`SpaceHardware`] for a complete list.

        Returns:
            [`RepoUrl`]: URL to the newly created repo. Value is a subclass of `str` containing
            attributes like `endpoint`, `repo_type` and `repo_id`.
        """
        organization, name = repo_id.split("/") if "/" in repo_id else (None, repo_id)

        path = f"{self.endpoint}/api/repos/create"

        if repo_type not in REPO_TYPES:
            raise ValueError("Invalid repo type")

        json = {"name": name, "organization": organization, "private": private}
        if repo_type is not None:
            json["type"] = repo_type
        if repo_type == "space":
            if space_sdk is None:
                raise ValueError(
                    "No space_sdk provided. `create_repo` expects space_sdk to be one"
                    f" of {SPACES_SDK_TYPES} when repo_type is 'space'`"
                )
            if space_sdk not in SPACES_SDK_TYPES:
                raise ValueError(f"Invalid space_sdk. Please choose one of {SPACES_SDK_TYPES}.")
            json["sdk"] = space_sdk

        if space_sdk is not None and repo_type != "space":
            warnings.warn("Ignoring provided space_sdk because repo_type is not 'space'.")

        if space_hardware is not None:
            if repo_type == "space":
                json["hardware"] = space_hardware
            else:
                warnings.warn("Ignoring provided space_hardware because repo_type is not 'space'.")

        if getattr(self, "_lfsmultipartthresh", None):
            # Testing purposes only.
            # See https://github.com/huggingface/huggingface_hub/pull/733/files#r820604472
            json["lfsmultipartthresh"] = self._lfsmultipartthresh  # type: ignore
        headers = self._build_hf_headers(token=token, is_write_action=True)
        r = get_session().post(path, headers=headers, json=json)

        try:
            hf_raise_for_status(r)
        except HTTPError as err:
            if exist_ok and err.response.status_code == 409:
                # Repo already exists and `exist_ok=True`
                pass
            elif exist_ok and err.response.status_code == 403:
                # No write permission on the namespace but repo might already exist
                try:
                    self.repo_info(repo_id=repo_id, repo_type=repo_type, token=token)
                    if repo_type is None or repo_type == REPO_TYPE_MODEL:
                        return RepoUrl(f"{self.endpoint}/{repo_id}")
                    return RepoUrl(f"{self.endpoint}/{repo_type}/{repo_id}")
                except HfHubHTTPError:
                    raise
            else:
                raise

        d = r.json()
        return RepoUrl(d["url"], endpoint=self.endpoint)

    @validate_hf_hub_args
    def delete_repo(
        self,
        repo_id: str,
        *,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
    ):
        """
        Delete a repo from the HuggingFace Hub. CAUTION: this is irreversible.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model.

        <Tip>

        Raises the following errors:

            - [`~utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.

        </Tip>
        """
        organization, name = repo_id.split("/") if "/" in repo_id else (None, repo_id)

        path = f"{self.endpoint}/api/repos/delete"

        if repo_type not in REPO_TYPES:
            raise ValueError("Invalid repo type")

        json = {"name": name, "organization": organization}
        if repo_type is not None:
            json["type"] = repo_type

        headers = self._build_hf_headers(token=token, is_write_action=True)
        r = get_session().delete(path, headers=headers, json=json)
        hf_raise_for_status(r)

    @validate_hf_hub_args
    def update_repo_visibility(
        self,
        repo_id: str,
        private: bool = False,
        *,
        token: Optional[str] = None,
        organization: Optional[str] = None,
        repo_type: Optional[str] = None,
        name: Optional[str] = None,
    ) -> Dict[str, bool]:
        """Update the visibility setting of a repository.

        Args:
            repo_id (`str`, *optional*):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            private (`bool`, *optional*, defaults to `False`):
                Whether the model repo should be private.
            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model. Default is
                `None`.

        Returns:
            The HTTP response in json.

        <Tip>

        Raises the following errors:

            - [`~utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.

        </Tip>
        """
        if repo_type not in REPO_TYPES:
            raise ValueError("Invalid repo type")

        organization, name = repo_id.split("/") if "/" in repo_id else (None, repo_id)

        if organization is None:
            namespace = self.whoami(token)["name"]
        else:
            namespace = organization

        if repo_type is None:
            repo_type = REPO_TYPE_MODEL  # default repo type

        r = get_session().put(
            url=f"{self.endpoint}/api/{repo_type}s/{namespace}/{name}/settings",
            headers=self._build_hf_headers(token=token, is_write_action=True),
            json={"private": private},
        )
        hf_raise_for_status(r)
        return r.json()

    def move_repo(
        self,
        from_id: str,
        to_id: str,
        *,
        repo_type: Optional[str] = None,
        token: Optional[str] = None,
    ):
        """
        Moving a repository from namespace1/repo_name1 to namespace2/repo_name2

        Note there are certain limitations. For more information about moving
        repositories, please see
        https://hf.co/docs/hub/repositories-settings#renaming-or-transferring-a-repo.

        Args:
            from_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`. Original repository identifier.
            to_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`. Final repository identifier.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model. Default is
                `None`.
            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)

        <Tip>

        Raises the following errors:

            - [`~utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.

        </Tip>
        """
        if len(from_id.split("/")) != 2:
            raise ValueError(f"Invalid repo_id: {from_id}. It should have a namespace (:namespace:/:repo_name:)")

        if len(to_id.split("/")) != 2:
            raise ValueError(f"Invalid repo_id: {to_id}. It should have a namespace (:namespace:/:repo_name:)")

        if repo_type is None:
            repo_type = REPO_TYPE_MODEL  # Hub won't accept `None`.

        json = {"fromRepo": from_id, "toRepo": to_id, "type": repo_type}

        path = f"{self.endpoint}/api/repos/move"
        headers = self._build_hf_headers(token=token, is_write_action=True)
        r = get_session().post(path, headers=headers, json=json)
        try:
            hf_raise_for_status(r)
        except HfHubHTTPError as e:
            e.append_to_message(
                "\nFor additional documentation please see"
                " https://hf.co/docs/hub/repositories-settings#renaming-or-transferring-a-repo."
            )
            raise

    @overload
    def create_commit(  # type: ignore
        self,
        repo_id: str,
        operations: Iterable[CommitOperation],
        *,
        commit_message: str,
        commit_description: Optional[str] = None,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
        revision: Optional[str] = None,
        create_pr: Optional[bool] = None,
        num_threads: int = 5,
        parent_commit: Optional[str] = None,
        run_as_future: Literal[False] = ...,
    ) -> CommitInfo:
        ...

    @overload
    def create_commit(
        self,
        repo_id: str,
        operations: Iterable[CommitOperation],
        *,
        commit_message: str,
        commit_description: Optional[str] = None,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
        revision: Optional[str] = None,
        create_pr: Optional[bool] = None,
        num_threads: int = 5,
        parent_commit: Optional[str] = None,
        run_as_future: Literal[True] = ...,
    ) -> Future[CommitInfo]:
        ...

    @validate_hf_hub_args
    @future_compatible
    def create_commit(
        self,
        repo_id: str,
        operations: Iterable[CommitOperation],
        *,
        commit_message: str,
        commit_description: Optional[str] = None,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
        revision: Optional[str] = None,
        create_pr: Optional[bool] = None,
        num_threads: int = 5,
        parent_commit: Optional[str] = None,
        run_as_future: bool = False,
    ) -> Union[CommitInfo, Future[CommitInfo]]:
        """
        Creates a commit in the given repo, deleting & uploading files as needed.

        Args:
            repo_id (`str`):
                The repository in which the commit will be created, for example:
                `"username/custom_transformers"`

            operations (`Iterable` of [`~hf_api.CommitOperation`]):
                An iterable of operations to include in the commit, either:

                    - [`~hf_api.CommitOperationAdd`] to upload a file
                    - [`~hf_api.CommitOperationDelete`] to delete a file
                    - [`~hf_api.CommitOperationCopy`] to copy a file

            commit_message (`str`):
                The summary (first line) of the commit that will be created.

            commit_description (`str`, *optional*):
                The description of the commit that will be created

            token (`str`, *optional*):
                Authentication token, obtained with `HfApi.login` method. Will
                default to the stored token.

            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model. Default is
                `None`.

            revision (`str`, *optional*):
                The git revision to commit from. Defaults to the head of the `"main"` branch.

            create_pr (`boolean`, *optional*):
                Whether or not to create a Pull Request with that commit. Defaults to `False`.
                If `revision` is not set, PR is opened against the `"main"` branch. If
                `revision` is set and is a branch, PR is opened against this branch. If
                `revision` is set and is not a branch name (example: a commit oid), an
                `RevisionNotFoundError` is returned by the server.

            num_threads (`int`, *optional*):
                Number of concurrent threads for uploading files. Defaults to 5.
                Setting it to 2 means at most 2 files will be uploaded concurrently.

            parent_commit (`str`, *optional*):
                The OID / SHA of the parent commit, as a hexadecimal string.
                Shorthands (7 first characters) are also supported. If specified and `create_pr` is `False`,
                the commit will fail if `revision` does not point to `parent_commit`. If specified and `create_pr`
                is `True`, the pull request will be created from `parent_commit`. Specifying `parent_commit`
                ensures the repo has not changed before committing the changes, and can be especially useful
                if the repo is updated / committed to concurrently.
            run_as_future (`bool`, *optional*):
                Whether or not to run this method in the background. Background jobs are run sequentially without
                blocking the main thread. Passing `run_as_future=True` will return a [Future](https://docs.python.org/3/library/concurrent.futures.html#future-objects)
                object. Defaults to `False`.

        Returns:
            [`CommitInfo`] or `Future`:
                Instance of [`CommitInfo`] containing information about the newly created commit (commit hash, commit
                url, pr url, commit message,...). If `run_as_future=True` is passed, returns a Future object which will
                contain the result when executed.

        Raises:
            [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
                If commit message is empty.
            [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
                If parent commit is not a valid commit OID.
            [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
                If the Hub API returns an HTTP 400 error (bad request)
            [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
                If `create_pr` is `True` and revision is neither `None` nor `"main"`.
            [`~utils.RepositoryNotFoundError`]:
                If repository is not found (error 404): wrong repo_id/repo_type, private
                but not authenticated or repo does not exist.

        <Tip warning={true}>

        `create_commit` assumes that the repo already exists on the Hub. If you get a
        Client error 404, please make sure you are authenticated and that `repo_id` and
        `repo_type` are set correctly. If repo does not exist, create it first using
        [`~hf_api.create_repo`].

        </Tip>

        <Tip warning={true}>

        `create_commit` is limited to 25k LFS files and a 1GB payload for regular files.

        </Tip>
        """
        _CREATE_COMMIT_NO_REPO_ERROR_MESSAGE = (
            "\nNote: Creating a commit assumes that the repo already exists on the"
            " Huggingface Hub. Please use `create_repo` if it's not the case."
        )

        if parent_commit is not None and not REGEX_COMMIT_OID.fullmatch(parent_commit):
            raise ValueError(
                f"`parent_commit` is not a valid commit OID. It must match the following regex: {REGEX_COMMIT_OID}"
            )

        if commit_message is None or len(commit_message) == 0:
            raise ValueError("`commit_message` can't be empty, please pass a value.")

        commit_description = commit_description if commit_description is not None else ""
        repo_type = repo_type if repo_type is not None else REPO_TYPE_MODEL
        if repo_type not in REPO_TYPES:
            raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")
        revision = quote(revision, safe="") if revision is not None else DEFAULT_REVISION
        create_pr = create_pr if create_pr is not None else False

        operations = list(operations)
        additions = [op for op in operations if isinstance(op, CommitOperationAdd)]
        copies = [op for op in operations if isinstance(op, CommitOperationCopy)]
        nb_additions = len(additions)
        nb_copies = len(copies)
        nb_deletions = len(operations) - nb_additions - nb_copies

        logger.debug(
            f"About to commit to the hub: {len(additions)} addition(s), {len(copies)} copie(s) and"
            f" {nb_deletions} deletion(s)."
        )

        # If updating twice the same file or update then delete a file in a single commit
        warn_on_overwriting_operations(operations)

        try:
            upload_modes = fetch_upload_modes(
                additions=additions,
                repo_type=repo_type,
                repo_id=repo_id,
                token=token or self.token,
                revision=revision,
                endpoint=self.endpoint,
                create_pr=create_pr,
            )
        except RepositoryNotFoundError as e:
            e.append_to_message(_CREATE_COMMIT_NO_REPO_ERROR_MESSAGE)
            raise
        files_to_copy = fetch_lfs_files_to_copy(
            copies=copies,
            repo_type=repo_type,
            repo_id=repo_id,
            token=token or self.token,
            revision=revision,
            endpoint=self.endpoint,
        )
        upload_lfs_files(
            additions=[addition for addition in additions if upload_modes[addition.path_in_repo] == "lfs"],
            repo_type=repo_type,
            repo_id=repo_id,
            token=token or self.token,
            endpoint=self.endpoint,
            num_threads=num_threads,
        )
        commit_payload = prepare_commit_payload(
            operations=operations,
            upload_modes=upload_modes,
            files_to_copy=files_to_copy,
            commit_message=commit_message,
            commit_description=commit_description,
            parent_commit=parent_commit,
        )
        commit_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/commit/{revision}"

        def _payload_as_ndjson() -> Iterable[bytes]:
            for item in commit_payload:
                yield json.dumps(item).encode()
                yield b"\n"

        headers = {
            # See https://github.com/huggingface/huggingface_hub/issues/1085#issuecomment-1265208073
            "Content-Type": "application/x-ndjson",
            **self._build_hf_headers(token=token, is_write_action=True),
        }
        data = b"".join(_payload_as_ndjson())
        params = {"create_pr": "1"} if create_pr else None

        try:
            commit_resp = get_session().post(url=commit_url, headers=headers, data=data, params=params)
            hf_raise_for_status(commit_resp, endpoint_name="commit")
        except RepositoryNotFoundError as e:
            e.append_to_message(_CREATE_COMMIT_NO_REPO_ERROR_MESSAGE)
            raise
        except EntryNotFoundError as e:
            if nb_deletions > 0 and "A file with this name doesn't exist" in str(e):
                e.append_to_message(
                    "\nMake sure to differentiate file and folder paths in delete"
                    " operations with a trailing '/' or using `is_folder=True/False`."
                )
            raise

        commit_data = commit_resp.json()
        return CommitInfo(
            commit_url=commit_data["commitUrl"],
            commit_message=commit_message,
            commit_description=commit_description,
            oid=commit_data["commitOid"],
            pr_url=commit_data["pullRequestUrl"] if create_pr else None,
        )

    @experimental
    @validate_hf_hub_args
    def create_commits_on_pr(
        self,
        *,
        repo_id: str,
        addition_commits: List[List[CommitOperationAdd]],
        deletion_commits: List[List[CommitOperationDelete]],
        commit_message: str,
        commit_description: Optional[str] = None,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
        merge_pr: bool = True,
        num_threads: int = 5,  # TODO: use to multithread uploads
        verbose: bool = False,
    ) -> str:
        """Push changes to the Hub in multiple commits.

        Commits are pushed to a draft PR branch. If the upload fails or gets interrupted, it can be resumed. Progress
        is tracked in the PR description. At the end of the process, the PR is set as open and the title is updated to
        match the initial commit message. If `merge_pr=True` is passed, the PR is merged automatically.

        All deletion commits are pushed first, followed by the addition commits. The order of the commits is not
        guaranteed as we might implement parallel commits in the future. Be sure that your are not updating several
        times the same file.

        <Tip warning={true}>

        `create_commits_on_pr` is experimental.  Its API and behavior is subject to change in the future without prior notice.

        </Tip>

        Args:
            repo_id (`str`):
                The repository in which the commits will be pushed. Example: `"username/my-cool-model"`.

            addition_commits (`List` of `List` of [`~hf_api.CommitOperationAdd`]):
                A list containing lists of [`~hf_api.CommitOperationAdd`]. Each sublist will result in a commit on the
                PR.

            deletion_commits
                A list containing lists of [`~hf_api.CommitOperationDelete`]. Each sublist will result in a commit on
                the PR. Deletion commits are pushed before addition commits.

            commit_message (`str`):
                The summary (first line) of the commit that will be created. Will also be the title of the PR.

            commit_description (`str`, *optional*):
                The description of the commit that will be created. The description will be added to the PR.

            token (`str`, *optional*):
                Authentication token, obtained with `HfApi.login` method. Will default to the stored token.

            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or space, `None` or `"model"` if uploading to
                a model. Default is `None`.

            merge_pr (`bool`):
                If set to `True`, the Pull Request is merged at the end of the process. Defaults to `True`.

            num_threads (`int`, *optional*):
                Number of concurrent threads for uploading files. Defaults to 5.

            verbose (`bool`):
                If set to `True`, process will run on verbose mode i.e. print information about the ongoing tasks.
                Defaults to `False`.

        Returns:
            `str`: URL to the created PR.

        Example:
        ```python
        >>> from huggingface_hub import HfApi, plan_multi_commits
        >>> addition_commits, deletion_commits = plan_multi_commits(
        ...     operations=[
        ...          CommitOperationAdd(...),
        ...          CommitOperationAdd(...),
        ...          CommitOperationDelete(...),
        ...          CommitOperationDelete(...),
        ...          CommitOperationAdd(...),
        ...     ],
        ... )
        >>> HfApi().create_commits_on_pr(
        ...     repo_id="my-cool-model",
        ...     addition_commits=addition_commits,
        ...     deletion_commits=deletion_commits,
        ...     (...)
        ...     verbose=True,
        ... )
        ```

        Raises:
            [`MultiCommitException`]:
                If an unexpected issue occur in the process: empty commits, unexpected commits in a PR, unexpected PR
                description, etc.

        <Tip warning={true}>

        `create_commits_on_pr` assumes that the repo already exists on the Hub. If you get a Client error 404, please
        make sure you are authenticated and that `repo_id` and `repo_type` are set correctly. If repo does not exist,
        create it first using [`~hf_api.create_repo`].

        </Tip>
        """
        logger = logging.get_logger(__name__ + ".create_commits_on_pr")
        if verbose:
            logger.setLevel("INFO")

        # 1. Get strategy ID
        logger.info(
            f"Will create {len(deletion_commits)} deletion commit(s) and {len(addition_commits)} addition commit(s),"
            f" totalling {sum(len(ops) for ops in addition_commits+deletion_commits)} atomic operations."
        )
        strategy = MultiCommitStrategy(
            addition_commits=[MultiCommitStep(operations=operations) for operations in addition_commits],  # type: ignore
            deletion_commits=[MultiCommitStep(operations=operations) for operations in deletion_commits],  # type: ignore
        )
        logger.info(f"Multi-commits strategy with ID {strategy.id}.")

        # 2. Get or create a PR with this strategy ID
        for discussion in self.get_repo_discussions(repo_id=repo_id, repo_type=repo_type, token=token):
            # search for a draft PR with strategy ID
            if discussion.is_pull_request and discussion.status == "draft" and strategy.id in discussion.title:
                pr = self.get_discussion_details(
                    repo_id=repo_id, discussion_num=discussion.num, repo_type=repo_type, token=token
                )
                logger.info(f"PR already exists: {pr.url}. Will resume process where it stopped.")
                break
        else:
            # did not find a PR matching the strategy ID
            pr = multi_commit_create_pull_request(
                self,
                repo_id=repo_id,
                commit_message=commit_message,
                commit_description=commit_description,
                strategy=strategy,
                token=token,
                repo_type=repo_type,
            )
            logger.info(f"New PR created: {pr.url}")

        # 3. Parse PR description to check consistency with strategy (e.g. same commits are scheduled)
        for event in pr.events:
            if isinstance(event, DiscussionComment):
                pr_comment = event
                break
        else:
            raise MultiCommitException(f"PR #{pr.num} must have at least 1 comment")

        description_commits = multi_commit_parse_pr_description(pr_comment.content)
        if len(description_commits) != len(strategy.all_steps):
            raise MultiCommitException(
                f"Corrupted multi-commit PR #{pr.num}: got {len(description_commits)} steps in"
                f" description but {len(strategy.all_steps)} in strategy."
            )
        for step_id in strategy.all_steps:
            if step_id not in description_commits:
                raise MultiCommitException(
                    f"Corrupted multi-commit PR #{pr.num}: expected step {step_id} but didn't find"
                    f" it (have {', '.join(description_commits)})."
                )

        # 4. Retrieve commit history (and check consistency)
        commits_on_main_branch = {
            commit.commit_id
            for commit in self.list_repo_commits(
                repo_id=repo_id, repo_type=repo_type, token=token, revision=DEFAULT_REVISION
            )
        }
        pr_commits = [
            commit
            for commit in self.list_repo_commits(
                repo_id=repo_id, repo_type=repo_type, token=token, revision=pr.git_reference
            )
            if commit.commit_id not in commits_on_main_branch
        ]
        if len(pr_commits) > 0:
            logger.info(f"Found {len(pr_commits)} existing commits on the PR.")

        # At this point `pr_commits` is a list of commits pushed to the PR. We expect all of these commits (if any) to have
        # a step_id as title. We raise exception if an unexpected commit has been pushed.
        if len(pr_commits) > len(strategy.all_steps):
            raise MultiCommitException(
                f"Corrupted multi-commit PR #{pr.num}: scheduled {len(strategy.all_steps)} steps but"
                f" {len(pr_commits)} commits have already been pushed to the PR."
            )

        # Check which steps are already completed
        remaining_additions = {step.id: step for step in strategy.addition_commits}
        remaining_deletions = {step.id: step for step in strategy.deletion_commits}
        for commit in pr_commits:
            if commit.title in remaining_additions:
                step = remaining_additions.pop(commit.title)
                step.completed = True
            elif commit.title in remaining_deletions:
                step = remaining_deletions.pop(commit.title)
                step.completed = True

        if len(remaining_deletions) > 0 and len(remaining_additions) < len(strategy.addition_commits):
            raise MultiCommitException(
                f"Corrupted multi-commit PR #{pr.num}: some addition commits have already been pushed to the PR but"
                " deletion commits are not all completed yet."
            )
        nb_remaining = len(remaining_deletions) + len(remaining_additions)
        if len(pr_commits) > 0:
            logger.info(
                f"{nb_remaining} commits remaining ({len(remaining_deletions)} deletion commits and"
                f" {len(remaining_additions)} addition commits)"
            )

        # 5. Push remaining commits to the PR + update description
        # TODO: multi-thread this
        for step in list(remaining_deletions.values()) + list(remaining_additions.values()):
            # Push new commit
            self.create_commit(
                repo_id=repo_id,
                repo_type=repo_type,
                token=token,
                commit_message=step.id,
                revision=pr.git_reference,
                num_threads=num_threads,
                operations=step.operations,
                create_pr=False,
            )
            step.completed = True
            nb_remaining -= 1
            logger.info(f"  step {step.id} completed (still {nb_remaining} to go).")

            # Update PR description
            self.edit_discussion_comment(
                repo_id=repo_id,
                repo_type=repo_type,
                token=token,
                discussion_num=pr.num,
                comment_id=pr_comment.id,
                new_content=multi_commit_generate_comment(
                    commit_message=commit_message, commit_description=commit_description, strategy=strategy
                ),
            )
        logger.info("All commits have been pushed.")

        # 6. Update PR (and merge)
        self.rename_discussion(
            repo_id=repo_id,
            repo_type=repo_type,
            token=token,
            discussion_num=pr.num,
            new_title=commit_message,
        )
        self.change_discussion_status(
            repo_id=repo_id,
            repo_type=repo_type,
            token=token,
            discussion_num=pr.num,
            new_status="open",
            comment=MULTI_COMMIT_PR_COMPLETION_COMMENT_TEMPLATE,
        )
        logger.info("PR is now open for reviews.")

        if merge_pr:  # User don't want a PR => merge it
            try:
                self.merge_pull_request(
                    repo_id=repo_id,
                    repo_type=repo_type,
                    token=token,
                    discussion_num=pr.num,
                    comment=MULTI_COMMIT_PR_CLOSING_COMMENT_TEMPLATE,
                )
                logger.info("PR has been automatically merged (`merge_pr=True` was passed).")
            except BadRequestError as error:
                if error.server_message is not None and "no associated changes" in error.server_message:
                    # PR cannot be merged as no changes are associated. We close the PR without merging with a comment to
                    # explain.
                    self.change_discussion_status(
                        repo_id=repo_id,
                        repo_type=repo_type,
                        token=token,
                        discussion_num=pr.num,
                        comment=MULTI_COMMIT_PR_CLOSE_COMMENT_FAILURE_NO_CHANGES_TEMPLATE,
                        new_status="closed",
                    )
                    logger.warning("Couldn't merge the PR: no associated changes.")
                else:
                    # PR cannot be merged for another reason (conflicting files for example). We comment the PR to explain
                    # and re-raise the exception.
                    self.comment_discussion(
                        repo_id=repo_id,
                        repo_type=repo_type,
                        token=token,
                        discussion_num=pr.num,
                        comment=MULTI_COMMIT_PR_CLOSE_COMMENT_FAILURE_BAD_REQUEST_TEMPLATE.format(
                            error_message=error.server_message
                        ),
                    )
                    raise MultiCommitException(
                        f"Couldn't merge Pull Request in multi-commit: {error.server_message}"
                    ) from error

        return pr.url

    @overload
    def upload_file(  # type: ignore
        self,
        *,
        path_or_fileobj: Union[str, Path, bytes, BinaryIO],
        path_in_repo: str,
        repo_id: str,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
        revision: Optional[str] = None,
        commit_message: Optional[str] = None,
        commit_description: Optional[str] = None,
        create_pr: Optional[bool] = None,
        parent_commit: Optional[str] = None,
        run_as_future: Literal[False] = ...,
    ) -> str:
        ...

    @overload
    def upload_file(
        self,
        *,
        path_or_fileobj: Union[str, Path, bytes, BinaryIO],
        path_in_repo: str,
        repo_id: str,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
        revision: Optional[str] = None,
        commit_message: Optional[str] = None,
        commit_description: Optional[str] = None,
        create_pr: Optional[bool] = None,
        parent_commit: Optional[str] = None,
        run_as_future: Literal[True] = ...,
    ) -> Future[str]:
        ...

    @validate_hf_hub_args
    @future_compatible
    def upload_file(
        self,
        *,
        path_or_fileobj: Union[str, Path, bytes, BinaryIO],
        path_in_repo: str,
        repo_id: str,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
        revision: Optional[str] = None,
        commit_message: Optional[str] = None,
        commit_description: Optional[str] = None,
        create_pr: Optional[bool] = None,
        parent_commit: Optional[str] = None,
        run_as_future: bool = False,
    ) -> Union[str, Future[str]]:
        """
        Upload a local file (up to 50 GB) to the given repo. The upload is done
        through a HTTP post request, and doesn't require git or git-lfs to be
        installed.

        Args:
            path_or_fileobj (`str`, `Path`, `bytes`, or `IO`):
                Path to a file on the local machine or binary data stream /
                fileobj / buffer.
            path_in_repo (`str`):
                Relative filepath in the repo, for example:
                `"checkpoints/1fec34a/weights.bin"`
            repo_id (`str`):
                The repository to which the file will be uploaded, for example:
                `"username/custom_transformers"`
            token (`str`, *optional*):
                Authentication token, obtained with `HfApi.login` method. Will
                default to the stored token.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model. Default is
                `None`.
            revision (`str`, *optional*):
                The git revision to commit from. Defaults to the head of the `"main"` branch.
            commit_message (`str`, *optional*):
                The summary / title / first line of the generated commit
            commit_description (`str` *optional*)
                The description of the generated commit
            create_pr (`boolean`, *optional*):
                Whether or not to create a Pull Request with that commit. Defaults to `False`.
                If `revision` is not set, PR is opened against the `"main"` branch. If
                `revision` is set and is a branch, PR is opened against this branch. If
                `revision` is set and is not a branch name (example: a commit oid), an
                `RevisionNotFoundError` is returned by the server.
            parent_commit (`str`, *optional*):
                The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported.
                If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`.
                If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`.
                Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be
                especially useful if the repo is updated / committed to concurrently.
            run_as_future (`bool`, *optional*):
                Whether or not to run this method in the background. Background jobs are run sequentially without
                blocking the main thread. Passing `run_as_future=True` will return a [Future](https://docs.python.org/3/library/concurrent.futures.html#future-objects)
                object. Defaults to `False`.


        Returns:
            `str` or `Future`: The URL to visualize the uploaded file on the hub. If `run_as_future=True` is passed,
            returns a Future object which will contain the result when executed.

        <Tip>

        Raises the following errors:

            - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
              if the HuggingFace API returned an error
            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
              if some parameter value is invalid
            - [`~utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.
            - [`~utils.RevisionNotFoundError`]
              If the revision to download from cannot be found.

        </Tip>

        <Tip warning={true}>

        `upload_file` assumes that the repo already exists on the Hub. If you get a
        Client error 404, please make sure you are authenticated and that `repo_id` and
        `repo_type` are set correctly. If repo does not exist, create it first using
        [`~hf_api.create_repo`].

        </Tip>

        Example:

        ```python
        >>> from huggingface_hub import upload_file

        >>> with open("./local/filepath", "rb") as fobj:
        ...     upload_file(
        ...         path_or_fileobj=fileobj,
        ...         path_in_repo="remote/file/path.h5",
        ...         repo_id="username/my-dataset",
        ...         repo_type="dataset",
        ...         token="my_token",
        ...     )
        "https://huggingface.co/datasets/username/my-dataset/blob/main/remote/file/path.h5"

        >>> upload_file(
        ...     path_or_fileobj=".\\\\local\\\\file\\\\path",
        ...     path_in_repo="remote/file/path.h5",
        ...     repo_id="username/my-model",
        ...     token="my_token",
        ... )
        "https://huggingface.co/username/my-model/blob/main/remote/file/path.h5"

        >>> upload_file(
        ...     path_or_fileobj=".\\\\local\\\\file\\\\path",
        ...     path_in_repo="remote/file/path.h5",
        ...     repo_id="username/my-model",
        ...     token="my_token",
        ...     create_pr=True,
        ... )
        "https://huggingface.co/username/my-model/blob/refs%2Fpr%2F1/remote/file/path.h5"
        ```
        """
        if repo_type not in REPO_TYPES:
            raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")

        commit_message = (
            commit_message if commit_message is not None else f"Upload {path_in_repo} with huggingface_hub"
        )
        operation = CommitOperationAdd(
            path_or_fileobj=path_or_fileobj,
            path_in_repo=path_in_repo,
        )

        commit_info = self.create_commit(
            repo_id=repo_id,
            repo_type=repo_type,
            operations=[operation],
            commit_message=commit_message,
            commit_description=commit_description,
            token=token,
            revision=revision,
            create_pr=create_pr,
            parent_commit=parent_commit,
        )

        if commit_info.pr_url is not None:
            revision = quote(_parse_revision_from_pr_url(commit_info.pr_url), safe="")
        if repo_type in REPO_TYPES_URL_PREFIXES:
            repo_id = REPO_TYPES_URL_PREFIXES[repo_type] + repo_id
        revision = revision if revision is not None else DEFAULT_REVISION
        # Similar to `hf_hub_url` but it's "blob" instead of "resolve"
        return f"{self.endpoint}/{repo_id}/blob/{revision}/{path_in_repo}"

    @overload
    def upload_folder(  # type: ignore
        self,
        *,
        repo_id: str,
        folder_path: Union[str, Path],
        path_in_repo: Optional[str] = None,
        commit_message: Optional[str] = None,
        commit_description: Optional[str] = None,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
        revision: Optional[str] = None,
        create_pr: Optional[bool] = None,
        parent_commit: Optional[str] = None,
        allow_patterns: Optional[Union[List[str], str]] = None,
        ignore_patterns: Optional[Union[List[str], str]] = None,
        delete_patterns: Optional[Union[List[str], str]] = None,
        multi_commits: bool = False,
        multi_commits_verbose: bool = False,
        run_as_future: Literal[False] = ...,
    ) -> str:
        ...

    @overload
    def upload_folder(
        self,
        *,
        repo_id: str,
        folder_path: Union[str, Path],
        path_in_repo: Optional[str] = None,
        commit_message: Optional[str] = None,
        commit_description: Optional[str] = None,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
        revision: Optional[str] = None,
        create_pr: Optional[bool] = None,
        parent_commit: Optional[str] = None,
        allow_patterns: Optional[Union[List[str], str]] = None,
        ignore_patterns: Optional[Union[List[str], str]] = None,
        delete_patterns: Optional[Union[List[str], str]] = None,
        multi_commits: bool = False,
        multi_commits_verbose: bool = False,
        run_as_future: Literal[True] = ...,
    ) -> Future[str]:
        ...

    @validate_hf_hub_args
    @future_compatible
    def upload_folder(
        self,
        *,
        repo_id: str,
        folder_path: Union[str, Path],
        path_in_repo: Optional[str] = None,
        commit_message: Optional[str] = None,
        commit_description: Optional[str] = None,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
        revision: Optional[str] = None,
        create_pr: Optional[bool] = None,
        parent_commit: Optional[str] = None,
        allow_patterns: Optional[Union[List[str], str]] = None,
        ignore_patterns: Optional[Union[List[str], str]] = None,
        delete_patterns: Optional[Union[List[str], str]] = None,
        multi_commits: bool = False,
        multi_commits_verbose: bool = False,
        run_as_future: bool = False,
    ) -> Union[str, Future[str]]:
        """
        Upload a local folder to the given repo. The upload is done through a HTTP requests, and doesn't require git or
        git-lfs to be installed.

        The structure of the folder will be preserved. Files with the same name already present in the repository will
        be overwritten. Others will be left untouched.

        Use the `allow_patterns` and `ignore_patterns` arguments to specify which files to upload. These parameters
        accept either a single pattern or a list of patterns. Patterns are Standard Wildcards (globbing patterns) as
        documented [here](https://tldp.org/LDP/GNU-Linux-Tools-Summary/html/x11655.htm). If both `allow_patterns` and
        `ignore_patterns` are provided, both constraints apply. By default, all files from the folder are uploaded.

        Use the `delete_patterns` argument to specify remote files you want to delete. Input type is the same as for
        `allow_patterns` (see above). If `path_in_repo` is also provided, the patterns are matched against paths
        relative to this folder. For example, `upload_folder(..., path_in_repo="experiment", delete_patterns="logs/*")`
        will delete any remote file under `./experiment/logs/`. Note that the `.gitattributes` file will not be deleted
        even if it matches the patterns.

        Any `.git/` folder present in any subdirectory will be ignored. However, please be aware that the `.gitignore`
        file is not taken into account.

        Uses `HfApi.create_commit` under the hood.

        Args:
            repo_id (`str`):
                The repository to which the file will be uploaded, for example:
                `"username/custom_transformers"`
            folder_path (`str` or `Path`):
                Path to the folder to upload on the local file system
            path_in_repo (`str`, *optional*):
                Relative path of the directory in the repo, for example:
                `"checkpoints/1fec34a/results"`. Will default to the root folder of the repository.
            token (`str`, *optional*):
                Authentication token, obtained with `HfApi.login` method. Will
                default to the stored token.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model. Default is
                `None`.
            revision (`str`, *optional*):
                The git revision to commit from. Defaults to the head of the `"main"` branch.
            commit_message (`str`, *optional*):
                The summary / title / first line of the generated commit. Defaults to:
                `f"Upload {path_in_repo} with huggingface_hub"`
            commit_description (`str` *optional*):
                The description of the generated commit
            create_pr (`boolean`, *optional*):
                Whether or not to create a Pull Request with that commit. Defaults to `False`. If `revision` is not
                set, PR is opened against the `"main"` branch. If `revision` is set and is a branch, PR is opened
                against this branch. If `revision` is set and is not a branch name (example: a commit oid), an
                `RevisionNotFoundError` is returned by the server. If both `multi_commits` and `create_pr` are True,
                the PR created in the multi-commit process is kept opened.
            parent_commit (`str`, *optional*):
                The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported.
                If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`.
                If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`.
                Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be
                especially useful if the repo is updated / committed to concurrently.
            allow_patterns (`List[str]` or `str`, *optional*):
                If provided, only files matching at least one pattern are uploaded.
            ignore_patterns (`List[str]` or `str`, *optional*):
                If provided, files matching any of the patterns are not uploaded.
            delete_patterns (`List[str]` or `str`, *optional*):
                If provided, remote files matching any of the patterns will be deleted from the repo while committing
                new files. This is useful if you don't know which files have already been uploaded.
                Note: to avoid discrepancies the `.gitattributes` file is not deleted even if it matches the pattern.
            multi_commits (`bool`):
                If True, changes are pushed to a PR using a multi-commit process. Defaults to `False`.
            multi_commits_verbose (`bool`):
                If True and `multi_commits` is used, more information will be displayed to the user.
            run_as_future (`bool`, *optional*):
                Whether or not to run this method in the background. Background jobs are run sequentially without
                blocking the main thread. Passing `run_as_future=True` will return a [Future](https://docs.python.org/3/library/concurrent.futures.html#future-objects)
                object. Defaults to `False`.

        Returns:
            `str` or `Future[str]`: A URL to visualize the uploaded folder on the hub. If `run_as_future=True` is passed,
            returns a Future object which will contain the result when executed.

        <Tip>

        Raises the following errors:

            - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
            if the HuggingFace API returned an error
            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
            if some parameter value is invalid

        </Tip>

        <Tip warning={true}>

        `upload_folder` assumes that the repo already exists on the Hub. If you get a Client error 404, please make
        sure you are authenticated and that `repo_id` and `repo_type` are set correctly. If repo does not exist, create
        it first using [`~hf_api.create_repo`].

        </Tip>

        <Tip warning={true}>

        `multi_commits` is experimental. Its API and behavior is subject to change in the future without prior notice.

        </Tip>

        Example:

        ```python
        # Upload checkpoints folder except the log files
        >>> upload_folder(
        ...     folder_path="local/checkpoints",
        ...     path_in_repo="remote/experiment/checkpoints",
        ...     repo_id="username/my-dataset",
        ...     repo_type="datasets",
        ...     token="my_token",
        ...     ignore_patterns="**/logs/*.txt",
        ... )
        # "https://huggingface.co/datasets/username/my-dataset/tree/main/remote/experiment/checkpoints"

        # Upload checkpoints folder including logs while deleting existing logs from the repo
        # Useful if you don't know exactly which log files have already being pushed
        >>> upload_folder(
        ...     folder_path="local/checkpoints",
        ...     path_in_repo="remote/experiment/checkpoints",
        ...     repo_id="username/my-dataset",
        ...     repo_type="datasets",
        ...     token="my_token",
        ...     delete_patterns="**/logs/*.txt",
        ... )
        "https://huggingface.co/datasets/username/my-dataset/tree/main/remote/experiment/checkpoints"

        # Upload checkpoints folder while creating a PR
        >>> upload_folder(
        ...     folder_path="local/checkpoints",
        ...     path_in_repo="remote/experiment/checkpoints",
        ...     repo_id="username/my-dataset",
        ...     repo_type="datasets",
        ...     token="my_token",
        ...     create_pr=True,
        ... )
        "https://huggingface.co/datasets/username/my-dataset/tree/refs%2Fpr%2F1/remote/experiment/checkpoints"

        ```
        """
        if repo_type not in REPO_TYPES:
            raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")

        if multi_commits:
            if revision is not None and revision != DEFAULT_REVISION:
                raise ValueError("Cannot use `multi_commit` to commit changes other than the main branch.")

        # By default, upload folder to the root directory in repo.
        if path_in_repo is None:
            path_in_repo = ""

        # Do not upload .git folder
        if ignore_patterns is None:
            ignore_patterns = []
        elif isinstance(ignore_patterns, str):
            ignore_patterns = [ignore_patterns]
        ignore_patterns += IGNORE_GIT_FOLDER_PATTERNS

        delete_operations = self._prepare_upload_folder_deletions(
            repo_id=repo_id,
            repo_type=repo_type,
            revision=DEFAULT_REVISION if create_pr else revision,
            token=token,
            path_in_repo=path_in_repo,
            delete_patterns=delete_patterns,
        )
        add_operations = _prepare_upload_folder_additions(
            folder_path,
            path_in_repo,
            allow_patterns=allow_patterns,
            ignore_patterns=ignore_patterns,
        )

        # Optimize operations: if some files will be overwritten, we don't need to delete them first
        if len(add_operations) > 0:
            added_paths = set(op.path_in_repo for op in add_operations)
            delete_operations = [
                delete_op for delete_op in delete_operations if delete_op.path_in_repo not in added_paths
            ]
        commit_operations = delete_operations + add_operations

        pr_url: Optional[str]
        commit_message = commit_message or "Upload folder using huggingface_hub"
        if multi_commits:
            addition_commits, deletion_commits = plan_multi_commits(operations=commit_operations)
            pr_url = self.create_commits_on_pr(
                repo_id=repo_id,
                repo_type=repo_type,
                addition_commits=addition_commits,
                deletion_commits=deletion_commits,
                commit_message=commit_message,
                commit_description=commit_description,
                token=token,
                merge_pr=not create_pr,
                verbose=multi_commits_verbose,
            )
        else:
            commit_info = self.create_commit(
                repo_type=repo_type,
                repo_id=repo_id,
                operations=commit_operations,
                commit_message=commit_message,
                commit_description=commit_description,
                token=token,
                revision=revision,
                create_pr=create_pr,
                parent_commit=parent_commit,
            )
            pr_url = commit_info.pr_url

        if create_pr and pr_url is not None:
            revision = quote(_parse_revision_from_pr_url(pr_url), safe="")
        if repo_type in REPO_TYPES_URL_PREFIXES:
            repo_id = REPO_TYPES_URL_PREFIXES[repo_type] + repo_id
        revision = revision if revision is not None else DEFAULT_REVISION
        # Similar to `hf_hub_url` but it's "tree" instead of "resolve"
        return f"{self.endpoint}/{repo_id}/tree/{revision}/{path_in_repo}"

    @validate_hf_hub_args
    def delete_file(
        self,
        path_in_repo: str,
        repo_id: str,
        *,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
        revision: Optional[str] = None,
        commit_message: Optional[str] = None,
        commit_description: Optional[str] = None,
        create_pr: Optional[bool] = None,
        parent_commit: Optional[str] = None,
    ) -> CommitInfo:
        """
        Deletes a file in the given repo.

        Args:
            path_in_repo (`str`):
                Relative filepath in the repo, for example:
                `"checkpoints/1fec34a/weights.bin"`
            repo_id (`str`):
                The repository from which the file will be deleted, for example:
                `"username/custom_transformers"`
            token (`str`, *optional*):
                Authentication token, obtained with `HfApi.login` method. Will
                default to the stored token.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if the file is in a dataset or
                space, `None` or `"model"` if in a model. Default is `None`.
            revision (`str`, *optional*):
                The git revision to commit from. Defaults to the head of the `"main"` branch.
            commit_message (`str`, *optional*):
                The summary / title / first line of the generated commit. Defaults to
                `f"Delete {path_in_repo} with huggingface_hub"`.
            commit_description (`str` *optional*)
                The description of the generated commit
            create_pr (`boolean`, *optional*):
                Whether or not to create a Pull Request with that commit. Defaults to `False`.
                If `revision` is not set, PR is opened against the `"main"` branch. If
                `revision` is set and is a branch, PR is opened against this branch. If
                `revision` is set and is not a branch name (example: a commit oid), an
                `RevisionNotFoundError` is returned by the server.
            parent_commit (`str`, *optional*):
                The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported.
                If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`.
                If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`.
                Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be
                especially useful if the repo is updated / committed to concurrently.


        <Tip>

        Raises the following errors:

            - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
              if the HuggingFace API returned an error
            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
              if some parameter value is invalid
            - [`~utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.
            - [`~utils.RevisionNotFoundError`]
              If the revision to download from cannot be found.
            - [`~utils.EntryNotFoundError`]
              If the file to download cannot be found.

        </Tip>

        """
        commit_message = (
            commit_message if commit_message is not None else f"Delete {path_in_repo} with huggingface_hub"
        )

        operations = [CommitOperationDelete(path_in_repo=path_in_repo)]

        return self.create_commit(
            repo_id=repo_id,
            repo_type=repo_type,
            token=token,
            operations=operations,
            revision=revision,
            commit_message=commit_message,
            commit_description=commit_description,
            create_pr=create_pr,
            parent_commit=parent_commit,
        )

    @validate_hf_hub_args
    def delete_folder(
        self,
        path_in_repo: str,
        repo_id: str,
        *,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
        revision: Optional[str] = None,
        commit_message: Optional[str] = None,
        commit_description: Optional[str] = None,
        create_pr: Optional[bool] = None,
        parent_commit: Optional[str] = None,
    ) -> CommitInfo:
        """
        Deletes a folder in the given repo.

        Simple wrapper around [`create_commit`] method.

        Args:
            path_in_repo (`str`):
                Relative folder path in the repo, for example: `"checkpoints/1fec34a"`.
            repo_id (`str`):
                The repository from which the folder will be deleted, for example:
                `"username/custom_transformers"`
            token (`str`, *optional*):
                Authentication token, obtained with `HfApi.login` method. Will default
                to the stored token.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if the folder is in a dataset or
                space, `None` or `"model"` if in a model. Default is `None`.
            revision (`str`, *optional*):
                The git revision to commit from. Defaults to the head of the `"main"` branch.
            commit_message (`str`, *optional*):
                The summary / title / first line of the generated commit. Defaults to
                `f"Delete folder {path_in_repo} with huggingface_hub"`.
            commit_description (`str` *optional*)
                The description of the generated commit.
            create_pr (`boolean`, *optional*):
                Whether or not to create a Pull Request with that commit. Defaults to `False`.
                If `revision` is not set, PR is opened against the `"main"` branch. If
                `revision` is set and is a branch, PR is opened against this branch. If
                `revision` is set and is not a branch name (example: a commit oid), an
                `RevisionNotFoundError` is returned by the server.
            parent_commit (`str`, *optional*):
                The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported.
                If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`.
                If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`.
                Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be
                especially useful if the repo is updated / committed to concurrently.
        """
        return self.create_commit(
            repo_id=repo_id,
            repo_type=repo_type,
            token=token,
            operations=[CommitOperationDelete(path_in_repo=path_in_repo, is_folder=True)],
            revision=revision,
            commit_message=(
                commit_message if commit_message is not None else f"Delete folder {path_in_repo} with huggingface_hub"
            ),
            commit_description=commit_description,
            create_pr=create_pr,
            parent_commit=parent_commit,
        )

    @validate_hf_hub_args
    def create_branch(
        self,
        repo_id: str,
        *,
        branch: str,
        revision: Optional[str] = None,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
        exist_ok: bool = False,
    ) -> None:
        """
        Create a new branch for a repo on the Hub, starting from the specified revision (defaults to `main`).
        To find a revision suiting your needs, you can use [`list_repo_refs`] or [`list_repo_commits`].

        Args:
            repo_id (`str`):
                The repository in which the branch will be created.
                Example: `"user/my-cool-model"`.

            branch (`str`):
                The name of the branch to create.

            revision (`str`, *optional*):
                The git revision to create the branch from. It can be a branch name or
                the OID/SHA of a commit, as a hexadecimal string. Defaults to the head
                of the `"main"` branch.

            token (`str`, *optional*):
                Authentication token. Will default to the stored token.

            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if creating a branch on a dataset or
                space, `None` or `"model"` if tagging a model. Default is `None`.

            exist_ok (`bool`, *optional*, defaults to `False`):
                If `True`, do not raise an error if branch already exists.

        Raises:
            [`~utils.RepositoryNotFoundError`]:
                If repository is not found (error 404): wrong repo_id/repo_type, private
                but not authenticated or repo does not exist.
            [`~utils.BadRequestError`]:
                If invalid reference for a branch. Ex: `refs/pr/5` or 'refs/foo/bar'.
            [`~utils.HfHubHTTPError`]:
                If the branch already exists on the repo (error 409) and `exist_ok` is
                set to `False`.
        """
        if repo_type is None:
            repo_type = REPO_TYPE_MODEL
        branch = quote(branch, safe="")

        # Prepare request
        branch_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/branch/{branch}"
        headers = self._build_hf_headers(token=token, is_write_action=True)
        payload = {}
        if revision is not None:
            payload["startingPoint"] = revision

        # Create branch
        response = get_session().post(url=branch_url, headers=headers, json=payload)
        try:
            hf_raise_for_status(response)
        except HfHubHTTPError as e:
            if not (e.response.status_code == 409 and exist_ok):
                raise

    @validate_hf_hub_args
    def delete_branch(
        self,
        repo_id: str,
        *,
        branch: str,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
    ) -> None:
        """
        Delete a branch from a repo on the Hub.

        Args:
            repo_id (`str`):
                The repository in which a branch will be deleted.
                Example: `"user/my-cool-model"`.

            branch (`str`):
                The name of the branch to delete.

            token (`str`, *optional*):
                Authentication token. Will default to the stored token.

            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if creating a branch on a dataset or
                space, `None` or `"model"` if tagging a model. Default is `None`.

        Raises:
            [`~utils.RepositoryNotFoundError`]:
                If repository is not found (error 404): wrong repo_id/repo_type, private
                but not authenticated or repo does not exist.
            [`~utils.HfHubHTTPError`]:
                If trying to delete a protected branch. Ex: `main` cannot be deleted.
            [`~utils.HfHubHTTPError`]:
                If trying to delete a branch that does not exist.

        """
        if repo_type is None:
            repo_type = REPO_TYPE_MODEL
        branch = quote(branch, safe="")

        # Prepare request
        branch_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/branch/{branch}"
        headers = self._build_hf_headers(token=token, is_write_action=True)

        # Delete branch
        response = get_session().delete(url=branch_url, headers=headers)
        hf_raise_for_status(response)

    @validate_hf_hub_args
    def create_tag(
        self,
        repo_id: str,
        *,
        tag: str,
        tag_message: Optional[str] = None,
        revision: Optional[str] = None,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
        exist_ok: bool = False,
    ) -> None:
        """
        Tag a given commit of a repo on the Hub.

        Args:
            repo_id (`str`):
                The repository in which a commit will be tagged.
                Example: `"user/my-cool-model"`.

            tag (`str`):
                The name of the tag to create.

            tag_message (`str`, *optional*):
                The description of the tag to create.

            revision (`str`, *optional*):
                The git revision to tag. It can be a branch name or the OID/SHA of a
                commit, as a hexadecimal string. Shorthands (7 first characters) are
                also supported. Defaults to the head of the `"main"` branch.

            token (`str`, *optional*):
                Authentication token. Will default to the stored token.

            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if tagging a dataset or
                space, `None` or `"model"` if tagging a model. Default is
                `None`.

            exist_ok (`bool`, *optional*, defaults to `False`):
                If `True`, do not raise an error if tag already exists.

        Raises:
            [`~utils.RepositoryNotFoundError`]:
                If repository is not found (error 404): wrong repo_id/repo_type, private
                but not authenticated or repo does not exist.
            [`~utils.RevisionNotFoundError`]:
                If revision is not found (error 404) on the repo.
            [`~utils.HfHubHTTPError`]:
                If the branch already exists on the repo (error 409) and `exist_ok` is
                set to `False`.
        """
        if repo_type is None:
            repo_type = REPO_TYPE_MODEL
        revision = quote(revision, safe="") if revision is not None else DEFAULT_REVISION

        # Prepare request
        tag_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/tag/{revision}"
        headers = self._build_hf_headers(token=token, is_write_action=True)
        payload = {"tag": tag}
        if tag_message is not None:
            payload["message"] = tag_message

        # Tag
        response = get_session().post(url=tag_url, headers=headers, json=payload)
        try:
            hf_raise_for_status(response)
        except HfHubHTTPError as e:
            if not (e.response.status_code == 409 and exist_ok):
                raise

    @validate_hf_hub_args
    def delete_tag(
        self,
        repo_id: str,
        *,
        tag: str,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
    ) -> None:
        """
        Delete a tag from a repo on the Hub.

        Args:
            repo_id (`str`):
                The repository in which a tag will be deleted.
                Example: `"user/my-cool-model"`.

            tag (`str`):
                The name of the tag to delete.

            token (`str`, *optional*):
                Authentication token. Will default to the stored token.

            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if tagging a dataset or space, `None` or
                `"model"` if tagging a model. Default is `None`.

        Raises:
            [`~utils.RepositoryNotFoundError`]:
                If repository is not found (error 404): wrong repo_id/repo_type, private
                but not authenticated or repo does not exist.
            [`~utils.RevisionNotFoundError`]:
                If tag is not found.
        """
        if repo_type is None:
            repo_type = REPO_TYPE_MODEL
        tag = quote(tag, safe="")

        # Prepare request
        tag_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/tag/{tag}"
        headers = self._build_hf_headers(token=token, is_write_action=True)

        # Un-tag
        response = get_session().delete(url=tag_url, headers=headers)
        hf_raise_for_status(response)

    @validate_hf_hub_args
    def get_full_repo_name(
        self,
        model_id: str,
        *,
        organization: Optional[str] = None,
        token: Optional[Union[bool, str]] = None,
    ):
        """
        Returns the repository name for a given model ID and optional
        organization.

        Args:
            model_id (`str`):
                The name of the model.
            organization (`str`, *optional*):
                If passed, the repository name will be in the organization
                namespace instead of the user namespace.
            token (`bool` or `str`, *optional*):
                A valid authentication token (see https://huggingface.co/settings/token).
                If `None` or `True` and machine is logged in (through `huggingface-cli login`
                or [`~huggingface_hub.login`]), token will be retrieved from the cache.
                If `False`, token is not sent in the request header.

        Returns:
            `str`: The repository name in the user's namespace
            ({username}/{model_id}) if no organization is passed, and under the
            organization namespace ({organization}/{model_id}) otherwise.
        """
        if organization is None:
            if "/" in model_id:
                username = model_id.split("/")[0]
            else:
                username = self.whoami(token=token)["name"]  # type: ignore
            return f"{username}/{model_id}"
        else:
            return f"{organization}/{model_id}"

    @validate_hf_hub_args
    def get_repo_discussions(
        self,
        repo_id: str,
        *,
        repo_type: Optional[str] = None,
        token: Optional[str] = None,
    ) -> Iterator[Discussion]:
        """
        Fetches Discussions and Pull Requests for the given repo.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if fetching from a dataset or
                space, `None` or `"model"` if fetching from a model. Default is
                `None`.
            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token).

        Returns:
            `Iterator[Discussion]`: An iterator of [`Discussion`] objects.

        Example:
            Collecting all discussions of a repo in a list:

            ```python
            >>> from huggingface_hub import get_repo_discussions
            >>> discussions_list = list(get_repo_discussions(repo_id="bert-base-uncased"))
            ```

            Iterating over discussions of a repo:

            ```python
            >>> from huggingface_hub import get_repo_discussions
            >>> for discussion in get_repo_discussions(repo_id="bert-base-uncased"):
            ...     print(discussion.num, discussion.title)
            ```
        """
        if repo_type not in REPO_TYPES:
            raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")
        if repo_type is None:
            repo_type = REPO_TYPE_MODEL

        headers = self._build_hf_headers(token=token)

        def _fetch_discussion_page(page_index: int):
            path = f"{self.endpoint}/api/{repo_type}s/{repo_id}/discussions?p={page_index}"
            resp = get_session().get(path, headers=headers)
            hf_raise_for_status(resp)
            paginated_discussions = resp.json()
            total = paginated_discussions["count"]
            start = paginated_discussions["start"]
            discussions = paginated_discussions["discussions"]
            has_next = (start + len(discussions)) < total
            return discussions, has_next

        has_next, page_index = True, 0

        while has_next:
            discussions, has_next = _fetch_discussion_page(page_index=page_index)
            for discussion in discussions:
                yield Discussion(
                    title=discussion["title"],
                    num=discussion["num"],
                    author=discussion.get("author", {}).get("name", "deleted"),
                    created_at=parse_datetime(discussion["createdAt"]),
                    status=discussion["status"],
                    repo_id=discussion["repo"]["name"],
                    repo_type=discussion["repo"]["type"],
                    is_pull_request=discussion["isPullRequest"],
                    endpoint=self.endpoint,
                )
            page_index = page_index + 1

    @validate_hf_hub_args
    def get_discussion_details(
        self,
        repo_id: str,
        discussion_num: int,
        *,
        repo_type: Optional[str] = None,
        token: Optional[str] = None,
    ) -> DiscussionWithDetails:
        """Fetches a Discussion's / Pull Request 's details from the Hub.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            discussion_num (`int`):
                The number of the Discussion or Pull Request . Must be a strictly positive integer.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model. Default is
                `None`.
            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)

        Returns: [`DiscussionWithDetails`]

        <Tip>

        Raises the following errors:

            - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
              if the HuggingFace API returned an error
            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
              if some parameter value is invalid
            - [`~utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.

        </Tip>
        """
        if not isinstance(discussion_num, int) or discussion_num <= 0:
            raise ValueError("Invalid discussion_num, must be a positive integer")
        if repo_type not in REPO_TYPES:
            raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")
        if repo_type is None:
            repo_type = REPO_TYPE_MODEL

        path = f"{self.endpoint}/api/{repo_type}s/{repo_id}/discussions/{discussion_num}"
        headers = self._build_hf_headers(token=token)
        resp = get_session().get(path, params={"diff": "1"}, headers=headers)
        hf_raise_for_status(resp)

        discussion_details = resp.json()
        is_pull_request = discussion_details["isPullRequest"]

        target_branch = discussion_details["changes"]["base"] if is_pull_request else None
        conflicting_files = discussion_details["filesWithConflicts"] if is_pull_request else None
        merge_commit_oid = discussion_details["changes"].get("mergeCommitId", None) if is_pull_request else None

        return DiscussionWithDetails(
            title=discussion_details["title"],
            num=discussion_details["num"],
            author=discussion_details.get("author", {}).get("name", "deleted"),
            created_at=parse_datetime(discussion_details["createdAt"]),
            status=discussion_details["status"],
            repo_id=discussion_details["repo"]["name"],
            repo_type=discussion_details["repo"]["type"],
            is_pull_request=discussion_details["isPullRequest"],
            events=[deserialize_event(evt) for evt in discussion_details["events"]],
            conflicting_files=conflicting_files,
            target_branch=target_branch,
            merge_commit_oid=merge_commit_oid,
            diff=discussion_details.get("diff"),
            endpoint=self.endpoint,
        )

    @validate_hf_hub_args
    def create_discussion(
        self,
        repo_id: str,
        title: str,
        *,
        token: Optional[str] = None,
        description: Optional[str] = None,
        repo_type: Optional[str] = None,
        pull_request: bool = False,
    ) -> DiscussionWithDetails:
        """Creates a Discussion or Pull Request.

        Pull Requests created programmatically will be in `"draft"` status.

        Creating a Pull Request with changes can also be done at once with [`HfApi.create_commit`].

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            title (`str`):
                The title of the discussion. It can be up to 200 characters long,
                and must be at least 3 characters long. Leading and trailing whitespaces
                will be stripped.
            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)
            description (`str`, *optional*):
                An optional description for the Pull Request.
                Defaults to `"Discussion opened with the huggingface_hub Python library"`
            pull_request (`bool`, *optional*):
                Whether to create a Pull Request or discussion. If `True`, creates a Pull Request.
                If `False`, creates a discussion. Defaults to `False`.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model. Default is
                `None`.

        Returns: [`DiscussionWithDetails`]

        <Tip>

        Raises the following errors:

            - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
              if the HuggingFace API returned an error
            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
              if some parameter value is invalid
            - [`~utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.

        </Tip>"""
        if repo_type not in REPO_TYPES:
            raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")
        if repo_type is None:
            repo_type = REPO_TYPE_MODEL

        if description is not None:
            description = description.strip()
        description = (
            description
            if description
            else (
                f"{'Pull Request' if pull_request else 'Discussion'} opened with the"
                " [huggingface_hub Python"
                " library](https://huggingface.co/docs/huggingface_hub)"
            )
        )

        headers = self._build_hf_headers(token=token, is_write_action=True)
        resp = get_session().post(
            f"{self.endpoint}/api/{repo_type}s/{repo_id}/discussions",
            json={
                "title": title.strip(),
                "description": description,
                "pullRequest": pull_request,
            },
            headers=headers,
        )
        hf_raise_for_status(resp)
        num = resp.json()["num"]
        return self.get_discussion_details(
            repo_id=repo_id,
            repo_type=repo_type,
            discussion_num=num,
            token=token,
        )

    @validate_hf_hub_args
    def create_pull_request(
        self,
        repo_id: str,
        title: str,
        *,
        token: Optional[str] = None,
        description: Optional[str] = None,
        repo_type: Optional[str] = None,
    ) -> DiscussionWithDetails:
        """Creates a Pull Request . Pull Requests created programmatically will be in `"draft"` status.

        Creating a Pull Request with changes can also be done at once with [`HfApi.create_commit`];

        This is a wrapper around [`HfApi.create_discussion`].

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            title (`str`):
                The title of the discussion. It can be up to 200 characters long,
                and must be at least 3 characters long. Leading and trailing whitespaces
                will be stripped.
            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)
            description (`str`, *optional*):
                An optional description for the Pull Request.
                Defaults to `"Discussion opened with the huggingface_hub Python library"`
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model. Default is
                `None`.

        Returns: [`DiscussionWithDetails`]

        <Tip>

        Raises the following errors:

            - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
              if the HuggingFace API returned an error
            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
              if some parameter value is invalid
            - [`~utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.

        </Tip>"""
        return self.create_discussion(
            repo_id=repo_id,
            title=title,
            token=token,
            description=description,
            repo_type=repo_type,
            pull_request=True,
        )

    def _post_discussion_changes(
        self,
        *,
        repo_id: str,
        discussion_num: int,
        resource: str,
        body: Optional[dict] = None,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
    ) -> requests.Response:
        """Internal utility to POST changes to a Discussion or Pull Request"""
        if not isinstance(discussion_num, int) or discussion_num <= 0:
            raise ValueError("Invalid discussion_num, must be a positive integer")
        if repo_type not in REPO_TYPES:
            raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")
        if repo_type is None:
            repo_type = REPO_TYPE_MODEL
        repo_id = f"{repo_type}s/{repo_id}"

        path = f"{self.endpoint}/api/{repo_id}/discussions/{discussion_num}/{resource}"

        headers = self._build_hf_headers(token=token, is_write_action=True)
        resp = requests.post(path, headers=headers, json=body)
        hf_raise_for_status(resp)
        return resp

    @validate_hf_hub_args
    def comment_discussion(
        self,
        repo_id: str,
        discussion_num: int,
        comment: str,
        *,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
    ) -> DiscussionComment:
        """Creates a new comment on the given Discussion.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            discussion_num (`int`):
                The number of the Discussion or Pull Request . Must be a strictly positive integer.
            comment (`str`):
                The content of the comment to create. Comments support markdown formatting.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model. Default is
                `None`.
            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)

        Returns:
            [`DiscussionComment`]: the newly created comment


        Examples:
            ```python

            >>> comment = \"\"\"
            ... Hello @otheruser!
            ...
            ... # This is a title
            ...
            ... **This is bold**, *this is italic* and ~this is strikethrough~
            ... And [this](http://url) is a link
            ... \"\"\"

            >>> HfApi().comment_discussion(
            ...     repo_id="username/repo_name",
            ...     discussion_num=34
            ...     comment=comment
            ... )
            # DiscussionComment(id='deadbeef0000000', type='comment', ...)

            ```

        <Tip>

        Raises the following errors:

            - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
              if the HuggingFace API returned an error
            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
              if some parameter value is invalid
            - [`~utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.

        </Tip>
        """
        resp = self._post_discussion_changes(
            repo_id=repo_id,
            repo_type=repo_type,
            discussion_num=discussion_num,
            token=token,
            resource="comment",
            body={"comment": comment},
        )
        return deserialize_event(resp.json()["newMessage"])  # type: ignore

    @validate_hf_hub_args
    def rename_discussion(
        self,
        repo_id: str,
        discussion_num: int,
        new_title: str,
        *,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
    ) -> DiscussionTitleChange:
        """Renames a Discussion.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            discussion_num (`int`):
                The number of the Discussion or Pull Request . Must be a strictly positive integer.
            new_title (`str`):
                The new title for the discussion
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model. Default is
                `None`.
            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)

        Returns:
            [`DiscussionTitleChange`]: the title change event


        Examples:
            ```python
            >>> new_title = "New title, fixing a typo"
            >>> HfApi().rename_discussion(
            ...     repo_id="username/repo_name",
            ...     discussion_num=34
            ...     new_title=new_title
            ... )
            # DiscussionTitleChange(id='deadbeef0000000', type='title-change', ...)

            ```

        <Tip>

        Raises the following errors:

            - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
              if the HuggingFace API returned an error
            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
              if some parameter value is invalid
            - [`~utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.

        </Tip>
        """
        resp = self._post_discussion_changes(
            repo_id=repo_id,
            repo_type=repo_type,
            discussion_num=discussion_num,
            token=token,
            resource="title",
            body={"title": new_title},
        )
        return deserialize_event(resp.json()["newTitle"])  # type: ignore

    @validate_hf_hub_args
    def change_discussion_status(
        self,
        repo_id: str,
        discussion_num: int,
        new_status: Literal["open", "closed"],
        *,
        token: Optional[str] = None,
        comment: Optional[str] = None,
        repo_type: Optional[str] = None,
    ) -> DiscussionStatusChange:
        """Closes or re-opens a Discussion or Pull Request.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            discussion_num (`int`):
                The number of the Discussion or Pull Request . Must be a strictly positive integer.
            new_status (`str`):
                The new status for the discussion, either `"open"` or `"closed"`.
            comment (`str`, *optional*):
                An optional comment to post with the status change.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model. Default is
                `None`.
            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)

        Returns:
            [`DiscussionStatusChange`]: the status change event


        Examples:
            ```python
            >>> new_title = "New title, fixing a typo"
            >>> HfApi().rename_discussion(
            ...     repo_id="username/repo_name",
            ...     discussion_num=34
            ...     new_title=new_title
            ... )
            # DiscussionStatusChange(id='deadbeef0000000', type='status-change', ...)

            ```

        <Tip>

        Raises the following errors:

            - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
              if the HuggingFace API returned an error
            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
              if some parameter value is invalid
            - [`~utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.

        </Tip>
        """
        if new_status not in ["open", "closed"]:
            raise ValueError("Invalid status, valid statuses are: 'open' and 'closed'")
        body: Dict[str, str] = {"status": new_status}
        if comment and comment.strip():
            body["comment"] = comment.strip()
        resp = self._post_discussion_changes(
            repo_id=repo_id,
            repo_type=repo_type,
            discussion_num=discussion_num,
            token=token,
            resource="status",
            body=body,
        )
        return deserialize_event(resp.json()["newStatus"])  # type: ignore

    @validate_hf_hub_args
    def merge_pull_request(
        self,
        repo_id: str,
        discussion_num: int,
        *,
        token: Optional[str] = None,
        comment: Optional[str] = None,
        repo_type: Optional[str] = None,
    ):
        """Merges a Pull Request.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            discussion_num (`int`):
                The number of the Discussion or Pull Request . Must be a strictly positive integer.
            comment (`str`, *optional*):
                An optional comment to post with the status change.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model. Default is
                `None`.
            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)

        Returns:
            [`DiscussionStatusChange`]: the status change event

        <Tip>

        Raises the following errors:

            - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
              if the HuggingFace API returned an error
            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
              if some parameter value is invalid
            - [`~utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.

        </Tip>
        """
        self._post_discussion_changes(
            repo_id=repo_id,
            repo_type=repo_type,
            discussion_num=discussion_num,
            token=token,
            resource="merge",
            body={"comment": comment.strip()} if comment and comment.strip() else None,
        )

    @validate_hf_hub_args
    def edit_discussion_comment(
        self,
        repo_id: str,
        discussion_num: int,
        comment_id: str,
        new_content: str,
        *,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
    ) -> DiscussionComment:
        """Edits a comment on a Discussion / Pull Request.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            discussion_num (`int`):
                The number of the Discussion or Pull Request . Must be a strictly positive integer.
            comment_id (`str`):
                The ID of the comment to edit.
            new_content (`str`):
                The new content of the comment. Comments support markdown formatting.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model. Default is
                `None`.
            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)

        Returns:
            [`DiscussionComment`]: the edited comment

        <Tip>

        Raises the following errors:

            - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
              if the HuggingFace API returned an error
            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
              if some parameter value is invalid
            - [`~utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.

        </Tip>
        """
        resp = self._post_discussion_changes(
            repo_id=repo_id,
            repo_type=repo_type,
            discussion_num=discussion_num,
            token=token,
            resource=f"comment/{comment_id.lower()}/edit",
            body={"content": new_content},
        )
        return deserialize_event(resp.json()["updatedComment"])  # type: ignore

    @validate_hf_hub_args
    def hide_discussion_comment(
        self,
        repo_id: str,
        discussion_num: int,
        comment_id: str,
        *,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
    ) -> DiscussionComment:
        """Hides a comment on a Discussion / Pull Request.

        <Tip warning={true}>
        Hidden comments' content cannot be retrieved anymore. Hiding a comment is irreversible.
        </Tip>

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            discussion_num (`int`):
                The number of the Discussion or Pull Request . Must be a strictly positive integer.
            comment_id (`str`):
                The ID of the comment to edit.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model. Default is
                `None`.
            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)

        Returns:
            [`DiscussionComment`]: the hidden comment

        <Tip>

        Raises the following errors:

            - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
              if the HuggingFace API returned an error
            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
              if some parameter value is invalid
            - [`~utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.

        </Tip>
        """
        warnings.warn(
            "Hidden comments' content cannot be retrieved anymore. Hiding a comment is irreversible.",
            UserWarning,
        )
        resp = self._post_discussion_changes(
            repo_id=repo_id,
            repo_type=repo_type,
            discussion_num=discussion_num,
            token=token,
            resource=f"comment/{comment_id.lower()}/hide",
        )
        return deserialize_event(resp.json()["updatedComment"])  # type: ignore

    @validate_hf_hub_args
    def add_space_secret(self, repo_id: str, key: str, value: str, *, token: Optional[str] = None) -> None:
        """Adds or updates a secret in a Space.

        Secrets allow to set secret keys or tokens to a Space without hardcoding them.
        For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets.

        Args:
            repo_id (`str`):
                ID of the repo to update. Example: `"bigcode/in-the-stack"`.
            key (`str`):
                Secret key. Example: `"GITHUB_API_KEY"`
            value (`str`):
                Secret value. Example: `"your_github_api_key"`.
            token (`str`, *optional*):
                Hugging Face token. Will default to the locally saved token if not provided.
        """
        r = get_session().post(
            f"{self.endpoint}/api/spaces/{repo_id}/secrets",
            headers=self._build_hf_headers(token=token),
            json={"key": key, "value": value},
        )
        hf_raise_for_status(r)

    @validate_hf_hub_args
    def delete_space_secret(self, repo_id: str, key: str, *, token: Optional[str] = None) -> None:
        """Deletes a secret from a Space.

        Secrets allow to set secret keys or tokens to a Space without hardcoding them.
        For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets.

        Args:
            repo_id (`str`):
                ID of the repo to update. Example: `"bigcode/in-the-stack"`.
            key (`str`):
                Secret key. Example: `"GITHUB_API_KEY"`.
            token (`str`, *optional*):
                Hugging Face token. Will default to the locally saved token if not provided.
        """
        r = get_session().delete(
            f"{self.endpoint}/api/spaces/{repo_id}/secrets",
            headers=self._build_hf_headers(token=token),
            json={"key": key},
        )
        hf_raise_for_status(r)

    @validate_hf_hub_args
    def get_space_runtime(self, repo_id: str, *, token: Optional[str] = None) -> SpaceRuntime:
        """Gets runtime information about a Space.

        Args:
            repo_id (`str`):
                ID of the repo to update. Example: `"bigcode/in-the-stack"`.
            token (`str`, *optional*):
                Hugging Face token. Will default to the locally saved token if
                not provided.
        Returns:
            [`SpaceRuntime`]: Runtime information about a Space including Space stage and hardware.
        """
        r = get_session().get(
            f"{self.endpoint}/api/spaces/{repo_id}/runtime", headers=self._build_hf_headers(token=token)
        )
        hf_raise_for_status(r)
        return SpaceRuntime(r.json())

    @validate_hf_hub_args
    def request_space_hardware(
        self,
        repo_id: str,
        hardware: SpaceHardware,
        *,
        token: Optional[str] = None,
        sleep_time: Optional[int] = None,
    ) -> SpaceRuntime:
        """Request new hardware for a Space.

        Args:
            repo_id (`str`):
                ID of the repo to update. Example: `"bigcode/in-the-stack"`.
            hardware (`str` or [`SpaceHardware`]):
                Hardware on which to run the Space. Example: `"t4-medium"`.
            token (`str`, *optional*):
                Hugging Face token. Will default to the locally saved token if not provided.
            sleep_time (`int`, *optional*):
                Number of seconds of inactivity to wait before a Space is put to sleep. Set to `-1` if you don't want
                your Space to sleep (default behavior for upgraded hardware). For free hardware, you can't configure
                the sleep time (value is fixed to 48 hours of inactivity).
                See https://huggingface.co/docs/hub/spaces-gpus#sleep-time for more details.
        Returns:
            [`SpaceRuntime`]: Runtime information about a Space including Space stage and hardware.

        <Tip>

        It is also possible to request hardware directly when creating the Space repo! See [`create_repo`] for details.

        </Tip>
        """
        if sleep_time is not None and hardware == SpaceHardware.CPU_BASIC:
            warnings.warn(
                (
                    "If your Space runs on the default 'cpu-basic' hardware, it will go to sleep if inactive for more"
                    " than 48 hours. This value is not configurable. If you don't want your Space to deactivate or if"
                    " you want to set a custom sleep time, you need to upgrade to a paid Hardware."
                ),
                UserWarning,
            )
        payload: Dict[str, Any] = {"flavor": hardware}
        if sleep_time is not None:
            payload["sleepTimeSeconds"] = sleep_time
        r = get_session().post(
            f"{self.endpoint}/api/spaces/{repo_id}/hardware",
            headers=self._build_hf_headers(token=token),
            json=payload,
        )
        hf_raise_for_status(r)
        return SpaceRuntime(r.json())

    @validate_hf_hub_args
    def set_space_sleep_time(self, repo_id: str, sleep_time: int, *, token: Optional[str] = None) -> SpaceRuntime:
        """Set a custom sleep time for a Space running on upgraded hardware..

        Your Space will go to sleep after X seconds of inactivity. You are not billed when your Space is in "sleep"
        mode. If a new visitor lands on your Space, it will "wake it up". Only upgraded hardware can have a
        configurable sleep time. To know more about the sleep stage, please refer to
        https://huggingface.co/docs/hub/spaces-gpus#sleep-time.

        Args:
            repo_id (`str`):
                ID of the repo to update. Example: `"bigcode/in-the-stack"`.
            sleep_time (`int`, *optional*):
                Number of seconds of inactivity to wait before a Space is put to sleep. Set to `-1` if you don't want
                your Space to pause (default behavior for upgraded hardware). For free hardware, you can't configure
                the sleep time (value is fixed to 48 hours of inactivity).
                See https://huggingface.co/docs/hub/spaces-gpus#sleep-time for more details.
            token (`str`, *optional*):
                Hugging Face token. Will default to the locally saved token if not provided.
        Returns:
            [`SpaceRuntime`]: Runtime information about a Space including Space stage and hardware.

        <Tip>

        It is also possible to set a custom sleep time when requesting hardware with [`request_space_hardware`].

        </Tip>
        """
        r = get_session().post(
            f"{self.endpoint}/api/spaces/{repo_id}/sleeptime",
            headers=self._build_hf_headers(token=token),
            json={"seconds": sleep_time},
        )
        hf_raise_for_status(r)
        runtime = SpaceRuntime(r.json())

        hardware = runtime.requested_hardware or runtime.hardware
        if hardware == SpaceHardware.CPU_BASIC:
            warnings.warn(
                (
                    "If your Space runs on the default 'cpu-basic' hardware, it will go to sleep if inactive for more"
                    " than 48 hours. This value is not configurable. If you don't want your Space to deactivate or if"
                    " you want to set a custom sleep time, you need to upgrade to a paid Hardware."
                ),
                UserWarning,
            )
        return runtime

    @validate_hf_hub_args
    def pause_space(self, repo_id: str, *, token: Optional[str] = None) -> SpaceRuntime:
        """Pause your Space.

        A paused Space stops executing until manually restarted by its owner. This is different from the sleeping
        state in which free Spaces go after 48h of inactivity. Paused time is not billed to your account, no matter the
        hardware you've selected. To restart your Space, use [`restart_space`] and go to your Space settings page.

        For more details, please visit [the docs](https://huggingface.co/docs/hub/spaces-gpus#pause).

        Args:
            repo_id (`str`):
                ID of the Space to pause. Example: `"Salesforce/BLIP2"`.
            token (`str`, *optional*):
                Hugging Face token. Will default to the locally saved token if not provided.

        Returns:
            [`SpaceRuntime`]: Runtime information about your Space including `stage=PAUSED` and requested hardware.

        Raises:
            [`~utils.RepositoryNotFoundError`]:
                If your Space is not found (error 404). Most probably wrong repo_id or your space is private but you
                are not authenticated.
            [`~utils.HfHubHTTPError`]:
                403 Forbidden: only the owner of a Space can pause it. If you want to manage a Space that you don't
                own, either ask the owner by opening a Discussion or duplicate the Space.
            [`~utils.BadRequestError`]:
                If your Space is a static Space. Static Spaces are always running and never billed. If you want to hide
                a static Space, you can set it to private.
        """
        r = get_session().post(
            f"{self.endpoint}/api/spaces/{repo_id}/pause", headers=self._build_hf_headers(token=token)
        )
        hf_raise_for_status(r)
        return SpaceRuntime(r.json())

    @validate_hf_hub_args
    def restart_space(self, repo_id: str, *, token: Optional[str] = None) -> SpaceRuntime:
        """Restart your Space.

        This is the only way to programmatically restart a Space if you've put it on Pause (see [`pause_space`]). You
        must be the owner of the Space to restart it. If you are using an upgraded hardware, your account will be
        billed as soon as the Space is restarted. You can trigger a restart no matter the current state of a Space.

        For more details, please visit [the docs](https://huggingface.co/docs/hub/spaces-gpus#pause).

        Args:
            repo_id (`str`):
                ID of the Space to restart. Example: `"Salesforce/BLIP2"`.
            token (`str`, *optional*):
                Hugging Face token. Will default to the locally saved token if not provided.

        Returns:
            [`SpaceRuntime`]: Runtime information about your Space.

        Raises:
            [`~utils.RepositoryNotFoundError`]:
                If your Space is not found (error 404). Most probably wrong repo_id or your space is private but you
                are not authenticated.
            [`~utils.HfHubHTTPError`]:
                403 Forbidden: only the owner of a Space can restart it. If you want to restart a Space that you don't
                own, either ask the owner by opening a Discussion or duplicate the Space.
            [`~utils.BadRequestError`]:
                If your Space is a static Space. Static Spaces are always running and never billed. If you want to hide
                a static Space, you can set it to private.
        """
        r = get_session().post(
            f"{self.endpoint}/api/spaces/{repo_id}/restart", headers=self._build_hf_headers(token=token)
        )
        hf_raise_for_status(r)
        return SpaceRuntime(r.json())

    @validate_hf_hub_args
    def duplicate_space(
        self,
        from_id: str,
        to_id: Optional[str] = None,
        *,
        private: Optional[bool] = None,
        token: Optional[str] = None,
        exist_ok: bool = False,
    ) -> RepoUrl:
        """Duplicate a Space.

        Programmatically duplicate a Space. The new Space will be created in your account and will be in the same state
        as the original Space (running or paused). You can duplicate a Space no matter the current state of a Space.

        Args:
            from_id (`str`):
                ID of the Space to duplicate. Example: `"pharma/CLIP-Interrogator"`.
            to_id (`str`, *optional*):
                ID of the new Space. Example: `"dog/CLIP-Interrogator"`. If not provided, the new Space will have the same
                name as the original Space, but in your account.
            private (`bool`, *optional*):
                Whether the new Space should be private or not. Defaults to the same privacy as the original Space.
            token (`str`, *optional*):
                Hugging Face token. Will default to the locally saved token if not provided.
            exist_ok (`bool`, *optional*, defaults to `False`):
                If `True`, do not raise an error if repo already exists.

        Returns:
            [`RepoUrl`]: URL to the newly created repo. Value is a subclass of `str` containing
            attributes like `endpoint`, `repo_type` and `repo_id`.

        Raises:
            - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
              if the HuggingFace API returned an error
            - [`~utils.RepositoryNotFoundError`]
              If one of `from_id` or `to_id` cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.

        Example:
        ```python
        >>> from huggingface_hub import duplicate_space

        # Duplicate a Space to your account
        >>> duplicate_space("multimodalart/dreambooth-training")
        RepoUrl('https://huggingface.co/spaces/nateraw/dreambooth-training',...)

        # Can set custom destination id and visibility flag.
        >>> duplicate_space("multimodalart/dreambooth-training", to_id="my-dreambooth", private=True)
        RepoUrl('https://huggingface.co/spaces/nateraw/my-dreambooth',...)
        ```
        """
        # Parse to_id if provided
        parsed_to_id = RepoUrl(to_id) if to_id is not None else None

        # Infer target repo_id
        to_namespace = (  # set namespace manually or default to username
            parsed_to_id.namespace
            if parsed_to_id is not None and parsed_to_id.namespace is not None
            else self.whoami(token)["name"]
        )
        to_repo_name = parsed_to_id.repo_name if to_id is not None else RepoUrl(from_id).repo_name  # type: ignore

        # repository must be a valid repo_id (namespace/repo_name).
        payload: Dict[str, Any] = {"repository": f"{to_namespace}/{to_repo_name}"}

        # private is optional with this endpoint, with None defaulting to the original space's privacy.
        if private is not None:
            payload["private"] = private

        r = get_session().post(
            f"{self.endpoint}/api/spaces/{from_id}/duplicate",
            headers=self._build_hf_headers(token=token, is_write_action=True),
            json=payload,
        )

        try:
            hf_raise_for_status(r)
        except HTTPError as err:
            if exist_ok and err.response.status_code == 409:
                # Repo already exists and `exist_ok=True`
                pass
            else:
                raise

        return RepoUrl(r.json()["url"], endpoint=self.endpoint)

    def _build_hf_headers(
        self,
        token: Optional[Union[bool, str]] = None,
        is_write_action: bool = False,
        library_name: Optional[str] = None,
        library_version: Optional[str] = None,
        user_agent: Union[Dict, str, None] = None,
    ) -> Dict[str, str]:
        """
        Alias for [`build_hf_headers`] that uses the token from [`HfApi`] client
        when `token` is not provided.
        """
        if token is None:
            # Cannot do `token = token or self.token` as token can be `False`.
            token = self.token
        return build_hf_headers(
            token=token,
            is_write_action=is_write_action,
            library_name=library_name or self.library_name,
            library_version=library_version or self.library_version,
            user_agent=user_agent or self.user_agent,
        )

    def _prepare_upload_folder_deletions(
        self,
        repo_id: str,
        repo_type: Optional[str],
        revision: Optional[str],
        token: Optional[str],
        path_in_repo: str,
        delete_patterns: Optional[Union[List[str], str]],
    ) -> List[CommitOperationDelete]:
        """Generate the list of Delete operations for a commit to delete files from a repo.

        List remote files and match them against the `delete_patterns` constraints. Returns a list of [`CommitOperationDelete`]
        with the matching items.

        Note: `.gitattributes` file is essential to make a repo work properly on the Hub. This file will always be
              kept even if it matches the `delete_patterns` constraints.
        """
        if delete_patterns is None:
            # If no delete patterns, no need to list and filter remote files
            return []

        # List remote files
        filenames = self.list_repo_files(repo_id=repo_id, revision=revision, repo_type=repo_type, token=token)

        # Compute relative path in repo
        if path_in_repo:
            path_in_repo = path_in_repo.strip("/") + "/"  # harmonize
            relpath_to_abspath = {
                file[len(path_in_repo) :]: file for file in filenames if file.startswith(path_in_repo)
            }
        else:
            relpath_to_abspath = {file: file for file in filenames}

        # Apply filter on relative paths and return
        return [
            CommitOperationDelete(path_in_repo=relpath_to_abspath[relpath], is_folder=False)
            for relpath in filter_repo_objects(relpath_to_abspath.keys(), allow_patterns=delete_patterns)
            if relpath_to_abspath[relpath] != ".gitattributes"
        ]


def _prepare_upload_folder_additions(
    folder_path: Union[str, Path],
    path_in_repo: str,
    allow_patterns: Optional[Union[List[str], str]] = None,
    ignore_patterns: Optional[Union[List[str], str]] = None,
) -> List[CommitOperationAdd]:
    """Generate the list of Add operations for a commit to upload a folder.

    Files not matching the `allow_patterns` (allowlist) and `ignore_patterns` (denylist)
    constraints are discarded.
    """
    folder_path = Path(folder_path).expanduser().resolve()
    if not folder_path.is_dir():
        raise ValueError(f"Provided path: '{folder_path}' is not a directory")

    # List files from folder
    relpath_to_abspath = {
        path.relative_to(folder_path).as_posix(): path
        for path in sorted(folder_path.glob("**/*"))  # sorted to be deterministic
        if path.is_file()
    }

    # Filter files and return
    # Patterns are applied on the path relative to `folder_path`. `path_in_repo` is prefixed after the filtering.
    prefix = f"{path_in_repo.strip('/')}/" if path_in_repo else ""
    return [
        CommitOperationAdd(
            path_or_fileobj=relpath_to_abspath[relpath],  # absolute path on disk
            path_in_repo=prefix + relpath,  # "absolute" path in repo
        )
        for relpath in filter_repo_objects(
            relpath_to_abspath.keys(), allow_patterns=allow_patterns, ignore_patterns=ignore_patterns
        )
    ]


def _parse_revision_from_pr_url(pr_url: str) -> str:
    """Safely parse revision number from a PR url.

    Example:
    ```py
    >>> _parse_revision_from_pr_url("https://huggingface.co/bigscience/bloom/discussions/2")
    "refs/pr/2"
    ```
    """
    re_match = re.match(_REGEX_DISCUSSION_URL, pr_url)
    if re_match is None:
        raise RuntimeError(f"Unexpected response from the hub, expected a Pull Request URL but got: '{pr_url}'")
    return f"refs/pr/{re_match[1]}"


api = HfApi()

whoami = api.whoami
get_token_permission = api.get_token_permission

list_models = api.list_models
model_info = api.model_info

list_datasets = api.list_datasets
dataset_info = api.dataset_info

list_spaces = api.list_spaces
space_info = api.space_info

repo_info = api.repo_info
list_repo_files = api.list_repo_files
list_repo_refs = api.list_repo_refs
list_repo_commits = api.list_repo_commits
list_files_info = api.list_files_info

list_metrics = api.list_metrics

get_model_tags = api.get_model_tags
get_dataset_tags = api.get_dataset_tags

create_commit = api.create_commit
create_repo = api.create_repo
delete_repo = api.delete_repo
update_repo_visibility = api.update_repo_visibility
move_repo = api.move_repo
upload_file = api.upload_file
upload_folder = api.upload_folder
delete_file = api.delete_file
delete_folder = api.delete_folder
create_commits_on_pr = api.create_commits_on_pr
create_branch = api.create_branch
delete_branch = api.delete_branch
create_tag = api.create_tag
delete_tag = api.delete_tag
get_full_repo_name = api.get_full_repo_name

# Background jobs
run_as_future = api.run_as_future

# Activity API
list_liked_repos = api.list_liked_repos
like = api.like
unlike = api.unlike

# Community API
get_discussion_details = api.get_discussion_details
get_repo_discussions = api.get_repo_discussions
create_discussion = api.create_discussion
create_pull_request = api.create_pull_request
change_discussion_status = api.change_discussion_status
comment_discussion = api.comment_discussion
edit_discussion_comment = api.edit_discussion_comment
rename_discussion = api.rename_discussion
merge_pull_request = api.merge_pull_request

# Space API
add_space_secret = api.add_space_secret
delete_space_secret = api.delete_space_secret
get_space_runtime = api.get_space_runtime
request_space_hardware = api.request_space_hardware
set_space_sleep_time = api.set_space_sleep_time
pause_space = api.pause_space
restart_space = api.restart_space
duplicate_space = api.duplicate_space


# src/huggingface_hub/hf_file_system.py

import itertools
import os
import tempfile
from dataclasses import dataclass
from datetime import datetime
from glob import has_magic
from typing import Any, Dict, List, Optional, Tuple, Union
from urllib.parse import quote, unquote

import fsspec

from ._commit_api import CommitOperationCopy, CommitOperationDelete
from .constants import DEFAULT_REVISION, ENDPOINT, REPO_TYPE_MODEL, REPO_TYPES_MAPPING, REPO_TYPES_URL_PREFIXES
from .hf_api import HfApi
from .utils import (
    EntryNotFoundError,
    HFValidationError,
    RepositoryNotFoundError,
    RevisionNotFoundError,
    hf_raise_for_status,
    http_backoff,
    paginate,
    parse_datetime,
)


@dataclass
class HfFileSystemResolvedPath:
    """Data structure containing information about a resolved Hugging Face file system path."""

    repo_type: str
    repo_id: str
    revision: str
    path_in_repo: str

    def unresolve(self) -> str:
        return (
            f"{REPO_TYPES_URL_PREFIXES.get(self.repo_type, '') + self.repo_id}@{safe_quote(self.revision)}/{self.path_in_repo}"
            .rstrip("/")
        )


class HfFileSystem(fsspec.AbstractFileSystem):
    """
    Access a remote Hugging Face Hub repository as if were a local file system.

    Args:
        endpoint (`str`, *optional*):
            The endpoint to use. If not provided, the default one (https://huggingface.co) is used.
        token (`str`, *optional*):
            Authentication token, obtained with [`HfApi.login`] method. Will default to the stored token.

    Usage:

    ```python
    >>> from huggingface_hub import HfFileSystem

    >>> fs = HfFileSystem()

    >>> # List files
    >>> fs.glob("my-username/my-model/*.bin")
    ['my-username/my-model/pytorch_model.bin']
    >>> fs.ls("datasets/my-username/my-dataset", detail=False)
    ['datasets/my-username/my-dataset/.gitattributes', 'datasets/my-username/my-dataset/README.md', 'datasets/my-username/my-dataset/data.json']

    >>> # Read/write files
    >>> with fs.open("my-username/my-model/pytorch_model.bin") as f:
    ...     data = f.read()
    >>> with fs.open("my-username/my-model/pytorch_model.bin", "wb") as f:
    ...     f.write(data)
    ```
    """

    root_marker = ""
    protocol = "hf"

    def __init__(
        self,
        *args,
        endpoint: Optional[str] = None,
        token: Optional[str] = None,
        **storage_options,
    ):
        super().__init__(*args, **storage_options)
        self.endpoint = endpoint or ENDPOINT
        self.token = token
        self._api = HfApi(endpoint=endpoint, token=token)
        # Maps (repo_type, repo_id, revision) to a 2-tuple with:
        #  * the 1st element indicating whether the repositoy and the revision exist
        #  * the 2nd element being the exception raised if the repository or revision doesn't exist
        self._repo_and_revision_exists_cache: Dict[
            Tuple[str, str, Optional[str]], Tuple[bool, Optional[Exception]]
        ] = {}

    def _repo_and_revision_exist(
        self, repo_type: str, repo_id: str, revision: Optional[str]
    ) -> Tuple[bool, Optional[Exception]]:
        if (repo_type, repo_id, revision) not in self._repo_and_revision_exists_cache:
            try:
                self._api.repo_info(repo_id, revision=revision, repo_type=repo_type)
            except (RepositoryNotFoundError, HFValidationError) as e:
                self._repo_and_revision_exists_cache[(repo_type, repo_id, revision)] = False, e
                self._repo_and_revision_exists_cache[(repo_type, repo_id, None)] = False, e
            except RevisionNotFoundError as e:
                self._repo_and_revision_exists_cache[(repo_type, repo_id, revision)] = False, e
                self._repo_and_revision_exists_cache[(repo_type, repo_id, None)] = True, None
            else:
                self._repo_and_revision_exists_cache[(repo_type, repo_id, revision)] = True, None
                self._repo_and_revision_exists_cache[(repo_type, repo_id, None)] = True, None
        return self._repo_and_revision_exists_cache[(repo_type, repo_id, revision)]

    def resolve_path(self, path: str, revision: Optional[str] = None) -> HfFileSystemResolvedPath:
        def _align_revision_in_path_with_revision(
            revision_in_path: Optional[str], revision: Optional[str]
        ) -> Optional[str]:
            if revision is not None:
                if revision_in_path is not None and revision_in_path != revision:
                    raise ValueError(
                        f'Revision specified in path ("{revision_in_path}") and in `revision` argument ("{revision}")'
                        " are not the same."
                    )
            else:
                revision = revision_in_path
            return revision

        path = self._strip_protocol(path)
        if not path:
            # can't list repositories at root
            raise NotImplementedError("Access to repositories lists is not implemented.")
        elif path.split("/")[0] + "/" in REPO_TYPES_URL_PREFIXES.values():
            if "/" not in path:
                # can't list repositories at the repository type level
                raise NotImplementedError("Acces to repositories lists is not implemented.")
            repo_type, path = path.split("/", 1)
            repo_type = REPO_TYPES_MAPPING[repo_type]
        else:
            repo_type = REPO_TYPE_MODEL
        if path.count("/") > 0:
            if "@" in path:
                repo_id, revision_in_path = path.split("@", 1)
                if "/" in revision_in_path:
                    revision_in_path, path_in_repo = revision_in_path.split("/", 1)
                else:
                    path_in_repo = ""
                revision_in_path = unquote(revision_in_path)
                revision = _align_revision_in_path_with_revision(revision_in_path, revision)
                repo_and_revision_exist, err = self._repo_and_revision_exist(repo_type, repo_id, revision)
                if not repo_and_revision_exist:
                    raise FileNotFoundError(path) from err
            else:
                repo_id_with_namespace = "/".join(path.split("/")[:2])
                path_in_repo_with_namespace = "/".join(path.split("/")[2:])
                repo_id_without_namespace = path.split("/")[0]
                path_in_repo_without_namespace = "/".join(path.split("/")[1:])
                repo_id = repo_id_with_namespace
                path_in_repo = path_in_repo_with_namespace
                repo_and_revision_exist, err = self._repo_and_revision_exist(repo_type, repo_id, revision)
                if not repo_and_revision_exist:
                    if isinstance(err, (RepositoryNotFoundError, HFValidationError)):
                        repo_id = repo_id_without_namespace
                        path_in_repo = path_in_repo_without_namespace
                        repo_and_revision_exist, _ = self._repo_and_revision_exist(repo_type, repo_id, revision)
                        if not repo_and_revision_exist:
                            raise FileNotFoundError(path) from err
                    else:
                        raise FileNotFoundError(path) from err
        else:
            repo_id = path
            path_in_repo = ""
            if "@" in path:
                repo_id, revision_in_path = path.split("@", 1)
                revision_in_path = unquote(revision_in_path)
                revision = _align_revision_in_path_with_revision(revision_in_path, revision)
            repo_and_revision_exist, _ = self._repo_and_revision_exist(repo_type, repo_id, revision)
            if not repo_and_revision_exist:
                raise NotImplementedError("Acces to repositories lists is not implemented.")

        revision = revision if revision is not None else DEFAULT_REVISION
        return HfFileSystemResolvedPath(repo_type, repo_id, revision, path_in_repo)

    def invalidate_cache(self, path: Optional[str] = None) -> None:
        if not path:
            self.dircache.clear()
            self._repository_type_and_id_exists_cache.clear()
        else:
            path = self.resolve_path(path).unresolve()
            while path:
                self.dircache.pop(path, None)
                path = self._parent(path)

    def _open(
        self,
        path: str,
        mode: str = "rb",
        revision: Optional[str] = None,
        **kwargs,
    ) -> "HfFileSystemFile":
        if mode == "ab":
            raise NotImplementedError("Appending to remote files is not yet supported.")
        return HfFileSystemFile(self, path, mode=mode, revision=revision, **kwargs)

    def _rm(self, path: str, revision: Optional[str] = None, **kwargs) -> None:
        resolved_path = self.resolve_path(path, revision=revision)
        self._api.delete_file(
            path_in_repo=resolved_path.path_in_repo,
            repo_id=resolved_path.repo_id,
            token=self.token,
            repo_type=resolved_path.repo_type,
            revision=resolved_path.revision,
            commit_message=kwargs.get("commit_message"),
            commit_description=kwargs.get("commit_description"),
        )
        self.invalidate_cache(path=resolved_path.unresolve())

    def rm(
        self,
        path: str,
        recursive: bool = False,
        maxdepth: Optional[int] = None,
        revision: Optional[str] = None,
        **kwargs,
    ) -> None:
        resolved_path = self.resolve_path(path, revision=revision)
        root_path = REPO_TYPES_URL_PREFIXES.get(resolved_path.repo_type, "") + resolved_path.repo_id
        paths = self.expand_path(path, recursive=recursive, maxdepth=maxdepth, revision=resolved_path.revision)
        paths_in_repo = [path[len(root_path) + 1 :] for path in paths if not self.isdir(path)]
        operations = [CommitOperationDelete(path_in_repo=path_in_repo) for path_in_repo in paths_in_repo]
        commit_message = f"Delete {path} "
        commit_message += "recursively " if recursive else ""
        commit_message += f"up to depth {maxdepth} " if maxdepth is not None else ""
        # TODO: use `commit_description` to list all the deleted paths?
        self._api.create_commit(
            repo_id=resolved_path.repo_id,
            repo_type=resolved_path.repo_type,
            token=self.token,
            operations=operations,
            revision=resolved_path.revision,
            commit_message=kwargs.get("commit_message", commit_message),
            commit_description=kwargs.get("commit_description"),
        )
        self.invalidate_cache(path=resolved_path.unresolve())

    def ls(
        self, path: str, detail: bool = True, refresh: bool = False, revision: Optional[str] = None, **kwargs
    ) -> List[Union[str, Dict[str, Any]]]:
        """List the contents of a directory."""
        resolved_path = self.resolve_path(path, revision=revision)
        revision_in_path = "@" + safe_quote(resolved_path.revision)
        has_revision_in_path = revision_in_path in path
        path = resolved_path.unresolve()
        if path not in self.dircache or refresh:
            path_prefix = (
                HfFileSystemResolvedPath(
                    resolved_path.repo_type, resolved_path.repo_id, resolved_path.revision, ""
                ).unresolve()
                + "/"
            )
            tree_path = path
            tree_iter = self._iter_tree(tree_path, revision=resolved_path.revision)
            try:
                tree_item = next(tree_iter)
            except EntryNotFoundError:
                if "/" in resolved_path.path_in_repo:
                    tree_path = self._parent(path)
                    tree_iter = self._iter_tree(tree_path, revision=resolved_path.revision)
                else:
                    raise
            else:
                tree_iter = itertools.chain([tree_item], tree_iter)
            child_infos = []
            for tree_item in tree_iter:
                child_info = {
                    "name": path_prefix + tree_item["path"],
                    "size": tree_item["size"],
                    "type": tree_item["type"],
                }
                if tree_item["type"] == "file":
                    child_info.update(
                        {
                            "blob_id": tree_item["oid"],
                            "lfs": tree_item.get("lfs"),
                            "last_modified": parse_datetime(tree_item["lastCommit"]["date"]),
                        },
                    )
                child_infos.append(child_info)
            self.dircache[tree_path] = child_infos
        out = self._ls_from_cache(path)
        if not has_revision_in_path:
            out = [{**o, "name": o["name"].replace(revision_in_path, "", 1)} for o in out]
        return out if detail else [o["name"] for o in out]

    def _iter_tree(self, path: str, revision: Optional[str] = None):
        # TODO: use HfApi.list_files_info instead when it supports "lastCommit" and "expand=True"
        # See https://github.com/huggingface/moon-landing/issues/5993
        resolved_path = self.resolve_path(path, revision=revision)
        path = f"{self._api.endpoint}/api/{resolved_path.repo_type}s/{resolved_path.repo_id}/tree/{safe_quote(resolved_path.revision)}/{resolved_path.path_in_repo}".rstrip(
            "/"
        )
        headers = self._api._build_hf_headers()
        yield from paginate(path, params={"expand": True}, headers=headers)

    def cp_file(self, path1: str, path2: str, revision: Optional[str] = None, **kwargs) -> None:
        resolved_path1 = self.resolve_path(path1, revision=revision)
        resolved_path2 = self.resolve_path(path2, revision=revision)

        same_repo = (
            resolved_path1.repo_type == resolved_path2.repo_type and resolved_path1.repo_id == resolved_path2.repo_id
        )

        # TODO: Wait for https://github.com/huggingface/huggingface_hub/issues/1083 to be resolved to simplify this logic
        if same_repo and self.info(path1, revision=resolved_path1.revision)["lfs"] is not None:
            commit_message = f"Copy {path1} to {path2}"
            self._api.create_commit(
                repo_id=resolved_path1.repo_id,
                repo_type=resolved_path1.repo_type,
                revision=resolved_path2.revision,
                commit_message=kwargs.get("commit_message", commit_message),
                commit_description=kwargs.get("commit_description", ""),
                operations=[
                    CommitOperationCopy(
                        src_path_in_repo=resolved_path1.path_in_repo,
                        path_in_repo=resolved_path2.path_in_repo,
                        src_revision=resolved_path1.revision,
                    )
                ],
            )
        else:
            with self.open(path1, "rb", revision=resolved_path1.revision) as f:
                content = f.read()
            commit_message = f"Copy {path1} to {path2}"
            self._api.upload_file(
                path_or_fileobj=content,
                path_in_repo=resolved_path2.path_in_repo,
                repo_id=resolved_path2.repo_id,
                token=self.token,
                repo_type=resolved_path2.repo_type,
                revision=resolved_path2.revision,
                commit_message=kwargs.get("commit_message", commit_message),
                commit_description=kwargs.get("commit_description"),
            )
        self.invalidate_cache(path=resolved_path1.unresolve())
        self.invalidate_cache(path=resolved_path2.unresolve())

    def modified(self, path: str, **kwargs) -> datetime:
        info = self.info(path, **kwargs)
        if "last_modified" not in info:
            raise IsADirectoryError(path)
        return info["last_modified"]

    def info(self, path: str, **kwargs) -> Dict[str, Any]:
        resolved_path = self.resolve_path(path)
        if not resolved_path.path_in_repo:
            revision_in_path = "@" + safe_quote(resolved_path.revision)
            has_revision_in_path = revision_in_path in path
            name = resolved_path.unresolve()
            name = name.replace(revision_in_path, "", 1) if not has_revision_in_path else name
            return {"name": name, "size": 0, "type": "directory"}
        return super().info(path, **kwargs)

    def expand_path(
        self, path: Union[str, List[str]], recursive: bool = False, maxdepth: Optional[int] = None, **kwargs
    ) -> List[str]:
        # The default implementation does not allow passing custom kwargs (e.g., we use these kwargs to propagate the `revision`)
        if maxdepth is not None and maxdepth < 1:
            raise ValueError("maxdepth must be at least 1")

        if isinstance(path, str):
            return self.expand_path([path], recursive, maxdepth)

        out = set()
        path = [self._strip_protocol(p) for p in path]
        for p in path:
            if has_magic(p):
                bit = set(self.glob(p, **kwargs))
                out |= bit
                if recursive:
                    out |= set(self.expand_path(list(bit), recursive=recursive, maxdepth=maxdepth, **kwargs))
                continue
            elif recursive:
                rec = set(self.find(p, maxdepth=maxdepth, withdirs=True, detail=False, **kwargs))
                out |= rec
            if p not in out and (recursive is False or self.exists(p)):
                # should only check once, for the root
                out.add(p)
        if not out:
            raise FileNotFoundError(path)
        return list(sorted(out))


class HfFileSystemFile(fsspec.spec.AbstractBufferedFile):
    def __init__(self, fs: HfFileSystem, path: str, revision: Optional[str] = None, **kwargs):
        super().__init__(fs, path, **kwargs)
        self.fs: HfFileSystem
        self.resolved_path = fs.resolve_path(path, revision=revision)

    def _fetch_range(self, start: int, end: int) -> bytes:
        headers = {
            "range": f"bytes={start}-{end - 1}",
            **self.fs._api._build_hf_headers(),
        }
        url = (
            f"{self.fs.endpoint}/{REPO_TYPES_URL_PREFIXES.get(self.resolved_path.repo_type, '') + self.resolved_path.repo_id}/resolve/{safe_quote(self.resolved_path.revision)}/{safe_quote(self.resolved_path.path_in_repo)}"
        )
        r = http_backoff("GET", url, headers=headers)
        hf_raise_for_status(r)
        return r.content

    def _initiate_upload(self) -> None:
        self.temp_file = tempfile.NamedTemporaryFile(prefix="hffs-", delete=False)

    def _upload_chunk(self, final: bool = False) -> None:
        self.buffer.seek(0)
        block = self.buffer.read()
        self.temp_file.write(block)
        if final:
            self.temp_file.close()
            self.fs._api.upload_file(
                path_or_fileobj=self.temp_file.name,
                path_in_repo=self.resolved_path.path_in_repo,
                repo_id=self.resolved_path.repo_id,
                token=self.fs.token,
                repo_type=self.resolved_path.repo_type,
                revision=self.resolved_path.revision,
                commit_message=self.kwargs.get("commit_message"),
                commit_description=self.kwargs.get("commit_description"),
            )
            os.remove(self.temp_file.name)
            self.fs.invalidate_cache(
                path=self.resolved_path.unresolve(),
            )


def safe_quote(s: str) -> str:
    return quote(s, safe="")


# src/huggingface_hub/hub_mixin.py

import json
import os
from pathlib import Path
from typing import Dict, List, Optional, Type, TypeVar, Union

import requests

from .constants import CONFIG_NAME, PYTORCH_WEIGHTS_NAME
from .file_download import hf_hub_download, is_torch_available
from .hf_api import HfApi
from .utils import SoftTemporaryDirectory, logging, validate_hf_hub_args


if is_torch_available():
    import torch  # type: ignore

logger = logging.get_logger(__name__)

# Generic variable that is either ModelHubMixin or a subclass thereof
T = TypeVar("T", bound="ModelHubMixin")


class ModelHubMixin:
    """
    A generic mixin to integrate ANY machine learning framework with the Hub.

    To integrate your framework, your model class must inherit from this class. Custom logic for saving/loading models
    have to be overwritten in  [`_from_pretrained`] and [`_save_pretrained`]. [`PyTorchModelHubMixin`] is a good example
    of mixin integration with the Hub. Check out our [integration guide](../guides/integrations) for more instructions.
    """

    def save_pretrained(
        self,
        save_directory: Union[str, Path],
        *,
        config: Optional[dict] = None,
        repo_id: Optional[str] = None,
        push_to_hub: bool = False,
        **kwargs,
    ) -> Optional[str]:
        """
        Save weights in local directory.

        Args:
            save_directory (`str` or `Path`):
                Path to directory in which the model weights and configuration will be saved.
            config (`dict`, *optional*):
                Model configuration specified as a key/value dictionary.
            push_to_hub (`bool`, *optional*, defaults to `False`):
                Whether or not to push your model to the Huggingface Hub after saving it.
            repo_id (`str`, *optional*):
                ID of your repository on the Hub. Used only if `push_to_hub=True`. Will default to the folder name if
                not provided.
            kwargs:
                Additional key word arguments passed along to the [`~ModelHubMixin._from_pretrained`] method.
        """
        save_directory = Path(save_directory)
        save_directory.mkdir(parents=True, exist_ok=True)

        # saving model weights/files
        self._save_pretrained(save_directory)

        # saving config
        if isinstance(config, dict):
            (save_directory / CONFIG_NAME).write_text(json.dumps(config))

        if push_to_hub:
            kwargs = kwargs.copy()  # soft-copy to avoid mutating input
            if config is not None:  # kwarg for `push_to_hub`
                kwargs["config"] = config
            if repo_id is None:
                repo_id = save_directory.name  # Defaults to `save_directory` name
            return self.push_to_hub(repo_id=repo_id, **kwargs)
        return None

    def _save_pretrained(self, save_directory: Path) -> None:
        """
        Overwrite this method in subclass to define how to save your model.
        Check out our [integration guide](../guides/integrations) for instructions.

        Args:
            save_directory (`str` or `Path`):
                Path to directory in which the model weights and configuration will be saved.
        """
        raise NotImplementedError

    @classmethod
    @validate_hf_hub_args
    def from_pretrained(
        cls: Type[T],
        pretrained_model_name_or_path: Union[str, Path],
        *,
        force_download: bool = False,
        resume_download: bool = False,
        proxies: Optional[Dict] = None,
        token: Optional[Union[str, bool]] = None,
        cache_dir: Optional[Union[str, Path]] = None,
        local_files_only: bool = False,
        revision: Optional[str] = None,
        **model_kwargs,
    ) -> T:
        """
        Download a model from the Huggingface Hub and instantiate it.

        Args:
            pretrained_model_name_or_path (`str`, `Path`):
                - Either the `model_id` (string) of a model hosted on the Hub, e.g. `bigscience/bloom`.
                - Or a path to a `directory` containing model weights saved using
                    [`~transformers.PreTrainedModel.save_pretrained`], e.g., `../path/to/my_model_directory/`.
            revision (`str`, *optional*):
                Revision of the model on the Hub. Can be a branch name, a git tag or any commit id.
                Defaults to the latest commit on `main` branch.
            force_download (`bool`, *optional*, defaults to `False`):
                Whether to force (re-)downloading the model weights and configuration files from the Hub, overriding
                the existing cache.
            resume_download (`bool`, *optional*, defaults to `False`):
                Whether to delete incompletely received files. Will attempt to resume the download if such a file exists.
            proxies (`Dict[str, str]`, *optional*):
                A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
                'http://hostname': 'foo.bar:4012'}`. The proxies are used on every request.
            token (`str` or `bool`, *optional*):
                The token to use as HTTP bearer authorization for remote files. By default, it will use the token
                cached when running `huggingface-cli login`.
            cache_dir (`str`, `Path`, *optional*):
                Path to the folder where cached files are stored.
            local_files_only (`bool`, *optional*, defaults to `False`):
                If `True`, avoid downloading the file and return the path to the local cached file if it exists.
            model_kwargs (`Dict`, *optional*):
                Additional kwargs to pass to the model during initialization.
        """
        model_id = pretrained_model_name_or_path
        config_file: Optional[str] = None
        if os.path.isdir(model_id):
            if CONFIG_NAME in os.listdir(model_id):
                config_file = os.path.join(model_id, CONFIG_NAME)
            else:
                logger.warning(f"{CONFIG_NAME} not found in {Path(model_id).resolve()}")
        elif isinstance(model_id, str):
            try:
                config_file = hf_hub_download(
                    repo_id=str(model_id),
                    filename=CONFIG_NAME,
                    revision=revision,
                    cache_dir=cache_dir,
                    force_download=force_download,
                    proxies=proxies,
                    resume_download=resume_download,
                    token=token,
                    local_files_only=local_files_only,
                )
            except requests.exceptions.RequestException:
                logger.warning(f"{CONFIG_NAME} not found in HuggingFace Hub.")

        if config_file is not None:
            with open(config_file, "r", encoding="utf-8") as f:
                config = json.load(f)
            model_kwargs.update({"config": config})

        return cls._from_pretrained(
            model_id=str(model_id),
            revision=revision,
            cache_dir=cache_dir,
            force_download=force_download,
            proxies=proxies,
            resume_download=resume_download,
            local_files_only=local_files_only,
            token=token,
            **model_kwargs,
        )

    @classmethod
    def _from_pretrained(
        cls: Type[T],
        *,
        model_id: str,
        revision: Optional[str],
        cache_dir: Optional[Union[str, Path]],
        force_download: bool,
        proxies: Optional[Dict],
        resume_download: bool,
        local_files_only: bool,
        token: Optional[Union[str, bool]],
        **model_kwargs,
    ) -> T:
        """Overwrite this method in subclass to define how to load your model from pretrained.

        Use [`hf_hub_download`] or [`snapshot_download`] to download files from the Hub before loading them. Most
        args taken as input can be directly passed to those 2 methods. If needed, you can add more arguments to this
        method using "model_kwargs". For example [`PyTorchModelHubMixin._from_pretrained`] takes as input a `map_location`
        parameter to set on which device the model should be loaded.

        Check out our [integration guide](../guides/integrations) for more instructions.

        Args:
            model_id (`str`):
                ID of the model to load from the Huggingface Hub (e.g. `bigscience/bloom`).
            revision (`str`, *optional*):
                Revision of the model on the Hub. Can be a branch name, a git tag or any commit id. Defaults to the
                latest commit on `main` branch.
            force_download (`bool`, *optional*, defaults to `False`):
                Whether to force (re-)downloading the model weights and configuration files from the Hub, overriding
                the existing cache.
            resume_download (`bool`, *optional*, defaults to `False`):
                Whether to delete incompletely received files. Will attempt to resume the download if such a file exists.
            proxies (`Dict[str, str]`, *optional*):
                A dictionary of proxy servers to use by protocol or endpoint (e.g., `{'http': 'foo.bar:3128',
                'http://hostname': 'foo.bar:4012'}`).
            token (`str` or `bool`, *optional*):
                The token to use as HTTP bearer authorization for remote files. By default, it will use the token
                cached when running `huggingface-cli login`.
            cache_dir (`str`, `Path`, *optional*):
                Path to the folder where cached files are stored.
            local_files_only (`bool`, *optional*, defaults to `False`):
                If `True`, avoid downloading the file and return the path to the local cached file if it exists.
            model_kwargs:
                Additional keyword arguments passed along to the [`~ModelHubMixin._from_pretrained`] method.
        """
        raise NotImplementedError

    @validate_hf_hub_args
    def push_to_hub(
        self,
        repo_id: str,
        *,
        config: Optional[dict] = None,
        commit_message: str = "Push model using huggingface_hub.",
        private: bool = False,
        api_endpoint: Optional[str] = None,
        token: Optional[str] = None,
        branch: Optional[str] = None,
        create_pr: Optional[bool] = None,
        allow_patterns: Optional[Union[List[str], str]] = None,
        ignore_patterns: Optional[Union[List[str], str]] = None,
        delete_patterns: Optional[Union[List[str], str]] = None,
    ) -> str:
        """
        Upload model checkpoint to the Hub.

        Use `allow_patterns` and `ignore_patterns` to precisely filter which files should be pushed to the hub. Use
        `delete_patterns` to delete existing remote files in the same commit. See [`upload_folder`] reference for more
        details.


        Args:
            repo_id (`str`):
                ID of the repository to push to (example: `"username/my-model"`).
            config (`dict`, *optional*):
                Configuration object to be saved alongside the model weights.
            commit_message (`str`, *optional*):
                Message to commit while pushing.
            private (`bool`, *optional*, defaults to `False`):
                Whether the repository created should be private.
            api_endpoint (`str`, *optional*):
                The API endpoint to use when pushing the model to the hub.
            token (`str`, *optional*):
                The token to use as HTTP bearer authorization for remote files. By default, it will use the token
                cached when running `huggingface-cli login`.
            branch (`str`, *optional*):
                The git branch on which to push the model. This defaults to `"main"`.
            create_pr (`boolean`, *optional*):
                Whether or not to create a Pull Request from `branch` with that commit. Defaults to `False`.
            allow_patterns (`List[str]` or `str`, *optional*):
                If provided, only files matching at least one pattern are pushed.
            ignore_patterns (`List[str]` or `str`, *optional*):
                If provided, files matching any of the patterns are not pushed.
            delete_patterns (`List[str]` or `str`, *optional*):
                If provided, remote files matching any of the patterns will be deleted from the repo.

        Returns:
            The url of the commit of your model in the given repository.
        """
        api = HfApi(endpoint=api_endpoint, token=token)
        repo_id = api.create_repo(repo_id=repo_id, private=private, exist_ok=True).repo_id

        # Push the files to the repo in a single commit
        with SoftTemporaryDirectory() as tmp:
            saved_path = Path(tmp) / repo_id
            self.save_pretrained(saved_path, config=config)
            return api.upload_folder(
                repo_id=repo_id,
                repo_type="model",
                folder_path=saved_path,
                commit_message=commit_message,
                revision=branch,
                create_pr=create_pr,
                allow_patterns=allow_patterns,
                ignore_patterns=ignore_patterns,
                delete_patterns=delete_patterns,
            )


class PyTorchModelHubMixin(ModelHubMixin):
    """
    Implementation of [`ModelHubMixin`] to provide model Hub upload/download capabilities to PyTorch models. The model
    is set in evaluation mode by default using `model.eval()` (dropout modules are deactivated). To train the model,
    you should first set it back in training mode with `model.train()`.

    Example:

    ```python
    >>> import torch
    >>> import torch.nn as nn
    >>> from huggingface_hub import PyTorchModelHubMixin


    >>> class MyModel(nn.Module, PyTorchModelHubMixin):
    ...     def __init__(self):
    ...         super().__init__()
    ...         self.param = nn.Parameter(torch.rand(3, 4))
    ...         self.linear = nn.Linear(4, 5)

    ...     def forward(self, x):
    ...         return self.linear(x + self.param)
    >>> model = MyModel()

    # Save model weights to local directory
    >>> model.save_pretrained("my-awesome-model")

    # Push model weights to the Hub
    >>> model.push_to_hub("my-awesome-model")

    # Download and initialize weights from the Hub
    >>> model = MyModel.from_pretrained("username/my-awesome-model")
    ```
    """

    def _save_pretrained(self, save_directory: Path) -> None:
        """Save weights from a Pytorch model to a local directory."""
        model_to_save = self.module if hasattr(self, "module") else self  # type: ignore
        torch.save(model_to_save.state_dict(), save_directory / PYTORCH_WEIGHTS_NAME)

    @classmethod
    def _from_pretrained(
        cls,
        *,
        model_id: str,
        revision: Optional[str],
        cache_dir: Optional[Union[str, Path]],
        force_download: bool,
        proxies: Optional[Dict],
        resume_download: bool,
        local_files_only: bool,
        token: Union[str, bool, None],
        map_location: str = "cpu",
        strict: bool = False,
        **model_kwargs,
    ):
        """Load Pytorch pretrained weights and return the loaded model."""
        if os.path.isdir(model_id):
            print("Loading weights from local directory")
            model_file = os.path.join(model_id, PYTORCH_WEIGHTS_NAME)
        else:
            model_file = hf_hub_download(
                repo_id=model_id,
                filename=PYTORCH_WEIGHTS_NAME,
                revision=revision,
                cache_dir=cache_dir,
                force_download=force_download,
                proxies=proxies,
                resume_download=resume_download,
                token=token,
                local_files_only=local_files_only,
            )
        model = cls(**model_kwargs)

        state_dict = torch.load(model_file, map_location=torch.device(map_location))
        model.load_state_dict(state_dict, strict=strict)  # type: ignore
        model.eval()  # type: ignore

        return model


# src/huggingface_hub/inference_api.py

import io
from typing import Any, Dict, List, Optional, Union

from .constants import INFERENCE_ENDPOINT
from .hf_api import HfApi
from .utils import build_hf_headers, get_session, is_pillow_available, logging, validate_hf_hub_args


logger = logging.get_logger(__name__)


ALL_TASKS = [
    # NLP
    "text-classification",
    "token-classification",
    "table-question-answering",
    "question-answering",
    "zero-shot-classification",
    "translation",
    "summarization",
    "conversational",
    "feature-extraction",
    "text-generation",
    "text2text-generation",
    "fill-mask",
    "sentence-similarity",
    # Audio
    "text-to-speech",
    "automatic-speech-recognition",
    "audio-to-audio",
    "audio-classification",
    "voice-activity-detection",
    # Computer vision
    "image-classification",
    "object-detection",
    "image-segmentation",
    "text-to-image",
    "image-to-image",
    # Others
    "tabular-classification",
    "tabular-regression",
]


class InferenceApi:
    """Client to configure requests and make calls to the HuggingFace Inference API.

    Example:

    ```python
    >>> from huggingface_hub.inference_api import InferenceApi

    >>> # Mask-fill example
    >>> inference = InferenceApi("bert-base-uncased")
    >>> inference(inputs="The goal of life is [MASK].")
    [{'sequence': 'the goal of life is life.', 'score': 0.10933292657136917, 'token': 2166, 'token_str': 'life'}]

    >>> # Question Answering example
    >>> inference = InferenceApi("deepset/roberta-base-squad2")
    >>> inputs = {
    ...     "question": "What's my name?",
    ...     "context": "My name is Clara and I live in Berkeley.",
    ... }
    >>> inference(inputs)
    {'score': 0.9326569437980652, 'start': 11, 'end': 16, 'answer': 'Clara'}

    >>> # Zero-shot example
    >>> inference = InferenceApi("typeform/distilbert-base-uncased-mnli")
    >>> inputs = "Hi, I recently bought a device from your company but it is not working as advertised and I would like to get reimbursed!"
    >>> params = {"candidate_labels": ["refund", "legal", "faq"]}
    >>> inference(inputs, params)
    {'sequence': 'Hi, I recently bought a device from your company but it is not working as advertised and I would like to get reimbursed!', 'labels': ['refund', 'faq', 'legal'], 'scores': [0.9378499388694763, 0.04914155602455139, 0.013008488342165947]}

    >>> # Overriding configured task
    >>> inference = InferenceApi("bert-base-uncased", task="feature-extraction")

    >>> # Text-to-image
    >>> inference = InferenceApi("stabilityai/stable-diffusion-2-1")
    >>> inference("cat")
    <PIL.PngImagePlugin.PngImageFile image (...)>

    >>> # Return as raw response to parse the output yourself
    >>> inference = InferenceApi("mio/amadeus")
    >>> response = inference("hello world", raw_response=True)
    >>> response.headers
    {"Content-Type": "audio/flac", ...}
    >>> response.content # raw bytes from server
    b'(...)'
    ```
    """

    @validate_hf_hub_args
    # TODO: add deprecation starting from version v0.16.0 so that we can proactively adapt external scripts.
    # @_deprecate_method(
    #     version="0.18.0",
    #     message=(
    #         "`InferenceApi` client is deprecated in favor of the more feature-complete `InferenceClient`. Check out"
    #         " this guide to learn how to convert your script to use it:"
    #         " https://huggingface.co/docs/huggingface_hub/guides/inference#legacy-inferenceapi-client."
    #     ),
    # )
    def __init__(
        self,
        repo_id: str,
        task: Optional[str] = None,
        token: Optional[str] = None,
        gpu: bool = False,
    ):
        """Inits headers and API call information.

        Args:
            repo_id (``str``):
                Id of repository (e.g. `user/bert-base-uncased`).
            task (``str``, `optional`, defaults ``None``):
                Whether to force a task instead of using task specified in the
                repository.
            token (`str`, `optional`):
                The API token to use as HTTP bearer authorization. This is not
                the authentication token. You can find the token in
                https://huggingface.co/settings/token. Alternatively, you can
                find both your organizations and personal API tokens using
                `HfApi().whoami(token)`.
            gpu (`bool`, `optional`, defaults `False`):
                Whether to use GPU instead of CPU for inference(requires Startup
                plan at least).
        """
        self.options = {"wait_for_model": True, "use_gpu": gpu}
        self.headers = build_hf_headers(token=token)

        # Configure task
        model_info = HfApi(token=token).model_info(repo_id=repo_id)
        if not model_info.pipeline_tag and not task:
            raise ValueError(
                "Task not specified in the repository. Please add it to the model card"
                " using pipeline_tag"
                " (https://huggingface.co/docs#how-is-a-models-type-of-inference-api-and-widget-determined)"
            )

        if task and task != model_info.pipeline_tag:
            if task not in ALL_TASKS:
                raise ValueError(f"Invalid task {task}. Make sure it's valid.")

            logger.warning(
                "You're using a different task than the one specified in the"
                " repository. Be sure to know what you're doing :)"
            )
            self.task = task
        else:
            assert model_info.pipeline_tag is not None, "Pipeline tag cannot be None"
            self.task = model_info.pipeline_tag

        self.api_url = f"{INFERENCE_ENDPOINT}/pipeline/{self.task}/{repo_id}"

    def __repr__(self):
        # Do not add headers to repr to avoid leaking token.
        return f"InferenceAPI(api_url='{self.api_url}', task='{self.task}', options={self.options})"

    def __call__(
        self,
        inputs: Optional[Union[str, Dict, List[str], List[List[str]]]] = None,
        params: Optional[Dict] = None,
        data: Optional[bytes] = None,
        raw_response: bool = False,
    ) -> Any:
        """Make a call to the Inference API.

        Args:
            inputs (`str` or `Dict` or `List[str]` or `List[List[str]]`, *optional*):
                Inputs for the prediction.
            params (`Dict`, *optional*):
                Additional parameters for the models. Will be sent as `parameters` in the
                payload.
            data (`bytes`, *optional*):
                Bytes content of the request. In this case, leave `inputs` and `params` empty.
            raw_response (`bool`, defaults to `False`):
                If `True`, the raw `Response` object is returned. You can parse its content
                as preferred. By default, the content is parsed into a more practical format
                (json dictionary or PIL Image for example).
        """
        # Build payload
        payload: Dict[str, Any] = {
            "options": self.options,
        }
        if inputs:
            payload["inputs"] = inputs
        if params:
            payload["parameters"] = params

        # Make API call
        response = get_session().post(self.api_url, headers=self.headers, json=payload, data=data)

        # Let the user handle the response
        if raw_response:
            return response

        # By default, parse the response for the user.
        content_type = response.headers.get("Content-Type") or ""
        if content_type.startswith("image"):
            if not is_pillow_available():
                raise ImportError(
                    f"Task '{self.task}' returned as image but Pillow is not installed."
                    " Please install it (`pip install Pillow`) or pass"
                    " `raw_response=True` to get the raw `Response` object and parse"
                    " the image by yourself."
                )

            from PIL import Image

            return Image.open(io.BytesIO(response.content))
        elif content_type == "application/json":
            return response.json()
        else:
            raise NotImplementedError(
                f"{content_type} output type is not implemented yet. You can pass"
                " `raw_response=True` to get the raw `Response` object and parse the"
                " output by yourself."
            )


# src/huggingface_hub/keras_mixin.py

import collections.abc as collections
import json
import os
import warnings
from pathlib import Path
from shutil import copytree
from typing import Any, Dict, List, Optional, Union

from huggingface_hub import ModelHubMixin, snapshot_download
from huggingface_hub.utils import (
    get_tf_version,
    is_graphviz_available,
    is_pydot_available,
    is_tf_available,
    yaml_dump,
)

from .constants import CONFIG_NAME
from .hf_api import HfApi
from .utils import SoftTemporaryDirectory, logging, validate_hf_hub_args


logger = logging.get_logger(__name__)

if is_tf_available():
    import tensorflow as tf  # type: ignore


def _flatten_dict(dictionary, parent_key=""):
    """Flatten a nested dictionary.
    Reference: https://stackoverflow.com/a/6027615/10319735

    Args:
        dictionary (`dict`):
            The nested dictionary to be flattened.
        parent_key (`str`):
            The parent key to be prefixed to the children keys.
            Necessary for recursing over the nested dictionary.

    Returns:
        The flattened dictionary.
    """
    items = []
    for key, value in dictionary.items():
        new_key = f"{parent_key}.{key}" if parent_key else key
        if isinstance(value, collections.MutableMapping):
            items.extend(
                _flatten_dict(
                    value,
                    new_key,
                ).items()
            )
        else:
            items.append((new_key, value))
    return dict(items)


def _create_hyperparameter_table(model):
    """Parse hyperparameter dictionary into a markdown table."""
    if model.optimizer is not None:
        optimizer_params = model.optimizer.get_config()
        # flatten the configuration
        optimizer_params = _flatten_dict(optimizer_params)
        optimizer_params["training_precision"] = tf.keras.mixed_precision.global_policy().name
        table = "| Hyperparameters | Value |\n| :-- | :-- |\n"
        for key, value in optimizer_params.items():
            table += f"| {key} | {value} |\n"
    else:
        table = None
    return table


def _plot_network(model, save_directory):
    tf.keras.utils.plot_model(
        model,
        to_file=f"{save_directory}/model.png",
        show_shapes=False,
        show_dtype=False,
        show_layer_names=True,
        rankdir="TB",
        expand_nested=False,
        dpi=96,
        layer_range=None,
    )


def _create_model_card(
    model,
    repo_dir: Path,
    plot_model: bool = True,
    metadata: Optional[dict] = None,
):
    """
    Creates a model card for the repository.
    """
    hyperparameters = _create_hyperparameter_table(model)
    if plot_model and is_graphviz_available() and is_pydot_available():
        _plot_network(model, repo_dir)
    if metadata is None:
        metadata = {}
    readme_path = f"{repo_dir}/README.md"
    metadata["library_name"] = "keras"
    model_card: str = "---\n"
    model_card += yaml_dump(metadata, default_flow_style=False)
    model_card += "---\n"
    model_card += "\n## Model description\n\nMore information needed\n"
    model_card += "\n## Intended uses & limitations\n\nMore information needed\n"
    model_card += "\n## Training and evaluation data\n\nMore information needed\n"
    if hyperparameters is not None:
        model_card += "\n## Training procedure\n"
        model_card += "\n### Training hyperparameters\n"
        model_card += "\nThe following hyperparameters were used during training:\n\n"
        model_card += hyperparameters
        model_card += "\n"
    if plot_model and os.path.exists(f"{repo_dir}/model.png"):
        model_card += "\n ## Model Plot\n"
        model_card += "\n<details>"
        model_card += "\n<summary>View Model Plot</summary>\n"
        path_to_plot = "./model.png"
        model_card += f"\n![Model Image]({path_to_plot})\n"
        model_card += "\n</details>"

    if os.path.exists(readme_path):
        with open(readme_path, "r", encoding="utf8") as f:
            readme = f.read()
    else:
        readme = model_card
    with open(readme_path, "w", encoding="utf-8") as f:
        f.write(readme)


def save_pretrained_keras(
    model,
    save_directory: Union[str, Path],
    config: Optional[Dict[str, Any]] = None,
    include_optimizer: bool = False,
    plot_model: bool = True,
    tags: Optional[Union[list, str]] = None,
    **model_save_kwargs,
):
    """
    Saves a Keras model to save_directory in SavedModel format. Use this if
    you're using the Functional or Sequential APIs.

    Args:
        model (`Keras.Model`):
            The [Keras
            model](https://www.tensorflow.org/api_docs/python/tf/keras/Model)
            you'd like to save. The model must be compiled and built.
        save_directory (`str` or `Path`):
            Specify directory in which you want to save the Keras model.
        config (`dict`, *optional*):
            Configuration object to be saved alongside the model weights.
        include_optimizer(`bool`, *optional*, defaults to `False`):
            Whether or not to include optimizer in serialization.
        plot_model (`bool`, *optional*, defaults to `True`):
            Setting this to `True` will plot the model and put it in the model
            card. Requires graphviz and pydot to be installed.
        tags (Union[`str`,`list`], *optional*):
            List of tags that are related to model or string of a single tag. See example tags
            [here](https://github.com/huggingface/hub-docs/blame/main/modelcard.md).
        model_save_kwargs(`dict`, *optional*):
            model_save_kwargs will be passed to
            [`tf.keras.models.save_model()`](https://www.tensorflow.org/api_docs/python/tf/keras/models/save_model).
    """
    if is_tf_available():
        import tensorflow as tf
    else:
        raise ImportError("Called a Tensorflow-specific function but could not import it.")

    if not model.built:
        raise ValueError("Model should be built before trying to save")

    save_directory = Path(save_directory)
    save_directory.mkdir(parents=True, exist_ok=True)

    # saving config
    if config:
        if not isinstance(config, dict):
            raise RuntimeError(f"Provided config to save_pretrained_keras should be a dict. Got: '{type(config)}'")

        with (save_directory / CONFIG_NAME).open("w") as f:
            json.dump(config, f)

    metadata = {}
    if isinstance(tags, list):
        metadata["tags"] = tags
    elif isinstance(tags, str):
        metadata["tags"] = [tags]

    task_name = model_save_kwargs.pop("task_name", None)
    if task_name is not None:
        warnings.warn(
            "`task_name` input argument is deprecated. Pass `tags` instead.",
            FutureWarning,
        )
        if "tags" in metadata:
            metadata["tags"].append(task_name)
        else:
            metadata["tags"] = [task_name]

    if model.history is not None:
        if model.history.history != {}:
            path = save_directory / "history.json"
            if path.exists():
                warnings.warn(
                    "`history.json` file already exists, it will be overwritten by the history of this version.",
                    UserWarning,
                )
            with path.open("w", encoding="utf-8") as f:
                json.dump(model.history.history, f, indent=2, sort_keys=True)

    _create_model_card(model, save_directory, plot_model, metadata)
    tf.keras.models.save_model(model, save_directory, include_optimizer=include_optimizer, **model_save_kwargs)


def from_pretrained_keras(*args, **kwargs) -> "KerasModelHubMixin":
    r"""
    Instantiate a pretrained Keras model from a pre-trained model from the Hub.
    The model is expected to be in `SavedModel` format.

    Args:
        pretrained_model_name_or_path (`str` or `os.PathLike`):
            Can be either:
                - A string, the `model id` of a pretrained model hosted inside a
                  model repo on huggingface.co. Valid model ids can be located
                  at the root-level, like `bert-base-uncased`, or namespaced
                  under a user or organization name, like
                  `dbmdz/bert-base-german-cased`.
                - You can add `revision` by appending `@` at the end of model_id
                  simply like this: `dbmdz/bert-base-german-cased@main` Revision
                  is the specific model version to use. It can be a branch name,
                  a tag name, or a commit id, since we use a git-based system
                  for storing models and other artifacts on huggingface.co, so
                  `revision` can be any identifier allowed by git.
                - A path to a `directory` containing model weights saved using
                  [`~transformers.PreTrainedModel.save_pretrained`], e.g.,
                  `./my_model_directory/`.
                - `None` if you are both providing the configuration and state
                  dictionary (resp. with keyword arguments `config` and
                  `state_dict`).
        force_download (`bool`, *optional*, defaults to `False`):
            Whether to force the (re-)download of the model weights and
            configuration files, overriding the cached versions if they exist.
        resume_download (`bool`, *optional*, defaults to `False`):
            Whether to delete incompletely received files. Will attempt to
            resume the download if such a file exists.
        proxies (`Dict[str, str]`, *optional*):
            A dictionary of proxy servers to use by protocol or endpoint, e.g.,
            `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The
            proxies are used on each request.
        token (`str` or `bool`, *optional*):
            The token to use as HTTP bearer authorization for remote files. If
            `True`, will use the token generated when running `transformers-cli
            login` (stored in `~/.huggingface`).
        cache_dir (`Union[str, os.PathLike]`, *optional*):
            Path to a directory in which a downloaded pretrained model
            configuration should be cached if the standard cache should not be
            used.
        local_files_only(`bool`, *optional*, defaults to `False`):
            Whether to only look at local files (i.e., do not try to download
            the model).
        model_kwargs (`Dict`, *optional*):
            model_kwargs will be passed to the model during initialization

    <Tip>

    Passing `token=True` is required when you want to use a private
    model.

    </Tip>
    """
    return KerasModelHubMixin.from_pretrained(*args, **kwargs)


@validate_hf_hub_args
def push_to_hub_keras(
    model,
    repo_id: str,
    *,
    config: Optional[dict] = None,
    commit_message: str = "Push Keras model using huggingface_hub.",
    private: bool = False,
    api_endpoint: Optional[str] = None,
    token: Optional[str] = None,
    branch: Optional[str] = None,
    create_pr: Optional[bool] = None,
    allow_patterns: Optional[Union[List[str], str]] = None,
    ignore_patterns: Optional[Union[List[str], str]] = None,
    delete_patterns: Optional[Union[List[str], str]] = None,
    log_dir: Optional[str] = None,
    include_optimizer: bool = False,
    tags: Optional[Union[list, str]] = None,
    plot_model: bool = True,
    **model_save_kwargs,
):
    """
    Upload model checkpoint to the Hub.

    Use `allow_patterns` and `ignore_patterns` to precisely filter which files should be pushed to the hub. Use
    `delete_patterns` to delete existing remote files in the same commit. See [`upload_folder`] reference for more
    details.

    Args:
        model (`Keras.Model`):
            The [Keras model](`https://www.tensorflow.org/api_docs/python/tf/keras/Model`) you'd like to push to the
            Hub. The model must be compiled and built.
        repo_id (`str`):
                ID of the repository to push to (example: `"username/my-model"`).
        commit_message (`str`, *optional*, defaults to "Add Keras model"):
            Message to commit while pushing.
        private (`bool`, *optional*, defaults to `False`):
            Whether the repository created should be private.
        api_endpoint (`str`, *optional*):
            The API endpoint to use when pushing the model to the hub.
        token (`str`, *optional*):
            The token to use as HTTP bearer authorization for remote files. If
            not set, will use the token set when logging in with
            `huggingface-cli login` (stored in `~/.huggingface`).
        branch (`str`, *optional*):
            The git branch on which to push the model. This defaults to
            the default branch as specified in your repository, which
            defaults to `"main"`.
        create_pr (`boolean`, *optional*):
            Whether or not to create a Pull Request from `branch` with that commit.
            Defaults to `False`.
        config (`dict`, *optional*):
            Configuration object to be saved alongside the model weights.
        allow_patterns (`List[str]` or `str`, *optional*):
            If provided, only files matching at least one pattern are pushed.
        ignore_patterns (`List[str]` or `str`, *optional*):
            If provided, files matching any of the patterns are not pushed.
        delete_patterns (`List[str]` or `str`, *optional*):
            If provided, remote files matching any of the patterns will be deleted from the repo.
        log_dir (`str`, *optional*):
            TensorBoard logging directory to be pushed. The Hub automatically
            hosts and displays a TensorBoard instance if log files are included
            in the repository.
        include_optimizer (`bool`, *optional*, defaults to `False`):
            Whether or not to include optimizer during serialization.
        tags (Union[`list`, `str`], *optional*):
            List of tags that are related to model or string of a single tag. See example tags
            [here](https://github.com/huggingface/hub-docs/blame/main/modelcard.md).
        plot_model (`bool`, *optional*, defaults to `True`):
            Setting this to `True` will plot the model and put it in the model
            card. Requires graphviz and pydot to be installed.
        model_save_kwargs(`dict`, *optional*):
            model_save_kwargs will be passed to
            [`tf.keras.models.save_model()`](https://www.tensorflow.org/api_docs/python/tf/keras/models/save_model).

    Returns:
        The url of the commit of your model in the given repository.
    """
    api = HfApi(endpoint=api_endpoint)
    repo_id = api.create_repo(repo_id=repo_id, token=token, private=private, exist_ok=True).repo_id

    # Push the files to the repo in a single commit
    with SoftTemporaryDirectory() as tmp:
        saved_path = Path(tmp) / repo_id
        save_pretrained_keras(
            model,
            saved_path,
            config=config,
            include_optimizer=include_optimizer,
            tags=tags,
            plot_model=plot_model,
            **model_save_kwargs,
        )

        # If `log_dir` provided, delete remote logs and upload new ones
        if log_dir is not None:
            delete_patterns = (
                []
                if delete_patterns is None
                else (
                    [delete_patterns]  # convert `delete_patterns` to a list
                    if isinstance(delete_patterns, str)
                    else delete_patterns
                )
            )
            delete_patterns.append("logs/*")
            copytree(log_dir, saved_path / "logs")

        return api.upload_folder(
            repo_type="model",
            repo_id=repo_id,
            folder_path=saved_path,
            commit_message=commit_message,
            token=token,
            revision=branch,
            create_pr=create_pr,
            allow_patterns=allow_patterns,
            ignore_patterns=ignore_patterns,
            delete_patterns=delete_patterns,
        )


class KerasModelHubMixin(ModelHubMixin):
    """
    Implementation of [`ModelHubMixin`] to provide model Hub upload/download
    capabilities to Keras models.


    ```python
    >>> import tensorflow as tf
    >>> from huggingface_hub import KerasModelHubMixin


    >>> class MyModel(tf.keras.Model, KerasModelHubMixin):
    ...     def __init__(self, **kwargs):
    ...         super().__init__()
    ...         self.config = kwargs.pop("config", None)
    ...         self.dummy_inputs = ...
    ...         self.layer = ...

    ...     def call(self, *args):
    ...         return ...


    >>> # Initialize and compile the model as you normally would
    >>> model = MyModel()
    >>> model.compile(...)
    >>> # Build the graph by training it or passing dummy inputs
    >>> _ = model(model.dummy_inputs)
    >>> # Save model weights to local directory
    >>> model.save_pretrained("my-awesome-model")
    >>> # Push model weights to the Hub
    >>> model.push_to_hub("my-awesome-model")
    >>> # Download and initialize weights from the Hub
    >>> model = MyModel.from_pretrained("username/super-cool-model")
    ```
    """

    def _save_pretrained(self, save_directory):
        save_pretrained_keras(self, save_directory)

    @classmethod
    def _from_pretrained(
        cls,
        model_id,
        revision,
        cache_dir,
        force_download,
        proxies,
        resume_download,
        local_files_only,
        token,
        **model_kwargs,
    ):
        """Here we just call [`from_pretrained_keras`] function so both the mixin and
        functional APIs stay in sync.

                TODO - Some args above aren't used since we are calling
                snapshot_download instead of hf_hub_download.
        """
        if is_tf_available():
            import tensorflow as tf
        else:
            raise ImportError("Called a TensorFlow-specific function but could not import it.")

        # TODO - Figure out what to do about these config values. Config is not going to be needed to load model
        cfg = model_kwargs.pop("config", None)

        # Root is either a local filepath matching model_id or a cached snapshot
        if not os.path.isdir(model_id):
            storage_folder = snapshot_download(
                repo_id=model_id,
                revision=revision,
                cache_dir=cache_dir,
                library_name="keras",
                library_version=get_tf_version(),
            )
        else:
            storage_folder = model_id

        model = tf.keras.models.load_model(storage_folder, **model_kwargs)

        # For now, we add a new attribute, config, to store the config loaded from the hub/a local dir.
        model.config = cfg

        return model


# src/huggingface_hub/lfs.py

# coding=utf-8
# Copyright 2019-present, the HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Git LFS related type definitions and utilities"""
import io
import os
import re
import warnings
from contextlib import AbstractContextManager
from dataclasses import dataclass
from math import ceil
from os.path import getsize
from pathlib import Path
from typing import TYPE_CHECKING, BinaryIO, Dict, Iterable, List, Optional, Tuple

from requests.auth import HTTPBasicAuth

from huggingface_hub.constants import ENDPOINT, HF_HUB_ENABLE_HF_TRANSFER, REPO_TYPES_URL_PREFIXES
from huggingface_hub.utils import get_session

from .utils import get_token_to_send, hf_raise_for_status, http_backoff, logging, validate_hf_hub_args
from .utils._typing import TypedDict
from .utils.sha import sha256, sha_fileobj


if TYPE_CHECKING:
    from ._commit_api import CommitOperationAdd

logger = logging.get_logger(__name__)

OID_REGEX = re.compile(r"^[0-9a-f]{40}$")

LFS_MULTIPART_UPLOAD_COMMAND = "lfs-multipart-upload"

LFS_HEADERS = {
    "Accept": "application/vnd.git-lfs+json",
    "Content-Type": "application/vnd.git-lfs+json",
}


@dataclass
class UploadInfo:
    """
    Dataclass holding required information to determine whether a blob
    should be uploaded to the hub using the LFS protocol or the regular protocol

    Args:
        sha256 (`bytes`):
            SHA256 hash of the blob
        size (`int`):
            Size in bytes of the blob
        sample (`bytes`):
            First 512 bytes of the blob
    """

    sha256: bytes
    size: int
    sample: bytes

    @classmethod
    def from_path(cls, path: str):
        size = getsize(path)
        with io.open(path, "rb") as file:
            sample = file.peek(512)[:512]
            sha = sha_fileobj(file)
        return cls(size=size, sha256=sha, sample=sample)

    @classmethod
    def from_bytes(cls, data: bytes):
        sha = sha256(data).digest()
        return cls(size=len(data), sample=data[:512], sha256=sha)

    @classmethod
    def from_fileobj(cls, fileobj: BinaryIO):
        sample = fileobj.read(512)
        fileobj.seek(0, io.SEEK_SET)
        sha = sha_fileobj(fileobj)
        size = fileobj.tell()
        fileobj.seek(0, io.SEEK_SET)
        return cls(size=size, sha256=sha, sample=sample)


@validate_hf_hub_args
def post_lfs_batch_info(
    upload_infos: Iterable[UploadInfo],
    token: Optional[str],
    repo_type: str,
    repo_id: str,
    endpoint: Optional[str] = None,
) -> Tuple[List[dict], List[dict]]:
    """
    Requests the LFS batch endpoint to retrieve upload instructions

    Learn more: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md

    Args:
        upload_infos (`Iterable` of `UploadInfo`):
            `UploadInfo` for the files that are being uploaded, typically obtained
            from `CommitOperationAdd.upload_info`
        repo_type (`str`):
            Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.
        repo_id (`str`):
            A namespace (user or an organization) and a repo name separated
            by a `/`.
        token (`str`, *optional*):
            An authentication token ( See https://huggingface.co/settings/tokens )

    Returns:
        `LfsBatchInfo`: 2-tuple:
            - First element is the list of upload instructions from the server
            - Second element is an list of errors, if any

    Raises:
        `ValueError`: If an argument is invalid or the server response is malformed

        `HTTPError`: If the server returned an error
    """
    endpoint = endpoint if endpoint is not None else ENDPOINT
    url_prefix = ""
    if repo_type in REPO_TYPES_URL_PREFIXES:
        url_prefix = REPO_TYPES_URL_PREFIXES[repo_type]
    batch_url = f"{endpoint}/{url_prefix}{repo_id}.git/info/lfs/objects/batch"
    resp = get_session().post(
        batch_url,
        headers=LFS_HEADERS,
        json={
            "operation": "upload",
            "transfers": ["basic", "multipart"],
            "objects": [
                {
                    "oid": upload.sha256.hex(),
                    "size": upload.size,
                }
                for upload in upload_infos
            ],
            "hash_algo": "sha256",
        },
        auth=HTTPBasicAuth(
            "access_token",
            get_token_to_send(token or True),  # type: ignore  # Token must be provided or retrieved
        ),
    )
    hf_raise_for_status(resp)
    batch_info = resp.json()

    objects = batch_info.get("objects", None)
    if not isinstance(objects, list):
        raise ValueError("Malformed response from server")

    return (
        [_validate_batch_actions(obj) for obj in objects if "error" not in obj],
        [_validate_batch_error(obj) for obj in objects if "error" in obj],
    )


class PayloadPartT(TypedDict):
    partNumber: int
    etag: str


class CompletionPayloadT(TypedDict):
    """Payload that will be sent to the Hub when uploading multi-part."""

    oid: str
    parts: List[PayloadPartT]


def lfs_upload(operation: "CommitOperationAdd", lfs_batch_action: Dict, token: Optional[str]) -> None:
    """
    Handles uploading a given object to the Hub with the LFS protocol.

    Can be a No-op if the content of the file is already present on the hub large file storage.

    Args:
        operation (`CommitOperationAdd`):
            The add operation triggering this upload.
        lfs_batch_action (`dict`):
            Upload instructions from the LFS batch endpoint for this object. See [`~utils.lfs.post_lfs_batch_info`] for
            more details.
        token (`str`, *optional*):
            A [user access token](https://hf.co/settings/tokens) to authenticate requests against the Hub

    Raises:
        - `ValueError` if `lfs_batch_action` is improperly formatted
        - `HTTPError` if the upload resulted in an error
    """
    # 0. If LFS file is already present, skip upload
    _validate_batch_actions(lfs_batch_action)
    actions = lfs_batch_action.get("actions")
    if actions is None:
        # The file was already uploaded
        logger.debug(f"Content of file {operation.path_in_repo} is already present upstream - skipping upload")
        return

    # 1. Validate server response (check required keys in dict)
    upload_action = lfs_batch_action["actions"]["upload"]
    _validate_lfs_action(upload_action)
    verify_action = lfs_batch_action["actions"].get("verify")
    if verify_action is not None:
        _validate_lfs_action(verify_action)

    # 2. Upload file (either single part or multi-part)
    header = upload_action.get("header", {})
    chunk_size = header.get("chunk_size")
    if chunk_size is not None:
        try:
            chunk_size = int(chunk_size)
        except (ValueError, TypeError):
            raise ValueError(
                f"Malformed response from LFS batch endpoint: `chunk_size` should be an integer. Got '{chunk_size}'."
            )
        _upload_multi_part(operation=operation, header=header, chunk_size=chunk_size, upload_url=upload_action["href"])
    else:
        _upload_single_part(operation=operation, upload_url=upload_action["href"])

    # 3. Verify upload went well
    if verify_action is not None:
        _validate_lfs_action(verify_action)
        verify_resp = get_session().post(
            verify_action["href"],
            auth=HTTPBasicAuth(username="USER", password=get_token_to_send(token or True)),  # type: ignore
            json={"oid": operation.upload_info.sha256.hex(), "size": operation.upload_info.size},
        )
        hf_raise_for_status(verify_resp)
    logger.debug(f"{operation.path_in_repo}: Upload successful")


def _validate_lfs_action(lfs_action: dict):
    """validates response from the LFS batch endpoint"""
    if not (
        isinstance(lfs_action.get("href"), str)
        and (lfs_action.get("header") is None or isinstance(lfs_action.get("header"), dict))
    ):
        raise ValueError("lfs_action is improperly formatted")
    return lfs_action


def _validate_batch_actions(lfs_batch_actions: dict):
    """validates response from the LFS batch endpoint"""
    if not (isinstance(lfs_batch_actions.get("oid"), str) and isinstance(lfs_batch_actions.get("size"), int)):
        raise ValueError("lfs_batch_actions is improperly formatted")

    upload_action = lfs_batch_actions.get("actions", {}).get("upload")
    verify_action = lfs_batch_actions.get("actions", {}).get("verify")
    if upload_action is not None:
        _validate_lfs_action(upload_action)
    if verify_action is not None:
        _validate_lfs_action(verify_action)
    return lfs_batch_actions


def _validate_batch_error(lfs_batch_error: dict):
    """validates response from the LFS batch endpoint"""
    if not (isinstance(lfs_batch_error.get("oid"), str) and isinstance(lfs_batch_error.get("size"), int)):
        raise ValueError("lfs_batch_error is improperly formatted")
    error_info = lfs_batch_error.get("error")
    if not (
        isinstance(error_info, dict)
        and isinstance(error_info.get("message"), str)
        and isinstance(error_info.get("code"), int)
    ):
        raise ValueError("lfs_batch_error is improperly formatted")
    return lfs_batch_error


def _upload_single_part(operation: "CommitOperationAdd", upload_url: str) -> None:
    """
    Uploads `fileobj` as a single PUT HTTP request (basic LFS transfer protocol)

    Args:
        upload_url (`str`):
            The URL to PUT the file to.
        fileobj:
            The file-like object holding the data to upload.

    Returns: `requests.Response`

    Raises: `requests.HTTPError` if the upload resulted in an error
    """
    with operation.as_file(with_tqdm=True) as fileobj:
        response = http_backoff("PUT", upload_url, data=fileobj)
        hf_raise_for_status(response)


def _upload_multi_part(operation: "CommitOperationAdd", header: Dict, chunk_size: int, upload_url: str) -> None:
    """
    Uploads file using HF multipart LFS transfer protocol.
    """
    # 1. Get upload URLs for each part
    sorted_parts_urls = _get_sorted_parts_urls(header=header, upload_info=operation.upload_info, chunk_size=chunk_size)

    # 2. Upload parts (either with hf_transfer or in pure Python)
    use_hf_transfer = HF_HUB_ENABLE_HF_TRANSFER
    if (
        HF_HUB_ENABLE_HF_TRANSFER
        and not isinstance(operation.path_or_fileobj, str)
        and not isinstance(operation.path_or_fileobj, Path)
    ):
        warnings.warn(
            "hf_transfer is enabled but does not support uploading from bytes or BinaryIO, falling back to regular"
            " upload"
        )
        use_hf_transfer = False

    response_headers = (
        _upload_parts_hf_transfer(operation=operation, sorted_parts_urls=sorted_parts_urls, chunk_size=chunk_size)
        if use_hf_transfer
        else _upload_parts_iteratively(operation=operation, sorted_parts_urls=sorted_parts_urls, chunk_size=chunk_size)
    )

    # 3. Send completion request
    completion_res = get_session().post(
        upload_url,
        json=_get_completion_payload(response_headers, operation.upload_info.sha256.hex()),
        headers=LFS_HEADERS,
    )
    hf_raise_for_status(completion_res)


def _get_sorted_parts_urls(header: Dict, upload_info: UploadInfo, chunk_size: int) -> List[str]:
    sorted_part_upload_urls = [
        upload_url
        for _, upload_url in sorted(
            [
                (int(part_num, 10), upload_url)
                for part_num, upload_url in header.items()
                if part_num.isdigit() and len(part_num) > 0
            ],
            key=lambda t: t[0],
        )
    ]
    num_parts = len(sorted_part_upload_urls)
    if num_parts != ceil(upload_info.size / chunk_size):
        raise ValueError("Invalid server response to upload large LFS file")
    return sorted_part_upload_urls


def _get_completion_payload(response_headers: List[Dict], oid: str) -> CompletionPayloadT:
    parts: List[PayloadPartT] = []
    for part_number, header in enumerate(response_headers):
        etag = header.get("etag")
        if etag is None or etag == "":
            raise ValueError(f"Invalid etag (`{etag}`) returned for part {part_number + 1}")
        parts.append(
            {
                "partNumber": part_number + 1,
                "etag": etag,
            }
        )
    return {"oid": oid, "parts": parts}


def _upload_parts_iteratively(
    operation: "CommitOperationAdd", sorted_parts_urls: List[str], chunk_size: int
) -> List[Dict]:
    headers = []
    with operation.as_file(with_tqdm=True) as fileobj:
        for part_idx, part_upload_url in enumerate(sorted_parts_urls):
            with SliceFileObj(
                fileobj,
                seek_from=chunk_size * part_idx,
                read_limit=chunk_size,
            ) as fileobj_slice:
                part_upload_res = http_backoff("PUT", part_upload_url, data=fileobj_slice)
                hf_raise_for_status(part_upload_res)
                headers.append(part_upload_res.headers)
    return headers  # type: ignore


def _upload_parts_hf_transfer(
    operation: "CommitOperationAdd", sorted_parts_urls: List[str], chunk_size: int
) -> List[Dict]:
    # Upload file using an external Rust-based package. Upload is faster but support less features (no progress bars).
    try:
        from hf_transfer import multipart_upload
    except ImportError:
        raise ValueError(
            "Fast uploading using 'hf_transfer' is enabled (HF_HUB_ENABLE_HF_TRANSFER=1) but 'hf_transfer' package is"
            " not available in your environment. Try `pip install hf_transfer`."
        )

    try:
        return multipart_upload(
            file_path=operation.path_or_fileobj,
            parts_urls=sorted_parts_urls,
            chunk_size=chunk_size,
            max_files=128,
            parallel_failures=127,  # could be removed
            max_retries=5,
        )
    except Exception as e:
        raise RuntimeError(
            "An error occurred while uploading using `hf_transfer`. Consider disabling HF_HUB_ENABLE_HF_TRANSFER for"
            " better error handling."
        ) from e


class SliceFileObj(AbstractContextManager):
    """
    Utility context manager to read a *slice* of a seekable file-like object as a seekable, file-like object.

    This is NOT thread safe

    Inspired by stackoverflow.com/a/29838711/593036

    Credits to @julien-c

    Args:
        fileobj (`BinaryIO`):
            A file-like object to slice. MUST implement `tell()` and `seek()` (and `read()` of course).
            `fileobj` will be reset to its original position when exiting the context manager.
        seek_from (`int`):
            The start of the slice (offset from position 0 in bytes).
        read_limit (`int`):
            The maximum number of bytes to read from the slice.

    Attributes:
        previous_position (`int`):
            The previous position

    Examples:

    Reading 200 bytes with an offset of 128 bytes from a file (ie bytes 128 to 327):
    ```python
    >>> with open("path/to/file", "rb") as file:
    ...     with SliceFileObj(file, seek_from=128, read_limit=200) as fslice:
    ...         fslice.read(...)
    ```

    Reading a file in chunks of 512 bytes
    ```python
    >>> import os
    >>> chunk_size = 512
    >>> file_size = os.getsize("path/to/file")
    >>> with open("path/to/file", "rb") as file:
    ...     for chunk_idx in range(ceil(file_size / chunk_size)):
    ...         with SliceFileObj(file, seek_from=chunk_idx * chunk_size, read_limit=chunk_size) as fslice:
    ...             chunk = fslice.read(...)

    ```
    """

    def __init__(self, fileobj: BinaryIO, seek_from: int, read_limit: int):
        self.fileobj = fileobj
        self.seek_from = seek_from
        self.read_limit = read_limit

    def __enter__(self):
        self._previous_position = self.fileobj.tell()
        end_of_stream = self.fileobj.seek(0, os.SEEK_END)
        self._len = min(self.read_limit, end_of_stream - self.seek_from)
        # ^^ The actual number of bytes that can be read from the slice
        self.fileobj.seek(self.seek_from, io.SEEK_SET)
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.fileobj.seek(self._previous_position, io.SEEK_SET)

    def read(self, n: int = -1):
        pos = self.tell()
        if pos >= self._len:
            return b""
        remaining_amount = self._len - pos
        data = self.fileobj.read(remaining_amount if n < 0 else min(n, remaining_amount))
        return data

    def tell(self) -> int:
        return self.fileobj.tell() - self.seek_from

    def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
        start = self.seek_from
        end = start + self._len
        if whence in (os.SEEK_SET, os.SEEK_END):
            offset = start + offset if whence == os.SEEK_SET else end + offset
            offset = max(start, min(offset, end))
            whence = os.SEEK_SET
        elif whence == os.SEEK_CUR:
            cur_pos = self.fileobj.tell()
            offset = max(start - cur_pos, min(offset, end - cur_pos))
        else:
            raise ValueError(f"whence value {whence} is not supported")
        return self.fileobj.seek(offset, whence) - self.seek_from

    def __iter__(self):
        yield self.read(n=4 * 1024 * 1024)


# src/huggingface_hub/repocard.py

import os
import re
from pathlib import Path
from typing import Any, Dict, Optional, Type, Union

import requests
import yaml

from huggingface_hub.file_download import hf_hub_download
from huggingface_hub.hf_api import upload_file
from huggingface_hub.repocard_data import (
    CardData,
    DatasetCardData,
    EvalResult,
    ModelCardData,
    SpaceCardData,
    eval_results_to_model_index,
    model_index_to_eval_results,
)
from huggingface_hub.utils import get_session, is_jinja_available, yaml_dump

from .constants import REPOCARD_NAME
from .utils import EntryNotFoundError, SoftTemporaryDirectory, validate_hf_hub_args
from .utils._typing import Literal
from .utils.logging import get_logger


TEMPLATE_MODELCARD_PATH = Path(__file__).parent / "templates" / "modelcard_template.md"
TEMPLATE_DATASETCARD_PATH = Path(__file__).parent / "templates" / "datasetcard_template.md"

# exact same regex as in the Hub server. Please keep in sync.
# See https://github.com/huggingface/moon-landing/blob/main/server/lib/ViewMarkdown.ts#L18
REGEX_YAML_BLOCK = re.compile(r"^(\s*---[\r\n]+)([\S\s]*?)([\r\n]+---(\r\n|\n|$))")

logger = get_logger(__name__)


class RepoCard:
    card_data_class = CardData
    default_template_path = TEMPLATE_MODELCARD_PATH
    repo_type = "model"

    def __init__(self, content: str, ignore_metadata_errors: bool = False):
        """Initialize a RepoCard from string content. The content should be a
        Markdown file with a YAML block at the beginning and a Markdown body.

        Args:
            content (`str`): The content of the Markdown file.

        Example:
            ```python
            >>> from huggingface_hub.repocard import RepoCard
            >>> text = '''
            ... ---
            ... language: en
            ... license: mit
            ... ---
            ...
            ... # My repo
            ... '''
            >>> card = RepoCard(text)
            >>> card.data.to_dict()
            {'language': 'en', 'license': 'mit'}
            >>> card.text
            '\\n# My repo\\n'

            ```
        <Tip>
        Raises the following error:

            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
              when the content of the repo card metadata is not a dictionary.

        </Tip>
        """

        # Set the content of the RepoCard, as well as underlying .data and .text attributes.
        # See the `content` property setter for more details.
        self.ignore_metadata_errors = ignore_metadata_errors
        self.content = content

    @property
    def content(self):
        """The content of the RepoCard, including the YAML block and the Markdown body."""
        line_break = _detect_line_ending(self._content) or "\n"
        return f"---{line_break}{self.data.to_yaml(line_break=line_break)}{line_break}---{line_break}{self.text}"

    @content.setter
    def content(self, content: str):
        """Set the content of the RepoCard."""
        self._content = content

        match = REGEX_YAML_BLOCK.search(content)
        if match:
            # Metadata found in the YAML block
            yaml_block = match.group(2)
            self.text = content[match.end() :]
            data_dict = yaml.safe_load(yaml_block)

            # The YAML block's data should be a dictionary
            if not isinstance(data_dict, dict):
                raise ValueError("repo card metadata block should be a dict")
        else:
            # Model card without metadata... create empty metadata
            logger.warning("Repo card metadata block was not found. Setting CardData to empty.")
            data_dict = {}
            self.text = content

        self.data = self.card_data_class(**data_dict, ignore_metadata_errors=self.ignore_metadata_errors)

    def __str__(self):
        return self.content

    def save(self, filepath: Union[Path, str]):
        r"""Save a RepoCard to a file.

        Args:
            filepath (`Union[Path, str]`): Filepath to the markdown file to save.

        Example:
            ```python
            >>> from huggingface_hub.repocard import RepoCard
            >>> card = RepoCard("---\nlanguage: en\n---\n# This is a test repo card")
            >>> card.save("/tmp/test.md")

            ```
        """
        filepath = Path(filepath)
        filepath.parent.mkdir(parents=True, exist_ok=True)
        # Preserve newlines as in the existing file.
        with open(filepath, mode="w", newline="", encoding="utf-8") as f:
            f.write(str(self))

    @classmethod
    def load(
        cls,
        repo_id_or_path: Union[str, Path],
        repo_type: Optional[str] = None,
        token: Optional[str] = None,
        ignore_metadata_errors: bool = False,
    ):
        """Initialize a RepoCard from a Hugging Face Hub repo's README.md or a local filepath.

        Args:
            repo_id_or_path (`Union[str, Path]`):
                The repo ID associated with a Hugging Face Hub repo or a local filepath.
            repo_type (`str`, *optional*):
                The type of Hugging Face repo to push to. Defaults to None, which will use use "model". Other options
                are "dataset" and "space". Not used when loading from a local filepath. If this is called from a child
                class, the default value will be the child class's `repo_type`.
            token (`str`, *optional*):
                Authentication token, obtained with `huggingface_hub.HfApi.login` method. Will default to the stored token.
            ignore_metadata_errors (`str`):
                If True, errors while parsing the metadata section will be ignored. Some information might be lost during
                the process. Use it at your own risk.

        Returns:
            [`huggingface_hub.repocard.RepoCard`]: The RepoCard (or subclass) initialized from the repo's
                README.md file or filepath.

        Example:
            ```python
            >>> from huggingface_hub.repocard import RepoCard
            >>> card = RepoCard.load("nateraw/food")
            >>> assert card.data.tags == ["generated_from_trainer", "image-classification", "pytorch"]

            ```
        """

        if Path(repo_id_or_path).exists():
            card_path = Path(repo_id_or_path)
        elif isinstance(repo_id_or_path, str):
            card_path = Path(
                hf_hub_download(
                    repo_id_or_path,
                    REPOCARD_NAME,
                    repo_type=repo_type or cls.repo_type,
                    token=token,
                )
            )
        else:
            raise ValueError(f"Cannot load RepoCard: path not found on disk ({repo_id_or_path}).")

        # Preserve newlines in the existing file.
        with card_path.open(mode="r", newline="", encoding="utf-8") as f:
            return cls(f.read(), ignore_metadata_errors=ignore_metadata_errors)

    def validate(self, repo_type: Optional[str] = None):
        """Validates card against Hugging Face Hub's card validation logic.
        Using this function requires access to the internet, so it is only called
        internally by [`huggingface_hub.repocard.RepoCard.push_to_hub`].

        Args:
            repo_type (`str`, *optional*, defaults to "model"):
                The type of Hugging Face repo to push to. Options are "model", "dataset", and "space".
                If this function is called from a child class, the default will be the child class's `repo_type`.

        <Tip>
        Raises the following errors:

            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
              if the card fails validation checks.
            - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
              if the request to the Hub API fails for any other reason.

        </Tip>
        """

        # If repo type is provided, otherwise, use the repo type of the card.
        repo_type = repo_type or self.repo_type

        body = {
            "repoType": repo_type,
            "content": str(self),
        }
        headers = {"Accept": "text/plain"}

        try:
            r = get_session().post("https://huggingface.co/api/validate-yaml", body, headers=headers)
            r.raise_for_status()
        except requests.exceptions.HTTPError as exc:
            if r.status_code == 400:
                raise ValueError(r.text)
            else:
                raise exc

    def push_to_hub(
        self,
        repo_id: str,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
        commit_message: Optional[str] = None,
        commit_description: Optional[str] = None,
        revision: Optional[str] = None,
        create_pr: Optional[bool] = None,
        parent_commit: Optional[str] = None,
    ):
        """Push a RepoCard to a Hugging Face Hub repo.

        Args:
            repo_id (`str`):
                The repo ID of the Hugging Face Hub repo to push to. Example: "nateraw/food".
            token (`str`, *optional*):
                Authentication token, obtained with `huggingface_hub.HfApi.login` method. Will default to
                the stored token.
            repo_type (`str`, *optional*, defaults to "model"):
                The type of Hugging Face repo to push to. Options are "model", "dataset", and "space". If this
                function is called by a child class, it will default to the child class's `repo_type`.
            commit_message (`str`, *optional*):
                The summary / title / first line of the generated commit.
            commit_description (`str`, *optional*)
                The description of the generated commit.
            revision (`str`, *optional*):
                The git revision to commit from. Defaults to the head of the `"main"` branch.
            create_pr (`bool`, *optional*):
                Whether or not to create a Pull Request with this commit. Defaults to `False`.
            parent_commit (`str`, *optional*):
                The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported.
                If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`.
                If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`.
                Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be
                especially useful if the repo is updated / committed to concurrently.
        Returns:
            `str`: URL of the commit which updated the card metadata.
        """

        # If repo type is provided, otherwise, use the repo type of the card.
        repo_type = repo_type or self.repo_type

        # Validate card before pushing to hub
        self.validate(repo_type=repo_type)

        with SoftTemporaryDirectory() as tmpdir:
            tmp_path = Path(tmpdir) / REPOCARD_NAME
            tmp_path.write_text(str(self))
            url = upload_file(
                path_or_fileobj=str(tmp_path),
                path_in_repo=REPOCARD_NAME,
                repo_id=repo_id,
                token=token,
                repo_type=repo_type,
                commit_message=commit_message,
                commit_description=commit_description,
                create_pr=create_pr,
                revision=revision,
                parent_commit=parent_commit,
            )
        return url

    @classmethod
    def from_template(
        cls,
        card_data: CardData,
        template_path: Optional[str] = None,
        **template_kwargs,
    ):
        """Initialize a RepoCard from a template. By default, it uses the default template.

        Templates are Jinja2 templates that can be customized by passing keyword arguments.

        Args:
            card_data (`huggingface_hub.CardData`):
                A huggingface_hub.CardData instance containing the metadata you want to include in the YAML
                header of the repo card on the Hugging Face Hub.
            template_path (`str`, *optional*):
                A path to a markdown file with optional Jinja template variables that can be filled
                in with `template_kwargs`. Defaults to the default template.

        Returns:
            [`huggingface_hub.repocard.RepoCard`]: A RepoCard instance with the specified card data and content from the
            template.
        """
        if is_jinja_available():
            import jinja2
        else:
            raise ImportError(
                "Using RepoCard.from_template requires Jinja2 to be installed. Please"
                " install it with `pip install Jinja2`."
            )

        kwargs = card_data.to_dict().copy()
        kwargs.update(template_kwargs)  # Template_kwargs have priority
        template = jinja2.Template(Path(template_path or cls.default_template_path).read_text())
        content = template.render(card_data=card_data.to_yaml(), **kwargs)
        return cls(content)


class ModelCard(RepoCard):
    card_data_class = ModelCardData
    default_template_path = TEMPLATE_MODELCARD_PATH
    repo_type = "model"

    @classmethod
    def from_template(  # type: ignore # violates Liskov property but easier to use
        cls,
        card_data: ModelCardData,
        template_path: Optional[str] = None,
        **template_kwargs,
    ):
        """Initialize a ModelCard from a template. By default, it uses the default template, which can be found here:
        https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/templates/modelcard_template.md

        Templates are Jinja2 templates that can be customized by passing keyword arguments.

        Args:
            card_data (`huggingface_hub.ModelCardData`):
                A huggingface_hub.ModelCardData instance containing the metadata you want to include in the YAML
                header of the model card on the Hugging Face Hub.
            template_path (`str`, *optional*):
                A path to a markdown file with optional Jinja template variables that can be filled
                in with `template_kwargs`. Defaults to the default template.

        Returns:
            [`huggingface_hub.ModelCard`]: A ModelCard instance with the specified card data and content from the
            template.

        Example:
            ```python
            >>> from huggingface_hub import ModelCard, ModelCardData, EvalResult

            >>> # Using the Default Template
            >>> card_data = ModelCardData(
            ...     language='en',
            ...     license='mit',
            ...     library_name='timm',
            ...     tags=['image-classification', 'resnet'],
            ...     datasets=['beans'],
            ...     metrics=['accuracy'],
            ... )
            >>> card = ModelCard.from_template(
            ...     card_data,
            ...     model_description='This model does x + y...'
            ... )

            >>> # Including Evaluation Results
            >>> card_data = ModelCardData(
            ...     language='en',
            ...     tags=['image-classification', 'resnet'],
            ...     eval_results=[
            ...         EvalResult(
            ...             task_type='image-classification',
            ...             dataset_type='beans',
            ...             dataset_name='Beans',
            ...             metric_type='accuracy',
            ...             metric_value=0.9,
            ...         ),
            ...     ],
            ...     model_name='my-cool-model',
            ... )
            >>> card = ModelCard.from_template(card_data)

            >>> # Using a Custom Template
            >>> card_data = ModelCardData(
            ...     language='en',
            ...     tags=['image-classification', 'resnet']
            ... )
            >>> card = ModelCard.from_template(
            ...     card_data=card_data,
            ...     template_path='./src/huggingface_hub/templates/modelcard_template.md',
            ...     custom_template_var='custom value',  # will be replaced in template if it exists
            ... )

            ```
        """
        return super().from_template(card_data, template_path, **template_kwargs)


class DatasetCard(RepoCard):
    card_data_class = DatasetCardData
    default_template_path = TEMPLATE_DATASETCARD_PATH
    repo_type = "dataset"

    @classmethod
    def from_template(  # type: ignore # violates Liskov property but easier to use
        cls,
        card_data: DatasetCardData,
        template_path: Optional[str] = None,
        **template_kwargs,
    ):
        """Initialize a DatasetCard from a template. By default, it uses the default template, which can be found here:
        https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/templates/datasetcard_template.md

        Templates are Jinja2 templates that can be customized by passing keyword arguments.

        Args:
            card_data (`huggingface_hub.DatasetCardData`):
                A huggingface_hub.DatasetCardData instance containing the metadata you want to include in the YAML
                header of the dataset card on the Hugging Face Hub.
            template_path (`str`, *optional*):
                A path to a markdown file with optional Jinja template variables that can be filled
                in with `template_kwargs`. Defaults to the default template.

        Returns:
            [`huggingface_hub.DatasetCard`]: A DatasetCard instance with the specified card data and content from the
            template.

        Example:
            ```python
            >>> from huggingface_hub import DatasetCard, DatasetCardData

            >>> # Using the Default Template
            >>> card_data = DatasetCardData(
            ...     language='en',
            ...     license='mit',
            ...     annotations_creators='crowdsourced',
            ...     task_categories=['text-classification'],
            ...     task_ids=['sentiment-classification', 'text-scoring'],
            ...     multilinguality='monolingual',
            ...     pretty_name='My Text Classification Dataset',
            ... )
            >>> card = DatasetCard.from_template(
            ...     card_data,
            ...     pretty_name=card_data.pretty_name,
            ... )

            >>> # Using a Custom Template
            >>> card_data = DatasetCardData(
            ...     language='en',
            ...     license='mit',
            ... )
            >>> card = DatasetCard.from_template(
            ...     card_data=card_data,
            ...     template_path='./src/huggingface_hub/templates/datasetcard_template.md',
            ...     custom_template_var='custom value',  # will be replaced in template if it exists
            ... )

            ```
        """
        return super().from_template(card_data, template_path, **template_kwargs)


class SpaceCard(RepoCard):
    card_data_class = SpaceCardData
    default_template_path = TEMPLATE_MODELCARD_PATH
    repo_type = "space"


def _detect_line_ending(content: str) -> Literal["\r", "\n", "\r\n", None]:  # noqa: F722
    """Detect the line ending of a string. Used by RepoCard to avoid making huge diff on newlines.

    Uses same implementation as in Hub server, keep it in sync.

    Returns:
        str: The detected line ending of the string.
    """
    cr = content.count("\r")
    lf = content.count("\n")
    crlf = content.count("\r\n")
    if cr + lf == 0:
        return None
    if crlf == cr and crlf == lf:
        return "\r\n"
    if cr > lf:
        return "\r"
    else:
        return "\n"


def metadata_load(local_path: Union[str, Path]) -> Optional[Dict]:
    content = Path(local_path).read_text()
    match = REGEX_YAML_BLOCK.search(content)
    if match:
        yaml_block = match.group(2)
        data = yaml.safe_load(yaml_block)
        if isinstance(data, dict):
            return data
        else:
            raise ValueError("repo card metadata block should be a dict")
    else:
        return None


def metadata_save(local_path: Union[str, Path], data: Dict) -> None:
    """
    Save the metadata dict in the upper YAML part Trying to preserve newlines as
    in the existing file. Docs about open() with newline="" parameter:
    https://docs.python.org/3/library/functions.html?highlight=open#open Does
    not work with "^M" linebreaks, which are replaced by \n
    """
    line_break = "\n"
    content = ""
    # try to detect existing newline character
    if os.path.exists(local_path):
        with open(local_path, "r", newline="", encoding="utf8") as readme:
            content = readme.read()
            if isinstance(readme.newlines, tuple):
                line_break = readme.newlines[0]
            elif isinstance(readme.newlines, str):
                line_break = readme.newlines

    # creates a new file if it not
    with open(local_path, "w", newline="", encoding="utf8") as readme:
        data_yaml = yaml_dump(data, sort_keys=False, line_break=line_break)
        # sort_keys: keep dict order
        match = REGEX_YAML_BLOCK.search(content)
        if match:
            output = content[: match.start()] + f"---{line_break}{data_yaml}---{line_break}" + content[match.end() :]
        else:
            output = f"---{line_break}{data_yaml}---{line_break}{content}"

        readme.write(output)
        readme.close()


def metadata_eval_result(
    *,
    model_pretty_name: str,
    task_pretty_name: str,
    task_id: str,
    metrics_pretty_name: str,
    metrics_id: str,
    metrics_value: Any,
    dataset_pretty_name: str,
    dataset_id: str,
    metrics_config: Optional[str] = None,
    metrics_verified: bool = False,
    dataset_config: Optional[str] = None,
    dataset_split: Optional[str] = None,
    dataset_revision: Optional[str] = None,
    metrics_verification_token: Optional[str] = None,
) -> Dict:
    """
    Creates a metadata dict with the result from a model evaluated on a dataset.

    Args:
        model_pretty_name (`str`):
            The name of the model in natural language.
        task_pretty_name (`str`):
            The name of a task in natural language.
        task_id (`str`):
            Example: automatic-speech-recognition. A task id.
        metrics_pretty_name (`str`):
            A name for the metric in natural language. Example: Test WER.
        metrics_id (`str`):
            Example: wer. A metric id from https://hf.co/metrics.
        metrics_value (`Any`):
            The value from the metric. Example: 20.0 or "20.0 ± 1.2".
        dataset_pretty_name (`str`):
            The name of the dataset in natural language.
        dataset_id (`str`):
            Example: common_voice. A dataset id from https://hf.co/datasets.
        metrics_config (`str`, *optional*):
            The name of the metric configuration used in `load_metric()`.
            Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`.
        metrics_verified (`bool`, *optional*, defaults to `False`):
            Indicates whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. Automatically computed by Hugging Face, do not set.
        dataset_config (`str`, *optional*):
            Example: fr. The name of the dataset configuration used in `load_dataset()`.
        dataset_split (`str`, *optional*):
            Example: test. The name of the dataset split used in `load_dataset()`.
        dataset_revision (`str`, *optional*):
            Example: 5503434ddd753f426f4b38109466949a1217c2bb. The name of the dataset dataset revision
            used in `load_dataset()`.
        metrics_verification_token (`bool`, *optional*):
            A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not.

    Returns:
        `dict`: a metadata dict with the result from a model evaluated on a dataset.

    Example:
        ```python
        >>> from huggingface_hub import metadata_eval_result
        >>> results = metadata_eval_result(
        ...         model_pretty_name="RoBERTa fine-tuned on ReactionGIF",
        ...         task_pretty_name="Text Classification",
        ...         task_id="text-classification",
        ...         metrics_pretty_name="Accuracy",
        ...         metrics_id="accuracy",
        ...         metrics_value=0.2662102282047272,
        ...         dataset_pretty_name="ReactionJPEG",
        ...         dataset_id="julien-c/reactionjpeg",
        ...         dataset_config="default",
        ...         dataset_split="test",
        ... )
        >>> results == {
        ...     'model-index': [
        ...         {
        ...             'name': 'RoBERTa fine-tuned on ReactionGIF',
        ...             'results': [
        ...                 {
        ...                     'task': {
        ...                         'type': 'text-classification',
        ...                         'name': 'Text Classification'
        ...                     },
        ...                     'dataset': {
        ...                         'name': 'ReactionJPEG',
        ...                         'type': 'julien-c/reactionjpeg',
        ...                         'config': 'default',
        ...                         'split': 'test'
        ...                     },
        ...                     'metrics': [
        ...                         {
        ...                             'type': 'accuracy',
        ...                             'value': 0.2662102282047272,
        ...                             'name': 'Accuracy',
        ...                             'verified': False
        ...                         }
        ...                     ]
        ...                 }
        ...             ]
        ...         }
        ...     ]
        ... }
        True

        ```
    """

    return {
        "model-index": eval_results_to_model_index(
            model_name=model_pretty_name,
            eval_results=[
                EvalResult(
                    task_name=task_pretty_name,
                    task_type=task_id,
                    metric_name=metrics_pretty_name,
                    metric_type=metrics_id,
                    metric_value=metrics_value,
                    dataset_name=dataset_pretty_name,
                    dataset_type=dataset_id,
                    metric_config=metrics_config,
                    verified=metrics_verified,
                    verify_token=metrics_verification_token,
                    dataset_config=dataset_config,
                    dataset_split=dataset_split,
                    dataset_revision=dataset_revision,
                )
            ],
        )
    }


@validate_hf_hub_args
def metadata_update(
    repo_id: str,
    metadata: Dict,
    *,
    repo_type: Optional[str] = None,
    overwrite: bool = False,
    token: Optional[str] = None,
    commit_message: Optional[str] = None,
    commit_description: Optional[str] = None,
    revision: Optional[str] = None,
    create_pr: bool = False,
    parent_commit: Optional[str] = None,
) -> str:
    """
    Updates the metadata in the README.md of a repository on the Hugging Face Hub.
    If the README.md file doesn't exist yet, a new one is created with metadata and an
    the default ModelCard or DatasetCard template. For `space` repo, an error is thrown
    as a Space cannot exist without a `README.md` file.

    Args:
        repo_id (`str`):
            The name of the repository.
        metadata (`dict`):
            A dictionary containing the metadata to be updated.
        repo_type (`str`, *optional*):
            Set to `"dataset"` or `"space"` if updating to a dataset or space,
            `None` or `"model"` if updating to a model. Default is `None`.
        overwrite (`bool`, *optional*, defaults to `False`):
            If set to `True` an existing field can be overwritten, otherwise
            attempting to overwrite an existing field will cause an error.
        token (`str`, *optional*):
            The Hugging Face authentication token.
        commit_message (`str`, *optional*):
            The summary / title / first line of the generated commit. Defaults to
            `f"Update metadata with huggingface_hub"`
        commit_description (`str` *optional*)
            The description of the generated commit
        revision (`str`, *optional*):
            The git revision to commit from. Defaults to the head of the
            `"main"` branch.
        create_pr (`boolean`, *optional*):
            Whether or not to create a Pull Request from `revision` with that commit.
            Defaults to `False`.
        parent_commit (`str`, *optional*):
            The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported.
            If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`.
            If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`.
            Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be
            especially useful if the repo is updated / committed to concurrently.
    Returns:
        `str`: URL of the commit which updated the card metadata.

    Example:
        ```python
        >>> from huggingface_hub import metadata_update
        >>> metadata = {'model-index': [{'name': 'RoBERTa fine-tuned on ReactionGIF',
        ...             'results': [{'dataset': {'name': 'ReactionGIF',
        ...                                      'type': 'julien-c/reactiongif'},
        ...                           'metrics': [{'name': 'Recall',
        ...                                        'type': 'recall',
        ...                                        'value': 0.7762102282047272}],
        ...                          'task': {'name': 'Text Classification',
        ...                                   'type': 'text-classification'}}]}]}
        >>> url = metadata_update("hf-internal-testing/reactiongif-roberta-card", metadata)

        ```
    """
    commit_message = commit_message if commit_message is not None else "Update metadata with huggingface_hub"

    # Card class given repo_type
    card_class: Type[RepoCard]
    if repo_type is None or repo_type == "model":
        card_class = ModelCard
    elif repo_type == "dataset":
        card_class = DatasetCard
    elif repo_type == "space":
        card_class = RepoCard
    else:
        raise ValueError(f"Unknown repo_type: {repo_type}")

    # Either load repo_card from the Hub or create an empty one.
    # NOTE: Will not create the repo if it doesn't exist.
    try:
        card = card_class.load(repo_id, token=token, repo_type=repo_type)
    except EntryNotFoundError:
        if repo_type == "space":
            raise ValueError("Cannot update metadata on a Space that doesn't contain a `README.md` file.")

        # Initialize a ModelCard or DatasetCard from default template and no data.
        card = card_class.from_template(CardData())

    for key, value in metadata.items():
        if key == "model-index":
            # if the new metadata doesn't include a name, either use existing one or repo name
            if "name" not in value[0]:
                value[0]["name"] = getattr(card, "model_name", repo_id)
            model_name, new_results = model_index_to_eval_results(value)
            if card.data.eval_results is None:
                card.data.eval_results = new_results
                card.data.model_name = model_name
            else:
                existing_results = card.data.eval_results

                # Iterate over new results
                #   Iterate over existing results
                #       If both results describe the same metric but value is different:
                #           If overwrite=True: overwrite the metric value
                #           Else: raise ValueError
                #       Else: append new result to existing ones.
                for new_result in new_results:
                    result_found = False
                    for existing_result in existing_results:
                        if new_result.is_equal_except_value(existing_result):
                            if new_result != existing_result and not overwrite:
                                raise ValueError(
                                    "You passed a new value for the existing metric"
                                    f" 'name: {new_result.metric_name}, type: "
                                    f"{new_result.metric_type}'. Set `overwrite=True`"
                                    " to overwrite existing metrics."
                                )
                            result_found = True
                            existing_result.metric_value = new_result.metric_value
                            if existing_result.verified is True:
                                existing_result.verify_token = new_result.verify_token
                    if not result_found:
                        card.data.eval_results.append(new_result)
        else:
            # Any metadata that is not a result metric
            if card.data.get(key) is not None and not overwrite and card.data.get(key) != value:
                raise ValueError(
                    f"You passed a new value for the existing meta data field '{key}'."
                    " Set `overwrite=True` to overwrite existing metadata."
                )
            else:
                card.data[key] = value

    return card.push_to_hub(
        repo_id,
        token=token,
        repo_type=repo_type,
        commit_message=commit_message,
        commit_description=commit_description,
        create_pr=create_pr,
        revision=revision,
        parent_commit=parent_commit,
    )


# src/huggingface_hub/repocard_data.py

import copy
from collections import defaultdict
from dataclasses import dataclass
from typing import Any, Dict, List, Optional, Tuple, Union

from huggingface_hub.utils import yaml_dump

from .utils.logging import get_logger


logger = get_logger(__name__)


@dataclass
class EvalResult:
    """
    Flattened representation of individual evaluation results found in model-index of Model Cards.

    For more information on the model-index spec, see https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1.

    Args:
        task_type (`str`):
            The task identifier. Example: "image-classification".
        dataset_type (`str`):
            The dataset identifier. Example: "common_voice". Use dataset id from https://hf.co/datasets.
        dataset_name (`str`):
            A pretty name for the dataset. Example: "Common Voice (French)".
        metric_type (`str`):
            The metric identifier. Example: "wer". Use metric id from https://hf.co/metrics.
        metric_value (`Any`):
            The metric value. Example: 0.9 or "20.0 ± 1.2".
        task_name (`str`, *optional*):
            A pretty name for the task. Example: "Speech Recognition".
        dataset_config (`str`, *optional*):
            The name of the dataset configuration used in `load_dataset()`.
            Example: fr in `load_dataset("common_voice", "fr")`. See the `datasets` docs for more info:
            https://hf.co/docs/datasets/package_reference/loading_methods#datasets.load_dataset.name
        dataset_split (`str`, *optional*):
            The split used in `load_dataset()`. Example: "test".
        dataset_revision (`str`, *optional*):
            The revision (AKA Git Sha) of the dataset used in `load_dataset()`.
            Example: 5503434ddd753f426f4b38109466949a1217c2bb
        dataset_args (`Dict[str, Any]`, *optional*):
            The arguments passed during `Metric.compute()`. Example for `bleu`: `{"max_order": 4}`
        metric_name (`str`, *optional*):
            A pretty name for the metric. Example: "Test WER".
        metric_config (`str`, *optional*):
            The name of the metric configuration used in `load_metric()`.
            Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`.
            See the `datasets` docs for more info: https://huggingface.co/docs/datasets/v2.1.0/en/loading#load-configurations
        metric_args (`Dict[str, Any]`, *optional*):
            The arguments passed during `Metric.compute()`. Example for `bleu`: max_order: 4
        verified (`bool`, *optional*):
            Indicates whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. Automatically computed by Hugging Face, do not set.
        verify_token (`str`, *optional*):
            A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not.
    """

    # Required

    # The task identifier
    # Example: automatic-speech-recognition
    task_type: str

    # The dataset identifier
    # Example: common_voice. Use dataset id from https://hf.co/datasets
    dataset_type: str

    # A pretty name for the dataset.
    # Example: Common Voice (French)
    dataset_name: str

    # The metric identifier
    # Example: wer. Use metric id from https://hf.co/metrics
    metric_type: str

    # Value of the metric.
    # Example: 20.0 or "20.0 ± 1.2"
    metric_value: Any

    # Optional

    # A pretty name for the task.
    # Example: Speech Recognition
    task_name: Optional[str] = None

    # The name of the dataset configuration used in `load_dataset()`.
    # Example: fr in `load_dataset("common_voice", "fr")`.
    # See the `datasets` docs for more info:
    # https://huggingface.co/docs/datasets/package_reference/loading_methods#datasets.load_dataset.name
    dataset_config: Optional[str] = None

    # The split used in `load_dataset()`.
    # Example: test
    dataset_split: Optional[str] = None

    # The revision (AKA Git Sha) of the dataset used in `load_dataset()`.
    # Example: 5503434ddd753f426f4b38109466949a1217c2bb
    dataset_revision: Optional[str] = None

    # The arguments passed during `Metric.compute()`.
    # Example for `bleu`: max_order: 4
    dataset_args: Optional[Dict[str, Any]] = None

    # A pretty name for the metric.
    # Example: Test WER
    metric_name: Optional[str] = None

    # The name of the metric configuration used in `load_metric()`.
    # Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`.
    # See the `datasets` docs for more info: https://huggingface.co/docs/datasets/v2.1.0/en/loading#load-configurations
    metric_config: Optional[str] = None

    # The arguments passed during `Metric.compute()`.
    # Example for `bleu`: max_order: 4
    metric_args: Optional[Dict[str, Any]] = None

    # Indicates whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. Automatically computed by Hugging Face, do not set.
    verified: Optional[bool] = None

    # A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not.
    verify_token: Optional[str] = None

    @property
    def unique_identifier(self) -> tuple:
        """Returns a tuple that uniquely identifies this evaluation."""
        return (
            self.task_type,
            self.dataset_type,
            self.dataset_config,
            self.dataset_split,
            self.dataset_revision,
        )

    def is_equal_except_value(self, other: "EvalResult") -> bool:
        """
        Return True if `self` and `other` describe exactly the same metric but with a
        different value.
        """
        for key, _ in self.__dict__.items():
            if key == "metric_value":
                continue
            # For metrics computed by Hugging Face's evaluation service, `verify_token` is derived from `metric_value`,
            # so we exclude it here in the comparison.
            if key != "verify_token" and getattr(self, key) != getattr(other, key):
                return False
        return True


@dataclass
class CardData:
    """Structure containing metadata from a RepoCard.

    [`CardData`] is the parent class of [`ModelCardData`] and [`DatasetCardData`].

    Metadata can be exported as a dictionary or YAML. Export can be customized to alter the representation of the data
    (example: flatten evaluation results). `CardData` behaves as a dictionary (can get, pop, set values) but do not
    inherit from `dict` to allow this export step.
    """

    def __init__(self, ignore_metadata_errors: bool = False, **kwargs):
        self.__dict__.update(kwargs)

    def to_dict(self) -> Dict[str, Any]:
        """Converts CardData to a dict.

        Returns:
            `dict`: CardData represented as a dictionary ready to be dumped to a YAML
            block for inclusion in a README.md file.
        """

        data_dict = copy.deepcopy(self.__dict__)
        self._to_dict(data_dict)
        return _remove_none(data_dict)

    def _to_dict(self, data_dict):
        """Use this method in child classes to alter the dict representation of the data. Alter the dict in-place.

        Args:
            data_dict (`dict`): The raw dict representation of the card data.
        """
        pass

    def to_yaml(self, line_break=None) -> str:
        """Dumps CardData to a YAML block for inclusion in a README.md file.

        Args:
            line_break (str, *optional*):
                The line break to use when dumping to yaml.

        Returns:
            `str`: CardData represented as a YAML block.
        """
        return yaml_dump(self.to_dict(), sort_keys=False, line_break=line_break).strip()

    def __repr__(self):
        return self.to_yaml()

    def get(self, key: str, default: Any = None) -> Any:
        """Get value for a given metadata key."""
        return self.__dict__.get(key, default)

    def pop(self, key: str, default: Any = None) -> Any:
        """Pop value for a given metadata key."""
        return self.__dict__.pop(key, default)

    def __getitem__(self, key: str) -> Any:
        """Get value for a given metadata key."""
        return self.__dict__[key]

    def __setitem__(self, key: str, value: Any) -> None:
        """Set value for a given metadata key."""
        self.__dict__[key] = value

    def __contains__(self, key: str) -> bool:
        """Check if a given metadata key is set."""
        return key in self.__dict__


class ModelCardData(CardData):
    """Model Card Metadata that is used by Hugging Face Hub when included at the top of your README.md

    Args:
        language (`Union[str, List[str]]`, *optional*):
            Language of model's training data or metadata. It must be an ISO 639-1, 639-2 or
            639-3 code (two/three letters), or a special value like "code", "multilingual". Defaults to `None`.
        license (`str`, *optional*):
            License of this model. Example: apache-2.0 or any license from
            https://huggingface.co/docs/hub/repositories-licenses. Defaults to None.
        library_name (`str`, *optional*):
            Name of library used by this model. Example: keras or any library from
            https://github.com/huggingface/hub-docs/blob/main/js/src/lib/interfaces/Libraries.ts.
            Defaults to None.
        tags (`List[str]`, *optional*):
            List of tags to add to your model that can be used when filtering on the Hugging
            Face Hub. Defaults to None.
        datasets (`List[str]`, *optional*):
            List of datasets that were used to train this model. Should be a dataset ID
            found on https://hf.co/datasets. Defaults to None.
        metrics (`List[str]`, *optional*):
            List of metrics used to evaluate this model. Should be a metric name that can be found
            at https://hf.co/metrics. Example: 'accuracy'. Defaults to None.
        eval_results (`Union[List[EvalResult], EvalResult]`, *optional*):
            List of `huggingface_hub.EvalResult` that define evaluation results of the model. If provided,
            `model_name` is used to as a name on PapersWithCode's leaderboards. Defaults to `None`.
        model_name (`str`, *optional*):
            A name for this model. It is used along with
            `eval_results` to construct the `model-index` within the card's metadata. The name
            you supply here is what will be used on PapersWithCode's leaderboards. If None is provided
            then the repo name is used as a default. Defaults to None.
        ignore_metadata_errors (`str`):
            If True, errors while parsing the metadata section will be ignored. Some information might be lost during
            the process. Use it at your own risk.
        kwargs (`dict`, *optional*):
            Additional metadata that will be added to the model card. Defaults to None.

    Example:
        ```python
        >>> from huggingface_hub import ModelCardData
        >>> card_data = ModelCardData(
        ...     language="en",
        ...     license="mit",
        ...     library_name="timm",
        ...     tags=['image-classification', 'resnet'],
        ... )
        >>> card_data.to_dict()
        {'language': 'en', 'license': 'mit', 'library_name': 'timm', 'tags': ['image-classification', 'resnet']}

        ```
    """

    def __init__(
        self,
        *,
        language: Optional[Union[str, List[str]]] = None,
        license: Optional[str] = None,
        library_name: Optional[str] = None,
        tags: Optional[List[str]] = None,
        datasets: Optional[List[str]] = None,
        metrics: Optional[List[str]] = None,
        eval_results: Optional[List[EvalResult]] = None,
        model_name: Optional[str] = None,
        ignore_metadata_errors: bool = False,
        **kwargs,
    ):
        self.language = language
        self.license = license
        self.library_name = library_name
        self.tags = tags
        self.datasets = datasets
        self.metrics = metrics
        self.eval_results = eval_results
        self.model_name = model_name

        model_index = kwargs.pop("model-index", None)
        if model_index:
            try:
                model_name, eval_results = model_index_to_eval_results(model_index)
                self.model_name = model_name
                self.eval_results = eval_results
            except KeyError as error:
                if ignore_metadata_errors:
                    logger.warning("Invalid model-index. Not loading eval results into CardData.")
                else:
                    raise ValueError(
                        f"Invalid `model_index` in metadata cannot be parsed: KeyError {error}. Pass"
                        " `ignore_metadata_errors=True` to ignore this error while loading a Model Card. Warning:"
                        " some information will be lost. Use it at your own risk."
                    )

        super().__init__(**kwargs)

        if self.eval_results:
            if type(self.eval_results) == EvalResult:
                self.eval_results = [self.eval_results]
            if self.model_name is None:
                raise ValueError("Passing `eval_results` requires `model_name` to be set.")

    def _to_dict(self, data_dict):
        """Format the internal data dict. In this case, we convert eval results to a valid model index"""
        if self.eval_results is not None:
            data_dict["model-index"] = eval_results_to_model_index(self.model_name, self.eval_results)
            del data_dict["eval_results"], data_dict["model_name"]


class DatasetCardData(CardData):
    """Dataset Card Metadata that is used by Hugging Face Hub when included at the top of your README.md

    Args:
        language (`List[str]`, *optional*):
            Language of dataset's data or metadata. It must be an ISO 639-1, 639-2 or
            639-3 code (two/three letters), or a special value like "code", "multilingual".
        license (`Union[str, List[str]]`, *optional*):
            License(s) of this dataset. Example: apache-2.0 or any license from
            https://huggingface.co/docs/hub/repositories-licenses.
        annotations_creators (`Union[str, List[str]]`, *optional*):
            How the annotations for the dataset were created.
            Options are: 'found', 'crowdsourced', 'expert-generated', 'machine-generated', 'no-annotation', 'other'.
        language_creators (`Union[str, List[str]]`, *optional*):
            How the text-based data in the dataset was created.
            Options are: 'found', 'crowdsourced', 'expert-generated', 'machine-generated', 'other'
        multilinguality (`Union[str, List[str]]`, *optional*):
            Whether the dataset is multilingual.
            Options are: 'monolingual', 'multilingual', 'translation', 'other'.
        size_categories (`Union[str, List[str]]`, *optional*):
            The number of examples in the dataset. Options are: 'n<1K', '1K<n<10K', '10K<n<100K',
            '100K<n<1M', '1M<n<10M', '10M<n<100M', '100M<n<1B', '1B<n<10B', '10B<n<100B', '100B<n<1T', 'n>1T', and 'other'.
        source_datasets (`List[str]]`, *optional*):
            Indicates whether the dataset is an original dataset or extended from another existing dataset.
            Options are: 'original' and 'extended'.
        task_categories (`Union[str, List[str]]`, *optional*):
            What categories of task does the dataset support?
        task_ids (`Union[str, List[str]]`, *optional*):
            What specific tasks does the dataset support?
        paperswithcode_id (`str`, *optional*):
            ID of the dataset on PapersWithCode.
        pretty_name (`str`, *optional*):
            A more human-readable name for the dataset. (ex. "Cats vs. Dogs")
        train_eval_index (`Dict`, *optional*):
            A dictionary that describes the necessary spec for doing evaluation on the Hub.
            If not provided, it will be gathered from the 'train-eval-index' key of the kwargs.
        config_names (`Union[str, List[str]]`, *optional*):
            A list of the available dataset configs for the dataset.
    """

    def __init__(
        self,
        *,
        language: Optional[Union[str, List[str]]] = None,
        license: Optional[Union[str, List[str]]] = None,
        annotations_creators: Optional[Union[str, List[str]]] = None,
        language_creators: Optional[Union[str, List[str]]] = None,
        multilinguality: Optional[Union[str, List[str]]] = None,
        size_categories: Optional[Union[str, List[str]]] = None,
        source_datasets: Optional[List[str]] = None,
        task_categories: Optional[Union[str, List[str]]] = None,
        task_ids: Optional[Union[str, List[str]]] = None,
        paperswithcode_id: Optional[str] = None,
        pretty_name: Optional[str] = None,
        train_eval_index: Optional[Dict] = None,
        config_names: Optional[Union[str, List[str]]] = None,
        ignore_metadata_errors: bool = False,
        **kwargs,
    ):
        self.annotations_creators = annotations_creators
        self.language_creators = language_creators
        self.language = language
        self.license = license
        self.multilinguality = multilinguality
        self.size_categories = size_categories
        self.source_datasets = source_datasets
        self.task_categories = task_categories
        self.task_ids = task_ids
        self.paperswithcode_id = paperswithcode_id
        self.pretty_name = pretty_name
        self.config_names = config_names

        # TODO - maybe handle this similarly to EvalResult?
        self.train_eval_index = train_eval_index or kwargs.pop("train-eval-index", None)
        super().__init__(**kwargs)

    def _to_dict(self, data_dict):
        data_dict["train-eval-index"] = data_dict.pop("train_eval_index")


class SpaceCardData(CardData):
    """Space Card Metadata that is used by Hugging Face Hub when included at the top of your README.md

    To get an exhaustive reference of Spaces configuration, please visit https://huggingface.co/docs/hub/spaces-config-reference#spaces-configuration-reference.

    Args:
        title (`str`, *optional*)
            Title of the Space.
        sdk (`str`, *optional*)
            SDK of the Space (one of `gradio`, `streamlit`, `docker`, or `static`).
        sdk_version (`str`, *optional*)
            Version of the used SDK (if Gradio/Streamlit sdk).
        python_version (`str`, *optional*)
            Python version used in the Space (if Gradio/Streamlit sdk).
        app_file (`str`, *optional*)
            Path to your main application file (which contains either gradio or streamlit Python code, or static html code).
            Path is relative to the root of the repository.
        app_port (`str`, *optional*)
            Port on which your application is running. Used only if sdk is `docker`.
        license (`str`, *optional*)
            License of this model. Example: apache-2.0 or any license from
            https://huggingface.co/docs/hub/repositories-licenses.
        duplicated_from (`str`, *optional*)
            ID of the original Space if this is a duplicated Space.
        models (List[`str`], *optional*)
            List of models related to this Space. Should be a dataset ID found on https://hf.co/models.
        datasets (`List[str]`, *optional*)
            List of datasets related to this Space. Should be a dataset ID found on https://hf.co/datasets.
        tags (`List[str]`, *optional*)
            List of tags to add to your Space that can be used when filtering on the Hub.
        ignore_metadata_errors (`str`):
            If True, errors while parsing the metadata section will be ignored. Some information might be lost during
            the process. Use it at your own risk.
        kwargs (`dict`, *optional*):
            Additional metadata that will be added to the space card.

    Example:
        ```python
        >>> from huggingface_hub import SpaceCardData
        >>> card_data = SpaceCardData(
        ...     title="Dreambooth Training",
        ...     license="mit",
        ...     sdk="gradio",
        ...     duplicated_from="multimodalart/dreambooth-training"
        ... )
        >>> card_data.to_dict()
        {'title': 'Dreambooth Training', 'sdk': 'gradio', 'license': 'mit', 'duplicated_from': 'multimodalart/dreambooth-training'}
        ```
    """

    def __init__(
        self,
        *,
        title: Optional[str] = None,
        sdk: Optional[str] = None,
        sdk_version: Optional[str] = None,
        python_version: Optional[str] = None,
        app_file: Optional[str] = None,
        app_port: Optional[int] = None,
        license: Optional[str] = None,
        duplicated_from: Optional[str] = None,
        models: Optional[List[str]] = None,
        datasets: Optional[List[str]] = None,
        tags: Optional[List[str]] = None,
        ignore_metadata_errors: bool = False,
        **kwargs,
    ):
        self.title = title
        self.sdk = sdk
        self.sdk_version = sdk_version
        self.python_version = python_version
        self.app_file = app_file
        self.app_port = app_port
        self.license = license
        self.duplicated_from = duplicated_from
        self.models = models
        self.datasets = datasets
        self.tags = tags
        super().__init__(**kwargs)


def model_index_to_eval_results(model_index: List[Dict[str, Any]]) -> Tuple[str, List[EvalResult]]:
    """Takes in a model index and returns the model name and a list of `huggingface_hub.EvalResult` objects.

    A detailed spec of the model index can be found here:
    https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1

    Args:
        model_index (`List[Dict[str, Any]]`):
            A model index data structure, likely coming from a README.md file on the
            Hugging Face Hub.

    Returns:
        model_name (`str`):
            The name of the model as found in the model index. This is used as the
            identifier for the model on leaderboards like PapersWithCode.
        eval_results (`List[EvalResult]`):
            A list of `huggingface_hub.EvalResult` objects containing the metrics
            reported in the provided model_index.

    Example:
        ```python
        >>> from huggingface_hub.repocard_data import model_index_to_eval_results
        >>> # Define a minimal model index
        >>> model_index = [
        ...     {
        ...         "name": "my-cool-model",
        ...         "results": [
        ...             {
        ...                 "task": {
        ...                     "type": "image-classification"
        ...                 },
        ...                 "dataset": {
        ...                     "type": "beans",
        ...                     "name": "Beans"
        ...                 },
        ...                 "metrics": [
        ...                     {
        ...                         "type": "accuracy",
        ...                         "value": 0.9
        ...                     }
        ...                 ]
        ...             }
        ...         ]
        ...     }
        ... ]
        >>> model_name, eval_results = model_index_to_eval_results(model_index)
        >>> model_name
        'my-cool-model'
        >>> eval_results[0].task_type
        'image-classification'
        >>> eval_results[0].metric_type
        'accuracy'

        ```
    """

    eval_results = []
    for elem in model_index:
        name = elem["name"]
        results = elem["results"]
        for result in results:
            task_type = result["task"]["type"]
            task_name = result["task"].get("name")
            dataset_type = result["dataset"]["type"]
            dataset_name = result["dataset"]["name"]
            dataset_config = result["dataset"].get("config")
            dataset_split = result["dataset"].get("split")
            dataset_revision = result["dataset"].get("revision")
            dataset_args = result["dataset"].get("args")

            for metric in result["metrics"]:
                metric_type = metric["type"]
                metric_value = metric["value"]
                metric_name = metric.get("name")
                metric_args = metric.get("args")
                metric_config = metric.get("config")
                verified = metric.get("verified")
                verify_token = metric.get("verifyToken")

                eval_result = EvalResult(
                    task_type=task_type,  # Required
                    dataset_type=dataset_type,  # Required
                    dataset_name=dataset_name,  # Required
                    metric_type=metric_type,  # Required
                    metric_value=metric_value,  # Required
                    task_name=task_name,
                    dataset_config=dataset_config,
                    dataset_split=dataset_split,
                    dataset_revision=dataset_revision,
                    dataset_args=dataset_args,
                    metric_name=metric_name,
                    metric_args=metric_args,
                    metric_config=metric_config,
                    verified=verified,
                    verify_token=verify_token,
                )
                eval_results.append(eval_result)
    return name, eval_results


def _remove_none(obj):
    """
    Recursively remove `None` values from a dict. Borrowed from: https://stackoverflow.com/a/20558778
    """
    if isinstance(obj, (list, tuple, set)):
        return type(obj)(_remove_none(x) for x in obj if x is not None)
    elif isinstance(obj, dict):
        return type(obj)((_remove_none(k), _remove_none(v)) for k, v in obj.items() if k is not None and v is not None)
    else:
        return obj


def eval_results_to_model_index(model_name: str, eval_results: List[EvalResult]) -> List[Dict[str, Any]]:
    """Takes in given model name and list of `huggingface_hub.EvalResult` and returns a
    valid model-index that will be compatible with the format expected by the
    Hugging Face Hub.

    Args:
        model_name (`str`):
            Name of the model (ex. "my-cool-model"). This is used as the identifier
            for the model on leaderboards like PapersWithCode.
        eval_results (`List[EvalResult]`):
            List of `huggingface_hub.EvalResult` objects containing the metrics to be
            reported in the model-index.

    Returns:
        model_index (`List[Dict[str, Any]]`): The eval_results converted to a model-index.

    Example:
        ```python
        >>> from huggingface_hub.repocard_data import eval_results_to_model_index, EvalResult
        >>> # Define minimal eval_results
        >>> eval_results = [
        ...     EvalResult(
        ...         task_type="image-classification",  # Required
        ...         dataset_type="beans",  # Required
        ...         dataset_name="Beans",  # Required
        ...         metric_type="accuracy",  # Required
        ...         metric_value=0.9,  # Required
        ...     )
        ... ]
        >>> eval_results_to_model_index("my-cool-model", eval_results)
        [{'name': 'my-cool-model', 'results': [{'task': {'type': 'image-classification'}, 'dataset': {'name': 'Beans', 'type': 'beans'}, 'metrics': [{'type': 'accuracy', 'value': 0.9}]}]}]

        ```
    """

    # Metrics are reported on a unique task-and-dataset basis.
    # Here, we make a map of those pairs and the associated EvalResults.
    task_and_ds_types_map = defaultdict(list)
    for eval_result in eval_results:
        task_and_ds_types_map[eval_result.unique_identifier].append(eval_result)

    # Use the map from above to generate the model index data.
    model_index_data = []
    for results in task_and_ds_types_map.values():
        # All items from `results` share same metadata
        sample_result = results[0]
        data = {
            "task": {
                "type": sample_result.task_type,
                "name": sample_result.task_name,
            },
            "dataset": {
                "name": sample_result.dataset_name,
                "type": sample_result.dataset_type,
                "config": sample_result.dataset_config,
                "split": sample_result.dataset_split,
                "revision": sample_result.dataset_revision,
                "args": sample_result.dataset_args,
            },
            "metrics": [
                {
                    "type": result.metric_type,
                    "value": result.metric_value,
                    "name": result.metric_name,
                    "config": result.metric_config,
                    "args": result.metric_args,
                    "verified": result.verified,
                    "verifyToken": result.verify_token,
                }
                for result in results
            ],
        }
        model_index_data.append(data)

    # TODO - Check if there cases where this list is longer than one?
    # Finally, the model index itself is list of dicts.
    model_index = [
        {
            "name": model_name,
            "results": model_index_data,
        }
    ]
    return _remove_none(model_index)


# src/huggingface_hub/repository.py

import atexit
import os
import re
import subprocess
import threading
import time
from contextlib import contextmanager
from pathlib import Path
from typing import Callable, Dict, Iterator, List, Optional, Tuple, Union
from urllib.parse import urlparse

from huggingface_hub.constants import REPO_TYPES_URL_PREFIXES, REPOCARD_NAME
from huggingface_hub.repocard import metadata_load, metadata_save

from .hf_api import HfApi, repo_type_and_id_from_hf_id
from .lfs import LFS_MULTIPART_UPLOAD_COMMAND
from .utils import (
    HfFolder,
    SoftTemporaryDirectory,
    logging,
    run_subprocess,
    tqdm,
    validate_hf_hub_args,
)
from .utils._typing import TypedDict


logger = logging.get_logger(__name__)


class CommandInProgress:
    """
    Utility to follow commands launched asynchronously.
    """

    def __init__(
        self,
        title: str,
        is_done_method: Callable,
        status_method: Callable,
        process: subprocess.Popen,
        post_method: Optional[Callable] = None,
    ):
        self.title = title
        self._is_done = is_done_method
        self._status = status_method
        self._process = process
        self._stderr = ""
        self._stdout = ""
        self._post_method = post_method

    @property
    def is_done(self) -> bool:
        """
        Whether the process is done.
        """
        result = self._is_done()

        if result and self._post_method is not None:
            self._post_method()
            self._post_method = None

        return result

    @property
    def status(self) -> int:
        """
        The exit code/status of the current action. Will return `0` if the
        command has completed successfully, and a number between 1 and 255 if
        the process errored-out.

        Will return -1 if the command is still ongoing.
        """
        return self._status()

    @property
    def failed(self) -> bool:
        """
        Whether the process errored-out.
        """
        return self.status > 0

    @property
    def stderr(self) -> str:
        """
        The current output message on the standard error.
        """
        if self._process.stderr is not None:
            self._stderr += self._process.stderr.read()
        return self._stderr

    @property
    def stdout(self) -> str:
        """
        The current output message on the standard output.
        """
        if self._process.stdout is not None:
            self._stdout += self._process.stdout.read()
        return self._stdout

    def __repr__(self):
        status = self.status

        if status == -1:
            status = "running"

        return (
            f"[{self.title} command, status code: {status},"
            f" {'in progress.' if not self.is_done else 'finished.'} PID:"
            f" {self._process.pid}]"
        )


def is_git_repo(folder: Union[str, Path]) -> bool:
    """
    Check if the folder is the root or part of a git repository

    Args:
        folder (`str`):
            The folder in which to run the command.

    Returns:
        `bool`: `True` if the repository is part of a repository, `False`
        otherwise.
    """
    folder_exists = os.path.exists(os.path.join(folder, ".git"))
    git_branch = subprocess.run("git branch".split(), cwd=folder, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    return folder_exists and git_branch.returncode == 0


def is_local_clone(folder: Union[str, Path], remote_url: str) -> bool:
    """
    Check if the folder is a local clone of the remote_url

    Args:
        folder (`str` or `Path`):
            The folder in which to run the command.
        remote_url (`str`):
            The url of a git repository.

    Returns:
        `bool`: `True` if the repository is a local clone of the remote
        repository specified, `False` otherwise.
    """
    if not is_git_repo(folder):
        return False

    remotes = run_subprocess("git remote -v", folder).stdout

    # Remove token for the test with remotes.
    remote_url = re.sub(r"https://.*@", "https://", remote_url)
    remotes = [re.sub(r"https://.*@", "https://", remote) for remote in remotes.split()]
    return remote_url in remotes


def is_tracked_with_lfs(filename: Union[str, Path]) -> bool:
    """
    Check if the file passed is tracked with git-lfs.

    Args:
        filename (`str` or `Path`):
            The filename to check.

    Returns:
        `bool`: `True` if the file passed is tracked with git-lfs, `False`
        otherwise.
    """
    folder = Path(filename).parent
    filename = Path(filename).name

    try:
        p = run_subprocess("git check-attr -a".split() + [filename], folder)
        attributes = p.stdout.strip()
    except subprocess.CalledProcessError as exc:
        if not is_git_repo(folder):
            return False
        else:
            raise OSError(exc.stderr)

    if len(attributes) == 0:
        return False

    found_lfs_tag = {"diff": False, "merge": False, "filter": False}

    for attribute in attributes.split("\n"):
        for tag in found_lfs_tag.keys():
            if tag in attribute and "lfs" in attribute:
                found_lfs_tag[tag] = True

    return all(found_lfs_tag.values())


def is_git_ignored(filename: Union[str, Path]) -> bool:
    """
    Check if file is git-ignored. Supports nested .gitignore files.

    Args:
        filename (`str` or `Path`):
            The filename to check.

    Returns:
        `bool`: `True` if the file passed is ignored by `git`, `False`
        otherwise.
    """
    folder = Path(filename).parent
    filename = Path(filename).name

    try:
        p = run_subprocess("git check-ignore".split() + [filename], folder, check=False)
        # Will return exit code 1 if not gitignored
        is_ignored = not bool(p.returncode)
    except subprocess.CalledProcessError as exc:
        raise OSError(exc.stderr)

    return is_ignored


def is_binary_file(filename: Union[str, Path]) -> bool:
    """
    Check if file is a binary file.

    Args:
        filename (`str` or `Path`):
            The filename to check.

    Returns:
        `bool`: `True` if the file passed is a binary file, `False` otherwise.
    """
    try:
        with open(filename, "rb") as f:
            content = f.read(10 * (1024**2))  # Read a maximum of 10MB

        # Code sample taken from the following stack overflow thread
        # https://stackoverflow.com/questions/898669/how-can-i-detect-if-a-file-is-binary-non-text-in-python/7392391#7392391
        text_chars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7F})
        return bool(content.translate(None, text_chars))
    except UnicodeDecodeError:
        return True


def files_to_be_staged(pattern: str = ".", folder: Union[str, Path, None] = None) -> List[str]:
    """
    Returns a list of filenames that are to be staged.

    Args:
        pattern (`str` or `Path`):
            The pattern of filenames to check. Put `.` to get all files.
        folder (`str` or `Path`):
            The folder in which to run the command.

    Returns:
        `List[str]`: List of files that are to be staged.
    """
    try:
        p = run_subprocess("git ls-files --exclude-standard -mo".split() + [pattern], folder)
        if len(p.stdout.strip()):
            files = p.stdout.strip().split("\n")
        else:
            files = []
    except subprocess.CalledProcessError as exc:
        raise EnvironmentError(exc.stderr)

    return files


def is_tracked_upstream(folder: Union[str, Path]) -> bool:
    """
    Check if the current checked-out branch is tracked upstream.

    Args:
        folder (`str` or `Path`):
            The folder in which to run the command.

    Returns:
        `bool`: `True` if the current checked-out branch is tracked upstream,
        `False` otherwise.
    """
    try:
        run_subprocess("git rev-parse --symbolic-full-name --abbrev-ref @{u}", folder)
        return True
    except subprocess.CalledProcessError as exc:
        if "HEAD" in exc.stderr:
            raise OSError("No branch checked out")

        return False


def commits_to_push(folder: Union[str, Path], upstream: Optional[str] = None) -> int:
    """
        Check the number of commits that would be pushed upstream

        Args:
            folder (`str` or `Path`):
                The folder in which to run the command.
            upstream (`str`, *optional*):
    The name of the upstream repository with which the comparison should be
    made.

        Returns:
            `int`: Number of commits that would be pushed upstream were a `git
            push` to proceed.
    """
    try:
        result = run_subprocess(f"git cherry -v {upstream or ''}", folder)
        return len(result.stdout.split("\n")) - 1
    except subprocess.CalledProcessError as exc:
        raise EnvironmentError(exc.stderr)


class PbarT(TypedDict):
    # Used to store an opened progress bar in `_lfs_log_progress`
    bar: tqdm
    past_bytes: int


@contextmanager
def _lfs_log_progress():
    """
    This is a context manager that will log the Git LFS progress of cleaning,
    smudging, pulling and pushing.
    """

    if logger.getEffectiveLevel() >= logging.ERROR:
        try:
            yield
        except Exception:
            pass
        return

    def output_progress(stopping_event: threading.Event):
        """
        To be launched as a separate thread with an event meaning it should stop
        the tail.
        """
        # Key is tuple(state, filename), value is a dict(tqdm bar and a previous value)
        pbars: Dict[Tuple[str, str], PbarT] = {}

        def close_pbars():
            for pbar in pbars.values():
                pbar["bar"].update(pbar["bar"].total - pbar["past_bytes"])
                pbar["bar"].refresh()
                pbar["bar"].close()

        def tail_file(filename) -> Iterator[str]:
            """
            Creates a generator to be iterated through, which will return each
            line one by one. Will stop tailing the file if the stopping_event is
            set.
            """
            with open(filename, "r") as file:
                current_line = ""
                while True:
                    if stopping_event.is_set():
                        close_pbars()
                        break

                    line_bit = file.readline()
                    if line_bit is not None and not len(line_bit.strip()) == 0:
                        current_line += line_bit
                        if current_line.endswith("\n"):
                            yield current_line
                            current_line = ""
                    else:
                        time.sleep(1)

        # If the file isn't created yet, wait for a few seconds before trying again.
        # Can be interrupted with the stopping_event.
        while not os.path.exists(os.environ["GIT_LFS_PROGRESS"]):
            if stopping_event.is_set():
                close_pbars()
                return

            time.sleep(2)

        for line in tail_file(os.environ["GIT_LFS_PROGRESS"]):
            try:
                state, file_progress, byte_progress, filename = line.split()
            except ValueError as error:
                # Try/except to ease debugging. See https://github.com/huggingface/huggingface_hub/issues/1373.
                raise ValueError(f"Cannot unpack LFS progress line:\n{line}") from error
            description = f"{state.capitalize()} file {filename}"

            current_bytes, total_bytes = byte_progress.split("/")
            current_bytes_int = int(current_bytes)
            total_bytes_int = int(total_bytes)

            pbar = pbars.get((state, filename))
            if pbar is None:
                # Initialize progress bar
                pbars[(state, filename)] = {
                    "bar": tqdm(
                        desc=description,
                        initial=current_bytes_int,
                        total=total_bytes_int,
                        unit="B",
                        unit_scale=True,
                        unit_divisor=1024,
                    ),
                    "past_bytes": int(current_bytes),
                }
            else:
                # Update progress bar
                pbar["bar"].update(current_bytes_int - pbar["past_bytes"])
                pbar["past_bytes"] = current_bytes_int

    current_lfs_progress_value = os.environ.get("GIT_LFS_PROGRESS", "")

    with SoftTemporaryDirectory() as tmpdir:
        os.environ["GIT_LFS_PROGRESS"] = os.path.join(tmpdir, "lfs_progress")
        logger.debug(f"Following progress in {os.environ['GIT_LFS_PROGRESS']}")

        exit_event = threading.Event()
        x = threading.Thread(target=output_progress, args=(exit_event,), daemon=True)
        x.start()

        try:
            yield
        finally:
            exit_event.set()
            x.join()

            os.environ["GIT_LFS_PROGRESS"] = current_lfs_progress_value


class Repository:
    """
    Helper class to wrap the git and git-lfs commands.

    The aim is to facilitate interacting with huggingface.co hosted model or
    dataset repos, though not a lot here (if any) is actually specific to
    huggingface.co.
    """

    command_queue: List[CommandInProgress]

    @validate_hf_hub_args
    def __init__(
        self,
        local_dir: Union[str, Path],
        clone_from: Optional[str] = None,
        repo_type: Optional[str] = None,
        token: Union[bool, str] = True,
        git_user: Optional[str] = None,
        git_email: Optional[str] = None,
        revision: Optional[str] = None,
        skip_lfs_files: bool = False,
        client: Optional[HfApi] = None,
    ):
        """
        Instantiate a local clone of a git repo.

        If `clone_from` is set, the repo will be cloned from an existing remote repository.
        If the remote repo does not exist, a `EnvironmentError` exception will be thrown.
        Please create the remote repo first using [`create_repo`].

        `Repository` uses the local git credentials by default. If explicitly set, the `token`
        or the `git_user`/`git_email` pair will be used instead.

        Args:
            local_dir (`str` or `Path`):
                path (e.g. `'my_trained_model/'`) to the local directory, where
                the `Repository` will be initialized.
            clone_from (`str`, *optional*):
                Either a repository url or `repo_id`.
                Example:
                - `"https://huggingface.co/philschmid/playground-tests"`
                - `"philschmid/playground-tests"`
            repo_type (`str`, *optional*):
                To set when cloning a repo from a repo_id. Default is model.
            token (`bool` or `str`, *optional*):
                A valid authentication token (see https://huggingface.co/settings/token).
                If `None` or `True` and machine is logged in (through `huggingface-cli login`
                or [`~huggingface_hub.login`]), token will be retrieved from the cache.
                If `False`, token is not sent in the request header.
            git_user (`str`, *optional*):
                will override the `git config user.name` for committing and
                pushing files to the hub.
            git_email (`str`, *optional*):
                will override the `git config user.email` for committing and
                pushing files to the hub.
            revision (`str`, *optional*):
                Revision to checkout after initializing the repository. If the
                revision doesn't exist, a branch will be created with that
                revision name from the default branch's current HEAD.
            skip_lfs_files (`bool`, *optional*, defaults to `False`):
                whether to skip git-LFS files or not.
            client (`HfApi`, *optional*):
                Instance of [`HfApi`] to use when calling the HF Hub API. A new
                instance will be created if this is left to `None`.

        Raises:
            - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
              if the remote repository set in `clone_from` does not exist.
        """
        if isinstance(local_dir, Path):
            local_dir = str(local_dir)
        os.makedirs(local_dir, exist_ok=True)
        self.local_dir = os.path.join(os.getcwd(), local_dir)
        self._repo_type = repo_type
        self.command_queue = []
        self.skip_lfs_files = skip_lfs_files
        self.client = client if client is not None else HfApi()

        self.check_git_versions()

        if isinstance(token, str):
            self.huggingface_token: Optional[str] = token
        elif token is False:
            self.huggingface_token = None
        else:
            # if `True` -> explicit use of the cached token
            # if `None` -> implicit use of the cached token
            self.huggingface_token = HfFolder.get_token()

        if clone_from is not None:
            self.clone_from(repo_url=clone_from)
        else:
            if is_git_repo(self.local_dir):
                logger.debug("[Repository] is a valid git repo")
            else:
                raise ValueError("If not specifying `clone_from`, you need to pass Repository a valid git clone.")

        if self.huggingface_token is not None and (git_email is None or git_user is None):
            user = self.client.whoami(self.huggingface_token)

            if git_email is None:
                git_email = user["email"]

            if git_user is None:
                git_user = user["fullname"]

        if git_user is not None or git_email is not None:
            self.git_config_username_and_email(git_user, git_email)

        self.lfs_enable_largefiles()
        self.git_credential_helper_store()

        if revision is not None:
            self.git_checkout(revision, create_branch_ok=True)

        # This ensures that all commands exit before exiting the Python runtime.
        # This will ensure all pushes register on the hub, even if other errors happen in subsequent operations.
        atexit.register(self.wait_for_commands)

    @property
    def current_branch(self) -> str:
        """
        Returns the current checked out branch.

        Returns:
            `str`: Current checked out branch.
        """
        try:
            result = run_subprocess("git rev-parse --abbrev-ref HEAD", self.local_dir).stdout.strip()
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

        return result

    def check_git_versions(self):
        """
        Checks that `git` and `git-lfs` can be run.

        Raises:
            - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
              if `git` or `git-lfs` are not installed.
        """
        try:
            git_version = run_subprocess("git --version", self.local_dir).stdout.strip()
        except FileNotFoundError:
            raise EnvironmentError("Looks like you do not have git installed, please install.")

        try:
            lfs_version = run_subprocess("git-lfs --version", self.local_dir).stdout.strip()
        except FileNotFoundError:
            raise EnvironmentError(
                "Looks like you do not have git-lfs installed, please install."
                " You can install from https://git-lfs.github.com/."
                " Then run `git lfs install` (you only have to do this once)."
            )
        logger.info(git_version + "\n" + lfs_version)

    @validate_hf_hub_args
    def clone_from(self, repo_url: str, token: Union[bool, str, None] = None):
        """
        Clone from a remote. If the folder already exists, will try to clone the
        repository within it.

        If this folder is a git repository with linked history, will try to
        update the repository.

        Args:
            repo_url (`str`):
                The URL from which to clone the repository
            token (`Union[str, bool]`, *optional*):
                Whether to use the authentication token. It can be:
                 - a string which is the token itself
                 - `False`, which would not use the authentication token
                 - `True`, which would fetch the authentication token from the
                   local folder and use it (you should be logged in for this to
                   work).
                - `None`, which would retrieve the value of
                  `self.huggingface_token`.

        <Tip>

        Raises the following error:

            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
              if an organization token (starts with "api_org") is passed. Use must use
              your own personal access token (see https://hf.co/settings/tokens).

            - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
              if you are trying to clone the repository in a non-empty folder, or if the
              `git` operations raise errors.

        </Tip>
        """
        token = (
            token  # str -> use it
            if isinstance(token, str)
            else (
                None  # `False` -> explicit no token
                if token is False
                else self.huggingface_token  # `None` or `True` -> use default
            )
        )
        if token is not None and token.startswith("api_org"):
            raise ValueError(
                "You must use your personal access token, not an Organization token"
                " (see https://hf.co/settings/tokens)."
            )

        hub_url = self.client.endpoint
        if hub_url in repo_url or ("http" not in repo_url and len(repo_url.split("/")) <= 2):
            repo_type, namespace, repo_name = repo_type_and_id_from_hf_id(repo_url, hub_url=hub_url)
            repo_id = f"{namespace}/{repo_name}" if namespace is not None else repo_name

            if repo_type is not None:
                self._repo_type = repo_type

            repo_url = hub_url + "/"

            if self._repo_type in REPO_TYPES_URL_PREFIXES:
                repo_url += REPO_TYPES_URL_PREFIXES[self._repo_type]

            if token is not None:
                # Add token in git url when provided
                scheme = urlparse(repo_url).scheme
                repo_url = repo_url.replace(f"{scheme}://", f"{scheme}://user:{token}@")

            repo_url += repo_id

        # For error messages, it's cleaner to show the repo url without the token.
        clean_repo_url = re.sub(r"(https?)://.*@", r"\1://", repo_url)
        try:
            run_subprocess("git lfs install", self.local_dir)

            # checks if repository is initialized in a empty repository or in one with files
            if len(os.listdir(self.local_dir)) == 0:
                logger.warning(f"Cloning {clean_repo_url} into local empty directory.")

                with _lfs_log_progress():
                    env = os.environ.copy()

                    if self.skip_lfs_files:
                        env.update({"GIT_LFS_SKIP_SMUDGE": "1"})

                    run_subprocess(
                        # 'git lfs clone' is deprecated (will display a warning in the terminal)
                        # but we still use it as it provides a nicer UX when downloading large
                        # files (shows progress).
                        f"{'git clone' if self.skip_lfs_files else 'git lfs clone'} {repo_url} .",
                        self.local_dir,
                        env=env,
                    )
            else:
                # Check if the folder is the root of a git repository
                if not is_git_repo(self.local_dir):
                    raise EnvironmentError(
                        "Tried to clone a repository in a non-empty folder that isn't"
                        f" a git repository ('{self.local_dir}'). If you really want to"
                        f" do this, do it manually:\n cd {self.local_dir} && git init"
                        " && git remote add origin && git pull origin main\n or clone"
                        " repo to a new folder and move your existing files there"
                        " afterwards."
                    )

                if is_local_clone(self.local_dir, repo_url):
                    logger.warning(
                        f"{self.local_dir} is already a clone of {clean_repo_url}."
                        " Make sure you pull the latest changes with"
                        " `repo.git_pull()`."
                    )
                else:
                    output = run_subprocess("git remote get-url origin", self.local_dir, check=False)

                    error_msg = (
                        f"Tried to clone {clean_repo_url} in an unrelated git"
                        " repository.\nIf you believe this is an error, please add"
                        f" a remote with the following URL: {clean_repo_url}."
                    )
                    if output.returncode == 0:
                        clean_local_remote_url = re.sub(r"https://.*@", "https://", output.stdout)
                        error_msg += f"\nLocal path has its origin defined as: {clean_local_remote_url}"
                    raise EnvironmentError(error_msg)

        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def git_config_username_and_email(self, git_user: Optional[str] = None, git_email: Optional[str] = None):
        """
        Sets git username and email (only in the current repo).

        Args:
            git_user (`str`, *optional*):
                The username to register through `git`.
            git_email (`str`, *optional*):
                The email to register through `git`.
        """
        try:
            if git_user is not None:
                run_subprocess("git config user.name".split() + [git_user], self.local_dir)

            if git_email is not None:
                run_subprocess(f"git config user.email {git_email}".split(), self.local_dir)
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def git_credential_helper_store(self):
        """
        Sets the git credential helper to `store`
        """
        try:
            run_subprocess("git config credential.helper store", self.local_dir)
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def git_head_hash(self) -> str:
        """
        Get commit sha on top of HEAD.

        Returns:
            `str`: The current checked out commit SHA.
        """
        try:
            p = run_subprocess("git rev-parse HEAD", self.local_dir)
            return p.stdout.strip()
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def git_remote_url(self) -> str:
        """
        Get URL to origin remote.

        Returns:
            `str`: The URL of the `origin` remote.
        """
        try:
            p = run_subprocess("git config --get remote.origin.url", self.local_dir)
            url = p.stdout.strip()
            # Strip basic auth info.
            return re.sub(r"https://.*@", "https://", url)
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def git_head_commit_url(self) -> str:
        """
        Get URL to last commit on HEAD. We assume it's been pushed, and the url
        scheme is the same one as for GitHub or HuggingFace.

        Returns:
            `str`: The URL to the current checked-out commit.
        """
        sha = self.git_head_hash()
        url = self.git_remote_url()
        if url.endswith("/"):
            url = url[:-1]
        return f"{url}/commit/{sha}"

    def list_deleted_files(self) -> List[str]:
        """
        Returns a list of the files that are deleted in the working directory or
        index.

        Returns:
            `List[str]`: A list of files that have been deleted in the working
            directory or index.
        """
        try:
            git_status = run_subprocess("git status -s", self.local_dir).stdout.strip()
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

        if len(git_status) == 0:
            return []

        # Receives a status like the following
        #  D .gitignore
        #  D new_file.json
        # AD new_file1.json
        # ?? new_file2.json
        # ?? new_file4.json

        # Strip each line of whitespaces
        modified_files_statuses = [status.strip() for status in git_status.split("\n")]

        # Only keep files that are deleted using the D prefix
        deleted_files_statuses = [status for status in modified_files_statuses if "D" in status.split()[0]]

        # Remove the D prefix and strip to keep only the relevant filename
        deleted_files = [status.split()[-1].strip() for status in deleted_files_statuses]

        return deleted_files

    def lfs_track(self, patterns: Union[str, List[str]], filename: bool = False):
        """
        Tell git-lfs to track files according to a pattern.

        Setting the `filename` argument to `True` will treat the arguments as
        literal filenames, not as patterns. Any special glob characters in the
        filename will be escaped when writing to the `.gitattributes` file.

        Args:
            patterns (`Union[str, List[str]]`):
                The pattern, or list of patterns, to track with git-lfs.
            filename (`bool`, *optional*, defaults to `False`):
                Whether to use the patterns as literal filenames.
        """
        if isinstance(patterns, str):
            patterns = [patterns]
        try:
            for pattern in patterns:
                run_subprocess(
                    f"git lfs track {'--filename' if filename else ''} {pattern}",
                    self.local_dir,
                )
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def lfs_untrack(self, patterns: Union[str, List[str]]):
        """
        Tell git-lfs to untrack those files.

        Args:
            patterns (`Union[str, List[str]]`):
                The pattern, or list of patterns, to untrack with git-lfs.
        """
        if isinstance(patterns, str):
            patterns = [patterns]
        try:
            for pattern in patterns:
                run_subprocess("git lfs untrack".split() + [pattern], self.local_dir)
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def lfs_enable_largefiles(self):
        """
        HF-specific. This enables upload support of files >5GB.
        """
        try:
            lfs_config = "git config lfs.customtransfer.multipart"
            run_subprocess(f"{lfs_config}.path huggingface-cli", self.local_dir)
            run_subprocess(
                f"{lfs_config}.args {LFS_MULTIPART_UPLOAD_COMMAND}",
                self.local_dir,
            )
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def auto_track_binary_files(self, pattern: str = ".") -> List[str]:
        """
        Automatically track binary files with git-lfs.

        Args:
            pattern (`str`, *optional*, defaults to "."):
                The pattern with which to track files that are binary.

        Returns:
            `List[str]`: List of filenames that are now tracked due to being
            binary files
        """
        files_to_be_tracked_with_lfs = []

        deleted_files = self.list_deleted_files()

        for filename in files_to_be_staged(pattern, folder=self.local_dir):
            if filename in deleted_files:
                continue

            path_to_file = os.path.join(os.getcwd(), self.local_dir, filename)

            if not (is_tracked_with_lfs(path_to_file) or is_git_ignored(path_to_file)):
                size_in_mb = os.path.getsize(path_to_file) / (1024 * 1024)

                if size_in_mb >= 10:
                    logger.warning(
                        "Parsing a large file to check if binary or not. Tracking large"
                        " files using `repository.auto_track_large_files` is"
                        " recommended so as to not load the full file in memory."
                    )

                is_binary = is_binary_file(path_to_file)

                if is_binary:
                    self.lfs_track(filename)
                    files_to_be_tracked_with_lfs.append(filename)

        # Cleanup the .gitattributes if files were deleted
        self.lfs_untrack(deleted_files)

        return files_to_be_tracked_with_lfs

    def auto_track_large_files(self, pattern: str = ".") -> List[str]:
        """
        Automatically track large files (files that weigh more than 10MBs) with
        git-lfs.

        Args:
            pattern (`str`, *optional*, defaults to "."):
                The pattern with which to track files that are above 10MBs.

        Returns:
            `List[str]`: List of filenames that are now tracked due to their
            size.
        """
        files_to_be_tracked_with_lfs = []

        deleted_files = self.list_deleted_files()

        for filename in files_to_be_staged(pattern, folder=self.local_dir):
            if filename in deleted_files:
                continue

            path_to_file = os.path.join(os.getcwd(), self.local_dir, filename)
            size_in_mb = os.path.getsize(path_to_file) / (1024 * 1024)

            if size_in_mb >= 10 and not is_tracked_with_lfs(path_to_file) and not is_git_ignored(path_to_file):
                self.lfs_track(filename)
                files_to_be_tracked_with_lfs.append(filename)

        # Cleanup the .gitattributes if files were deleted
        self.lfs_untrack(deleted_files)

        return files_to_be_tracked_with_lfs

    def lfs_prune(self, recent=False):
        """
        git lfs prune

        Args:
            recent (`bool`, *optional*, defaults to `False`):
                Whether to prune files even if they were referenced by recent
                commits. See the following
                [link](https://github.com/git-lfs/git-lfs/blob/f3d43f0428a84fc4f1e5405b76b5a73ec2437e65/docs/man/git-lfs-prune.1.ronn#recent-files)
                for more information.
        """
        try:
            with _lfs_log_progress():
                result = run_subprocess(f"git lfs prune {'--recent' if recent else ''}", self.local_dir)
                logger.info(result.stdout)
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def git_pull(self, rebase: bool = False, lfs: bool = False):
        """
        git pull

        Args:
            rebase (`bool`, *optional*, defaults to `False`):
                Whether to rebase the current branch on top of the upstream
                branch after fetching.
            lfs (`bool`, *optional*, defaults to `False`):
                Whether to fetch the LFS files too. This option only changes the
                behavior when a repository was cloned without fetching the LFS
                files; calling `repo.git_pull(lfs=True)` will then fetch the LFS
                file from the remote repository.
        """
        command = "git pull" if not lfs else "git lfs pull"
        if rebase:
            command += " --rebase"
        try:
            with _lfs_log_progress():
                result = run_subprocess(command, self.local_dir)
                logger.info(result.stdout)
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def git_add(self, pattern: str = ".", auto_lfs_track: bool = False):
        """
        git add

        Setting the `auto_lfs_track` parameter to `True` will automatically
        track files that are larger than 10MB with `git-lfs`.

        Args:
            pattern (`str`, *optional*, defaults to "."):
                The pattern with which to add files to staging.
            auto_lfs_track (`bool`, *optional*, defaults to `False`):
                Whether to automatically track large and binary files with
                git-lfs. Any file over 10MB in size, or in binary format, will
                be automatically tracked.
        """
        if auto_lfs_track:
            # Track files according to their size (>=10MB)
            tracked_files = self.auto_track_large_files(pattern)

            # Read the remaining files and track them if they're binary
            tracked_files.extend(self.auto_track_binary_files(pattern))

            if tracked_files:
                logger.warning(
                    f"Adding files tracked by Git LFS: {tracked_files}. This may take a"
                    " bit of time if the files are large."
                )

        try:
            result = run_subprocess("git add -v".split() + [pattern], self.local_dir)
            logger.info(f"Adding to index:\n{result.stdout}\n")
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def git_commit(self, commit_message: str = "commit files to HF hub"):
        """
        git commit

        Args:
            commit_message (`str`, *optional*, defaults to "commit files to HF hub"):
                The message attributed to the commit.
        """
        try:
            result = run_subprocess("git commit -v -m".split() + [commit_message], self.local_dir)
            logger.info(f"Committed:\n{result.stdout}\n")
        except subprocess.CalledProcessError as exc:
            if len(exc.stderr) > 0:
                raise EnvironmentError(exc.stderr)
            else:
                raise EnvironmentError(exc.stdout)

    def git_push(
        self,
        upstream: Optional[str] = None,
        blocking: bool = True,
        auto_lfs_prune: bool = False,
    ) -> Union[str, Tuple[str, CommandInProgress]]:
        """
        git push

        If used without setting `blocking`, will return url to commit on remote
        repo. If used with `blocking=True`, will return a tuple containing the
        url to commit and the command object to follow for information about the
        process.

        Args:
            upstream (`str`, *optional*):
                Upstream to which this should push. If not specified, will push
                to the lastly defined upstream or to the default one (`origin
                main`).
            blocking (`bool`, *optional*, defaults to `True`):
                Whether the function should return only when the push has
                finished. Setting this to `False` will return an
                `CommandInProgress` object which has an `is_done` property. This
                property will be set to `True` when the push is finished.
            auto_lfs_prune (`bool`, *optional*, defaults to `False`):
                Whether to automatically prune files once they have been pushed
                to the remote.
        """
        command = "git push"

        if upstream:
            command += f" --set-upstream {upstream}"

        number_of_commits = commits_to_push(self.local_dir, upstream)

        if number_of_commits > 1:
            logger.warning(f"Several commits ({number_of_commits}) will be pushed upstream.")
            if blocking:
                logger.warning("The progress bars may be unreliable.")

        try:
            with _lfs_log_progress():
                process = subprocess.Popen(
                    command.split(),
                    stderr=subprocess.PIPE,
                    stdout=subprocess.PIPE,
                    encoding="utf-8",
                    cwd=self.local_dir,
                )

                if blocking:
                    stdout, stderr = process.communicate()
                    return_code = process.poll()
                    process.kill()

                    if len(stderr):
                        logger.warning(stderr)

                    if return_code:
                        raise subprocess.CalledProcessError(return_code, process.args, output=stdout, stderr=stderr)

        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

        if not blocking:

            def status_method():
                status = process.poll()
                if status is None:
                    return -1
                else:
                    return status

            command_in_progress = CommandInProgress(
                "push",
                is_done_method=lambda: process.poll() is not None,
                status_method=status_method,
                process=process,
                post_method=self.lfs_prune if auto_lfs_prune else None,
            )

            self.command_queue.append(command_in_progress)

            return self.git_head_commit_url(), command_in_progress

        if auto_lfs_prune:
            self.lfs_prune()

        return self.git_head_commit_url()

    def git_checkout(self, revision: str, create_branch_ok: bool = False):
        """
        git checkout a given revision

        Specifying `create_branch_ok` to `True` will create the branch to the
        given revision if that revision doesn't exist.

        Args:
            revision (`str`):
                The revision to checkout.
            create_branch_ok (`str`, *optional*, defaults to `False`):
                Whether creating a branch named with the `revision` passed at
                the current checked-out reference if `revision` isn't an
                existing revision is allowed.
        """
        try:
            result = run_subprocess(f"git checkout {revision}", self.local_dir)
            logger.warning(f"Checked out {revision} from {self.current_branch}.")
            logger.warning(result.stdout)
        except subprocess.CalledProcessError as exc:
            if not create_branch_ok:
                raise EnvironmentError(exc.stderr)
            else:
                try:
                    result = run_subprocess(f"git checkout -b {revision}", self.local_dir)
                    logger.warning(
                        f"Revision `{revision}` does not exist. Created and checked out branch `{revision}`."
                    )
                    logger.warning(result.stdout)
                except subprocess.CalledProcessError as exc:
                    raise EnvironmentError(exc.stderr)

    def tag_exists(self, tag_name: str, remote: Optional[str] = None) -> bool:
        """
        Check if a tag exists or not.

        Args:
            tag_name (`str`):
                The name of the tag to check.
            remote (`str`, *optional*):
                Whether to check if the tag exists on a remote. This parameter
                should be the identifier of the remote.

        Returns:
            `bool`: Whether the tag exists.
        """
        if remote:
            try:
                result = run_subprocess(f"git ls-remote origin refs/tags/{tag_name}", self.local_dir).stdout.strip()
            except subprocess.CalledProcessError as exc:
                raise EnvironmentError(exc.stderr)

            return len(result) != 0
        else:
            try:
                git_tags = run_subprocess("git tag", self.local_dir).stdout.strip()
            except subprocess.CalledProcessError as exc:
                raise EnvironmentError(exc.stderr)

            git_tags = git_tags.split("\n")
            return tag_name in git_tags

    def delete_tag(self, tag_name: str, remote: Optional[str] = None) -> bool:
        """
        Delete a tag, both local and remote, if it exists

        Args:
            tag_name (`str`):
                The tag name to delete.
            remote (`str`, *optional*):
                The remote on which to delete the tag.

        Returns:
             `bool`: `True` if deleted, `False` if the tag didn't exist.
                If remote is not passed, will just be updated locally
        """
        delete_locally = True
        delete_remotely = True

        if not self.tag_exists(tag_name):
            delete_locally = False

        if not self.tag_exists(tag_name, remote=remote):
            delete_remotely = False

        if delete_locally:
            try:
                run_subprocess(["git", "tag", "-d", tag_name], self.local_dir).stdout.strip()
            except subprocess.CalledProcessError as exc:
                raise EnvironmentError(exc.stderr)

        if remote and delete_remotely:
            try:
                run_subprocess(f"git push {remote} --delete {tag_name}", self.local_dir).stdout.strip()
            except subprocess.CalledProcessError as exc:
                raise EnvironmentError(exc.stderr)

        return True

    def add_tag(self, tag_name: str, message: Optional[str] = None, remote: Optional[str] = None):
        """
        Add a tag at the current head and push it

        If remote is None, will just be updated locally

        If no message is provided, the tag will be lightweight. if a message is
        provided, the tag will be annotated.

        Args:
            tag_name (`str`):
                The name of the tag to be added.
            message (`str`, *optional*):
                The message that accompanies the tag. The tag will turn into an
                annotated tag if a message is passed.
            remote (`str`, *optional*):
                The remote on which to add the tag.
        """
        if message:
            tag_args = ["git", "tag", "-a", tag_name, "-m", message]
        else:
            tag_args = ["git", "tag", tag_name]

        try:
            run_subprocess(tag_args, self.local_dir).stdout.strip()
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

        if remote:
            try:
                run_subprocess(f"git push {remote} {tag_name}", self.local_dir).stdout.strip()
            except subprocess.CalledProcessError as exc:
                raise EnvironmentError(exc.stderr)

    def is_repo_clean(self) -> bool:
        """
        Return whether or not the git status is clean or not

        Returns:
            `bool`: `True` if the git status is clean, `False` otherwise.
        """
        try:
            git_status = run_subprocess("git status --porcelain", self.local_dir).stdout.strip()
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

        return len(git_status) == 0

    def push_to_hub(
        self,
        commit_message: str = "commit files to HF hub",
        blocking: bool = True,
        clean_ok: bool = True,
        auto_lfs_prune: bool = False,
    ) -> Union[None, str, Tuple[str, CommandInProgress]]:
        """
        Helper to add, commit, and push files to remote repository on the
        HuggingFace Hub. Will automatically track large files (>10MB).

        Args:
            commit_message (`str`):
                Message to use for the commit.
            blocking (`bool`, *optional*, defaults to `True`):
                Whether the function should return only when the `git push` has
                finished.
            clean_ok (`bool`, *optional*, defaults to `True`):
                If True, this function will return None if the repo is
                untouched. Default behavior is to fail because the git command
                fails.
            auto_lfs_prune (`bool`, *optional*, defaults to `False`):
                Whether to automatically prune files once they have been pushed
                to the remote.
        """
        if clean_ok and self.is_repo_clean():
            logger.info("Repo currently clean. Ignoring push_to_hub")
            return None
        self.git_add(auto_lfs_track=True)
        self.git_commit(commit_message)
        return self.git_push(
            upstream=f"origin {self.current_branch}",
            blocking=blocking,
            auto_lfs_prune=auto_lfs_prune,
        )

    @contextmanager
    def commit(
        self,
        commit_message: str,
        branch: Optional[str] = None,
        track_large_files: bool = True,
        blocking: bool = True,
        auto_lfs_prune: bool = False,
    ):
        """
        Context manager utility to handle committing to a repository. This
        automatically tracks large files (>10Mb) with git-lfs. Set the
        `track_large_files` argument to `False` if you wish to ignore that
        behavior.

        Args:
            commit_message (`str`):
                Message to use for the commit.
            branch (`str`, *optional*):
                The branch on which the commit will appear. This branch will be
                checked-out before any operation.
            track_large_files (`bool`, *optional*, defaults to `True`):
                Whether to automatically track large files or not. Will do so by
                default.
            blocking (`bool`, *optional*, defaults to `True`):
                Whether the function should return only when the `git push` has
                finished.
            auto_lfs_prune (`bool`, defaults to `True`):
                Whether to automatically prune files once they have been pushed
                to the remote.

        Examples:

        ```python
        >>> with Repository(
        ...     "text-files",
        ...     clone_from="<user>/text-files",
        ...     token=True,
        >>> ).commit("My first file :)"):
        ...     with open("file.txt", "w+") as f:
        ...         f.write(json.dumps({"hey": 8}))

        >>> import torch

        >>> model = torch.nn.Transformer()
        >>> with Repository(
        ...     "torch-model",
        ...     clone_from="<user>/torch-model",
        ...     token=True,
        >>> ).commit("My cool model :)"):
        ...     torch.save(model.state_dict(), "model.pt")
        ```

        """

        files_to_stage = files_to_be_staged(".", folder=self.local_dir)

        if len(files_to_stage):
            if len(files_to_stage) > 5:
                files_in_msg = str(files_to_stage[:5])[:-1] + ", ...]"

            logger.error(
                "There exists some updated files in the local repository that are not"
                f" committed: {files_in_msg}. This may lead to errors if checking out"
                " a branch. These files and their modifications will be added to the"
                " current commit."
            )

        if branch is not None:
            self.git_checkout(branch, create_branch_ok=True)

        if is_tracked_upstream(self.local_dir):
            logger.warning("Pulling changes ...")
            self.git_pull(rebase=True)
        else:
            logger.warning(f"The current branch has no upstream branch. Will push to 'origin {self.current_branch}'")

        current_working_directory = os.getcwd()
        os.chdir(os.path.join(current_working_directory, self.local_dir))

        try:
            yield self
        finally:
            self.git_add(auto_lfs_track=track_large_files)

            try:
                self.git_commit(commit_message)
            except OSError as e:
                # If no changes are detected, there is nothing to commit.
                if "nothing to commit" not in str(e):
                    raise e

            try:
                self.git_push(
                    upstream=f"origin {self.current_branch}",
                    blocking=blocking,
                    auto_lfs_prune=auto_lfs_prune,
                )
            except OSError as e:
                # If no changes are detected, there is nothing to commit.
                if "could not read Username" in str(e):
                    raise OSError("Couldn't authenticate user for push. Did you set `token` to `True`?") from e
                else:
                    raise e

            os.chdir(current_working_directory)

    def repocard_metadata_load(self) -> Optional[Dict]:
        filepath = os.path.join(self.local_dir, REPOCARD_NAME)
        if os.path.isfile(filepath):
            return metadata_load(filepath)
        return None

    def repocard_metadata_save(self, data: Dict) -> None:
        return metadata_save(os.path.join(self.local_dir, REPOCARD_NAME), data)

    @property
    def commands_failed(self):
        """
        Returns the asynchronous commands that failed.
        """
        return [c for c in self.command_queue if c.status > 0]

    @property
    def commands_in_progress(self):
        """
        Returns the asynchronous commands that are currently in progress.
        """
        return [c for c in self.command_queue if not c.is_done]

    def wait_for_commands(self):
        """
        Blocking method: blocks all subsequent execution until all commands have
        been processed.
        """
        index = 0
        for command_failed in self.commands_failed:
            logger.error(f"The {command_failed.title} command with PID {command_failed._process.pid} failed.")
            logger.error(command_failed.stderr)

        while self.commands_in_progress:
            if index % 10 == 0:
                logger.warning(
                    f"Waiting for the following commands to finish before shutting down: {self.commands_in_progress}."
                )

            index += 1

            time.sleep(1)


# contrib/conftest.py

import time
import uuid
from typing import Generator

import pytest

from huggingface_hub import HfFolder, delete_repo


@pytest.fixture(scope="session")
def token() -> str:
    # Not critical, only usable on the sandboxed CI instance.
    return "hf_94wBhPGp6KrrTH3KDchhKpRxZwd6dmHWLL"


@pytest.fixture(scope="session")
def user() -> str:
    return "__DUMMY_TRANSFORMERS_USER__"


@pytest.fixture(autouse=True, scope="session")
def login_as_dummy_user(token: str) -> Generator:
    """Login with dummy user token on machine

    Once all tests are completed, set back previous token."""
    # Remove registered token
    old_token = HfFolder().get_token()
    HfFolder().save_token(token)

    yield  # Run all tests

    # Set back token once all tests have passed
    if old_token is not None:
        HfFolder().save_token(old_token)


@pytest.fixture
def repo_name(request) -> None:
    """
    Return a readable pseudo-unique repository name for tests.

    Example: "repo-2fe93f-16599646671840"
    """
    prefix = request.module.__name__  # example: `test_timm`
    id = uuid.uuid4().hex[:6]
    ts = int(time.time() * 10e3)
    return f"repo-{prefix}-{id}-{ts}"


@pytest.fixture
def cleanup_repo(user: str, repo_name: str) -> None:
    """Delete the repo at the end of the tests.

    TODO: Adapt to handle `repo_type` as well
    """
    yield  # run test
    delete_repo(repo_id=f"{user}/{repo_name}")


# contrib/utils.py

import contextlib
from typing import Generator
from unittest.mock import patch


@contextlib.contextmanager
def production_endpoint() -> Generator:
    """Patch huggingface_hub to connect to production server in a context manager.

    Ugly way to patch all constants at once.
    TODO: refactor when https://github.com/huggingface/huggingface_hub/issues/1172 is fixed.

    Example:
    ```py
    def test_push_to_hub():
        # Pull from production Hub
        with production_endpoint():
            model = ...from_pretrained("modelname")

        # Push to staging Hub
        model.push_to_hub()
    ```
    """
    PROD_ENDPOINT = "https://huggingface.co"
    ENDPOINT_TARGETS = [
        "huggingface_hub.constants",
        "huggingface_hub._commit_api",
        "huggingface_hub.hf_api",
        "huggingface_hub.lfs",
        "huggingface_hub.commands.user",
        "huggingface_hub.utils._git_credential",
    ]

    PROD_URL_TEMPLATE = PROD_ENDPOINT + "/{repo_id}/resolve/{revision}/{filename}"
    URL_TEMPLATE_TARGETS = [
        "huggingface_hub.constants",
        "huggingface_hub.file_download",
    ]

    from huggingface_hub.hf_api import api

    patchers = (
        [patch(target + ".ENDPOINT", PROD_ENDPOINT) for target in ENDPOINT_TARGETS]
        + [patch(target + ".HUGGINGFACE_CO_URL_TEMPLATE", PROD_URL_TEMPLATE) for target in URL_TEMPLATE_TARGETS]
        + [patch.object(api, "endpoint", PROD_URL_TEMPLATE)]
    )

    # Start all patches
    for patcher in patchers:
        patcher.start()

    yield

    # Stop all patches
    for patcher in patchers:
        patcher.stop()


# tests/conftest.py

import os
import shutil
from pathlib import Path
from typing import Generator

import pytest
from _pytest.fixtures import SubRequest

import huggingface_hub
from huggingface_hub import HfApi, HfFolder
from huggingface_hub.utils import SoftTemporaryDirectory

from .testing_constants import ENDPOINT_PRODUCTION, PRODUCTION_TOKEN
from .testing_utils import repo_name, set_write_permission_and_retry


@pytest.fixture
def fx_cache_dir(request: SubRequest) -> Generator[None, None, None]:
    """Add a `cache_dir` attribute pointing to a temporary directory in tests.

    Example:
    ```py
    @pytest.mark.usefixtures("fx_cache_dir")
    class TestWithCache(unittest.TestCase):
        cache_dir: Path

        def test_cache_dir(self) -> None:
            self.assertTrue(self.cache_dir.is_dir())
    ```
    """
    with SoftTemporaryDirectory() as cache_dir:
        request.cls.cache_dir = Path(cache_dir).resolve()
        yield
        # TemporaryDirectory is not super robust on Windows when a git repository is
        # cloned in it. See https://www.scivision.dev/python-tempfile-permission-error-windows/.
        shutil.rmtree(cache_dir, onerror=set_write_permission_and_retry)


@pytest.fixture(autouse=True, scope="session")
def clean_hf_folder_token_for_tests() -> Generator:
    """Clean token stored on machine before all tests and reset it back at the end.

    Useful to avoid token deletion when running tests locally.
    """
    # Remove registered token
    token = HfFolder().get_token()
    HfFolder().delete_token()

    yield  # Run all tests

    # Set back token once all tests have passed
    if token is not None:
        HfFolder().save_token(token)


@pytest.fixture(autouse=True)
def disable_symlinks_on_windows_ci(monkeypatch: pytest.MonkeyPatch) -> None:
    class FakeSymlinkDict(dict):
        def __contains__(self, __o: object) -> bool:
            return True  # consider any `cache_dir` to be already checked

        def __getitem__(self, __key: str) -> bool:
            return False  # symlinks are never supported

    if os.name == "nt" and os.environ.get("DISABLE_SYMLINKS_IN_WINDOWS_TESTS"):
        monkeypatch.setattr(
            huggingface_hub.file_download,
            "_are_symlinks_supported_in_dir",
            FakeSymlinkDict(),
        )


@pytest.fixture(autouse=True)
def disable_experimental_warnings(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_DISABLE_EXPERIMENTAL_WARNING", True)


@pytest.fixture
def fx_production_space(request: SubRequest) -> Generator[None, None, None]:
    """Add a `repo_id` attribute referencing a Space repo on the production Hub.

    Fully testing Spaces is not currently possible on staging so we need to use the production
    environment for it. Tests are skipped if we can't find a `HUGGINGFACE_PRODUCTION_USER_TOKEN`
    environment variable.

    Example:
    ```py
    @pytest.mark.usefixtures("fx_production_space")
    class TestSpaceAPI(unittest.TestCase):
        repo_id: str
        api: HfApi

        def test_space(self) -> None:
            api.repo_info(repo_id, repo_type="space")
    ```
    """
    # Check if production token exists
    if PRODUCTION_TOKEN is None:
        pytest.skip("Skip Space tests. `HUGGINGFACE_PRODUCTION_USER_TOKEN` environment variable is not set.")

    # Generate repo id from prod token
    api = HfApi(token=PRODUCTION_TOKEN, endpoint=ENDPOINT_PRODUCTION)
    user = api.whoami()["name"]
    repo_id = f"{user}/{repo_name(prefix='tmp_test_space')}"
    request.cls.api = api
    request.cls.repo_id = repo_id

    # Create and clean space repo
    api.create_repo(repo_id=repo_id, repo_type="space", space_sdk="gradio", private=True)
    api.upload_file(
        path_or_fileobj=_BASIC_APP_PY_TEMPLATE,
        repo_id=repo_id,
        repo_type="space",
        path_in_repo="app.py",
    )
    yield
    api.delete_repo(repo_id=repo_id, repo_type="space")


_BASIC_APP_PY_TEMPLATE = """
import gradio as gr


def greet(name):
    return "Hello " + name + "!!"

iface = gr.Interface(fn=greet, inputs="text", outputs="text")
iface.launch()
""".encode()


# tests/test_cache_layout.py

import os
import time
import unittest
from io import BytesIO

from huggingface_hub import HfApi, hf_hub_download, snapshot_download
from huggingface_hub.utils import SoftTemporaryDirectory, logging
from huggingface_hub.utils._errors import EntryNotFoundError

from .testing_constants import ENDPOINT_STAGING, TOKEN
from .testing_utils import (
    repo_name,
    with_production_testing,
    xfail_on_windows,
)


logger = logging.get_logger(__name__)
MODEL_IDENTIFIER = "hf-internal-testing/hfh-cache-layout"


def get_file_contents(path):
    with open(path) as f:
        content = f.read()

    return content


@with_production_testing
class CacheFileLayoutHfHubDownload(unittest.TestCase):
    @xfail_on_windows(reason="Symlinks are deactivated in Windows tests.")
    def test_file_downloaded_in_cache(self):
        for revision, expected_reference in (
            (None, "main"),
            ("file-2", "file-2"),
        ):
            with self.subTest(revision), SoftTemporaryDirectory() as cache:
                hf_hub_download(
                    MODEL_IDENTIFIER,
                    "file_0.txt",
                    cache_dir=cache,
                    revision=revision,
                )

                expected_directory_name = f'models--{MODEL_IDENTIFIER.replace("/", "--")}'
                expected_path = os.path.join(cache, expected_directory_name)

                refs = os.listdir(os.path.join(expected_path, "refs"))
                snapshots = os.listdir(os.path.join(expected_path, "snapshots"))

                # Only reference should be the expected one.
                self.assertListEqual(refs, [expected_reference])

                with open(os.path.join(expected_path, "refs", expected_reference)) as f:
                    snapshot_name = f.readline().strip()

                # The `main` reference should point to the only snapshot we have downloaded
                self.assertListEqual(snapshots, [snapshot_name])

                snapshot_path = os.path.join(expected_path, "snapshots", snapshot_name)
                snapshot_content = os.listdir(snapshot_path)

                # Only a single file in the snapshot
                self.assertEqual(len(snapshot_content), 1)

                snapshot_content_path = os.path.join(snapshot_path, snapshot_content[0])

                # The snapshot content should link to a blob
                self.assertTrue(os.path.islink(snapshot_content_path))

                resolved_blob_relative = os.readlink(snapshot_content_path)
                resolved_blob_absolute = os.path.normpath(os.path.join(snapshot_path, resolved_blob_relative))

                with open(resolved_blob_absolute) as f:
                    blob_contents = f.readline().strip()

                # The contents of the file should be 'File 0'.
                self.assertEqual(blob_contents, "File 0")

    def test_no_exist_file_is_cached(self):
        revisions = [None, "file-2"]
        expected_references = ["main", "file-2"]
        for revision, expected_reference in zip(revisions, expected_references):
            with self.subTest(revision), SoftTemporaryDirectory() as cache:
                filename = "this_does_not_exist.txt"
                with self.assertRaises(EntryNotFoundError):
                    # The file does not exist, so we get an exception.
                    hf_hub_download(MODEL_IDENTIFIER, filename, cache_dir=cache, revision=revision)

                expected_directory_name = f'models--{MODEL_IDENTIFIER.replace("/", "--")}'
                expected_path = os.path.join(cache, expected_directory_name)

                refs = os.listdir(os.path.join(expected_path, "refs"))
                no_exist_snapshots = os.listdir(os.path.join(expected_path, ".no_exist"))

                # Only reference should be `main`.
                self.assertListEqual(refs, [expected_reference])

                with open(os.path.join(expected_path, "refs", expected_reference)) as f:
                    snapshot_name = f.readline().strip()

                # The `main` reference should point to the only snapshot we have downloaded
                self.assertListEqual(no_exist_snapshots, [snapshot_name])

                no_exist_path = os.path.join(expected_path, ".no_exist", snapshot_name)
                no_exist_content = os.listdir(no_exist_path)

                # Only a single file in the no_exist snapshot
                self.assertEqual(len(no_exist_content), 1)

                # The no_exist content should be our file
                self.assertEqual(no_exist_content[0], filename)

                with open(os.path.join(no_exist_path, filename)) as f:
                    content = f.read().strip()

                # The contents of the file should be empty.
                self.assertEqual(content, "")

    def test_file_download_happens_once(self):
        # Tests that a file is only downloaded once if it's not updated.
        with SoftTemporaryDirectory() as cache:
            path = hf_hub_download(MODEL_IDENTIFIER, "file_0.txt", cache_dir=cache)
            creation_time_0 = os.path.getmtime(path)

            time.sleep(2)

            path = hf_hub_download(MODEL_IDENTIFIER, "file_0.txt", cache_dir=cache)
            creation_time_1 = os.path.getmtime(path)

            self.assertEqual(creation_time_0, creation_time_1)

    @xfail_on_windows(reason="Symlinks are deactivated in Windows tests.")
    def test_file_download_happens_once_intra_revision(self):
        # Tests that a file is only downloaded once if it's not updated, even across different revisions.

        with SoftTemporaryDirectory() as cache:
            path = hf_hub_download(MODEL_IDENTIFIER, "file_0.txt", cache_dir=cache)
            creation_time_0 = os.path.getmtime(path)

            time.sleep(2)

            path = hf_hub_download(MODEL_IDENTIFIER, "file_0.txt", cache_dir=cache, revision="file-2")
            creation_time_1 = os.path.getmtime(path)

            self.assertEqual(creation_time_0, creation_time_1)

    @xfail_on_windows(reason="Symlinks are deactivated in Windows tests.")
    def test_multiple_refs_for_same_file(self):
        with SoftTemporaryDirectory() as cache:
            hf_hub_download(MODEL_IDENTIFIER, "file_0.txt", cache_dir=cache)
            hf_hub_download(MODEL_IDENTIFIER, "file_0.txt", cache_dir=cache, revision="file-2")

            expected_directory_name = f'models--{MODEL_IDENTIFIER.replace("/", "--")}'
            expected_path = os.path.join(cache, expected_directory_name)

            refs = os.listdir(os.path.join(expected_path, "refs"))
            refs.sort()

            snapshots = os.listdir(os.path.join(expected_path, "snapshots"))
            snapshots.sort()

            # Directory should contain two revisions
            self.assertListEqual(refs, ["file-2", "main"])

            refs_contents = [get_file_contents(os.path.join(expected_path, "refs", f)) for f in refs]
            refs_contents.sort()

            # snapshots directory should contain two snapshots
            self.assertListEqual(refs_contents, snapshots)

            snapshot_links = [
                os.readlink(os.path.join(expected_path, "snapshots", filename, "file_0.txt")) for filename in snapshots
            ]

            # All snapshot links should point to the same file.
            self.assertEqual(*snapshot_links)


@with_production_testing
class CacheFileLayoutSnapshotDownload(unittest.TestCase):
    @xfail_on_windows(reason="Symlinks are deactivated in Windows tests.")
    def test_file_downloaded_in_cache(self):
        with SoftTemporaryDirectory() as cache:
            snapshot_download(MODEL_IDENTIFIER, cache_dir=cache)

            expected_directory_name = f'models--{MODEL_IDENTIFIER.replace("/", "--")}'
            expected_path = os.path.join(cache, expected_directory_name)

            refs = os.listdir(os.path.join(expected_path, "refs"))

            snapshots = os.listdir(os.path.join(expected_path, "snapshots"))
            snapshots.sort()

            # Directory should contain two revisions
            self.assertListEqual(refs, ["main"])

            ref_content = get_file_contents(os.path.join(expected_path, "refs", refs[0]))

            # snapshots directory should contain two snapshots
            self.assertListEqual([ref_content], snapshots)

            snapshot_path = os.path.join(expected_path, "snapshots", snapshots[0])

            files_in_snapshot = os.listdir(snapshot_path)

            snapshot_links = [os.readlink(os.path.join(snapshot_path, filename)) for filename in files_in_snapshot]

            resolved_snapshot_links = [os.path.normpath(os.path.join(snapshot_path, link)) for link in snapshot_links]

            self.assertTrue(all([os.path.isfile(link) for link in resolved_snapshot_links]))

    @xfail_on_windows(reason="Symlinks are deactivated in Windows tests.")
    def test_file_downloaded_in_cache_several_revisions(self):
        with SoftTemporaryDirectory() as cache:
            snapshot_download(MODEL_IDENTIFIER, cache_dir=cache, revision="file-3")
            snapshot_download(MODEL_IDENTIFIER, cache_dir=cache, revision="file-2")

            expected_directory_name = f'models--{MODEL_IDENTIFIER.replace("/", "--")}'
            expected_path = os.path.join(cache, expected_directory_name)

            refs = os.listdir(os.path.join(expected_path, "refs"))
            refs.sort()

            snapshots = os.listdir(os.path.join(expected_path, "snapshots"))
            snapshots.sort()

            # Directory should contain two revisions
            self.assertListEqual(refs, ["file-2", "file-3"])

            refs_content = [get_file_contents(os.path.join(expected_path, "refs", ref)) for ref in refs]
            refs_content.sort()

            # snapshots directory should contain two snapshots
            self.assertListEqual(refs_content, snapshots)

            snapshots_paths = [os.path.join(expected_path, "snapshots", s) for s in snapshots]

            files_in_snapshots = {s: os.listdir(s) for s in snapshots_paths}
            links_in_snapshots = {
                k: [os.readlink(os.path.join(k, _v)) for _v in v] for k, v in files_in_snapshots.items()
            }

            resolved_snapshots_links = {
                k: [os.path.normpath(os.path.join(k, link)) for link in v] for k, v in links_in_snapshots.items()
            }

            all_links = [b for a in resolved_snapshots_links.values() for b in a]
            all_unique_links = set(all_links)

            # [ 100]  .
            # ├── [ 140]  blobs
            # │   ├── [   7]  4475433e279a71203927cbe80125208a3b5db560
            # │   ├── [   7]  50fcd26d6ce3000f9d5f12904e80eccdc5685dd1
            # │   ├── [   7]  80146afc836c60e70ba67933fec439ab05b478f6
            # │   ├── [   7]  8cf9e18f080becb674b31c21642538269fe886a4
            # │   └── [1.1K]  ac481c8eb05e4d2496fbe076a38a7b4835dd733d
            # ├── [  80]  refs
            # │   ├── [  40]  file-2
            # │   └── [  40]  file-3
            # └── [  80]  snapshots
            #     ├── [ 120]  5e23cb3ae7f904919a442e1b27dcddae6c6bc292
            #     │   ├── [  52]  file_0.txt -> ../../blobs/80146afc836c60e70ba67933fec439ab05b478f6
            #     │   ├── [  52]  file_1.txt -> ../../blobs/50fcd26d6ce3000f9d5f12904e80eccdc5685dd1
            #     │   ├── [  52]  file_2.txt -> ../../blobs/4475433e279a71203927cbe80125208a3b5db560
            #     │   └── [  52]  .gitattributes -> ../../blobs/ac481c8eb05e4d2496fbe076a38a7b4835dd733d
            #     └── [ 120]  78aa2ebdb60bba086496a8792ba506e58e587b4c
            #         ├── [  52]  file_0.txt -> ../../blobs/80146afc836c60e70ba67933fec439ab05b478f6
            #         ├── [  52]  file_1.txt -> ../../blobs/50fcd26d6ce3000f9d5f12904e80eccdc5685dd1
            #         ├── [  52]  file_3.txt -> ../../blobs/8cf9e18f080becb674b31c21642538269fe886a4
            #         └── [  52]  .gitattributes -> ../../blobs/ac481c8eb05e4d2496fbe076a38a7b4835dd733d

            # Across the two revisions, there should be 8 total links
            self.assertEqual(len(all_links), 8)

            # Across the two revisions, there should only be 5 unique files.
            self.assertEqual(len(all_unique_links), 5)


class ReferenceUpdates(unittest.TestCase):
    _api = HfApi(endpoint=ENDPOINT_STAGING, token=TOKEN)

    def test_update_reference(self):
        repo_id = self._api.create_repo(repo_name(), exist_ok=True).repo_id

        try:
            self._api.upload_file(path_or_fileobj=BytesIO(b"Some string"), path_in_repo="file.txt", repo_id=repo_id)

            with SoftTemporaryDirectory() as cache:
                hf_hub_download(repo_id, "file.txt", cache_dir=cache)

                expected_directory_name = f'models--{repo_id.replace("/", "--")}'
                expected_path = os.path.join(cache, expected_directory_name)

                refs = os.listdir(os.path.join(expected_path, "refs"))

                # Directory should contain two revisions
                self.assertListEqual(refs, ["main"])

                initial_ref_content = get_file_contents(os.path.join(expected_path, "refs", refs[0]))

                # Upload a new file on the same branch
                self._api.upload_file(
                    path_or_fileobj=BytesIO(b"Some new string"),
                    path_in_repo="file.txt",
                    repo_id=repo_id,
                )

                hf_hub_download(repo_id, "file.txt", cache_dir=cache)

                final_ref_content = get_file_contents(os.path.join(expected_path, "refs", refs[0]))

                # The `main` reference should point to two different, but existing snapshots which contain
                # a 'file.txt'
                self.assertNotEqual(initial_ref_content, final_ref_content)
                self.assertTrue(os.path.isdir(os.path.join(expected_path, "snapshots", initial_ref_content)))
                self.assertTrue(
                    os.path.isfile(os.path.join(expected_path, "snapshots", initial_ref_content, "file.txt"))
                )
                self.assertTrue(os.path.isdir(os.path.join(expected_path, "snapshots", final_ref_content)))
                self.assertTrue(
                    os.path.isfile(os.path.join(expected_path, "snapshots", final_ref_content, "file.txt"))
                )
        except Exception:
            raise
        finally:
            self._api.delete_repo(repo_id)


# tests/test_cache_no_symlinks.py

import unittest
import warnings
from pathlib import Path
from unittest.mock import Mock, patch

import pytest

from huggingface_hub import hf_hub_download, scan_cache_dir
from huggingface_hub.constants import CONFIG_NAME, HUGGINGFACE_HUB_CACHE
from huggingface_hub.file_download import are_symlinks_supported

from .testing_constants import TOKEN
from .testing_utils import DUMMY_MODEL_ID, with_production_testing


@with_production_testing
@pytest.mark.usefixtures("fx_cache_dir")
class TestCacheLayoutIfSymlinksNotSupported(unittest.TestCase):
    cache_dir: Path

    @patch(
        "huggingface_hub.file_download._are_symlinks_supported_in_dir",
        {HUGGINGFACE_HUB_CACHE: True},
    )
    def test_are_symlinks_supported_default(self) -> None:
        self.assertTrue(are_symlinks_supported())

    @patch("huggingface_hub.file_download.os.symlink")
    @patch("huggingface_hub.file_download._are_symlinks_supported_in_dir", {})
    def test_are_symlinks_supported_windows_specific_dir(self, mock_symlink: Mock) -> None:
        mock_symlink.side_effect = [OSError(), None]  # First dir not supported then yes
        this_dir = Path(__file__).parent

        # First time in `this_dir`: warning is raised
        with self.assertWarns(UserWarning):
            self.assertFalse(are_symlinks_supported(this_dir))

        with warnings.catch_warnings():
            # Assert no warnings raised
            # Taken from https://stackoverflow.com/a/45671804
            warnings.simplefilter("error")

            # Second time in `this_dir` but with absolute path: value is still cached
            self.assertFalse(are_symlinks_supported(this_dir.absolute()))

            # Try with another directory: symlinks are supported, no warnings
            self.assertTrue(are_symlinks_supported())  # True

    @patch("huggingface_hub.file_download.are_symlinks_supported")
    def test_download_no_symlink_new_file(self, mock_are_symlinks_supported: Mock) -> None:
        mock_are_symlinks_supported.return_value = False
        filepath = Path(
            hf_hub_download(
                DUMMY_MODEL_ID,
                filename=CONFIG_NAME,
                cache_dir=self.cache_dir,
                local_files_only=False,
                use_auth_token=TOKEN,
            )
        )
        # Not a symlink !
        self.assertFalse(filepath.is_symlink())
        self.assertTrue(filepath.is_file())

        # Blobs directory is empty
        self.assertEqual(len(list((Path(filepath).parents[2] / "blobs").glob("*"))), 0)

    @patch("huggingface_hub.file_download.are_symlinks_supported")
    def test_download_no_symlink_existing_file(self, mock_are_symlinks_supported: Mock) -> None:
        mock_are_symlinks_supported.return_value = True
        filepath = Path(
            hf_hub_download(
                DUMMY_MODEL_ID,
                filename=CONFIG_NAME,
                cache_dir=self.cache_dir,
                local_files_only=False,
                use_auth_token=TOKEN,
            )
        )
        self.assertTrue(filepath.is_symlink())
        blob_path = filepath.resolve()
        self.assertTrue(blob_path.is_file())

        # Delete file in snapshot
        filepath.unlink()

        # Re-download but symlinks are not supported anymore (example: not an admin)
        mock_are_symlinks_supported.return_value = False
        new_filepath = Path(
            hf_hub_download(
                DUMMY_MODEL_ID,
                filename=CONFIG_NAME,
                cache_dir=self.cache_dir,
                local_files_only=False,
                use_auth_token=TOKEN,
            )
        )
        # File exist but is not a symlink
        self.assertFalse(new_filepath.is_symlink())
        self.assertTrue(new_filepath.is_file())

        # Blob file still exists as well (has not been deleted)
        # => duplicate file on disk
        self.assertTrue(blob_path.is_file())

    @patch("huggingface_hub.file_download.are_symlinks_supported")
    def test_scan_and_delete_cache_no_symlinks(self, mock_are_symlinks_supported: Mock) -> None:
        """Test scan_cache_dir works as well when cache-system doesn't use symlinks."""
        OLDER_REVISION = "44c70f043cfe8162efc274ff531575e224a0e6f0"

        # Symlinks not supported
        mock_are_symlinks_supported.return_value = False

        # Download config.json from main
        hf_hub_download(
            DUMMY_MODEL_ID,
            filename=CONFIG_NAME,
            cache_dir=self.cache_dir,
            use_auth_token=TOKEN,
        )

        # Download README.md from main
        hf_hub_download(
            DUMMY_MODEL_ID,
            filename="README.md",
            cache_dir=self.cache_dir,
            use_auth_token=TOKEN,
        )

        # Download config.json from older revision
        hf_hub_download(
            DUMMY_MODEL_ID,
            filename=CONFIG_NAME,
            cache_dir=self.cache_dir,
            use_auth_token=TOKEN,
            revision=OLDER_REVISION,
        )

        # Now symlinks work: user has rerun the script as admin
        mock_are_symlinks_supported.return_value = True

        # Download merges.txt from older revision with symlinks
        hf_hub_download(
            DUMMY_MODEL_ID,
            filename="merges.txt",
            cache_dir=self.cache_dir,
            use_auth_token=TOKEN,
            revision=OLDER_REVISION,
        )

        # Scan cache directory
        report = scan_cache_dir(self.cache_dir)

        # 1 repo found, no warnings
        self.assertEqual(len(report.repos), 1)
        self.assertEqual(len(report.warnings), 0)
        repo = list(report.repos)[0]

        # 2 revisions found
        self.assertEqual(len(repo.revisions), 2)
        self.assertEqual(repo.nb_files, 4)
        self.assertEqual(len(repo.refs), 1)  # only `main`
        main_revision = repo.refs["main"]
        main_ref = main_revision.commit_hash
        older_revision = [rev for rev in repo.revisions if rev is not main_revision][0]

        # 2 files in `main` revisions, both are not symlinks
        self.assertEqual(main_revision.nb_files, 2)
        for file in main_revision.files:
            # No symlinks means the files are in the snapshot dir itself
            self.assertTrue(main_revision.snapshot_path in file.blob_path.parents)

        # 2 files in older revision, only 1 as symlink
        for file in older_revision.files:
            if file.file_name == CONFIG_NAME:
                # In snapshot dir
                self.assertTrue(older_revision.snapshot_path in file.blob_path.parents)
            else:
                # In blob dir
                self.assertFalse(older_revision.snapshot_path in file.blob_path.parents)
                self.assertTrue("blobs" in str(file.blob_path))

        # Since files are not shared (README.md is duplicated in cache), the total size
        # of the repo is the sum of each revision size. If symlinks were used, the total
        # size of the repo would be lower.
        self.assertEqual(repo.size_on_disk, main_revision.size_on_disk + older_revision.size_on_disk)

        # Test delete repo strategy
        strategy_delete_repo = report.delete_revisions(main_ref, OLDER_REVISION)
        self.assertEqual(strategy_delete_repo.expected_freed_size, repo.size_on_disk)
        self.assertEqual(len(strategy_delete_repo.blobs), 0)
        self.assertEqual(len(strategy_delete_repo.snapshots), 0)
        self.assertEqual(len(strategy_delete_repo.refs), 0)
        self.assertEqual(len(strategy_delete_repo.repos), 1)

        # Test delete older revision strategy
        strategy_delete_revision = report.delete_revisions(OLDER_REVISION)
        self.assertEqual(
            strategy_delete_revision.blobs,
            {file.blob_path for file in older_revision.files},
        )
        self.assertEqual(strategy_delete_revision.snapshots, {older_revision.snapshot_path})
        self.assertEqual(len(strategy_delete_revision.refs), 0)
        self.assertEqual(len(strategy_delete_revision.repos), 0)
        strategy_delete_revision.execute()  # Execute without error


# tests/test_cli.py

import unittest
from argparse import ArgumentParser

from huggingface_hub.commands.delete_cache import DeleteCacheCommand
from huggingface_hub.commands.scan_cache import ScanCacheCommand


class TestCLI(unittest.TestCase):
    def setUp(self) -> None:
        """
        Set up CLI as in `src/huggingface_hub/commands/huggingface_cli.py`.

        TODO: add other subcommands.
        """
        self.parser = ArgumentParser("huggingface-cli", usage="huggingface-cli <command> [<args>]")
        commands_parser = self.parser.add_subparsers()
        ScanCacheCommand.register_subcommand(commands_parser)
        DeleteCacheCommand.register_subcommand(commands_parser)

    def test_scan_cache_basic(self) -> None:
        """Test `huggingface-cli scan-cache`."""
        args = self.parser.parse_args(["scan-cache"])
        self.assertEqual(args.dir, None)
        self.assertEqual(args.verbose, 0)
        self.assertEqual(args.func, ScanCacheCommand)

    def test_scan_cache_verbose(self) -> None:
        """Test `huggingface-cli scan-cache -v`."""
        args = self.parser.parse_args(["scan-cache", "-v"])
        self.assertEqual(args.dir, None)
        self.assertEqual(args.verbose, 1)
        self.assertEqual(args.func, ScanCacheCommand)

    def test_scan_cache_with_dir(self) -> None:
        """Test `huggingface-cli scan-cache --dir something`."""
        args = self.parser.parse_args(["scan-cache", "--dir", "something"])
        self.assertEqual(args.dir, "something")
        self.assertEqual(args.verbose, 0)
        self.assertEqual(args.func, ScanCacheCommand)

    def test_scan_cache_ultra_verbose(self) -> None:
        """Test `huggingface-cli scan-cache -vvv`."""
        args = self.parser.parse_args(["scan-cache", "-vvv"])
        self.assertEqual(args.dir, None)
        self.assertEqual(args.verbose, 3)
        self.assertEqual(args.func, ScanCacheCommand)

    def test_delete_cache_with_dir(self) -> None:
        """Test `huggingface-cli delete-cache --dir something`."""
        args = self.parser.parse_args(["delete-cache", "--dir", "something"])
        self.assertEqual(args.dir, "something")
        self.assertEqual(args.func, DeleteCacheCommand)


# tests/test_command_delete_cache.py

import os
import unittest
from pathlib import Path
from tempfile import mkstemp
from unittest.mock import Mock, patch

from InquirerPy.base.control import Choice
from InquirerPy.separator import Separator

from huggingface_hub.commands.delete_cache import (
    _CANCEL_DELETION_STR,
    DeleteCacheCommand,
    _ask_for_confirmation_no_tui,
    _get_expectations_str,
    _get_tui_choices_from_scan,
    _manual_review_no_tui,
    _read_manual_review_tmp_file,
)
from huggingface_hub.utils import SoftTemporaryDirectory, capture_output

from .testing_utils import handle_injection


class TestDeleteCacheHelpers(unittest.TestCase):
    def test_get_tui_choices_from_scan_empty(self) -> None:
        choices = _get_tui_choices_from_scan(repos={}, preselected=[])
        self.assertEqual(len(choices), 1)
        self.assertIsInstance(choices[0], Choice)
        self.assertEqual(choices[0].value, _CANCEL_DELETION_STR)
        self.assertTrue(len(choices[0].name) != 0)  # Something displayed to the user
        self.assertFalse(choices[0].enabled)

    def test_get_tui_choices_from_scan_with_preselection(self) -> None:
        choices = _get_tui_choices_from_scan(
            repos=_get_cache_mock().repos,
            preselected=[
                "dataset_revision_hash_id",  # dataset_1 is preselected
                "a_revision_id_that_does_not_exist",  # unknown but will not complain
                "older_hash_id",  # only the oldest revision from model_2
            ],
        )
        self.assertEqual(len(choices), 8)

        # Item to cancel everything
        self.assertIsInstance(choices[0], Choice)
        self.assertEqual(choices[0].value, _CANCEL_DELETION_STR)
        self.assertTrue(len(choices[0].name) != 0)
        self.assertFalse(choices[0].enabled)

        # Dataset repo separator
        self.assertIsInstance(choices[1], Separator)
        self.assertEqual(choices[1]._line, "\nDataset dummy_dataset (8M, used 2 weeks ago)")

        # Only revision of `dummy_dataset`
        self.assertIsInstance(choices[2], Choice)
        self.assertEqual(choices[2].value, "dataset_revision_hash_id")
        self.assertEqual(
            choices[2].name,
            # truncated hash id + detached + last modified
            "dataset_: (detached) # modified 1 day ago",
        )
        self.assertTrue(choices[2].enabled)  # preselected

        # Model `dummy_model` separator
        self.assertIsInstance(choices[3], Separator)
        self.assertEqual(choices[3]._line, "\nModel dummy_model (1.4K, used 2 years ago)")

        # Oldest revision of `dummy_model`
        self.assertIsInstance(choices[4], Choice)
        self.assertEqual(choices[4].value, "older_hash_id")
        self.assertEqual(choices[4].name, "older_ha: (detached) # modified 3 years ago")
        self.assertTrue(choices[4].enabled)  # preselected

        # Newest revision of `dummy_model`
        self.assertIsInstance(choices[5], Choice)
        self.assertEqual(choices[5].value, "recent_hash_id")
        self.assertEqual(choices[5].name, "recent_h: main # modified 2 years ago")
        self.assertFalse(choices[5].enabled)

        # Model `gpt2` separator
        self.assertIsInstance(choices[6], Separator)
        self.assertEqual(choices[6]._line, "\nModel gpt2 (3.6G, used 2 hours ago)")

        # Only revision of `gpt2`
        self.assertIsInstance(choices[7], Choice)
        self.assertEqual(choices[7].value, "abcdef123456789")
        self.assertEqual(choices[7].name, "abcdef12: main, refs/pr/1 # modified 2 years ago")
        self.assertFalse(choices[7].enabled)

    def test_get_expectations_str_on_no_deletion_item(self) -> None:
        """Test `_get_instructions` when `_CANCEL_DELETION_STR` is passed."""
        self.assertEqual(
            _get_expectations_str(
                hf_cache_info=Mock(),
                selected_hashes=["hash_1", _CANCEL_DELETION_STR, "hash_2"],
            ),
            "Nothing will be deleted.",
        )

    def test_get_expectations_str_with_selection(self) -> None:
        """Test `_get_instructions` with 2 revisions selected."""
        strategy_mock = Mock()
        strategy_mock.expected_freed_size_str = "5.1M"

        cache_mock = Mock()
        cache_mock.delete_revisions.return_value = strategy_mock

        self.assertEqual(
            _get_expectations_str(
                hf_cache_info=cache_mock,
                selected_hashes=["hash_1", "hash_2"],
            ),
            "2 revisions selected counting for 5.1M.",
        )
        cache_mock.delete_revisions.assert_called_once_with("hash_1", "hash_2")

    def test_read_manual_review_tmp_file(self) -> None:
        """Test `_read_manual_review_tmp_file`."""

        with SoftTemporaryDirectory() as tmp_dir:
            tmp_path = Path(tmp_dir) / "file.txt"

            with tmp_path.open("w") as f:
                f.writelines(
                    [
                        "# something commented out\n",
                        "###\n",
                        "\n\n\n\n",  # some empty lines
                        "    # Something commented out after spaces\n",
                        "a_revision_hash\n",
                        "a_revision_hash_with_a_comment # 2 years ago\n",
                        "    a_revision_hash_after_spaces\n",
                        "  a_revision_hash_with_a_comment_after_spaces # 2years ago\n",
                        "    # hash_commented_out #  2 years ago\n",
                        "a_revision_hash\n",  # Duplicate
                        "",  # empty line
                    ]
                )

            # Only non-commented lines are returned
            # Order is kept and lines are not de-duplicated
            self.assertListEqual(
                _read_manual_review_tmp_file(tmp_path),
                [
                    "a_revision_hash",
                    "a_revision_hash_with_a_comment",
                    "a_revision_hash_after_spaces",
                    "a_revision_hash_with_a_comment_after_spaces",
                    "a_revision_hash",
                ],
            )

    @patch("huggingface_hub.commands.delete_cache.input")
    @patch("huggingface_hub.commands.delete_cache.mkstemp")
    def test_manual_review_no_tui(self, mock_mkstemp: Mock, mock_input: Mock) -> None:
        # Mock file creation so that we know the file location in test
        fd, tmp_path = mkstemp()
        mock_mkstemp.return_value = fd, tmp_path

        # Mock cache
        cache_mock = _get_cache_mock()

        # Mock input from user
        def _input_answers():
            self.assertTrue(os.path.isfile(tmp_path))  # not deleted yet
            with open(tmp_path) as f:
                content = f.read()
            self.assertTrue(content.startswith("# INSTRUCTIONS"))

            # older_hash_id is not commented
            self.assertIn("\n    older_hash_id # Refs: (detached)", content)
            # same for abcdef123456789
            self.assertIn("\n    abcdef123456789 # Refs: main, refs/pr/1", content)
            # dataset revision is not preselected
            self.assertIn("#    dataset_revision_hash_id", content)
            # same for recent_hash_id
            self.assertIn("#    recent_hash_id", content)

            # Select dataset revision
            content = content.replace("#    dataset_revision_hash_id", "dataset_revision_hash_id")
            # Deselect abcdef123456789
            content = content.replace("abcdef123456789", "# abcdef123456789")
            with open(tmp_path, "w") as f:
                f.write(content)

            yield "no"  # User edited the file and want to see the strategy diff
            yield "y"  # User confirms

        mock_input.side_effect = _input_answers()

        # Run manual review
        with capture_output() as output:
            selected_hashes = _manual_review_no_tui(
                hf_cache_info=cache_mock,
                preselected=["abcdef123456789", "older_hash_id"],
            )

        # Tmp file has been created but is now deleted
        mock_mkstemp.assert_called_once_with(suffix=".txt")
        self.assertFalse(os.path.isfile(tmp_path))  # now deleted

        # User changed the selection
        self.assertListEqual(selected_hashes, ["dataset_revision_hash_id", "older_hash_id"])

        # Check printed instructions
        printed = output.getvalue()
        self.assertTrue(printed.startswith("TUI is disabled. In order to"))  # ...
        self.assertIn(tmp_path, printed)

        # Check input called twice
        self.assertEqual(mock_input.call_count, 2)

    @patch("huggingface_hub.commands.delete_cache.input")
    def test_ask_for_confirmation_no_tui(self, mock_input: Mock) -> None:
        """Test `_ask_for_confirmation_no_tui`."""
        # Answer yes
        mock_input.side_effect = ("y",)
        value = _ask_for_confirmation_no_tui("custom message 1", default=True)
        mock_input.assert_called_with("custom message 1 (Y/n) ")
        self.assertTrue(value)

        # Answer no
        mock_input.side_effect = ("NO",)
        value = _ask_for_confirmation_no_tui("custom message 2", default=True)
        mock_input.assert_called_with("custom message 2 (Y/n) ")
        self.assertFalse(value)

        # Answer invalid, then default
        mock_input.side_effect = ("foo", "")
        with capture_output() as output:
            value = _ask_for_confirmation_no_tui("custom message 3", default=False)
        mock_input.assert_called_with("custom message 3 (y/N) ")
        self.assertFalse(value)
        self.assertEqual(
            output.getvalue(),
            "Invalid input. Must be one of ('y', 'yes', '1', 'n', 'no', '0', '')\n",
        )


@patch("huggingface_hub.commands.delete_cache._ask_for_confirmation_no_tui")
@patch("huggingface_hub.commands.delete_cache._get_expectations_str")
@patch("huggingface_hub.commands.delete_cache.inquirer.confirm")
@patch("huggingface_hub.commands.delete_cache._manual_review_tui")
@patch("huggingface_hub.commands.delete_cache._manual_review_no_tui")
@patch("huggingface_hub.commands.delete_cache.scan_cache_dir")
@handle_injection
class TestMockedDeleteCacheCommand(unittest.TestCase):
    """Test case with a patched `DeleteCacheCommand` to test `.run()` without testing
    the manual review.
    """

    args: Mock
    command: DeleteCacheCommand

    def setUp(self) -> None:
        self.args = Mock()
        self.command = DeleteCacheCommand(self.args)

    def test_run_and_delete_with_tui(
        self,
        mock_scan_cache_dir: Mock,
        mock__manual_review_tui: Mock,
        mock__get_expectations_str: Mock,
        mock_confirm: Mock,
    ) -> None:
        """Test command run with a mocked manual review step."""
        # Mock return values
        mock__manual_review_tui.return_value = ["hash_1", "hash_2"]
        mock__get_expectations_str.return_value = "Will delete A and B."
        mock_confirm.return_value.execute.return_value = True
        mock_scan_cache_dir.return_value = _get_cache_mock()

        # Run
        self.command.disable_tui = False
        with capture_output() as output:
            self.command.run()

        # Step 1: scan
        mock_scan_cache_dir.assert_called_once_with(self.args.dir)
        cache_mock = mock_scan_cache_dir.return_value

        # Step 2: manual review
        mock__manual_review_tui.assert_called_once_with(cache_mock, preselected=[])

        # Step 3: ask confirmation
        mock__get_expectations_str.assert_called_once_with(cache_mock, ["hash_1", "hash_2"])
        mock_confirm.assert_called_once_with("Will delete A and B. Confirm deletion ?", default=True)
        mock_confirm().execute.assert_called_once_with()

        # Step 4: delete
        cache_mock.delete_revisions.assert_called_once_with("hash_1", "hash_2")
        strategy_mock = cache_mock.delete_revisions.return_value
        strategy_mock.execute.assert_called_once_with()

        # Check output
        self.assertEqual(
            output.getvalue(),
            "Start deletion.\nDone. Deleted 0 repo(s) and 0 revision(s) for a total of 5.1M.\n",
        )

    def test_run_nothing_selected_with_tui(self, mock__manual_review_tui: Mock) -> None:
        """Test command run but nothing is selected in manual review."""
        # Mock return value
        mock__manual_review_tui.return_value = []

        # Run
        self.command.disable_tui = False
        with capture_output() as output:
            self.command.run()

        # Check output
        self.assertEqual(output.getvalue(), "Deletion is cancelled. Do nothing.\n")

    def test_run_stuff_selected_but_cancel_item_as_well_with_tui(self, mock__manual_review_tui: Mock) -> None:
        """Test command run when some are selected but "cancel item" as well."""
        # Mock return value
        mock__manual_review_tui.return_value = [
            "hash_1",
            "hash_2",
            _CANCEL_DELETION_STR,
        ]

        # Run
        self.command.disable_tui = False
        with capture_output() as output:
            self.command.run()

        # Check output
        self.assertEqual(output.getvalue(), "Deletion is cancelled. Do nothing.\n")

    def test_run_and_delete_no_tui(
        self,
        mock_scan_cache_dir: Mock,
        mock__manual_review_no_tui: Mock,
        mock__get_expectations_str: Mock,
        mock__ask_for_confirmation_no_tui: Mock,
    ) -> None:
        """Test command run with a mocked manual review step."""
        # Mock return values
        mock__manual_review_no_tui.return_value = ["hash_1", "hash_2"]
        mock__get_expectations_str.return_value = "Will delete A and B."
        mock__ask_for_confirmation_no_tui.return_value.return_value = True
        mock_scan_cache_dir.return_value = _get_cache_mock()

        # Run
        self.command.disable_tui = True
        with capture_output() as output:
            self.command.run()

        # Step 1: scan
        mock_scan_cache_dir.assert_called_once_with(self.args.dir)
        cache_mock = mock_scan_cache_dir.return_value

        # Step 2: manual review
        mock__manual_review_no_tui.assert_called_once_with(cache_mock, preselected=[])

        # Step 3: ask confirmation
        mock__get_expectations_str.assert_called_once_with(cache_mock, ["hash_1", "hash_2"])
        mock__ask_for_confirmation_no_tui.assert_called_once_with("Will delete A and B. Confirm deletion ?")

        # Step 4: delete
        cache_mock.delete_revisions.assert_called_once_with("hash_1", "hash_2")
        strategy_mock = cache_mock.delete_revisions.return_value
        strategy_mock.execute.assert_called_once_with()

        # Check output
        self.assertEqual(
            output.getvalue(),
            "Start deletion.\nDone. Deleted 0 repo(s) and 0 revision(s) for a total of 5.1M.\n",
        )


def _get_cache_mock() -> Mock:
    # First model with 1 revision
    model_1 = Mock()
    model_1.repo_type = "model"
    model_1.repo_id = "gpt2"
    model_1.size_on_disk_str = "3.6G"
    model_1.last_accessed = 1660000000
    model_1.last_accessed_str = "2 hours ago"

    model_1_revision_1 = Mock()
    model_1_revision_1.commit_hash = "abcdef123456789"
    model_1_revision_1.refs = {"main", "refs/pr/1"}
    # model_1_revision_1.last_modified = 123456789  # timestamp
    model_1_revision_1.last_modified_str = "2 years ago"

    model_1.revisions = {model_1_revision_1}

    # Second model with 2 revisions
    model_2 = Mock()
    model_2.repo_type = "model"
    model_2.repo_id = "dummy_model"
    model_2.size_on_disk_str = "1.4K"
    model_2.last_accessed = 1550000000
    model_2.last_accessed_str = "2 years ago"

    model_2_revision_1 = Mock()
    model_2_revision_1.commit_hash = "recent_hash_id"
    model_2_revision_1.refs = {"main"}
    model_2_revision_1.last_modified = 123456789  # newer timestamp
    model_2_revision_1.last_modified_str = "2 years ago"

    model_2_revision_2 = Mock()
    model_2_revision_2.commit_hash = "older_hash_id"
    model_2_revision_2.refs = {}
    model_2_revision_2.last_modified = 12345678  # older timestamp
    model_2_revision_2.last_modified_str = "3 years ago"

    model_2.revisions = {model_2_revision_1, model_2_revision_2}

    # And a dataset with 1 revision
    dataset_1 = Mock()
    dataset_1.repo_type = "dataset"
    dataset_1.repo_id = "dummy_dataset"
    dataset_1.size_on_disk_str = "8M"
    dataset_1.last_accessed = 1660000000
    dataset_1.last_accessed_str = "2 weeks ago"

    dataset_1_revision_1 = Mock()
    dataset_1_revision_1.commit_hash = "dataset_revision_hash_id"
    dataset_1_revision_1.refs = {}
    dataset_1_revision_1.last_modified_str = "1 day ago"

    dataset_1.revisions = {dataset_1_revision_1}

    # Fake cache
    strategy_mock = Mock()
    strategy_mock.repos = []
    strategy_mock.snapshots = []
    strategy_mock.expected_freed_size_str = "5.1M"

    cache_mock = Mock()
    cache_mock.repos = {model_1, model_2, dataset_1}
    cache_mock.delete_revisions.return_value = strategy_mock
    return cache_mock


# tests/test_commit_api.py

import unittest

from huggingface_hub._commit_api import (
    CommitOperationAdd,
    CommitOperationDelete,
    warn_on_overwriting_operations,
)


class TestCommitOperationDelete(unittest.TestCase):
    def test_implicit_file(self):
        self.assertFalse(CommitOperationDelete(path_in_repo="path/to/file").is_folder)
        self.assertFalse(CommitOperationDelete(path_in_repo="path/to/file.md").is_folder)

    def test_implicit_folder(self):
        self.assertTrue(CommitOperationDelete(path_in_repo="path/to/folder/").is_folder)
        self.assertTrue(CommitOperationDelete(path_in_repo="path/to/folder.md/").is_folder)

    def test_explicit_file(self):
        # Weird case: if user explicitly set as file (`is_folder`=False) but path has a
        # trailing "/" => user input has priority
        self.assertFalse(CommitOperationDelete(path_in_repo="path/to/folder/", is_folder=False).is_folder)
        self.assertFalse(CommitOperationDelete(path_in_repo="path/to/folder.md/", is_folder=False).is_folder)

    def test_explicit_folder(self):
        # No need for the trailing "/" is `is_folder` explicitly passed
        self.assertTrue(CommitOperationDelete(path_in_repo="path/to/folder", is_folder=True).is_folder)
        self.assertTrue(CommitOperationDelete(path_in_repo="path/to/folder.md", is_folder=True).is_folder)

    def test_is_folder_wrong_value(self):
        with self.assertRaises(ValueError):
            CommitOperationDelete(path_in_repo="path/to/folder", is_folder="any value")


class TestCommitOperationPathInRepo(unittest.TestCase):
    valid_values = {  # key is input, value is expected validated output
        "file.txt": "file.txt",
        ".file.txt": ".file.txt",
        "/file.txt": "file.txt",
        "./file.txt": "file.txt",
    }
    invalid_values = [".", "..", "../file.txt"]

    def test_path_in_repo_valid(self) -> None:
        for input, expected in self.valid_values.items():
            with self.subTest(f"Testing with valid input: '{input}'"):
                self.assertEqual(CommitOperationAdd(path_in_repo=input, path_or_fileobj=b"").path_in_repo, expected)
                self.assertEqual(CommitOperationDelete(path_in_repo=input).path_in_repo, expected)

    def test_path_in_repo_invalid(self) -> None:
        for input in self.invalid_values:
            with self.subTest(f"Testing with invalid input: '{input}'"):
                with self.assertRaises(ValueError):
                    CommitOperationAdd(path_in_repo=input, path_or_fileobj=b"")
                with self.assertRaises(ValueError):
                    CommitOperationDelete(path_in_repo=input)


class TestCommitOperationForbiddenPathInRepo(unittest.TestCase):
    """Commit operations must throw an error on files in the .git/ folder.

    Server would error anyway so it's best to prevent early.
    """

    INVALID_PATHS_IN_REPO = {
        ".git",
        ".git/path/to/file",
        "./.git/path/to/file",
        "subfolder/path/.git/to/file",
        "./subfolder/path/.git/to/file",
    }

    VALID_PATHS_IN_REPO = {
        ".gitignore",
        "path/to/.gitignore",
        "path/to/something.git",
        "path/to/something.git/more",
    }

    def test_cannot_update_file_in_git_folder(self):
        for path in self.INVALID_PATHS_IN_REPO:
            with self.subTest(msg=f"Add: '{path}'"):
                with self.assertRaises(ValueError):
                    CommitOperationAdd(path_in_repo=path, path_or_fileobj=b"content")
            with self.subTest(msg=f"Delete: '{path}'"):
                with self.assertRaises(ValueError):
                    CommitOperationDelete(path_in_repo=path)

    def test_valid_path_in_repo_containing_git(self):
        for path in self.VALID_PATHS_IN_REPO:
            with self.subTest(msg=f"Add: '{path}'"):
                CommitOperationAdd(path_in_repo=path, path_or_fileobj=b"content")
            with self.subTest(msg=f"Delete: '{path}'"):
                CommitOperationDelete(path_in_repo=path)


class TestWarnOnOverwritingOperations(unittest.TestCase):
    add_file_ab = CommitOperationAdd(path_in_repo="a/b.txt", path_or_fileobj=b"data")
    add_file_abc = CommitOperationAdd(path_in_repo="a/b/c.md", path_or_fileobj=b"data")
    add_file_abd = CommitOperationAdd(path_in_repo="a/b/d.md", path_or_fileobj=b"data")
    update_file_abc = CommitOperationAdd(path_in_repo="a/b/c.md", path_or_fileobj=b"updated data")
    delete_file_abc = CommitOperationDelete(path_in_repo="a/b/c.md")
    delete_folder_a = CommitOperationDelete(path_in_repo="a/")
    delete_folder_e = CommitOperationDelete(path_in_repo="e/")

    def test_no_overwrite(self) -> None:
        warn_on_overwriting_operations(
            [
                self.add_file_ab,
                self.add_file_abc,
                self.add_file_abd,
                self.delete_folder_e,
            ]
        )

    def test_add_then_update_file(self) -> None:
        with self.assertWarns(UserWarning):
            warn_on_overwriting_operations([self.add_file_abc, self.update_file_abc])

    def test_add_then_delete_file(self) -> None:
        with self.assertWarns(UserWarning):
            warn_on_overwriting_operations([self.add_file_abc, self.delete_file_abc])

    def test_add_then_delete_folder(self) -> None:
        with self.assertWarns(UserWarning):
            warn_on_overwriting_operations([self.add_file_abc, self.delete_folder_a])

        with self.assertWarns(UserWarning):
            warn_on_overwriting_operations([self.add_file_ab, self.delete_folder_a])

    def test_delete_file_then_add(self) -> None:
        warn_on_overwriting_operations([self.delete_file_abc, self.add_file_abc])

    def test_delete_folder_then_add(self) -> None:
        warn_on_overwriting_operations([self.delete_folder_a, self.add_file_ab, self.add_file_abc])


# tests/test_commit_scheduler.py

import time
import unittest
from io import SEEK_END
from pathlib import Path
from unittest.mock import MagicMock, patch

import pytest

from huggingface_hub import CommitOperationAdd, HfApi, hf_hub_download
from huggingface_hub._commit_scheduler import CommitScheduler, PartialFileIO

from .testing_constants import ENDPOINT_STAGING, TOKEN
from .testing_utils import repo_name


@pytest.mark.usefixtures("fx_cache_dir")
class TestCommitScheduler(unittest.TestCase):
    cache_dir: Path

    def setUp(self) -> None:
        self.api = HfApi(token=TOKEN, endpoint=ENDPOINT_STAGING)
        self.repo_name = repo_name()

    def tearDown(self) -> None:
        try:  # try stopping scheduler (if exists)
            self.scheduler.stop()
        except AttributeError:
            pass

        try:  # try delete temporary repo
            self.api.delete_repo(self.repo_name)
        except Exception:
            pass

    @patch("huggingface_hub._commit_scheduler.CommitScheduler.push_to_hub")
    def test_mocked_push_to_hub(self, push_to_hub_mock: MagicMock) -> None:
        self.scheduler = CommitScheduler(
            folder_path=self.cache_dir,
            repo_id=self.repo_name,
            every=1 / 60 / 10,  # every 0.1s
            hf_api=self.api,
        )
        time.sleep(0.3)

        # Triggered at least twice times (at 0.0s and then 0.1s, 0.2s,...)
        self.assertGreater(len(push_to_hub_mock.call_args_list), 2)

        # Can get the last upload result
        self.assertEqual(self.scheduler.last_future.result(), push_to_hub_mock.return_value)

    def test_invalid_folder_path_is_a_file(self) -> None:
        """Test cannot scheduler upload of a single file."""
        file_path = self.cache_dir / "file.txt"
        file_path.write_text("something")

        with self.assertRaises(ValueError):
            CommitScheduler(folder_path=file_path, repo_id=self.repo_name, hf_api=self.api)

    def test_missing_folder_is_created(self) -> None:
        folder_path = self.cache_dir / "folder" / "subfolder"
        self.scheduler = CommitScheduler(folder_path=folder_path, repo_id=self.repo_name, hf_api=self.api)
        self.assertTrue(folder_path.is_dir())

    def test_sync_local_folder(self) -> None:
        """Test sync local folder to remote repo."""
        watched_folder = self.cache_dir / "watched_folder"
        hub_cache = self.cache_dir / "hub"  # to download hub files

        file_path = watched_folder / "file.txt"
        lfs_path = watched_folder / "lfs.bin"

        self.scheduler = CommitScheduler(
            folder_path=watched_folder,
            repo_id=self.repo_name,
            every=1 / 60,  # every 1s
            hf_api=self.api,
        )

        # 1 push to hub triggered (empty commit not pushed)
        time.sleep(0.5)

        # write content to files
        with file_path.open("a") as f:
            f.write("first line\n")
        with lfs_path.open("a") as f:
            f.write("binary content")

        # 2 push to hub triggered (1 commit + 1 ignored)
        time.sleep(2)
        self.scheduler.last_future.result()

        # new content in file
        with file_path.open("a") as f:
            f.write("second line\n")

        # 1 push to hub triggered (1 commit)
        time.sleep(1)
        self.scheduler.last_future.result()

        with lfs_path.open("a") as f:
            f.write(" updated")

        # 5 push to hub triggered (1 commit)
        time.sleep(5)  # wait for every threads/uploads to complete
        self.scheduler.stop()
        self.scheduler.last_future.result()

        # 4 commits expected (initial commit + 3 push to hub)
        repo_id = self.scheduler.repo_id
        commits = self.api.list_repo_commits(repo_id)
        self.assertEqual(len(commits), 4)
        push_1 = commits[2].commit_id  # sorted by last first
        push_2 = commits[1].commit_id
        push_3 = commits[0].commit_id

        def _download(filename: str, revision: str) -> Path:
            return Path(hf_hub_download(repo_id=repo_id, filename=filename, cache_dir=hub_cache, revision=revision))

        # Check file.txt consistency
        file_push1 = _download(filename="file.txt", revision=push_1)
        file_push2 = _download(filename="file.txt", revision=push_2)
        file_push3 = _download(filename="file.txt", revision=push_3)

        self.assertEqual(file_push1.read_text(), "first line\n")
        self.assertEqual(file_push2.read_text(), "first line\nsecond line\n")
        self.assertEqual(file_push3.read_text(), "first line\nsecond line\n")

        # Check lfs.bin consistency
        lfs_push1 = _download(filename="lfs.bin", revision=push_1)
        lfs_push2 = _download(filename="lfs.bin", revision=push_2)
        lfs_push3 = _download(filename="lfs.bin", revision=push_3)

        self.assertEqual(lfs_push1.read_text(), "binary content")
        self.assertEqual(lfs_push2.read_text(), "binary content")
        self.assertEqual(lfs_push3.read_text(), "binary content updated")


@pytest.mark.usefixtures("fx_cache_dir")
class TestPartialFileIO(unittest.TestCase):
    """Test PartialFileIO object."""

    cache_dir: Path

    def setUp(self) -> None:
        """Set up a test file."""
        self.file_path = self.cache_dir / "file.txt"
        self.file_path.write_text("123456789")  # file size: 9 bytes

    def test_read_partial_file_twice(self) -> None:
        file = PartialFileIO(self.file_path, size_limit=5)
        self.assertEqual(file.read(), b"12345")
        self.assertEqual(file.read(), b"")  # End of file

    def test_read_partial_file_by_chunks(self) -> None:
        file = PartialFileIO(self.file_path, size_limit=5)
        self.assertEqual(file.read(2), b"12")
        self.assertEqual(file.read(2), b"34")
        self.assertEqual(file.read(2), b"5")
        self.assertEqual(file.read(2), b"")

    def test_read_partial_file_too_much(self) -> None:
        file = PartialFileIO(self.file_path, size_limit=5)
        self.assertEqual(file.read(20), b"12345")

    def test_partial_file_len(self) -> None:
        """Useful for `requests` internally."""
        file = PartialFileIO(self.file_path, size_limit=5)
        self.assertEqual(len(file), 5)

        file = PartialFileIO(self.file_path, size_limit=50)
        self.assertEqual(len(file), 9)

    def test_partial_file_seek_and_tell(self) -> None:
        file = PartialFileIO(self.file_path, size_limit=5)

        self.assertEqual(file.tell(), 0)

        file.read(2)
        self.assertEqual(file.tell(), 2)

        file.seek(0)
        self.assertEqual(file.tell(), 0)

        file.seek(2)
        self.assertEqual(file.tell(), 2)

        file.seek(50)
        self.assertEqual(file.tell(), 5)

        file.seek(-3, SEEK_END)
        self.assertEqual(file.tell(), 2)  # 5-3

    def test_methods_not_implemented(self) -> None:
        """Test `PartialFileIO` only implements a subset of the `io` interface. This is on-purpose to avoid misuse."""
        file = PartialFileIO(self.file_path, size_limit=5)

        with self.assertRaises(NotImplementedError):
            file.readline()

        with self.assertRaises(NotImplementedError):
            file.write(b"123")

    def test_append_to_file_then_read(self) -> None:
        file = PartialFileIO(self.file_path, size_limit=9)

        with self.file_path.open("ab") as f:
            f.write(b"abcdef")

        # Output is truncated even if new content appended to the wrapped file
        self.assertEqual(file.read(), b"123456789")

    def test_high_size_limit(self) -> None:
        file = PartialFileIO(self.file_path, size_limit=20)
        with self.file_path.open("ab") as f:
            f.write(b"abcdef")

        # File size limit is truncated to the actual file size at instance creation (not on the fly)
        self.assertEqual(len(file), 9)
        self.assertEqual(file._size_limit, 9)

    def test_with_commit_operation_add(self) -> None:
        # Truncated file
        op_truncated = CommitOperationAdd(
            path_or_fileobj=PartialFileIO(self.file_path, size_limit=5), path_in_repo="file.txt"
        )
        self.assertEqual(op_truncated.upload_info.size, 5)
        self.assertEqual(op_truncated.upload_info.sample, b"12345")

        with op_truncated.as_file() as f:
            self.assertEqual(f.read(), b"12345")

        # Full file
        op_full = CommitOperationAdd(
            path_or_fileobj=PartialFileIO(self.file_path, size_limit=9), path_in_repo="file.txt"
        )
        self.assertEqual(op_full.upload_info.size, 9)
        self.assertEqual(op_full.upload_info.sample, b"123456789")

        with op_full.as_file() as f:
            self.assertEqual(f.read(), b"123456789")

        # Truncated file has a different hash than the full file
        self.assertNotEqual(op_truncated.upload_info.sha256, op_full.upload_info.sha256)


# tests/test_endpoint_helpers.py

# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest

import requests

from huggingface_hub.hf_api import HfApi
from huggingface_hub.utils.endpoint_helpers import (
    AttributeDictionary,
    DatasetTags,
    GeneralTags,
    ModelTags,
)

from .testing_utils import with_production_testing


class AttributeDictionaryCommonTest(unittest.TestCase):
    _attrdict = AttributeDictionary()


class AttributeDictionaryTest(AttributeDictionaryCommonTest):
    def setUp(self):
        self._attrdict.clear()

    def test_adding_item(self):
        self._attrdict["itemA"] = 2
        self.assertEqual(self._attrdict.itemA, 2)
        self.assertEqual(self._attrdict["itemA"], 2)
        # We should be able to both set a property and a key
        self._attrdict.itemB = 3
        self.assertEqual(self._attrdict.itemB, 3)
        self.assertEqual(self._attrdict["itemB"], 3)

    def test_removing_item(self):
        self._attrdict["itemA"] = 2
        delattr(self._attrdict, "itemA")
        with self.assertRaises(KeyError):
            _ = self._attrdict["itemA"]

        self._attrdict.itemB = 3
        del self._attrdict["itemB"]
        with self.assertRaises(AttributeError):
            _ = self._attrdict.itemB

    def test_dir(self):
        # Since we subclass dict, dir should have everything
        # from dict and the attributes
        _dict_keys = dir(dict) + [
            "__dict__",
            "__getattr__",
            "__module__",
            "__weakref__",
        ]
        self._attrdict["itemA"] = 2
        self._attrdict.itemB = 3
        _dict_keys += ["itemA", "itemB"]
        _dict_keys.sort()

        full_dir = dir(self._attrdict)
        full_dir.sort()
        self.assertEqual(full_dir, _dict_keys)

    def test_dir_with_numbers(self):
        self._attrdict["1a"] = 4
        self.assertFalse("1a" in dir(self._attrdict))
        self.assertTrue("1a" in list(self._attrdict.keys()))

    def test_dir_with_special_characters(self):
        self._attrdict["1<2"] = 3
        self.assertFalse("1<2" in dir(self._attrdict))
        self.assertTrue("1<2" in list(self._attrdict.keys()))

        self._attrdict["?abc"] = 4
        self.assertFalse("?abc" in dir(self._attrdict))
        self.assertTrue("?abc" in list(self._attrdict.keys()))

    def test_repr(self):
        self._attrdict["itemA"] = 2
        self._attrdict.itemB = 3
        self._attrdict["1a"] = 2
        self._attrdict["itemA?"] = 4
        repr_string = "Available Attributes or Keys:\n * 1a (Key only)\n * itemA\n * itemA? (Key only)\n * itemB\n"
        self.assertEqual(repr_string, repr(self._attrdict))


class GeneralTagsCommonTest(unittest.TestCase):
    # Similar to the output from /api/***-tags-by-type
    # id = how we can search hfapi, such as `'id': 'language:en'`
    # label = A human readable version assigned to everything, such as `"label":"en"`
    _tag_dictionary = {
        "language": [
            {"id": "itemA", "label": "Item A"},
            {"id": "itemB", "label": "1Item-B"},
        ],
        "license": [
            {"id": "itemC", "label": "Item C"},
            {"id": "itemD", "label": "Item.D"},
        ],
    }


class GeneralTagsTest(GeneralTagsCommonTest):
    def test_init(self):
        _tags = GeneralTags(self._tag_dictionary)
        languages = _tags.language
        licenses = _tags.license
        # Ensure they have the right bits

        self.assertEqual(
            languages,
            AttributeDictionary({"ItemA": "itemA", "1Item_B": "itemB"}),
        )

        self.assertTrue("1Item_B" not in dir(languages))

        self.assertEqual(licenses, AttributeDictionary({"ItemC": "itemC", "Item_D": "itemD"}))

    def test_filter(self):
        _tags = GeneralTags(self._tag_dictionary, keys=["license"])
        self.assertTrue(hasattr(_tags, "license"))
        self.assertFalse(hasattr(_tags, "languages"))
        self.assertEqual(_tags.license, AttributeDictionary({"ItemC": "itemC", "Item_D": "itemD"}))


class ModelTagsTest(unittest.TestCase):
    @with_production_testing
    def test_tags(self):
        # ModelTags instantiation must not fail!
        res = requests.get(f"{HfApi().endpoint}/api/models-tags-by-type")
        res.raise_for_status()
        tags = ModelTags(res.json())

        # Check existing keys to get notified about server-side changes
        for existing_key in [
            "dataset",
            "language",
            "library",
            "license",
            "pipeline_tag",
        ]:
            self.assertGreater(len(getattr(tags, existing_key).keys()), 0)


class DatasetTagsTest(unittest.TestCase):
    @with_production_testing
    def test_tags(self):
        # DatasetTags instantiation must not fail!
        res = requests.get(f"{HfApi().endpoint}/api/datasets-tags-by-type")
        res.raise_for_status()
        tags = DatasetTags(res.json())

        # Some keys existed before but have been removed server-side
        for missing_key in (
            "language_creators",
            "multilinguality",
        ):
            self.assertEqual(len(getattr(tags, missing_key).keys()), 0)

        # Check existing keys to get notified about server-side changes
        for existing_key in [
            "benchmark",
            "language",
            "license",
            "size_categories",
            "task_categories",
            "task_ids",
        ]:
            self.assertGreater(len(getattr(tags, existing_key).keys()), 0)


# tests/test_fastai_integration.py

import os
from unittest import TestCase, skip

from huggingface_hub import HfApi
from huggingface_hub.fastai_utils import (
    _save_pretrained_fastai,
    from_pretrained_fastai,
    push_to_hub_fastai,
)
from huggingface_hub.utils import (
    SoftTemporaryDirectory,
    is_fastai_available,
    is_fastcore_available,
    is_torch_available,
)

from .testing_constants import ENDPOINT_STAGING, TOKEN, USER
from .testing_utils import repo_name


WORKING_REPO_SUBDIR = f"fixtures/working_repo_{__name__.split('.')[-1]}"
WORKING_REPO_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), WORKING_REPO_SUBDIR)

if is_fastai_available():
    from fastai.data.block import DataBlock
    from fastai.test_utils import synth_learner

if is_torch_available():
    import torch


def require_fastai_fastcore(test_case):
    """
    Decorator marking a test that requires fastai and fastcore.
    These tests are skipped when fastai and fastcore are not installed.
    """
    if not is_fastai_available():
        return skip("Test requires fastai")(test_case)
    elif not is_fastcore_available():
        return skip("Test requires fastcore")(test_case)
    else:
        return test_case


def fake_dataloaders(a=2, b=3, bs=16, n=10):
    def get_data(n):
        x = torch.randn(bs * n, 1)
        return torch.cat((x, a * x + b + 0.1 * torch.randn(bs * n, 1)), 1)

    ds = get_data(n)
    dblock = DataBlock()
    return dblock.dataloaders(ds)


if is_fastai_available():
    dummy_model = synth_learner(data=fake_dataloaders())
    dummy_config = dict(test="test_0")
else:
    dummy_model = None
    dummy_config = None


@require_fastai_fastcore
class TestFastaiUtils(TestCase):
    def test_save_pretrained_without_config(self):
        with SoftTemporaryDirectory() as tmpdir:
            _save_pretrained_fastai(dummy_model, tmpdir)
            files = os.listdir(tmpdir)

        self.assertTrue("model.pkl" in files)
        self.assertTrue("pyproject.toml" in files)
        self.assertTrue("README.md" in files)
        self.assertEqual(len(files), 3)

    def test_save_pretrained_with_config(self):
        with SoftTemporaryDirectory() as tmpdir:
            _save_pretrained_fastai(dummy_model, tmpdir, config=dummy_config)
            files = os.listdir(tmpdir)

        self.assertTrue("config.json" in files)
        self.assertEqual(len(files), 4)

    def test_push_to_hub_and_from_pretrained_fastai(self):
        api = HfApi(endpoint=ENDPOINT_STAGING, token=TOKEN)
        repo_id = f"{USER}/{repo_name()}"

        push_to_hub_fastai(learner=dummy_model, repo_id=repo_id, token=TOKEN, config=dummy_config)
        model_info = api.model_info(repo_id)
        self.assertEqual(model_info.modelId, repo_id)
        loaded_model = from_pretrained_fastai(repo_id)
        self.assertEqual(dummy_model.show_training_loop(), loaded_model.show_training_loop())

        api.delete_repo(repo_id=repo_id)


# tests/test_file_download.py

# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import re
import shutil
import stat
import unittest
from pathlib import Path
from unittest.mock import Mock, patch

import pytest
import requests
from requests import Response

import huggingface_hub.file_download
from huggingface_hub import HfApi
from huggingface_hub.constants import (
    CONFIG_NAME,
    HUGGINGFACE_HEADER_X_LINKED_ETAG,
    PYTORCH_WEIGHTS_NAME,
    REPO_TYPE_DATASET,
)
from huggingface_hub.file_download import (
    _CACHED_NO_EXIST,
    HfFileMetadata,
    _create_symlink,
    _get_pointer_path,
    _normalize_etag,
    _to_local_dir,
    cached_download,
    filename_to_url,
    get_hf_file_metadata,
    hf_hub_download,
    hf_hub_url,
    try_to_load_from_cache,
)
from huggingface_hub.utils import (
    EntryNotFoundError,
    GatedRepoError,
    LocalEntryNotFoundError,
    RepositoryNotFoundError,
    RevisionNotFoundError,
    SoftTemporaryDirectory,
)

from .testing_constants import ENDPOINT_STAGING, OTHER_TOKEN, TOKEN
from .testing_utils import (
    DUMMY_MODEL_ID,
    DUMMY_MODEL_ID_PINNED_SHA1,
    DUMMY_MODEL_ID_PINNED_SHA256,
    DUMMY_MODEL_ID_REVISION_INVALID,
    DUMMY_MODEL_ID_REVISION_ONE_SPECIFIC_COMMIT,
    DUMMY_RENAMED_NEW_MODEL_ID,
    DUMMY_RENAMED_OLD_MODEL_ID,
    SAMPLE_DATASET_IDENTIFIER,
    OfflineSimulationMode,
    expect_deprecation,
    offline,
    repo_name,
    with_production_testing,
    xfail_on_windows,
)


REVISION_ID_DEFAULT = "main"
# Default branch name

DATASET_ID = SAMPLE_DATASET_IDENTIFIER
# An actual dataset hosted on huggingface.co


DATASET_REVISION_ID_ONE_SPECIFIC_COMMIT = "e25d55a1c4933f987c46cc75d8ffadd67f257c61"
# One particular commit for DATASET_ID
DATASET_SAMPLE_PY_FILE = "custom_squad.py"


@with_production_testing
class CachedDownloadTests(unittest.TestCase):
    def test_bogus_url(self):
        url = "https://bogus"
        with self.assertRaisesRegex(ValueError, "Connection error"):
            _ = cached_download(url, legacy_cache_layout=True)

    def test_no_connection(self):
        invalid_url = hf_hub_url(
            DUMMY_MODEL_ID,
            filename=CONFIG_NAME,
            revision=DUMMY_MODEL_ID_REVISION_INVALID,
        )
        valid_url = hf_hub_url(DUMMY_MODEL_ID, filename=CONFIG_NAME, revision=REVISION_ID_DEFAULT)
        self.assertIsNotNone(cached_download(valid_url, force_download=True, legacy_cache_layout=True))
        for offline_mode in OfflineSimulationMode:
            with offline(mode=offline_mode):
                with self.assertRaisesRegex(ValueError, "Connection error"):
                    _ = cached_download(invalid_url, legacy_cache_layout=True)
                with self.assertRaisesRegex(ValueError, "Connection error"):
                    _ = cached_download(valid_url, force_download=True, legacy_cache_layout=True)
                self.assertIsNotNone(cached_download(valid_url, legacy_cache_layout=True))

    def test_file_not_found_on_repo(self):
        # Valid revision (None) but missing file on repo.
        url = hf_hub_url(DUMMY_MODEL_ID, filename="missing.bin")
        with self.assertRaisesRegex(
            EntryNotFoundError,
            re.compile("404 Client Error(.*)Entry Not Found", flags=re.DOTALL),
        ):
            _ = cached_download(url, legacy_cache_layout=True)

    def test_file_not_found_locally_and_network_disabled(self):
        # Valid file but missing locally and network is disabled.
        with SoftTemporaryDirectory() as tmpdir:
            # Download a first time to get the refs ok
            filepath = hf_hub_download(
                DUMMY_MODEL_ID,
                filename=CONFIG_NAME,
                cache_dir=tmpdir,
                local_files_only=False,
            )

            # Remove local file
            os.remove(filepath)

            # Get without network must fail
            with pytest.raises(LocalEntryNotFoundError):
                hf_hub_download(
                    DUMMY_MODEL_ID,
                    filename=CONFIG_NAME,
                    cache_dir=tmpdir,
                    local_files_only=True,
                )

    def test_file_not_found_locally_and_network_disabled_legacy(self):
        # Valid file but missing locally and network is disabled.
        url = hf_hub_url(DUMMY_MODEL_ID, filename=CONFIG_NAME)
        with SoftTemporaryDirectory() as tmpdir:
            # Get without network must fail
            with pytest.raises(LocalEntryNotFoundError):
                cached_download(
                    url,
                    legacy_cache_layout=True,
                    local_files_only=True,
                    cache_dir=tmpdir,
                )

    def test_file_cached_and_read_only_access(self):
        """Should works if file is already cached and user has read-only permission.

        Regression test for https://github.com/huggingface/huggingface_hub/issues/1216.
        """
        # Valid file but missing locally and network is disabled.
        with SoftTemporaryDirectory() as tmpdir:
            # Download a first time to get the refs ok
            hf_hub_download(DUMMY_MODEL_ID, filename=CONFIG_NAME, cache_dir=tmpdir)

            # Set read-only permission recursively
            _recursive_chmod(tmpdir, 0o555)

            # Get without write-access must succeed
            hf_hub_download(DUMMY_MODEL_ID, filename=CONFIG_NAME, cache_dir=tmpdir)

            # Set permission back for cleanup
            _recursive_chmod(tmpdir, 0o777)

    def test_revision_not_found(self):
        # Valid file but missing revision
        url = hf_hub_url(
            DUMMY_MODEL_ID,
            filename=CONFIG_NAME,
            revision=DUMMY_MODEL_ID_REVISION_INVALID,
        )
        with self.assertRaisesRegex(
            RevisionNotFoundError,
            re.compile("404 Client Error(.*)Revision Not Found", flags=re.DOTALL),
        ):
            _ = cached_download(url, legacy_cache_layout=True)

    def test_repo_not_found(self):
        # Invalid model file.
        url = hf_hub_url("bert-base", filename="pytorch_model.bin")
        with self.assertRaisesRegex(
            RepositoryNotFoundError,
            re.compile("401 Client Error(.*)Repository Not Found", flags=re.DOTALL),
        ):
            _ = cached_download(url, legacy_cache_layout=True)

    def test_standard_object(self):
        url = hf_hub_url(DUMMY_MODEL_ID, filename=CONFIG_NAME, revision=REVISION_ID_DEFAULT)
        filepath = cached_download(url, force_download=True, legacy_cache_layout=True)
        metadata = filename_to_url(filepath, legacy_cache_layout=True)
        self.assertEqual(metadata, (url, f'"{DUMMY_MODEL_ID_PINNED_SHA1}"'))

    def test_standard_object_rev(self):
        # Same object, but different revision
        url = hf_hub_url(
            DUMMY_MODEL_ID,
            filename=CONFIG_NAME,
            revision=DUMMY_MODEL_ID_REVISION_ONE_SPECIFIC_COMMIT,
        )
        filepath = cached_download(url, force_download=True, legacy_cache_layout=True)
        metadata = filename_to_url(filepath, legacy_cache_layout=True)
        self.assertNotEqual(metadata[1], f'"{DUMMY_MODEL_ID_PINNED_SHA1}"')
        # Caution: check that the etag is *not* equal to the one from `test_standard_object`

    def test_lfs_object(self):
        url = hf_hub_url(DUMMY_MODEL_ID, filename=PYTORCH_WEIGHTS_NAME, revision=REVISION_ID_DEFAULT)
        filepath = cached_download(url, force_download=True, legacy_cache_layout=True)
        metadata = filename_to_url(filepath, legacy_cache_layout=True)
        self.assertEqual(metadata, (url, f'"{DUMMY_MODEL_ID_PINNED_SHA256}"'))

    def test_dataset_standard_object_rev(self):
        url = hf_hub_url(
            DATASET_ID,
            filename=DATASET_SAMPLE_PY_FILE,
            repo_type=REPO_TYPE_DATASET,
            revision=DATASET_REVISION_ID_ONE_SPECIFIC_COMMIT,
        )
        # now let's download
        filepath = cached_download(url, force_download=True, legacy_cache_layout=True)
        metadata = filename_to_url(filepath, legacy_cache_layout=True)
        self.assertNotEqual(metadata[1], f'"{DUMMY_MODEL_ID_PINNED_SHA1}"')

    def test_dataset_lfs_object(self):
        url = hf_hub_url(
            DATASET_ID,
            filename="dev-v1.1.json",
            repo_type=REPO_TYPE_DATASET,
            revision=DATASET_REVISION_ID_ONE_SPECIFIC_COMMIT,
        )
        filepath = cached_download(url, force_download=True, legacy_cache_layout=True)
        metadata = filename_to_url(filepath, legacy_cache_layout=True)
        self.assertEqual(
            metadata,
            (url, '"95aa6a52d5d6a735563366753ca50492a658031da74f301ac5238b03966972c9"'),
        )

    @xfail_on_windows(reason="umask is UNIX-specific")
    def test_hf_hub_download_custom_cache_permission(self):
        """Checks `hf_hub_download` respect the cache dir permission.

        Regression test for #1141 #1215.
        https://github.com/huggingface/huggingface_hub/issues/1141
        https://github.com/huggingface/huggingface_hub/issues/1215
        """
        with SoftTemporaryDirectory() as tmpdir:
            # Equivalent to umask u=rwx,g=r,o=
            previous_umask = os.umask(0o037)
            try:
                filepath = hf_hub_download(DUMMY_RENAMED_OLD_MODEL_ID, "config.json", cache_dir=tmpdir)
                # Permissions are honored (640: u=rw,g=r,o=)
                self.assertEqual(stat.S_IMODE(os.stat(filepath).st_mode), 0o640)
            finally:
                os.umask(previous_umask)

    def test_download_from_a_renamed_repo_with_hf_hub_download(self):
        """Checks `hf_hub_download` works also on a renamed repo.

        Regression test for #981.
        https://github.com/huggingface/huggingface_hub/issues/981
        """
        with SoftTemporaryDirectory() as tmpdir:
            filepath = hf_hub_download(DUMMY_RENAMED_OLD_MODEL_ID, "config.json", cache_dir=tmpdir)
            self.assertTrue(os.path.exists(filepath))

    def test_download_from_a_renamed_repo_with_cached_download(self):
        """Checks `cached_download` works also on a renamed repo.

        Regression test for #981.
        https://github.com/huggingface/huggingface_hub/issues/981
        """
        with pytest.warns(FutureWarning):
            with SoftTemporaryDirectory() as tmpdir:
                filepath = cached_download(
                    hf_hub_url(
                        DUMMY_RENAMED_OLD_MODEL_ID,
                        filename="config.json",
                    ),
                    cache_dir=tmpdir,
                )
                self.assertTrue(os.path.exists(filepath))

    def test_hf_hub_download_with_empty_subfolder(self):
        """
        Check subfolder arg is processed correctly when empty string is passed to
        `hf_hub_download`.

        See https://github.com/huggingface/huggingface_hub/issues/1016.
        """
        filepath = Path(
            hf_hub_download(
                DUMMY_MODEL_ID,
                filename=CONFIG_NAME,
                subfolder="",  # Subfolder should be processed as `None`
            )
        )

        # Check file exists and is not in a subfolder in cache
        # e.g: "(...)/snapshots/<commit-id>/config.json"
        self.assertTrue(filepath.is_file())
        self.assertEqual(filepath.name, CONFIG_NAME)
        self.assertEqual(Path(filepath).parent.parent.name, "snapshots")

    def test_hf_hub_download_offline_no_refs(self):
        """Regression test for #1305.

        If "refs/" dir did not exists on "local_files_only" (or connection broken), a
        non-explicit `FileNotFoundError` was raised (for the "/refs/revision" file) instead
        of the documented `LocalEntryNotFoundError` (for the actual searched file).

        See https://github.com/huggingface/huggingface_hub/issues/1305.
        """
        with SoftTemporaryDirectory() as cache_dir:
            with self.assertRaises(LocalEntryNotFoundError):
                hf_hub_download(
                    DUMMY_MODEL_ID,
                    filename=CONFIG_NAME,
                    local_files_only=True,
                    cache_dir=cache_dir,
                )

    def test_hf_hub_url_with_empty_subfolder(self):
        """
        Check subfolder arg is processed correctly when empty string is passed to
        `hf_hub_url`.

        See https://github.com/huggingface/huggingface_hub/issues/1016.
        """
        url = hf_hub_url(
            DUMMY_MODEL_ID,
            filename=CONFIG_NAME,
            subfolder="",  # Subfolder should be processed as `None`
        )
        self.assertTrue(
            url.endswith(
                # "./resolve/main/config.json" and not "./resolve/main//config.json"
                f"{DUMMY_MODEL_ID}/resolve/main/config.json",
            )
        )

    def test_hf_hub_download_legacy(self):
        filepath = hf_hub_download(
            DUMMY_MODEL_ID,
            filename=CONFIG_NAME,
            revision=REVISION_ID_DEFAULT,
            force_download=True,
            legacy_cache_layout=True,
        )
        metadata = filename_to_url(filepath, legacy_cache_layout=True)
        self.assertEqual(metadata[1], f'"{DUMMY_MODEL_ID_PINNED_SHA1}"')

    def test_try_to_load_from_cache_exist(self):
        # Make sure the file is cached
        filepath = hf_hub_download(DUMMY_MODEL_ID, filename=CONFIG_NAME)

        new_file_path = try_to_load_from_cache(DUMMY_MODEL_ID, filename=CONFIG_NAME)
        self.assertEqual(filepath, new_file_path)

        new_file_path = try_to_load_from_cache(DUMMY_MODEL_ID, filename=CONFIG_NAME, revision="main")
        self.assertEqual(filepath, new_file_path)

        # If file is not cached, returns None
        self.assertIsNone(try_to_load_from_cache(DUMMY_MODEL_ID, filename="conf.json"))
        # Same for uncached revisions
        self.assertIsNone(
            try_to_load_from_cache(
                DUMMY_MODEL_ID,
                filename=CONFIG_NAME,
                revision="aaa",
            )
        )
        # Same for uncached models
        self.assertIsNone(try_to_load_from_cache("bert-base", filename=CONFIG_NAME))

    def test_try_to_load_from_cache_specific_pr_revision_exists(self):
        # Make sure the file is cached
        file_path = hf_hub_download(DUMMY_MODEL_ID, filename=CONFIG_NAME, revision="refs/pr/1")

        new_file_path = try_to_load_from_cache(DUMMY_MODEL_ID, filename=CONFIG_NAME, revision="refs/pr/1")
        self.assertEqual(file_path, new_file_path)

        # If file is not cached, returns None
        self.assertIsNone(try_to_load_from_cache(DUMMY_MODEL_ID, filename="conf.json", revision="refs/pr/1"))

        # If revision does not exist, returns None
        self.assertIsNone(try_to_load_from_cache(DUMMY_MODEL_ID, filename=CONFIG_NAME, revision="does-not-exist"))

    def test_try_to_load_from_cache_no_exist(self):
        # Make sure the file is cached
        with self.assertRaises(EntryNotFoundError):
            _ = hf_hub_download(DUMMY_MODEL_ID, filename="dummy")

        new_file_path = try_to_load_from_cache(DUMMY_MODEL_ID, filename="dummy")
        self.assertEqual(new_file_path, _CACHED_NO_EXIST)

        new_file_path = try_to_load_from_cache(DUMMY_MODEL_ID, filename="dummy", revision="main")
        self.assertEqual(new_file_path, _CACHED_NO_EXIST)

        # If file non-existence is not cached, returns None
        self.assertIsNone(try_to_load_from_cache(DUMMY_MODEL_ID, filename="dummy2"))

    def test_try_to_load_from_cache_specific_commit_id_exist(self):
        """Regression test for #1306.

        See https://github.com/huggingface/huggingface_hub/pull/1306."""
        with SoftTemporaryDirectory() as cache_dir:
            # Cache file from specific commit id (no "refs/"" folder)
            commit_id = HfApi().model_info(DUMMY_MODEL_ID).sha
            filepath = hf_hub_download(
                DUMMY_MODEL_ID,
                filename=CONFIG_NAME,
                revision=commit_id,
                cache_dir=cache_dir,
            )

            # Must be able to retrieve it "offline"
            attempt = try_to_load_from_cache(
                DUMMY_MODEL_ID,
                filename=CONFIG_NAME,
                revision=commit_id,
                cache_dir=cache_dir,
            )
            self.assertEqual(filepath, attempt)

    def test_try_to_load_from_cache_specific_commit_id_no_exist(self):
        """Regression test for #1306.

        See https://github.com/huggingface/huggingface_hub/pull/1306."""
        with SoftTemporaryDirectory() as cache_dir:
            # Cache file from specific commit id (no "refs/"" folder)
            commit_id = HfApi().model_info(DUMMY_MODEL_ID).sha
            with self.assertRaises(EntryNotFoundError):
                hf_hub_download(
                    DUMMY_MODEL_ID,
                    filename="missing_file",
                    revision=commit_id,
                    cache_dir=cache_dir,
                )

            # Must be able to retrieve it "offline"
            attempt = try_to_load_from_cache(
                DUMMY_MODEL_ID,
                filename="missing_file",
                revision=commit_id,
                cache_dir=cache_dir,
            )
            self.assertEqual(attempt, _CACHED_NO_EXIST)

    def test_get_hf_file_metadata_basic(self) -> None:
        """Test getting metadata from a file on the Hub."""
        url = hf_hub_url(
            DUMMY_MODEL_ID,
            filename=CONFIG_NAME,
            revision=DUMMY_MODEL_ID_REVISION_ONE_SPECIFIC_COMMIT,
        )
        metadata = get_hf_file_metadata(url)

        # Metadata
        self.assertEqual(metadata.commit_hash, DUMMY_MODEL_ID_REVISION_ONE_SPECIFIC_COMMIT)
        self.assertIsNotNone(metadata.etag)  # example: "85c2fc2dcdd86563aaa85ef4911..."
        self.assertEqual(metadata.location, url)  # no redirect
        self.assertEqual(metadata.size, 851)

    def test_get_hf_file_metadata_from_a_renamed_repo(self) -> None:
        """Test getting metadata from a file in a renamed repo on the Hub."""
        url = hf_hub_url(
            DUMMY_RENAMED_OLD_MODEL_ID,
            filename=CONFIG_NAME,
            subfolder="",  # Subfolder should be processed as `None`
        )
        metadata = get_hf_file_metadata(url)

        # Got redirected to renamed repo
        self.assertEqual(
            metadata.location,
            url.replace(DUMMY_RENAMED_OLD_MODEL_ID, DUMMY_RENAMED_NEW_MODEL_ID),
        )

    def test_get_hf_file_metadata_from_a_lfs_file(self) -> None:
        """Test getting metadata from an LFS file.

        Must get size of the LFS file, not size of the pointer file
        """
        url = hf_hub_url("gpt2", filename="tf_model.h5")
        metadata = get_hf_file_metadata(url)

        self.assertIn("cdn-lfs", metadata.location)  # Redirection
        self.assertEqual(metadata.size, 497933648)  # Size of LFS file, not pointer

    def test_file_consistency_check_fails_regular_file(self):
        """Regression test for #1396 (regular file).

        Download fails if file size is different than the expected one (from headers metadata).

        See https://github.com/huggingface/huggingface_hub/pull/1396."""
        with SoftTemporaryDirectory() as cache_dir:

            def _mocked_hf_file_metadata(*args, **kwargs):
                metadata = get_hf_file_metadata(*args, **kwargs)
                return HfFileMetadata(
                    commit_hash=metadata.commit_hash,
                    etag=metadata.etag,
                    location=metadata.location,
                    size=450,  # will expect 450 bytes but will download 496 bytes
                )

            with patch("huggingface_hub.file_download.get_hf_file_metadata", _mocked_hf_file_metadata):
                with self.assertRaises(EnvironmentError):
                    hf_hub_download(DUMMY_MODEL_ID, filename=CONFIG_NAME, cache_dir=cache_dir)

    def test_file_consistency_check_fails_LFS_file(self):
        """Regression test for #1396 (LFS file).

        Download fails if file size is different than the expected one (from headers metadata).

        See https://github.com/huggingface/huggingface_hub/pull/1396."""
        with SoftTemporaryDirectory() as cache_dir:

            def _mocked_hf_file_metadata(*args, **kwargs):
                metadata = get_hf_file_metadata(*args, **kwargs)
                return HfFileMetadata(
                    commit_hash=metadata.commit_hash,
                    etag=metadata.etag,
                    location=metadata.location,
                    size=65000,  # will expect 65000 bytes but will download 65074 bytes
                )

            with patch("huggingface_hub.file_download.get_hf_file_metadata", _mocked_hf_file_metadata):
                with self.assertRaises(EnvironmentError):
                    hf_hub_download(DUMMY_MODEL_ID, filename="pytorch_model.bin", cache_dir=cache_dir)

    @expect_deprecation("cached_download")
    def test_cached_download_from_github(self):
        """Regression test for #1449.

        File consistency check was failing due to compression in HTTP request which made the expected size smaller than
        the actual one. `cached_download` is deprecated but still heavily used so we need to make sure it works.

        See:
        - https://github.com/huggingface/huggingface_hub/issues/1449.
        - https://github.com/huggingface/diffusers/issues/3213.
        """
        with SoftTemporaryDirectory() as cache_dir:
            cached_download(
                url="https://raw.githubusercontent.com/huggingface/diffusers/v0.15.1/examples/community/lpw_stable_diffusion.py",
                token=None,
                cache_dir=cache_dir,
            )


@with_production_testing
@pytest.mark.usefixtures("fx_cache_dir")
class HfHubDownloadToLocalDir(unittest.TestCase):
    cache_dir: Path

    def test_with_local_dir_and_symlinks_and_file_cached(self) -> None:
        # File already cached
        hf_hub_download(DUMMY_MODEL_ID, filename=CONFIG_NAME, cache_dir=self.cache_dir)

        # Download to local dir
        with SoftTemporaryDirectory() as local_dir:
            returned_path = hf_hub_download(
                DUMMY_MODEL_ID,
                filename=CONFIG_NAME,
                cache_dir=self.cache_dir,
                local_dir=local_dir,
                local_dir_use_symlinks=True,
            )
            config_file = Path(local_dir) / CONFIG_NAME
            self.assertEqual(returned_path, str(config_file))
            self.assertTrue(config_file.is_file())
            self.assertTrue(  # File is symlink (except in Windows CI)
                config_file.is_symlink() if os.name != "nt" else not config_file.is_symlink()
            )

    def test_with_local_dir_and_symlinks_and_file_not_cached(self) -> None:
        # Download to local dir
        with SoftTemporaryDirectory() as local_dir:
            returned_path = hf_hub_download(
                DUMMY_MODEL_ID,
                filename=CONFIG_NAME,
                cache_dir=self.cache_dir,
                local_dir=local_dir,
                local_dir_use_symlinks=True,
            )
            config_file = Path(local_dir) / CONFIG_NAME
            self.assertEqual(returned_path, str(config_file))
            self.assertTrue(config_file.is_file())
            if os.name != "nt":  # File is symlink (except in Windows CI)
                self.assertTrue(config_file.is_symlink())
                blob_path = config_file.resolve()
                self.assertTrue(self.cache_dir in blob_path.parents)  # blob is cached!
            else:
                self.assertFalse(config_file.is_symlink())

    def test_with_local_dir_and_no_symlink_and_file_cached(self) -> None:
        # File already cached
        hf_hub_download(DUMMY_MODEL_ID, filename=CONFIG_NAME, cache_dir=self.cache_dir)

        # Download to local dir
        with SoftTemporaryDirectory() as local_dir:
            with patch.object(
                huggingface_hub.file_download, "http_get", wraps=huggingface_hub.file_download.http_get
            ) as mock:
                returned_path = hf_hub_download(
                    DUMMY_MODEL_ID,
                    filename=CONFIG_NAME,
                    cache_dir=self.cache_dir,
                    local_dir=local_dir,
                    local_dir_use_symlinks=False,  # no symlinks
                )
                mock.assert_not_called()  # reused file from cache

            config_file = Path(local_dir) / CONFIG_NAME
            self.assertEqual(returned_path, str(config_file))
            self.assertTrue(config_file.is_file())
            self.assertFalse(config_file.is_symlink())

    def test_with_local_dir_and_no_symlink_and_file_not_cached(self) -> None:
        # Download to local dir
        with SoftTemporaryDirectory() as local_dir:
            with patch.object(
                huggingface_hub.file_download, "http_get", wraps=huggingface_hub.file_download.http_get
            ) as mock:
                returned_path = hf_hub_download(
                    DUMMY_MODEL_ID,
                    filename=CONFIG_NAME,
                    cache_dir=self.cache_dir,
                    local_dir=local_dir,
                    local_dir_use_symlinks=False,  # no symlinks
                )
                mock.assert_called()  # no file cached => had to download it

            config_file = Path(local_dir) / CONFIG_NAME
            self.assertEqual(returned_path, str(config_file))
            self.assertTrue(config_file.is_file())
            self.assertFalse(config_file.is_symlink())

            # Cache directory not used (no blobs, no symlinks in it)
            for path in self.cache_dir.glob("**/blobs/**"):
                self.assertFalse(path.is_file())
            for path in self.cache_dir.glob("**/snapshots/**"):
                self.assertFalse(path.is_file())

    @patch("huggingface_hub.constants.HF_HUB_LOCAL_DIR_AUTO_SYMLINK_THRESHOLD", 1024)
    def test_with_local_dir_and_auto_symlinks_and_file_cached(self) -> None:
        # File already cached
        hf_hub_download(DUMMY_MODEL_ID, filename=CONFIG_NAME, cache_dir=self.cache_dir)  # 496 bytes -> small
        hf_hub_download(DUMMY_MODEL_ID, filename="README.md", cache_dir=self.cache_dir)  # 1.11kB -> "big"

        # Download to local dir
        with SoftTemporaryDirectory() as local_dir:
            config = hf_hub_download(
                DUMMY_MODEL_ID, filename=CONFIG_NAME, cache_dir=self.cache_dir, local_dir=local_dir
            )
            readme = hf_hub_download(
                DUMMY_MODEL_ID, filename="README.md", cache_dir=self.cache_dir, local_dir=local_dir
            )
            self.assertFalse(Path(config).is_symlink())  # 496b => small => duplicated
            if os.name != "nt":
                self.assertTrue(Path(readme).is_symlink())  # 1.11kB => big => symlink

    @patch("huggingface_hub.constants.HF_HUB_LOCAL_DIR_AUTO_SYMLINK_THRESHOLD", 1024)
    def test_with_local_dir_and_auto_symlinks_and_file_not_cached(self) -> None:
        # Download to local dir
        with SoftTemporaryDirectory() as local_dir:
            config = hf_hub_download(
                DUMMY_MODEL_ID, filename=CONFIG_NAME, cache_dir=self.cache_dir, local_dir=local_dir
            )
            readme = hf_hub_download(
                DUMMY_MODEL_ID, filename="README.md", cache_dir=self.cache_dir, local_dir=local_dir
            )
            self.assertFalse(Path(config).is_symlink())  # 496b => small => duplicated
            if os.name != "nt":
                self.assertTrue(Path(readme).is_symlink())  # 1.11kB => big => symlink

    def test_with_local_dir_and_symlinks_and_overwrite(self) -> None:
        # Download to local dir
        with SoftTemporaryDirectory() as local_dir:
            config_path = Path(local_dir) / CONFIG_NAME
            config_path.write_text("this will be overwritten")
            hf_hub_download(
                DUMMY_MODEL_ID,
                filename=CONFIG_NAME,
                cache_dir=self.cache_dir,
                local_dir=local_dir,
                local_dir_use_symlinks=True,
            )
            if os.name != "nt":
                self.assertTrue(config_path.is_symlink())
            self.assertNotEqual(config_path.read_text(), "this will be overwritten")

    def test_with_local_dir_and_no_symlinks_and_overwrite(self) -> None:
        # Download to local dir
        with SoftTemporaryDirectory() as local_dir:
            config_path = Path(local_dir) / CONFIG_NAME
            config_path.write_text("this will be overwritten")
            hf_hub_download(
                DUMMY_MODEL_ID,
                filename=CONFIG_NAME,
                cache_dir=self.cache_dir,
                local_dir=local_dir,
                local_dir_use_symlinks=False,
            )
            self.assertFalse(config_path.is_symlink())
            self.assertNotEquals(config_path.read_text(), "this will be overwritten")


class StagingCachedDownloadTest(unittest.TestCase):
    def test_download_from_a_gated_repo_with_hf_hub_download(self):
        """Checks `hf_hub_download` outputs error on gated repo.

        Regression test for #1121.
        https://github.com/huggingface/huggingface_hub/pull/1121
        """
        # Create a gated repo on the fly. Repo is created by "other user" so that the
        # usual CI user don't have access to it.
        api = HfApi(token=OTHER_TOKEN)
        repo_url = api.create_repo(repo_id="gated_repo_for_huggingface_hub_ci", exist_ok=True)
        requests.put(
            f"{repo_url.endpoint}/api/models/{repo_url.repo_id}/settings",
            headers=api._build_hf_headers(),
            json={"gated": "auto"},
        ).raise_for_status()

        # Cannot download file as repo is gated
        with SoftTemporaryDirectory() as tmpdir:
            with self.assertRaisesRegex(
                GatedRepoError,
                "Access to model .* is restricted and you are not in the authorized list",
            ):
                hf_hub_download(
                    repo_id=repo_url.repo_id,
                    filename=".gitattributes",
                    use_auth_token=TOKEN,
                    cache_dir=tmpdir,
                )


@pytest.mark.usefixtures("fx_cache_dir")
class StagingCachedDownloadOnAwfulFilenamesTest(unittest.TestCase):
    """Implement regression tests for #1161.

    Issue was on filename not url encoded by `hf_hub_download` and `hf_hub_url`.

    See https://github.com/huggingface/huggingface_hub/issues/1161
    """

    cache_dir: Path
    subfolder = "subfolder/to?"
    filename = "awful?filename%you:should,never.give"
    filepath = f"subfolder/to?/{filename}"

    @classmethod
    def setUpClass(cls):
        cls.api = HfApi(endpoint=ENDPOINT_STAGING, token=TOKEN)
        cls.repo_url = cls.api.create_repo(repo_id=repo_name("awful_filename"))
        cls.expected_resolve_url = (
            f"{cls.repo_url}/resolve/main/subfolder/to%3F/awful%3Ffilename%25you%3Ashould%2Cnever.give"
        )
        cls.api.upload_file(
            path_or_fileobj=b"content",
            path_in_repo=cls.filepath,
            repo_id=cls.repo_url.repo_id,
        )

    @classmethod
    def tearDownClass(cls) -> None:
        cls.api.delete_repo(repo_id=cls.repo_url.repo_id)

    def test_hf_hub_url_on_awful_filepath(self):
        self.assertEqual(hf_hub_url(self.repo_url.repo_id, self.filepath), self.expected_resolve_url)

    def test_hf_hub_url_on_awful_subfolder_and_filename(self):
        self.assertEqual(
            hf_hub_url(self.repo_url.repo_id, self.filename, subfolder=self.subfolder),
            self.expected_resolve_url,
        )

    @xfail_on_windows(reason="Windows paths cannot contain a '?'.")
    def test_hf_hub_download_on_awful_filepath(self):
        local_path = hf_hub_download(self.repo_url.repo_id, self.filepath, cache_dir=self.cache_dir)
        # Local path is not url-encoded
        self.assertTrue(local_path.endswith(self.filepath))

    @xfail_on_windows(reason="Windows paths cannot contain a '?'.")
    def test_hf_hub_download_on_awful_subfolder_and_filename(self):
        local_path = hf_hub_download(
            self.repo_url.repo_id,
            self.filename,
            subfolder=self.subfolder,
            cache_dir=self.cache_dir,
        )
        # Local path is not url-encoded
        self.assertTrue(local_path.endswith(self.filepath))


@pytest.mark.usefixtures("fx_cache_dir")
class TestHfHubDownloadRelativePaths(unittest.TestCase):
    """Regression test for HackerOne report 1928845.

    Issue was that any file outside of the local dir could be overwritten (Windows only).

    In the end, multiple protections have been added to prevent this (..\\ in filename forbidden on Windows, always check
    the filepath is in local_dir/snapshot_dir).
    """

    cache_dir: Path

    @classmethod
    def setUpClass(cls):
        cls.api = HfApi(endpoint=ENDPOINT_STAGING, token=TOKEN)
        cls.repo_id = cls.api.create_repo(repo_id=repo_name()).repo_id
        cls.api.upload_file(path_or_fileobj=b"content", path_in_repo="..\\ddd", repo_id=cls.repo_id)
        cls.api.upload_file(path_or_fileobj=b"content", path_in_repo="folder/..\\..\\..\\file", repo_id=cls.repo_id)

    @classmethod
    def tearDownClass(cls) -> None:
        cls.api.delete_repo(repo_id=cls.repo_id)

    @xfail_on_windows(reason="Windows paths cannot start with '..\\'.", raises=ValueError)
    def test_download_file_in_cache_dir(self) -> None:
        hf_hub_download(self.repo_id, "..\\ddd", cache_dir=self.cache_dir)

    @xfail_on_windows(reason="Windows paths cannot start with '..\\'.", raises=ValueError)
    def test_download_file_to_local_dir(self) -> None:
        with SoftTemporaryDirectory() as local_dir:
            hf_hub_download(self.repo_id, "..\\ddd", cache_dir=self.cache_dir, local_dir=local_dir)

    @xfail_on_windows(reason="Windows paths cannot contain '\\..\\'.", raises=ValueError)
    def test_download_folder_file_in_cache_dir(self) -> None:
        hf_hub_download(self.repo_id, "folder/..\\..\\..\\file", cache_dir=self.cache_dir)

    @xfail_on_windows(reason="Windows paths cannot contain '\\..\\'.", raises=ValueError)
    def test_download_folder_file_to_local_dir(self) -> None:
        with SoftTemporaryDirectory() as local_dir:
            hf_hub_download(self.repo_id, "folder/..\\..\\..\\file", cache_dir=self.cache_dir, local_dir=local_dir)

    def test_get_pointer_path_and_valid_relative_filename(self) -> None:
        # Cannot happen because of other protections, but just in case.
        self.assertEqual(
            _get_pointer_path("path/to/storage", "abcdef", "path/to/file.txt"),
            os.path.join("path/to/storage", "snapshots", "abcdef", "path/to/file.txt"),
        )

    def test_get_pointer_path_but_invalid_relative_filename(self) -> None:
        # Cannot happen because of other protections, but just in case.
        relative_filename = "folder\\..\\..\\..\\file.txt" if os.name == "nt" else "folder/../../../file.txt"
        with self.assertRaises(ValueError):
            _get_pointer_path("path/to/storage", "abcdef", relative_filename)

    def test_to_local_dir_but_invalid_relative_filename(self) -> None:
        # Cannot happen because of other protections, but just in case.
        relative_filename = "folder\\..\\..\\..\\file.txt" if os.name == "nt" else "folder/../../../file.txt"
        with self.assertRaises(ValueError):
            _to_local_dir(
                "path/to/file_to_copy", "path/to/local/dir", relative_filename=relative_filename, use_symlinks=False
            )


class CreateSymlinkTest(unittest.TestCase):
    @unittest.skipIf(os.name == "nt", "No symlinks on Windows")
    @patch("huggingface_hub.file_download.are_symlinks_supported")
    def test_create_symlink_concurrent_access(self, mock_are_symlinks_supported: Mock) -> None:
        with SoftTemporaryDirectory() as tmpdir:
            src = os.path.join(tmpdir, "source")
            other = os.path.join(tmpdir, "other")
            dst = os.path.join(tmpdir, "destination")

            # Normal case: symlink does not exist
            mock_are_symlinks_supported.return_value = True
            _create_symlink(src, dst)
            self.assertEqual(os.path.realpath(dst), os.path.realpath(src))

            # Symlink already exists when it tries to create it (most probably from a
            # concurrent access) but do not raise exception
            def _are_symlinks_supported(cache_dir: str) -> bool:
                os.symlink(src, dst)
                return True

            mock_are_symlinks_supported.side_effect = _are_symlinks_supported
            _create_symlink(src, dst)

            # Symlink already exists but pointing to a different source file. This should
            # never happen in the context of HF cache system -> raise exception
            def _are_symlinks_supported(cache_dir: str) -> bool:
                os.symlink(other, dst)
                return True

            mock_are_symlinks_supported.side_effect = _are_symlinks_supported
            with self.assertRaises(FileExistsError):
                _create_symlink(src, dst)

    def test_create_symlink_relative_src(self) -> None:
        """Regression test for #1388.

        See https://github.com/huggingface/huggingface_hub/issues/1388.
        """
        # Test dir has to be relative
        test_dir = Path(".") / "dir_for_create_symlink_test"
        test_dir.mkdir(parents=True, exist_ok=True)
        src = Path(test_dir) / "source"
        src.touch()
        dst = Path(test_dir) / "destination"

        _create_symlink(str(src), str(dst))
        self.assertTrue(dst.resolve().is_file())
        if os.name != "nt":
            self.assertEqual(dst.resolve(), src.resolve())
        shutil.rmtree(test_dir)


class TestNormalizeEtag(unittest.TestCase):
    """Unit tests implemented after a server-side change broke the ETag normalization once (see #1428).

    TL;DR: _normalize_etag was expecting only strong references, but the server started to return weak references after
    a config update. Problem was quickly fixed server-side but we prefer to make sure this doesn't happen again by
    supporting weak etags. For context, etags are used to build the cache-system structure.

    For more details, see https://github.com/huggingface/huggingface_hub/pull/1428 and related issues.
    """

    def test_strong_reference(self):
        self.assertEqual(
            _normalize_etag('"a16a55fda99d2f2e7b69cce5cf93ff4ad3049930"'), "a16a55fda99d2f2e7b69cce5cf93ff4ad3049930"
        )

    def test_weak_reference(self):
        self.assertEqual(
            _normalize_etag('W/"a16a55fda99d2f2e7b69cce5cf93ff4ad3049930"'), "a16a55fda99d2f2e7b69cce5cf93ff4ad3049930"
        )

    @with_production_testing
    def test_resolve_endpoint_on_regular_file(self):
        url = "https://huggingface.co/gpt2/resolve/e7da7f221d5bf496a48136c0cd264e630fe9fcc8/README.md"
        response = requests.head(url)
        self.assertEqual(self._get_etag_and_normalize(response), "a16a55fda99d2f2e7b69cce5cf93ff4ad3049930")

    @with_production_testing
    def test_resolve_endpoint_on_lfs_file(self):
        url = "https://huggingface.co/gpt2/resolve/e7da7f221d5bf496a48136c0cd264e630fe9fcc8/pytorch_model.bin"
        response = requests.head(url)
        self.assertEqual(
            self._get_etag_and_normalize(response), "7c5d3f4b8b76583b422fcb9189ad6c89d5d97a094541ce8932dce3ecabde1421"
        )

    @staticmethod
    def _get_etag_and_normalize(response: Response) -> str:
        response.raise_for_status()
        return _normalize_etag(response.headers.get(HUGGINGFACE_HEADER_X_LINKED_ETAG) or response.headers.get("ETag"))


def _recursive_chmod(path: str, mode: int) -> None:
    # Taken from https://stackoverflow.com/a/2853934
    for root, dirs, files in os.walk(path):
        for d in dirs:
            os.chmod(os.path.join(root, d), mode)
        for f in files:
            os.chmod(os.path.join(root, f), mode)


# tests/test_hf_api.py

# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import re
import shutil
import subprocess
import tempfile
import time
import types
import unittest
import warnings
from concurrent.futures import Future
from functools import partial
from io import BytesIO
from pathlib import Path
from typing import Any, Dict, List, Union
from unittest.mock import Mock, patch
from urllib.parse import quote

import pytest
import requests
from requests.exceptions import HTTPError

import huggingface_hub.lfs
from huggingface_hub import SpaceHardware, SpaceStage
from huggingface_hub._commit_api import (
    CommitOperationAdd,
    CommitOperationCopy,
    CommitOperationDelete,
    fetch_upload_modes,
)
from huggingface_hub.community import DiscussionComment, DiscussionWithDetails
from huggingface_hub.constants import (
    REPO_TYPE_DATASET,
    REPO_TYPE_MODEL,
    REPO_TYPE_SPACE,
    SPACES_SDK_TYPES,
)
from huggingface_hub.file_download import hf_hub_download
from huggingface_hub.hf_api import (
    CommitInfo,
    DatasetInfo,
    HfApi,
    MetricInfo,
    ModelInfo,
    ModelSearchArguments,
    RepoFile,
    RepoUrl,
    ReprMixin,
    SpaceInfo,
    repo_type_and_id_from_hf_id,
)
from huggingface_hub.utils import (
    BadRequestError,
    EntryNotFoundError,
    HfFolder,
    HfHubHTTPError,
    RepositoryNotFoundError,
    RevisionNotFoundError,
    SoftTemporaryDirectory,
    logging,
)
from huggingface_hub.utils.endpoint_helpers import (
    DatasetFilter,
    ModelFilter,
    _filter_emissions,
)

from .testing_constants import (
    ENDPOINT_STAGING,
    ENDPOINT_STAGING_BASIC_AUTH,
    FULL_NAME,
    OTHER_TOKEN,
    OTHER_USER,
    TOKEN,
    USER,
)
from .testing_utils import (
    DUMMY_DATASET_ID,
    DUMMY_DATASET_ID_REVISION_ONE_SPECIFIC_COMMIT,
    DUMMY_MODEL_ID,
    DUMMY_MODEL_ID_REVISION_ONE_SPECIFIC_COMMIT,
    SAMPLE_DATASET_IDENTIFIER,
    repo_name,
    require_git_lfs,
    retry_endpoint,
    rmtree_with_retry,
    use_tmp_repo,
    with_production_testing,
)


logger = logging.get_logger(__name__)

dataset_repo_name = partial(repo_name, prefix="my-dataset")
space_repo_name = partial(repo_name, prefix="my-space")
large_file_repo_name = partial(repo_name, prefix="my-model-largefiles")

WORKING_REPO_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/working_repo")
LARGE_FILE_14MB = "https://cdn-media.huggingface.co/lfs-largefiles/progit.epub"
LARGE_FILE_18MB = "https://cdn-media.huggingface.co/lfs-largefiles/progit.pdf"


class HfApiCommonTest(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        """Share the valid token in all tests below."""
        cls._token = TOKEN
        cls._api = HfApi(endpoint=ENDPOINT_STAGING, token=TOKEN)


@retry_endpoint
def test_repo_id_no_warning():
    # tests that passing repo_id as positional arg doesn't raise any warnings
    # for {create, delete}_repo and update_repo_visibility
    api = HfApi(endpoint=ENDPOINT_STAGING, token=TOKEN)
    REPO_NAME = repo_name("crud")

    args = [
        ("create_repo", {}),
        ("update_repo_visibility", {"private": False}),
        ("delete_repo", {}),
    ]

    for method, kwargs in args:
        with warnings.catch_warnings(record=True) as record:
            getattr(api, method)(REPO_NAME, repo_type=REPO_TYPE_MODEL, **kwargs)
        assert not len(record)


class HfApiEndpointsTest(HfApiCommonTest):
    def test_whoami_with_passing_token(self):
        info = self._api.whoami(token=self._token)
        self.assertEqual(info["name"], USER)
        self.assertEqual(info["fullname"], FULL_NAME)
        self.assertIsInstance(info["orgs"], list)
        valid_org = [org for org in info["orgs"] if org["name"] == "valid_org"][0]
        self.assertIsInstance(valid_org["apiToken"], str)

    @patch("huggingface_hub.utils._headers.HfFolder")
    def test_whoami_with_implicit_token_from_login(self, mock_HfFolder: Mock) -> None:
        """Test using `whoami` after a `huggingface-cli login`."""
        mock_HfFolder().get_token.return_value = self._token

        with patch.object(self._api, "token", None):  # no default token
            info = self._api.whoami()
        self.assertEqual(info["name"], USER)

    @patch("huggingface_hub.utils._headers.HfFolder")
    def test_whoami_with_implicit_token_from_hf_api(self, mock_HfFolder: Mock) -> None:
        """Test using `whoami` with token from the HfApi client."""
        info = self._api.whoami()
        self.assertEqual(info["name"], USER)
        mock_HfFolder().get_token.assert_not_called()

    @retry_endpoint
    def test_delete_repo_error_message(self):
        # test for #751
        # See https://github.com/huggingface/huggingface_hub/issues/751
        with self.assertRaisesRegex(
            requests.exceptions.HTTPError,
            re.compile(
                r"404 Client Error(.+)\(Request ID: .+\)(.*)Repository Not Found",
                flags=re.DOTALL,
            ),
        ):
            self._api.delete_repo("repo-that-does-not-exist")

    @retry_endpoint
    def test_create_update_and_delete_repo(self):
        REPO_NAME = repo_name("crud")
        self._api.create_repo(repo_id=REPO_NAME)
        res = self._api.update_repo_visibility(repo_id=REPO_NAME, private=True)
        self.assertTrue(res["private"])
        res = self._api.update_repo_visibility(repo_id=REPO_NAME, private=False)
        self.assertFalse(res["private"])
        self._api.delete_repo(repo_id=REPO_NAME)

    @retry_endpoint
    def test_create_update_and_delete_model_repo(self):
        REPO_NAME = repo_name("crud")
        self._api.create_repo(repo_id=REPO_NAME, repo_type=REPO_TYPE_MODEL)
        res = self._api.update_repo_visibility(repo_id=REPO_NAME, private=True, repo_type=REPO_TYPE_MODEL)
        self.assertTrue(res["private"])
        res = self._api.update_repo_visibility(repo_id=REPO_NAME, private=False, repo_type=REPO_TYPE_MODEL)
        self.assertFalse(res["private"])
        self._api.delete_repo(repo_id=REPO_NAME, repo_type=REPO_TYPE_MODEL)

    @retry_endpoint
    def test_create_update_and_delete_dataset_repo(self):
        DATASET_REPO_NAME = dataset_repo_name("crud")
        self._api.create_repo(repo_id=DATASET_REPO_NAME, repo_type=REPO_TYPE_DATASET)
        res = self._api.update_repo_visibility(repo_id=DATASET_REPO_NAME, private=True, repo_type=REPO_TYPE_DATASET)
        self.assertTrue(res["private"])
        res = self._api.update_repo_visibility(repo_id=DATASET_REPO_NAME, private=False, repo_type=REPO_TYPE_DATASET)
        self.assertFalse(res["private"])
        self._api.delete_repo(repo_id=DATASET_REPO_NAME, repo_type=REPO_TYPE_DATASET)

    @unittest.skip(
        "Create repo fails on staging endpoint. See"
        " https://huggingface.slack.com/archives/C02EMARJ65P/p1666795928977419"
        " (internal link)."
    )
    @retry_endpoint
    def test_create_update_and_delete_space_repo(self):
        SPACE_REPO_NAME = space_repo_name("failing")
        with pytest.raises(ValueError, match=r"No space_sdk provided.*"):
            self._api.create_repo(repo_id=SPACE_REPO_NAME, repo_type=REPO_TYPE_SPACE, space_sdk=None)
        with pytest.raises(ValueError, match=r"Invalid space_sdk.*"):
            self._api.create_repo(repo_id=SPACE_REPO_NAME, repo_type=REPO_TYPE_SPACE, space_sdk="something")

        for sdk in SPACES_SDK_TYPES:
            SPACE_REPO_NAME = space_repo_name(sdk)
            self._api.create_repo(repo_id=SPACE_REPO_NAME, repo_type=REPO_TYPE_SPACE, space_sdk=sdk)
            res = self._api.update_repo_visibility(repo_id=SPACE_REPO_NAME, private=True, repo_type=REPO_TYPE_SPACE)
            self.assertTrue(res["private"])
            res = self._api.update_repo_visibility(repo_id=SPACE_REPO_NAME, private=False, repo_type=REPO_TYPE_SPACE)
            self.assertFalse(res["private"])
            self._api.delete_repo(repo_id=SPACE_REPO_NAME, repo_type=REPO_TYPE_SPACE)

    @retry_endpoint
    def test_move_repo_normal_usage(self):
        repo_id = f"{USER}/{repo_name()}"
        new_repo_id = f"{USER}/{repo_name()}"

        # Spaces not tested on staging (error 500)
        for repo_type in [None, REPO_TYPE_MODEL, REPO_TYPE_DATASET]:
            self._api.create_repo(repo_id=repo_id, repo_type=repo_type)
            self._api.move_repo(from_id=repo_id, to_id=new_repo_id, repo_type=repo_type)
            self._api.delete_repo(repo_id=new_repo_id, repo_type=repo_type)

    def test_move_repo_target_already_exists(self) -> None:
        repo_id_1 = f"{USER}/{repo_name()}"
        repo_id_2 = f"{USER}/{repo_name()}"

        self._api.create_repo(repo_id=repo_id_1)
        self._api.create_repo(repo_id=repo_id_2)

        with pytest.raises(HfHubHTTPError, match=r"A model repository called .* already exists"):
            self._api.move_repo(from_id=repo_id_1, to_id=repo_id_2, repo_type=REPO_TYPE_MODEL)

        self._api.delete_repo(repo_id=repo_id_1)
        self._api.delete_repo(repo_id=repo_id_2)

    def test_move_repo_invalid_repo_id(self) -> None:
        """Test from_id and to_id must be in the form `"namespace/repo_name"`."""
        with pytest.raises(ValueError, match=r"Invalid repo_id*"):
            self._api.move_repo(from_id="namespace/repo_name", to_id="invalid_repo_id")

        with pytest.raises(ValueError, match=r"Invalid repo_id*"):
            self._api.move_repo(from_id="invalid_repo_id", to_id="namespace/repo_name")


class CommitApiTest(HfApiCommonTest):
    def setUp(self) -> None:
        super().setUp()
        self.tmp_dir = tempfile.mkdtemp()
        self.tmp_file = os.path.join(self.tmp_dir, "temp")
        self.tmp_file_content = "Content of the file"
        with open(self.tmp_file, "w+") as f:
            f.write(self.tmp_file_content)
        os.makedirs(os.path.join(self.tmp_dir, "nested"))
        self.nested_tmp_file = os.path.join(self.tmp_dir, "nested", "file.bin")
        with open(self.nested_tmp_file, "wb+") as f:
            f.truncate(1024 * 1024)

        self.addCleanup(rmtree_with_retry, self.tmp_dir)

    @retry_endpoint
    def test_upload_file_validation(self) -> None:
        with self.assertRaises(ValueError, msg="Wrong repo type"):
            self._api.upload_file(
                path_or_fileobj=self.tmp_file,
                path_in_repo="README.md",
                repo_id="something",
                repo_type="this type does not exist",
            )

    def test_commit_operation_validation(self):
        with open(self.tmp_file, "rt") as ftext:
            with self.assertRaises(
                ValueError,
                msg="If you passed a file-like object, make sure it is in binary mode",
            ):
                CommitOperationAdd(path_or_fileobj=ftext, path_in_repo="README.md")  # type: ignore

        with self.assertRaises(ValueError, msg="path_or_fileobj is str but does not point to a file"):
            CommitOperationAdd(
                path_or_fileobj=os.path.join(self.tmp_dir, "nofile.pth"),
                path_in_repo="README.md",
            )

    @retry_endpoint
    @use_tmp_repo()
    def test_upload_file_str_path(self, repo_url: RepoUrl) -> None:
        repo_id = repo_url.repo_id
        return_val = self._api.upload_file(
            path_or_fileobj=self.tmp_file,
            path_in_repo="temp/new_file.md",
            repo_id=repo_id,
        )
        self.assertEqual(return_val, f"{repo_url}/blob/main/temp/new_file.md")

        with SoftTemporaryDirectory() as cache_dir:
            with open(hf_hub_download(repo_id=repo_id, filename="temp/new_file.md", cache_dir=cache_dir)) as f:
                self.assertEqual(f.read(), self.tmp_file_content)

    @retry_endpoint
    @use_tmp_repo()
    def test_upload_file_pathlib_path(self, repo_url: RepoUrl) -> None:
        """Regression test for https://github.com/huggingface/huggingface_hub/issues/1246."""
        self._api.upload_file(path_or_fileobj=Path(self.tmp_file), path_in_repo="README.md", repo_id=repo_url.repo_id)
        self.assertIn("README.md", self._api.list_repo_files(repo_id=repo_url.repo_id))

    @retry_endpoint
    @use_tmp_repo()
    def test_upload_file_fileobj(self, repo_url: RepoUrl) -> None:
        repo_id = repo_url.repo_id
        with open(self.tmp_file, "rb") as filestream:
            return_val = self._api.upload_file(
                path_or_fileobj=filestream,
                path_in_repo="temp/new_file.md",
                repo_id=repo_id,
            )
        self.assertEqual(return_val, f"{repo_url}/blob/main/temp/new_file.md")

        with SoftTemporaryDirectory() as cache_dir:
            with open(hf_hub_download(repo_id=repo_id, filename="temp/new_file.md", cache_dir=cache_dir)) as f:
                self.assertEqual(f.read(), self.tmp_file_content)

    @retry_endpoint
    @use_tmp_repo()
    def test_upload_file_bytesio(self, repo_url: RepoUrl) -> None:
        repo_id = repo_url.repo_id
        content = BytesIO(b"File content, but in bytes IO")
        return_val = self._api.upload_file(
            path_or_fileobj=content,
            path_in_repo="temp/new_file.md",
            repo_id=repo_id,
        )
        self.assertEqual(return_val, f"{repo_url}/blob/main/temp/new_file.md")

        with SoftTemporaryDirectory() as cache_dir:
            with open(hf_hub_download(repo_id=repo_id, filename="temp/new_file.md", cache_dir=cache_dir)) as f:
                self.assertEqual(f.read(), content.getvalue().decode())

    @retry_endpoint
    def test_create_repo_return_value(self) -> None:
        REPO_NAME = repo_name("org")
        url = self._api.create_repo(repo_id=REPO_NAME)
        self.assertIsInstance(url, str)
        self.assertIsInstance(url, RepoUrl)
        self.assertEqual(url.repo_id, f"{USER}/{REPO_NAME}")
        self._api.delete_repo(repo_id=url.repo_id)

    @retry_endpoint
    def test_create_repo_org_token_fail(self):
        REPO_NAME = repo_name("org")
        with pytest.raises(ValueError, match="You must use your personal account token."):
            self._api.create_repo(repo_id=REPO_NAME, token="api_org_dummy_token")

    @retry_endpoint
    def test_create_repo_org_token_none_fail(self):
        HfFolder.save_token("api_org_dummy_token")
        with pytest.raises(ValueError, match="You must use your personal account token."):
            with patch.object(self._api, "token", None):  # no default token
                self._api.create_repo(repo_id=repo_name("org"))

    def test_create_repo_already_exists_but_no_write_permission(self):
        # Create under other user namespace
        repo_id = self._api.create_repo(repo_id=repo_name(), token=OTHER_TOKEN).repo_id

        # Try to create with our namespace -> should not fail as the repo already exists
        self._api.create_repo(repo_id=repo_id, token=TOKEN, exist_ok=True)

        # Clean up
        self._api.delete_repo(repo_id=repo_id, token=OTHER_TOKEN)

    @retry_endpoint
    @use_tmp_repo()
    def test_upload_file_create_pr(self, repo_url: RepoUrl) -> None:
        repo_id = repo_url.repo_id
        return_val = self._api.upload_file(
            path_or_fileobj=self.tmp_file_content.encode(),
            path_in_repo="temp/new_file.md",
            repo_id=repo_id,
            create_pr=True,
        )
        self.assertEqual(return_val, f"{repo_url}/blob/{quote('refs/pr/1', safe='')}/temp/new_file.md")

        with SoftTemporaryDirectory() as cache_dir:
            with open(
                hf_hub_download(
                    repo_id=repo_id, filename="temp/new_file.md", revision="refs/pr/1", cache_dir=cache_dir
                )
            ) as f:
                self.assertEqual(f.read(), self.tmp_file_content)

    @retry_endpoint
    @use_tmp_repo()
    def test_delete_file(self, repo_url: RepoUrl) -> None:
        self._api.upload_file(
            path_or_fileobj=self.tmp_file,
            path_in_repo="temp/new_file.md",
            repo_id=repo_url.repo_id,
        )
        self._api.delete_file(path_in_repo="temp/new_file.md", repo_id=repo_url.repo_id)

        with self.assertRaises(EntryNotFoundError):
            # Should raise a 404
            hf_hub_download(repo_url.repo_id, "temp/new_file.md")

    def test_get_full_repo_name(self):
        repo_name_with_no_org = self._api.get_full_repo_name("model")
        self.assertEqual(repo_name_with_no_org, f"{USER}/model")

        repo_name_with_no_org = self._api.get_full_repo_name("model", organization="org")
        self.assertEqual(repo_name_with_no_org, "org/model")

    @retry_endpoint
    def test_upload_folder(self):
        for private in (False, True):
            visibility = "private" if private else "public"
            with self.subTest(f"{visibility} repo"):
                REPO_NAME = repo_name(f"upload_folder_{visibility}")
                self._api.create_repo(repo_id=REPO_NAME, private=private, exist_ok=False)
                try:
                    url = self._api.upload_folder(
                        folder_path=self.tmp_dir,
                        path_in_repo="temp/dir",
                        repo_id=f"{USER}/{REPO_NAME}",
                    )
                    self.assertEqual(
                        url,
                        f"{self._api.endpoint}/{USER}/{REPO_NAME}/tree/main/temp/dir",
                    )
                    for rpath in ["temp", "nested/file.bin"]:
                        local_path = os.path.join(self.tmp_dir, rpath)
                        remote_path = f"temp/dir/{rpath}"
                        filepath = hf_hub_download(
                            repo_id=f"{USER}/{REPO_NAME}",
                            filename=remote_path,
                            revision="main",
                            use_auth_token=self._token,
                        )
                        assert filepath is not None
                        with open(filepath, "rb") as downloaded_file:
                            content = downloaded_file.read()
                        with open(local_path, "rb") as local_file:
                            expected_content = local_file.read()
                        self.assertEqual(content, expected_content)

                    # Re-uploading the same folder twice should be fine
                    self._api.upload_folder(
                        folder_path=self.tmp_dir,
                        path_in_repo="temp/dir",
                        repo_id=f"{USER}/{REPO_NAME}",
                    )
                except Exception as err:
                    self.fail(err)
                finally:
                    self._api.delete_repo(repo_id=REPO_NAME)

    @retry_endpoint
    def test_upload_folder_create_pr(self):
        pr_revision = quote("refs/pr/1", safe="")
        for private in (False, True):
            visibility = "private" if private else "public"
            with self.subTest(f"{visibility} repo"):
                REPO_NAME = repo_name(f"upload_folder_{visibility}")
                self._api.create_repo(repo_id=REPO_NAME, private=private, exist_ok=False)
                try:
                    return_val = self._api.upload_folder(
                        folder_path=self.tmp_dir,
                        path_in_repo="temp/dir",
                        repo_id=f"{USER}/{REPO_NAME}",
                        create_pr=True,
                    )
                    self.assertEqual(
                        return_val,
                        f"{self._api.endpoint}/{USER}/{REPO_NAME}/tree/{pr_revision}/temp/dir",
                    )
                    for rpath in ["temp", "nested/file.bin"]:
                        local_path = os.path.join(self.tmp_dir, rpath)
                        remote_path = f"temp/dir/{rpath}"
                        filepath = hf_hub_download(
                            repo_id=f"{USER}/{REPO_NAME}",
                            filename=remote_path,
                            revision="refs/pr/1",
                            use_auth_token=self._token,
                        )
                        assert filepath is not None
                        with open(filepath, "rb") as downloaded_file:
                            content = downloaded_file.read()
                        with open(local_path, "rb") as local_file:
                            expected_content = local_file.read()
                        self.assertEqual(content, expected_content)
                except Exception as err:
                    self.fail(err)
                finally:
                    self._api.delete_repo(repo_id=REPO_NAME)

    @retry_endpoint
    def test_upload_folder_default_path_in_repo(self):
        REPO_NAME = repo_name("upload_folder_to_root")
        self._api.create_repo(repo_id=REPO_NAME, exist_ok=False)
        url = self._api.upload_folder(folder_path=self.tmp_dir, repo_id=f"{USER}/{REPO_NAME}")
        # URL to root of repository
        self.assertEqual(url, f"{self._api.endpoint}/{USER}/{REPO_NAME}/tree/main/")

    @retry_endpoint
    @use_tmp_repo()
    def test_upload_folder_git_folder_excluded(self, repo_url: RepoUrl) -> None:
        # Simulate a folder with a .git folder
        def _create_file(*parts) -> None:
            path = Path(self.tmp_dir, *parts)
            path.parent.mkdir(parents=True, exist_ok=True)
            path.write_text("content")

        _create_file(".git", "file.txt")
        _create_file(".git", "folder", "file.txt")
        _create_file("folder", ".git", "file.txt")
        _create_file("folder", ".git", "folder", "file.txt")
        _create_file(".git_something", "file.txt")
        _create_file("file.git")

        # Upload folder and check that .git folder is excluded
        self._api.upload_folder(folder_path=self.tmp_dir, repo_id=repo_url.repo_id)
        self.assertEqual(
            set(self._api.list_repo_files(repo_id=repo_url.repo_id)),
            {".gitattributes", ".git_something/file.txt", "file.git", "temp", "nested/file.bin"},
        )

    @retry_endpoint
    def test_create_commit_create_pr(self):
        REPO_NAME = repo_name("create_commit_create_pr")
        self._api.create_repo(repo_id=REPO_NAME, exist_ok=False)
        try:
            self._api.upload_file(
                path_or_fileobj=self.tmp_file,
                path_in_repo="temp/new_file.md",
                repo_id=f"{USER}/{REPO_NAME}",
            )
            operations = [
                CommitOperationDelete(path_in_repo="temp/new_file.md"),
                CommitOperationAdd(path_in_repo="buffer", path_or_fileobj=b"Buffer data"),
            ]
            resp = self._api.create_commit(
                operations=operations,
                commit_message="Test create_commit",
                repo_id=f"{USER}/{REPO_NAME}",
                create_pr=True,
            )

            # Check commit info
            self.assertIsInstance(resp, CommitInfo)
            commit_id = resp.oid
            self.assertIn("pr_revision='refs/pr/1'", str(resp))
            self.assertIsInstance(commit_id, str)
            self.assertGreater(len(commit_id), 0)
            self.assertEqual(
                resp.commit_url,
                f"{self._api.endpoint}/{USER}/{REPO_NAME}/commit/{commit_id}",
            )
            self.assertEqual(resp.commit_message, "Test create_commit")
            self.assertEqual(resp.commit_description, "")
            self.assertEqual(
                resp.pr_url,
                f"{self._api.endpoint}/{USER}/{REPO_NAME}/discussions/1",
            )
            self.assertEqual(resp.pr_num, 1)
            self.assertEqual(resp.pr_revision, "refs/pr/1")

            with self.assertRaises(HTTPError) as ctx:
                # Should raise a 404
                hf_hub_download(f"{USER}/{REPO_NAME}", "buffer", use_auth_token=self._token)
                self.assertEqual(ctx.exception.response.status_code, 404)
            filepath = hf_hub_download(
                filename="buffer",
                repo_id=f"{USER}/{REPO_NAME}",
                use_auth_token=self._token,
                revision="refs/pr/1",
            )
            self.assertTrue(filepath is not None)
            with open(filepath, "rb") as downloaded_file:
                content = downloaded_file.read()
            self.assertEqual(content, b"Buffer data")
        except Exception as err:
            self.fail(err)
        finally:
            self._api.delete_repo(repo_id=REPO_NAME)

    @retry_endpoint
    def test_create_commit_create_pr_against_branch(self):
        repo_id = f"{USER}/{repo_name()}"

        # Create repo and create a non-main branch
        self._api.create_repo(repo_id=repo_id, exist_ok=False)
        self._api.create_branch(repo_id=repo_id, branch="test_branch")
        head = self._api.list_repo_refs(repo_id=repo_id).branches[0].target_commit

        # Create PR against non-main branch works
        resp = self._api.create_commit(
            operations=[],
            commit_message="PR against existing branch",
            repo_id=repo_id,
            revision="test_branch",
            create_pr=True,
        )
        self.assertIsInstance(resp, CommitInfo)

        # Create PR against a oid fails
        with self.assertRaises(RevisionNotFoundError):
            self._api.create_commit(
                operations=[],
                commit_message="PR against a oid",
                repo_id=repo_id,
                revision=head,
                create_pr=True,
            )

        # Create PR against a non-existing branch fails
        with self.assertRaises(RevisionNotFoundError):
            self._api.create_commit(
                operations=[],
                commit_message="PR against missing branch",
                repo_id=repo_id,
                revision="missing_branch",
                create_pr=True,
            )

        # Cleanup
        self._api.delete_repo(repo_id=repo_id)

    @retry_endpoint
    def test_create_commit_create_pr_on_foreign_repo(self):
        # Create a repo with another user. The normal CI user don't have rights on it.
        # We must be able to create a PR on it
        foreign_api = HfApi(token=OTHER_TOKEN)
        foreign_repo_url = foreign_api.create_repo(repo_id=repo_name("repo-for-hfh-ci"))

        self._api.create_commit(
            operations=[
                CommitOperationAdd(path_in_repo="regular.txt", path_or_fileobj=b"File content"),
                CommitOperationAdd(path_in_repo="lfs.pkl", path_or_fileobj=b"File content"),
            ],
            commit_message="PR on foreign repo",
            repo_id=foreign_repo_url.repo_id,
            create_pr=True,
        )

        foreign_api.delete_repo(repo_id=foreign_repo_url.repo_id)

    @retry_endpoint
    def test_create_commit(self):
        for private in (False, True):
            visibility = "private" if private else "public"
            with self.subTest(f"{visibility} repo"):
                REPO_NAME = repo_name(f"create_commit_{visibility}")
                self._api.create_repo(repo_id=REPO_NAME, private=private, exist_ok=False)
                try:
                    self._api.upload_file(
                        path_or_fileobj=self.tmp_file,
                        path_in_repo="temp/new_file.md",
                        repo_id=f"{USER}/{REPO_NAME}",
                    )
                    with open(self.tmp_file, "rb") as fileobj:
                        operations = [
                            CommitOperationDelete(path_in_repo="temp/new_file.md"),
                            CommitOperationAdd(path_in_repo="buffer", path_or_fileobj=b"Buffer data"),
                            CommitOperationAdd(
                                path_in_repo="bytesio",
                                path_or_fileobj=BytesIO(b"BytesIO data"),
                            ),
                            CommitOperationAdd(path_in_repo="fileobj", path_or_fileobj=fileobj),
                            CommitOperationAdd(
                                path_in_repo="nested/path",
                                path_or_fileobj=self.tmp_file,
                            ),
                        ]
                        resp = self._api.create_commit(
                            operations=operations,
                            commit_message="Test create_commit",
                            repo_id=f"{USER}/{REPO_NAME}",
                        )
                        # Check commit info
                        self.assertIsInstance(resp, CommitInfo)
                        self.assertIsNone(resp.pr_url)  # No pr created
                        self.assertIsNone(resp.pr_num)
                        self.assertIsNone(resp.pr_revision)
                    with self.assertRaises(HTTPError):
                        # Should raise a 404
                        hf_hub_download(
                            f"{USER}/{REPO_NAME}",
                            "temp/new_file.md",
                            use_auth_token=self._token,
                        )

                    for path, expected_content in [
                        ("buffer", b"Buffer data"),
                        ("bytesio", b"BytesIO data"),
                        ("fileobj", self.tmp_file_content.encode()),
                        ("nested/path", self.tmp_file_content.encode()),
                    ]:
                        filepath = hf_hub_download(
                            repo_id=f"{USER}/{REPO_NAME}",
                            filename=path,
                            revision="main",
                            use_auth_token=self._token,
                        )
                        self.assertTrue(filepath is not None)
                        with open(filepath, "rb") as downloaded_file:
                            content = downloaded_file.read()
                        self.assertEqual(content, expected_content)
                except Exception as err:
                    self.fail(err)
                finally:
                    self._api.delete_repo(repo_id=REPO_NAME)

    @retry_endpoint
    def test_create_commit_conflict(self):
        REPO_NAME = repo_name("create_commit_conflict")
        self._api.create_repo(repo_id=REPO_NAME, exist_ok=False)
        parent_commit = self._api.model_info(f"{USER}/{REPO_NAME}").sha
        try:
            self._api.upload_file(
                path_or_fileobj=self.tmp_file,
                path_in_repo="temp/new_file.md",
                repo_id=f"{USER}/{REPO_NAME}",
            )
            operations = [
                CommitOperationAdd(path_in_repo="buffer", path_or_fileobj=b"Buffer data"),
            ]
            with self.assertRaises(HTTPError) as exc_ctx:
                self._api.create_commit(
                    operations=operations,
                    commit_message="Test create_commit",
                    repo_id=f"{USER}/{REPO_NAME}",
                    parent_commit=parent_commit,
                )
            self.assertEqual(exc_ctx.exception.response.status_code, 412)
            self.assertIn(
                # Check the server message is added to the exception
                "A commit has happened since. Please refresh and try again.",
                str(exc_ctx.exception),
            )
        except Exception as err:
            self.fail(err)
        finally:
            self._api.delete_repo(repo_id=REPO_NAME)

    @retry_endpoint
    def test_create_commit_repo_does_not_exist(self) -> None:
        """Test error message is detailed when creating a commit on a missing repo."""
        # Test once with empty commit and once with an addition commit.
        for route, operations in (
            ("commit", []),
            ("preupload", [CommitOperationAdd("config.json", b"content")]),
        ):
            with self.subTest():
                with self.assertRaises(RepositoryNotFoundError) as context:
                    self._api.create_commit(
                        repo_id=f"{USER}/repo_that_do_not_exist",
                        operations=operations,
                        commit_message="fake_message",
                    )

                request_id = context.exception.response.headers.get("X-Request-Id")
                expected_message = (
                    f"404 Client Error. (Request ID: {request_id})\n\nRepository Not"
                    " Found for url:"
                    f" {self._api.endpoint}/api/models/{USER}/repo_that_do_not_exist/{route}/main.\nPlease"
                    " make sure you specified the correct `repo_id` and"
                    " `repo_type`.\nIf you are trying to access a private or gated"
                    " repo, make sure you are authenticated.\nNote: Creating a commit"
                    " assumes that the repo already exists on the Huggingface Hub."
                    " Please use `create_repo` if it's not the case."
                )

                self.assertEqual(str(context.exception), expected_message)

    @retry_endpoint
    def test_create_commit_lfs_file_implicit_token(self):
        """Test that uploading a file as LFS works with implicit token (from cache).

        Regression test for https://github.com/huggingface/huggingface_hub/pull/1084.
        """
        REPO_NAME = repo_name("create_commit_with_lfs")
        repo_id = f"{USER}/{REPO_NAME}"

        def _inner(mock: Mock) -> None:
            mock.return_value = self._token  # Set implicit token

            # Create repo
            self._api.create_repo(repo_id=REPO_NAME, exist_ok=False)

            # Set repo to track png files as LFS
            self._api.create_commit(
                operations=[
                    CommitOperationAdd(
                        path_in_repo=".gitattributes",
                        path_or_fileobj=b"*.png filter=lfs diff=lfs merge=lfs -text",
                    ),
                ],
                commit_message="Update .gitattributes",
                repo_id=repo_id,
            )

            # Upload a PNG file
            self._api.create_commit(
                operations=[
                    CommitOperationAdd(path_in_repo="image.png", path_or_fileobj=b"image data"),
                ],
                commit_message="Test upload lfs file",
                repo_id=repo_id,
            )

            # Check uploaded as LFS
            info = self._api.model_info(repo_id=repo_id, use_auth_token=self._token, files_metadata=True)
            siblings = {file.rfilename: file for file in info.siblings}
            self.assertIsInstance(siblings["image.png"].lfs, dict)  # LFS file

            # Delete repo
            self._api.delete_repo(repo_id=REPO_NAME, token=self._token)

        with patch.object(self._api, "token", None):  # no default token
            with patch("huggingface_hub.utils.HfFolder.get_token") as mock:
                _inner(mock)  # just to avoid indenting twice the code code

    @retry_endpoint
    def test_create_commit_huge_regular_files(self):
        """Test committing 12 text files (>100MB in total) at once.

        This was not possible when using `json` format instead of `ndjson`
        on the `/create-commit` endpoint.

        See https://github.com/huggingface/huggingface_hub/pull/1117.
        """
        REPO_NAME = repo_name("create_commit_huge_regular_files")
        self._api.create_repo(repo_id=REPO_NAME, exist_ok=False)
        try:
            operations = []
            for num in range(12):
                operations.append(
                    CommitOperationAdd(
                        path_in_repo=f"file-{num}.text",
                        path_or_fileobj=b"Hello regular " + b"a" * 1024 * 1024 * 9,
                    )
                )
            self._api.create_commit(
                operations=operations,  # 12*9MB regular => too much for "old" method
                commit_message="Test create_commit with huge regular files",
                repo_id=f"{USER}/{REPO_NAME}",
            )
        except Exception as err:
            self.fail(err)
        finally:
            self._api.delete_repo(repo_id=REPO_NAME)

    @retry_endpoint
    def test_commit_preflight_on_lots_of_lfs_files(self):
        """Test committing 1300 LFS files at once.

        This was not possible when `fetch_upload_modes` was not fetching metadata by
        chunks. We are not testing the full upload as it would require to upload 1300
        files which is unnecessary for the test. Having an overall large payload (for
        `/create-commit` endpoint) is tested in `test_create_commit_huge_regular_files`.

        There is also a 25k LFS files limit on the Hub but this is not tested.

        See https://github.com/huggingface/huggingface_hub/pull/1117.
        """
        REPO_NAME = repo_name("commit_preflight_lots_of_lfs_files")
        self._api.create_repo(repo_id=REPO_NAME, exist_ok=False)
        try:
            operations = []
            for num in range(1300):
                operations.append(
                    CommitOperationAdd(
                        path_in_repo=f"file-{num}.bin",  # considered as LFS
                        path_or_fileobj=b"Hello LFS" + b"a" * 2048,  # big enough sample
                    )
                )

            # Test `fetch_upload_modes` preflight ("are they regular or LFS files?")
            res = fetch_upload_modes(
                additions=operations,
                repo_type="model",
                repo_id=f"{USER}/{REPO_NAME}",
                token=TOKEN,
                revision="main",
                endpoint=ENDPOINT_STAGING,
            )
            self.assertEqual(len(res), 1300)
            for _, mode in res.items():
                self.assertEqual(mode, "lfs")
        except Exception as err:
            self.fail(err)
        finally:
            self._api.delete_repo(repo_id=REPO_NAME)

    @retry_endpoint
    def test_create_commit_repo_id_case_insensitive(self):
        """Test create commit but repo_id is lowercased.

        Regression test for #1371. Hub API is already case insensitive. Somehow the issue was with the `requests`
        streaming implementation when generating the ndjson payload "on the fly". It seems that the server was
        receiving only the first line which causes a confusing "400 Bad Request - Add a line with the key `lfsFile`,
        `file` or `deletedFile`". Passing raw bytes instead of a generator fixes the problem.

        See https://github.com/huggingface/huggingface_hub/issues/1371.
        """
        REPO_NAME = repo_name("CaSe_Is_ImPoRtAnT")
        repo_id = self._api.create_repo(repo_id=REPO_NAME, exist_ok=False).repo_id

        try:
            self._api.create_commit(
                repo_id=repo_id.lower(),  # API is case-insensitive!
                commit_message="Add 1 regular and 1 LFs files.",
                operations=[
                    CommitOperationAdd(path_in_repo="file.txt", path_or_fileobj=b"content"),
                    CommitOperationAdd(path_in_repo="lfs.bin", path_or_fileobj=b"LFS content"),
                ],
            )
            repo_files = self._api.list_repo_files(repo_id=repo_id)
            self.assertIn("file.txt", repo_files)
            self.assertIn("lfs.bin", repo_files)
        except Exception as err:
            self.fail(err)
        finally:
            self._api.delete_repo(repo_id=repo_id)

    @retry_endpoint
    @use_tmp_repo()
    def test_commit_copy_file(self, repo_url: RepoUrl) -> None:
        """Test CommitOperationCopy.

        Works only when copying an LFS file.
        """
        repo_id = repo_url.repo_id

        self._api.upload_file(path_or_fileobj=b"content", repo_id=repo_id, path_in_repo="file.txt")
        self._api.upload_file(path_or_fileobj=b"LFS content", repo_id=repo_id, path_in_repo="lfs.bin")

        self._api.create_commit(
            repo_id=repo_id,
            commit_message="Copy LFS file.",
            operations=[
                CommitOperationCopy(src_path_in_repo="lfs.bin", path_in_repo="lfs Copy.bin"),
                CommitOperationCopy(src_path_in_repo="lfs.bin", path_in_repo="lfs Copy (1).bin"),
            ],
        )
        with self.assertRaises(NotImplementedError):
            self._api.create_commit(
                repo_id=repo_id,
                commit_message="Copy regular file.",
                operations=[CommitOperationCopy(src_path_in_repo="file.txt", path_in_repo="file Copy.txt")],
            )
        with self.assertRaises(EntryNotFoundError):
            self._api.create_commit(
                repo_id=repo_id,
                commit_message="Copy a file that doesn't exist.",
                operations=[
                    CommitOperationCopy(src_path_in_repo="doesnt-exist.txt", path_in_repo="doesnt-exist Copy.txt")
                ],
            )

        # Check repo files
        repo_files = self._api.list_repo_files(repo_id=repo_id)
        self.assertIn("file.txt", repo_files)
        self.assertIn("lfs.bin", repo_files)
        self.assertIn("lfs Copy.bin", repo_files)
        self.assertIn("lfs Copy (1).bin", repo_files)

        # Check same LFS file
        repo_file1, repo_file2 = self._api.list_files_info(repo_id=repo_id, paths=["lfs.bin", "lfs Copy.bin"])
        self.assertEqual(repo_file1.lfs["sha256"], repo_file2.lfs["sha256"])


class HfApiUploadEmptyFileTest(HfApiCommonTest):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        # Create repo for all tests as they are not dependent on each other.
        cls.repo_id = f"{USER}/{repo_name('upload_empty_file')}"
        cls._api.create_repo(repo_id=cls.repo_id, exist_ok=False)

    @classmethod
    def tearDownClass(cls):
        cls._api.delete_repo(repo_id=cls.repo_id)
        super().tearDownClass()

    def test_upload_empty_lfs_file(self) -> None:
        # Should have been an LFS file, but uploaded as regular (would fail otherwise)
        self._api.upload_file(repo_id=self.repo_id, path_in_repo="empty.pkl", path_or_fileobj=b"")
        info = self._api.repo_info(repo_id=self.repo_id, files_metadata=True)

        repo_file = {file.rfilename: file for file in info.siblings}["empty.pkl"]
        self.assertEqual(repo_file.size, 0)
        self.assertIsNone(repo_file.lfs)  # As regular


class HfApiDeleteFolderTest(HfApiCommonTest):
    def setUp(self):
        self.repo_id = f"{USER}/{repo_name('create_commit_delete_folder')}"
        self._api.create_repo(repo_id=self.repo_id, exist_ok=False)

        self._api.create_commit(
            repo_id=self.repo_id,
            commit_message="Init repo",
            operations=[
                CommitOperationAdd(path_or_fileobj=b"data", path_in_repo="1/file_1.md"),
                CommitOperationAdd(path_or_fileobj=b"data", path_in_repo="1/file_2.md"),
                CommitOperationAdd(path_or_fileobj=b"data", path_in_repo="2/file_3.md"),
            ],
        )

    def tearDown(self):
        self._api.delete_repo(repo_id=self.repo_id)

    @retry_endpoint
    def test_create_commit_delete_folder_implicit(self):
        self._api.create_commit(
            operations=[CommitOperationDelete(path_in_repo="1/")],
            commit_message="Test delete folder implicit",
            repo_id=self.repo_id,
        )

        with self.assertRaises(EntryNotFoundError):
            hf_hub_download(self.repo_id, "1/file_1.md", use_auth_token=self._token)

        with self.assertRaises(EntryNotFoundError):
            hf_hub_download(self.repo_id, "1/file_2.md", use_auth_token=self._token)

        # Still exists
        hf_hub_download(self.repo_id, "2/file_3.md", use_auth_token=self._token)

    @retry_endpoint
    def test_create_commit_delete_folder_explicit(self):
        self._api.delete_folder(path_in_repo="1", repo_id=self.repo_id)
        with self.assertRaises(EntryNotFoundError):
            hf_hub_download(self.repo_id, "1/file_1.md", use_auth_token=self._token)

    @retry_endpoint
    def test_create_commit_failing_implicit_delete_folder(self):
        with self.assertRaisesRegex(
            EntryNotFoundError,
            'A file with the name "1" does not exist',
        ):
            self._api.create_commit(
                operations=[CommitOperationDelete(path_in_repo="1")],
                commit_message="Failing delete folder",
                repo_id=self.repo_id,
            )


class HfApiListFilesInfoTest(HfApiCommonTest):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.repo_id = cls._api.create_repo(repo_id=repo_name()).repo_id

        cls._api.create_commit(
            repo_id=cls.repo_id,
            commit_message="A first repo",
            operations=[
                CommitOperationAdd(path_or_fileobj=b"data", path_in_repo="file.md"),
                CommitOperationAdd(path_or_fileobj=b"data", path_in_repo="lfs.bin"),
                CommitOperationAdd(path_or_fileobj=b"data", path_in_repo="1/file_1.md"),
                CommitOperationAdd(path_or_fileobj=b"data", path_in_repo="1/2/file_1_2.md"),
                CommitOperationAdd(path_or_fileobj=b"data", path_in_repo="2/file_2.md"),
            ],
        )

        cls._api.create_commit(
            repo_id=cls.repo_id,
            commit_message="Another commit",
            operations=[
                CommitOperationAdd(path_or_fileobj=b"data2", path_in_repo="3/file_3.md"),
            ],
        )

    @classmethod
    def tearDownClass(cls):
        cls._api.delete_repo(repo_id=cls.repo_id)

    def test_get_regular_file_info(self):
        files = list(self._api.list_files_info(repo_id=self.repo_id, paths="file.md"))
        self.assertEqual(len(files), 1)
        file = files[0]

        self.assertEqual(file.rfilename, "file.md")
        self.assertIsNone(file.lfs)
        self.assertEqual(file.size, 4)
        self.assertEqual(file.blob_id, "6320cd248dd8aeaab759d5871f8781b5c0505172")

    def test_get_lfs_file_info(self):
        files = list(self._api.list_files_info(repo_id=self.repo_id, paths="lfs.bin"))
        self.assertEqual(len(files), 1)
        file = files[0]

        self.assertEqual(file.rfilename, "lfs.bin")
        self.assertEqual(
            file.lfs,
            {
                "size": 4,
                "sha256": "3a6eb0790f39ac87c94f3856b2dd2c5d110e6811602261a9a923d3bb23adc8b7",
                "pointer_size": 126,
            },
        )
        self.assertEqual(file.size, 4)
        self.assertEqual(file.blob_id, "0a828055346279420bd02a4221c177bbcdc045d8")

    def test_list_files(self):
        files = list(self._api.list_files_info(repo_id=self.repo_id, paths=["file.md", "lfs.bin", "2/file_2.md"]))
        self.assertEqual(len(files), 3)
        self.assertEqual({f.rfilename for f in files}, {"file.md", "lfs.bin", "2/file_2.md"})

    def test_list_files_and_folder(self):
        files = list(self._api.list_files_info(repo_id=self.repo_id, paths=["file.md", "lfs.bin", "2"]))
        self.assertEqual(len(files), 3)
        self.assertEqual({f.rfilename for f in files}, {"file.md", "lfs.bin", "2/file_2.md"})

    def test_list_unknown_path_among_other(self):
        files = list(self._api.list_files_info(repo_id=self.repo_id, paths=["file.md", "unknown"]))
        self.assertEqual(len(files), 1)

    def test_list_unknown_path_alone(self):
        files = list(self._api.list_files_info(repo_id=self.repo_id, paths="unknown"))
        self.assertEqual(len(files), 0)

    def test_list_folder_flat(self):
        files = list(self._api.list_files_info(repo_id=self.repo_id, paths=["2"]))
        self.assertEqual(len(files), 1)
        self.assertEqual(files[0].rfilename, "2/file_2.md")

    def test_list_folder_recursively(self):
        files = list(self._api.list_files_info(repo_id=self.repo_id, paths=["1"]))
        self.assertEqual(len(files), 2)
        self.assertEqual({f.rfilename for f in files}, {"1/2/file_1_2.md", "1/file_1.md"})

    def test_list_repo_files_manually(self):
        files = list(self._api.list_files_info(repo_id=self.repo_id))
        self.assertEqual(len(files), 7)
        self.assertEqual(
            {f.rfilename for f in files},
            {".gitattributes", "1/2/file_1_2.md", "1/file_1.md", "2/file_2.md", "3/file_3.md", "file.md", "lfs.bin"},
        )

    def test_list_repo_files_alias(self):
        self.assertEqual(
            set(self._api.list_repo_files(repo_id=self.repo_id)),
            {".gitattributes", "1/2/file_1_2.md", "1/file_1.md", "2/file_2.md", "3/file_3.md", "file.md", "lfs.bin"},
        )

    def test_list_with_root_path_is_ignored(self):
        # must use `paths=None`
        files = list(self._api.list_files_info(repo_id=self.repo_id, paths="/"))
        self.assertEqual(len(files), 0)

    def test_list_with_empty_path_is_invalid(self):
        # must use `paths=None`
        with self.assertRaises(BadRequestError):
            list(self._api.list_files_info(repo_id=self.repo_id, paths=""))

    @with_production_testing
    def test_list_files_with_expand(self):
        files = list(
            HfApi().list_files_info(
                repo_id="prompthero/openjourney-v4",
                expand=True,
                revision="c9211c53404dd6f4cfac5f04f33535892260668e",
            )
        )
        self.assertEqual(len(files), 22)

        # check lastCommit and security are present
        vae_model = next(file for file in files if file.rfilename == "vae/diffusion_pytorch_model.bin")
        self.assertIsNotNone(vae_model.lastCommit)
        self.assertEqual(vae_model.lastCommit["id"], "47b62b20b20e06b9de610e840282b7e6c3d51190")
        self.assertIsNotNone(vae_model.security)
        self.assertTrue(vae_model.security["safe"])
        self.assertTrue(isinstance(vae_model.security["avScan"], dict))  # all details in here

    @with_production_testing
    def test_list_files_without_expand(self):
        files = list(
            HfApi().list_files_info(
                repo_id="prompthero/openjourney-v4",
                expand=False,
                revision="c9211c53404dd6f4cfac5f04f33535892260668e",
            )
        )
        self.assertEqual(len(files), 22)

        # check lastCommit and security are missing
        vae_model = next(file for file in files if file.rfilename == "vae/diffusion_pytorch_model.bin")
        self.assertFalse(hasattr(vae_model, "lastCommit"))
        self.assertFalse(hasattr(vae_model, "security"))


class HfApiTagEndpointTest(HfApiCommonTest):
    @retry_endpoint
    @use_tmp_repo("model")
    def test_create_tag_on_main(self, repo_url: RepoUrl) -> None:
        """Check `create_tag` on default main branch works."""
        self._api.create_tag(repo_url.repo_id, tag="v0", tag_message="This is a tag message.")

        # Check tag  is on `main`
        tag_info = self._api.model_info(repo_url.repo_id, revision="v0")
        main_info = self._api.model_info(repo_url.repo_id, revision="main")
        self.assertEqual(tag_info.sha, main_info.sha)

    @retry_endpoint
    @use_tmp_repo("model")
    def test_create_tag_on_pr(self, repo_url: RepoUrl) -> None:
        """Check `create_tag` on a PR ref works."""
        # Create a PR with a readme
        commit_info: CommitInfo = self._api.create_commit(
            repo_id=repo_url.repo_id,
            create_pr=True,
            commit_message="upload readme",
            operations=[CommitOperationAdd(path_or_fileobj=b"this is a file content", path_in_repo="readme.md")],
        )

        # Tag the PR
        self._api.create_tag(repo_url.repo_id, tag="v0", revision=commit_info.pr_revision)

        # Check tag  is on `refs/pr/1`
        tag_info = self._api.model_info(repo_url.repo_id, revision="v0")
        pr_info = self._api.model_info(repo_url.repo_id, revision=commit_info.pr_revision)
        main_info = self._api.model_info(repo_url.repo_id)

        self.assertEqual(tag_info.sha, pr_info.sha)
        self.assertNotEqual(tag_info.sha, main_info.sha)

    @retry_endpoint
    @use_tmp_repo("dataset")
    def test_create_tag_on_commit_oid(self, repo_url: RepoUrl) -> None:
        """Check `create_tag` on specific commit oid works (both long and shorthands).

        Test it on a `dataset` repo.
        """
        # Create a PR with a readme
        commit_info_1: CommitInfo = self._api.create_commit(
            repo_id=repo_url.repo_id,
            repo_type="dataset",
            commit_message="upload readme",
            operations=[CommitOperationAdd(path_or_fileobj=b"this is a file content", path_in_repo="readme.md")],
        )
        commit_info_2: CommitInfo = self._api.create_commit(
            repo_id=repo_url.repo_id,
            repo_type="dataset",
            commit_message="upload config",
            operations=[CommitOperationAdd(path_or_fileobj=b"{'hello': 'world'}", path_in_repo="config.json")],
        )

        # Tag commits
        self._api.create_tag(
            repo_url.repo_id,
            tag="commit_1",
            repo_type="dataset",
            revision=commit_info_1.oid,  # long version
        )
        self._api.create_tag(
            repo_url.repo_id,
            tag="commit_2",
            repo_type="dataset",
            revision=commit_info_2.oid[:7],  # use shorthand !
        )

        # Check tags
        tag_1_info = self._api.dataset_info(repo_url.repo_id, revision="commit_1")
        tag_2_info = self._api.dataset_info(repo_url.repo_id, revision="commit_2")

        self.assertEqual(tag_1_info.sha, commit_info_1.oid)
        self.assertEqual(tag_2_info.sha, commit_info_2.oid)

    @retry_endpoint
    @use_tmp_repo("model")
    def test_invalid_tag_name(self, repo_url: RepoUrl) -> None:
        """Check `create_tag` with an invalid tag name."""
        with self.assertRaises(HTTPError):
            self._api.create_tag(repo_url.repo_id, tag="invalid tag")

    @retry_endpoint
    @use_tmp_repo("model")
    def test_create_tag_on_missing_revision(self, repo_url: RepoUrl) -> None:
        """Check `create_tag` on a missing revision."""
        with self.assertRaises(RevisionNotFoundError):
            self._api.create_tag(repo_url.repo_id, tag="invalid tag", revision="foobar")

    @retry_endpoint
    @use_tmp_repo("model")
    def test_create_tag_twice(self, repo_url: RepoUrl) -> None:
        """Check `create_tag` called twice on same tag should fail with HTTP 409."""
        self._api.create_tag(repo_url.repo_id, tag="tag_1")
        with self.assertRaises(HfHubHTTPError) as err:
            self._api.create_tag(repo_url.repo_id, tag="tag_1")
        self.assertEqual(err.exception.response.status_code, 409)

        # exist_ok=True => doesn't fail
        self._api.create_tag(repo_url.repo_id, tag="tag_1", exist_ok=True)

    @retry_endpoint
    @use_tmp_repo("model")
    def test_create_and_delete_tag(self, repo_url: RepoUrl) -> None:
        """Check `delete_tag` deletes the tag."""
        self._api.create_tag(repo_url.repo_id, tag="v0")
        self._api.model_info(repo_url.repo_id, revision="v0")

        self._api.delete_tag(repo_url.repo_id, tag="v0")
        with self.assertRaises(RevisionNotFoundError):
            self._api.model_info(repo_url.repo_id, revision="v0")

    @retry_endpoint
    @use_tmp_repo("model")
    def test_delete_tag_missing_tag(self, repo_url: RepoUrl) -> None:
        """Check cannot `delete_tag` if tag doesn't exist."""
        with self.assertRaises(RevisionNotFoundError):
            self._api.delete_tag(repo_url.repo_id, tag="v0")

    @retry_endpoint
    @use_tmp_repo("model")
    def test_delete_tag_with_branch_name(self, repo_url: RepoUrl) -> None:
        """Try to `delete_tag` if tag is a branch name.

        Currently getting a HTTP 500.
        See https://github.com/huggingface/moon-landing/issues/4223.
        """
        with self.assertRaises(HfHubHTTPError):
            self._api.delete_tag(repo_url.repo_id, tag="main")


class HfApiBranchEndpointTest(HfApiCommonTest):
    @retry_endpoint
    @use_tmp_repo()
    def test_create_and_delete_branch(self, repo_url: RepoUrl) -> None:
        """Test `create_branch` from main branch."""
        self._api.create_branch(repo_url.repo_id, branch="cool-branch")

        # Check `cool-branch` branch exists
        self._api.model_info(repo_url.repo_id, revision="cool-branch")

        # Delete it
        self._api.delete_branch(repo_url.repo_id, branch="cool-branch")

        # Check doesn't exist anymore
        with self.assertRaises(RevisionNotFoundError):
            self._api.model_info(repo_url.repo_id, revision="cool-branch")

    @retry_endpoint
    @use_tmp_repo()
    def test_create_branch_existing_branch_fails(self, repo_url: RepoUrl) -> None:
        """Test `create_branch` on existing branch."""
        self._api.create_branch(repo_url.repo_id, branch="cool-branch")

        with self.assertRaisesRegex(HfHubHTTPError, "Reference already exists"):
            self._api.create_branch(repo_url.repo_id, branch="cool-branch")

        with self.assertRaisesRegex(HfHubHTTPError, "Reference already exists"):
            self._api.create_branch(repo_url.repo_id, branch="main")

        # exist_ok=True => doesn't fail
        self._api.create_branch(repo_url.repo_id, branch="cool-branch", exist_ok=True)
        self._api.create_branch(repo_url.repo_id, branch="main", exist_ok=True)

    @retry_endpoint
    @use_tmp_repo()
    def test_create_branch_existing_tag_does_not_fail(self, repo_url: RepoUrl) -> None:
        """Test `create_branch` on existing tag."""
        self._api.create_tag(repo_url.repo_id, tag="tag")
        self._api.create_branch(repo_url.repo_id, branch="tag")

    @unittest.skip(
        "Test user is flagged as isHF which gives permissions to create invalid references."
        "Not relevant to test it anyway (i.e. it's more a server-side test)."
    )
    @retry_endpoint
    @use_tmp_repo()
    def test_create_branch_forbidden_ref_branch_fails(self, repo_url: RepoUrl) -> None:
        """Test `create_branch` on forbidden ref branch."""
        with self.assertRaisesRegex(BadRequestError, "Invalid reference for a branch"):
            self._api.create_branch(repo_url.repo_id, branch="refs/pr/5")

        with self.assertRaisesRegex(BadRequestError, "Invalid reference for a branch"):
            self._api.create_branch(repo_url.repo_id, branch="refs/something/random")

    @retry_endpoint
    @use_tmp_repo()
    def test_delete_branch_on_protected_branch_fails(self, repo_url: RepoUrl) -> None:
        """Test `delete_branch` fails on protected branch."""
        with self.assertRaisesRegex(HfHubHTTPError, "Cannot delete refs/heads/main"):
            self._api.delete_branch(repo_url.repo_id, branch="main")

    @retry_endpoint
    @use_tmp_repo()
    def test_delete_branch_on_missing_branch_fails(self, repo_url: RepoUrl) -> None:
        """Test `delete_branch` fails on missing branch."""
        with self.assertRaisesRegex(HfHubHTTPError, "Reference does not exist"):
            self._api.delete_branch(repo_url.repo_id, branch="cool-branch")

        # Using a tag instead of branch -> fails
        self._api.create_tag(repo_url.repo_id, tag="cool-tag")
        with self.assertRaisesRegex(HfHubHTTPError, "Reference does not exist"):
            self._api.delete_branch(repo_url.repo_id, branch="cool-tag")

    @retry_endpoint
    @use_tmp_repo()
    def test_create_branch_from_revision(self, repo_url: RepoUrl) -> None:
        """Test `create_branch` from a different revision than main HEAD."""
        # Create commit and remember initial/latest commit
        initial_commit = self._api.model_info(repo_url.repo_id).sha
        self._api.create_commit(
            repo_url.repo_id,
            operations=[CommitOperationAdd(path_in_repo="app.py", path_or_fileobj=b"content")],
            commit_message="test commit",
        )
        latest_commit = self._api.model_info(repo_url.repo_id).sha

        # Create branches
        self._api.create_branch(repo_url.repo_id, branch="from-head")
        self._api.create_branch(repo_url.repo_id, branch="from-initial", revision=initial_commit)
        self._api.create_branch(repo_url.repo_id, branch="from-branch", revision="from-initial")

        # Checks branches start from expected commits
        self.assertEqual(
            {
                "main": latest_commit,
                "from-head": latest_commit,
                "from-initial": initial_commit,
                "from-branch": initial_commit,
            },
            {ref.name: ref.target_commit for ref in self._api.list_repo_refs(repo_id=repo_url.repo_id).branches},
        )


class HfApiPublicStagingTest(unittest.TestCase):
    def setUp(self) -> None:
        self._api = HfApi()

    def test_staging_list_datasets(self):
        self._api.list_datasets()

    def test_staging_list_models(self):
        self._api.list_models()

    def test_staging_list_metrics(self):
        self._api.list_metrics()


class HfApiPublicProductionTest(unittest.TestCase):
    @with_production_testing
    def setUp(self) -> None:
        self._api = HfApi()

    def test_list_models(self):
        models = list(self._api.list_models(limit=500))
        self.assertGreater(len(models), 100)
        self.assertIsInstance(models[0], ModelInfo)

    def test_list_models_author(self):
        models = list(self._api.list_models(author="google"))
        self.assertGreater(len(models), 10)
        self.assertIsInstance(models[0], ModelInfo)
        for model in models:
            self.assertTrue(model.modelId.startswith("google/"))

    def test_list_models_search(self):
        models = list(self._api.list_models(search="bert"))
        self.assertGreater(len(models), 10)
        self.assertIsInstance(models[0], ModelInfo)
        for model in models[:10]:
            # Rough rule: at least first 10 will have "bert" in the name
            # Not optimal since it is dependent on how the Hub implements the search
            # (and changes it in the future) but for now it should do the trick.
            self.assertTrue("bert" in model.modelId.lower())

    def test_list_models_complex_query(self):
        # Let's list the 10 most recent models
        # with tags "bert" and "jax",
        # ordered by last modified date.
        models = list(self._api.list_models(filter=("bert", "jax"), sort="lastModified", direction=-1, limit=10))
        # we have at least 1 models
        self.assertGreater(len(models), 1)
        self.assertLessEqual(len(models), 10)
        model = models[0]
        self.assertIsInstance(model, ModelInfo)
        self.assertTrue(all(tag in model.tags for tag in ["bert", "jax"]))

    def test_list_models_with_config(self):
        for model in self._api.list_models(filter="adapter-transformers", fetch_config=True, limit=20):
            self.assertIsNotNone(model.config)

    def test_list_models_without_config(self):
        for model in self._api.list_models(filter="adapter-transformers", fetch_config=False, limit=20):
            self.assertIsNone(model.config)

    def test_model_info(self):
        model = self._api.model_info(repo_id=DUMMY_MODEL_ID)
        self.assertIsInstance(model, ModelInfo)
        self.assertNotEqual(model.sha, DUMMY_MODEL_ID_REVISION_ONE_SPECIFIC_COMMIT)
        # One particular commit (not the top of `main`)
        model = self._api.model_info(repo_id=DUMMY_MODEL_ID, revision=DUMMY_MODEL_ID_REVISION_ONE_SPECIFIC_COMMIT)
        self.assertIsInstance(model, ModelInfo)
        self.assertEqual(model.sha, DUMMY_MODEL_ID_REVISION_ONE_SPECIFIC_COMMIT)

    # TODO; un-skip this test once it's fixed.
    @unittest.skip(
        "Security status is currently unreliable on the server endpoint, so this"
        " test occasionally fails. Issue is tracked in"
        " https://github.com/huggingface/huggingface_hub/issues/1002 and"
        " https://github.com/huggingface/moon-landing/issues/3695. TODO: un-skip"
        " this test once it's fixed."
    )
    def test_model_info_with_security(self):
        model = self._api.model_info(
            repo_id=DUMMY_MODEL_ID,
            revision=DUMMY_MODEL_ID_REVISION_ONE_SPECIFIC_COMMIT,
            securityStatus=True,
        )
        self.assertEqual(model.securityStatus, {"containsInfected": False})

    def test_model_info_with_file_metadata(self):
        model = self._api.model_info(
            repo_id=DUMMY_MODEL_ID,
            revision=DUMMY_MODEL_ID_REVISION_ONE_SPECIFIC_COMMIT,
            files_metadata=True,
        )
        files = model.siblings
        assert files is not None
        self._check_siblings_metadata(files)

    def test_list_repo_files(self):
        files = self._api.list_repo_files(repo_id=DUMMY_MODEL_ID)
        expected_files = [
            ".gitattributes",
            "README.md",
            "config.json",
            "flax_model.msgpack",
            "merges.txt",
            "pytorch_model.bin",
            "tf_model.h5",
            "vocab.json",
        ]
        self.assertListEqual(files, expected_files)

    def test_list_datasets_no_filter(self):
        datasets = list(self._api.list_datasets(limit=500))
        self.assertGreater(len(datasets), 100)
        self.assertIsInstance(datasets[0], DatasetInfo)

    def test_filter_datasets_by_author_and_name(self):
        f = DatasetFilter(author="huggingface", dataset_name="DataMeasurementsFiles")
        datasets = list(self._api.list_datasets(filter=f))
        self.assertEqual(len(datasets), 1)
        self.assertTrue("huggingface" in datasets[0].author)
        self.assertTrue("DataMeasurementsFiles" in datasets[0].id)

    def test_filter_datasets_by_benchmark(self):
        f = DatasetFilter(benchmark="raft")
        datasets = list(self._api.list_datasets(filter=f))
        self.assertGreater(len(datasets), 0)
        self.assertTrue("benchmark:raft" in datasets[0].tags)

    def test_filter_datasets_by_language_creator(self):
        f = DatasetFilter(language_creators="crowdsourced")
        datasets = list(self._api.list_datasets(filter=f))
        self.assertGreater(len(datasets), 0)
        self.assertTrue("language_creators:crowdsourced" in datasets[0].tags)

    def test_filter_datasets_by_language_only(self):
        datasets = list(self._api.list_datasets(filter=DatasetFilter(language="en"), limit=100))
        self.assertGreater(len(datasets), 0)
        self.assertTrue("language:en" in datasets[0].tags)

        datasets = list(self._api.list_datasets(filter=DatasetFilter(language=("en", "fr")), limit=100))
        self.assertGreater(len(datasets), 0)
        self.assertTrue("language:en" in datasets[0].tags)
        self.assertTrue("language:fr" in datasets[0].tags)

    def test_filter_datasets_by_multilinguality(self):
        datasets = list(self._api.list_datasets(filter=DatasetFilter(multilinguality="multilingual")))
        self.assertGreater(len(datasets), 0)
        self.assertTrue("multilinguality:multilingual" in datasets[0].tags)

    def test_filter_datasets_by_size_categories(self):
        datasets = list(self._api.list_datasets(filter=DatasetFilter(size_categories="100K<n<1M")))
        self.assertGreater(len(datasets), 0)
        self.assertTrue("size_categories:100K<n<1M" in datasets[0].tags)

    def test_filter_datasets_by_task_categories(self):
        datasets = list(self._api.list_datasets(filter=DatasetFilter(task_categories="audio-classification")))
        self.assertGreater(len(datasets), 0)
        self.assertTrue("task_categories:audio-classification" in datasets[0].tags)

    def test_filter_datasets_by_task_ids(self):
        datasets = list(self._api.list_datasets(filter=DatasetFilter(task_ids="natural-language-inference")))
        self.assertGreater(len(datasets), 0)
        self.assertTrue("task_ids:natural-language-inference" in datasets[0].tags)

    def test_list_datasets_full(self):
        datasets = list(self._api.list_datasets(full=True, limit=500))
        self.assertGreater(len(datasets), 100)
        dataset = datasets[0]
        self.assertIsInstance(dataset, DatasetInfo)
        self.assertTrue(any(dataset.cardData for dataset in datasets))

    def test_list_datasets_author(self):
        datasets = list(self._api.list_datasets(author="huggingface"))
        self.assertGreater(len(datasets), 1)
        self.assertIsInstance(datasets[0], DatasetInfo)

    def test_list_datasets_search(self):
        datasets = list(self._api.list_datasets(search="wikipedia"))
        self.assertGreater(len(datasets), 10)
        self.assertIsInstance(datasets[0], DatasetInfo)

    def test_filter_datasets_with_cardData(self):
        datasets = list(self._api.list_datasets(full=True, limit=500))
        self.assertGreater(
            sum([getattr(dataset, "cardData", None) is not None for dataset in datasets]),
            0,
        )
        datasets = list(self._api.list_datasets(limit=500))
        self.assertTrue(all([getattr(dataset, "cardData", None) is None for dataset in datasets]))

    def test_dataset_info(self):
        dataset = self._api.dataset_info(repo_id=DUMMY_DATASET_ID)
        self.assertTrue(isinstance(dataset.cardData, dict) and len(dataset.cardData) > 0)
        self.assertTrue(isinstance(dataset.siblings, list) and len(dataset.siblings) > 0)
        self.assertIsInstance(dataset, DatasetInfo)
        self.assertNotEqual(dataset.sha, DUMMY_DATASET_ID_REVISION_ONE_SPECIFIC_COMMIT)
        dataset = self._api.dataset_info(
            repo_id=DUMMY_DATASET_ID,
            revision=DUMMY_DATASET_ID_REVISION_ONE_SPECIFIC_COMMIT,
        )
        self.assertIsInstance(dataset, DatasetInfo)
        self.assertEqual(dataset.sha, DUMMY_DATASET_ID_REVISION_ONE_SPECIFIC_COMMIT)

    def test_dataset_info_with_file_metadata(self):
        dataset = self._api.dataset_info(
            repo_id=SAMPLE_DATASET_IDENTIFIER,
            files_metadata=True,
        )
        files = dataset.siblings
        assert files is not None
        self._check_siblings_metadata(files)

    def _check_siblings_metadata(self, files: List[RepoFile]):
        """Check requested metadata has been received from the server."""
        at_least_one_lfs = False
        for file in files:
            self.assertTrue(isinstance(file.blob_id, str))
            self.assertTrue(isinstance(file.size, int))
            if file.lfs is not None:
                at_least_one_lfs = True
                self.assertTrue(isinstance(file.lfs, dict))
                self.assertTrue("sha256" in file.lfs)
        self.assertTrue(at_least_one_lfs)

    def test_list_metrics(self):
        metrics = self._api.list_metrics()
        self.assertGreater(len(metrics), 10)
        self.assertIsInstance(metrics[0], MetricInfo)
        self.assertTrue(any(metric.description for metric in metrics))

    def test_filter_models_by_author(self):
        models = list(self._api.list_models(filter=ModelFilter(author="muellerzr")))
        self.assertGreater(len(models), 0)
        self.assertTrue("muellerzr" in models[0].modelId)

    def test_filter_models_by_author_and_name(self):
        # Test we can search by an author and a name, but the model is not found
        models = list(self._api.list_models(filter=ModelFilter("facebook", model_name="bart-base")))
        self.assertTrue("facebook/bart-base" in models[0].modelId)

    def test_failing_filter_models_by_author_and_model_name(self):
        # Test we can search by an author and a name, but the model is not found
        models = list(self._api.list_models(filter=ModelFilter(author="muellerzr", model_name="testme")))
        self.assertEqual(len(models), 0)

    def test_filter_models_with_library(self):
        models = list(
            self._api.list_models(filter=ModelFilter("microsoft", model_name="wavlm-base-sd", library="tensorflow"))
        )
        self.assertEqual(len(models), 0)

        models = list(
            self._api.list_models(filter=ModelFilter("microsoft", model_name="wavlm-base-sd", library="pytorch"))
        )
        self.assertGreater(len(models), 0)

    def test_filter_models_with_task(self):
        models = list(self._api.list_models(filter=ModelFilter(task="fill-mask", model_name="albert-base-v2")))
        self.assertTrue("fill-mask" == models[0].pipeline_tag)
        self.assertTrue("albert-base-v2" in models[0].modelId)

        models = list(self._api.list_models(filter=ModelFilter(task="dummytask")))
        self.assertEqual(len(models), 0)

    def test_filter_models_by_language(self):
        res_fr = list(self._api.list_models(filter=ModelFilter(language="fr")))
        res_en = list(self._api.list_models(filter=ModelFilter(language="en")))
        self.assertGreater(len(res_en), len(res_fr))

    def test_filter_models_with_complex_query(self):
        args = ModelSearchArguments(api=self._api)
        f = ModelFilter(
            task=args.pipeline_tag.TextClassification,
            library=[args.library.PyTorch, args.library.TensorFlow],
        )
        models = list(self._api.list_models(filter=f))
        self.assertGreater(len(models), 1)
        self.assertTrue(
            ["text-classification" in model.pipeline_tag or "text-classification" in model.tags for model in models]
        )
        self.assertTrue(["pytorch" in model.tags and "tf" in model.tags for model in models])

    def test_filter_models_with_cardData(self):
        models = self._api.list_models(filter="co2_eq_emissions", cardData=True)
        self.assertTrue([hasattr(model, "cardData") for model in models])
        models = self._api.list_models(filter="co2_eq_emissions")
        self.assertTrue(all([not hasattr(model, "cardData") for model in models]))

    def test_filter_emissions_dict(self):
        # tests that dictionary is handled correctly as "emissions" and that
        # 17g is accepted and parsed correctly as a value
        # regression test for #753
        model = ModelInfo(cardData={"co2_eq_emissions": {"emissions": "17g"}})
        res = list(_filter_emissions([model], -1, 100))
        assert len(res) == 1

    def test_filter_emissions_with_max(self):
        models = self._api.list_models(emissions_thresholds=(None, 100), cardData=True, limit=1000)
        self.assertTrue(
            all(
                [
                    model.cardData["co2_eq_emissions"] <= 100
                    for model in models
                    if isinstance(model.cardData["co2_eq_emissions"], (float, int))
                ]
            )
        )

    def test_filter_emissions_with_min(self):
        models = self._api.list_models(emissions_thresholds=(5, None), cardData=True, limit=1000)
        self.assertTrue(
            all(
                [
                    model.cardData["co2_eq_emissions"] >= 5
                    for model in models
                    if isinstance(model.cardData["co2_eq_emissions"], (float, int))
                ]
            )
        )

    def test_filter_emissions_with_min_and_max(self):
        models = self._api.list_models(emissions_thresholds=(5, 100), cardData=True, limit=1000)
        self.assertTrue(
            all(
                [
                    model.cardData["co2_eq_emissions"] >= 5
                    for model in models
                    if isinstance(model.cardData["co2_eq_emissions"], (float, int))
                ]
            )
        )
        self.assertTrue(
            all(
                [
                    model.cardData["co2_eq_emissions"] <= 100
                    for model in models
                    if isinstance(model.cardData["co2_eq_emissions"], (float, int))
                ]
            )
        )

    def test_list_spaces_full(self):
        spaces = list(self._api.list_spaces(full=True, limit=500))
        self.assertGreater(len(spaces), 100)
        space = spaces[0]
        self.assertIsInstance(space, SpaceInfo)
        self.assertTrue(any(space.cardData for space in spaces))

    def test_list_spaces_author(self):
        spaces = list(self._api.list_spaces(author="evaluate-metric"))
        self.assertGreater(len(spaces), 10)
        self.assertTrue(
            set([space.id for space in spaces]).issuperset(
                set(["evaluate-metric/trec_eval", "evaluate-metric/perplexity"])
            )
        )

    def test_list_spaces_search(self):
        spaces = list(self._api.list_spaces(search="wikipedia"))
        space = spaces[0]
        self.assertTrue("wikipedia" in space.id.lower())

    def test_list_spaces_sort_and_direction(self):
        spaces_descending_likes = list(self._api.list_spaces(sort="likes", direction=-1))
        spaces_ascending_likes = list(self._api.list_spaces(sort="likes"))
        self.assertGreater(spaces_descending_likes[0].likes, spaces_descending_likes[1].likes)
        self.assertLess(spaces_ascending_likes[-2].likes, spaces_ascending_likes[-1].likes)

    def test_list_spaces_limit(self):
        spaces = list(self._api.list_spaces(limit=5))
        self.assertEqual(len(spaces), 5)

    def test_list_spaces_with_models(self):
        spaces = list(self._api.list_spaces(models="bert-base-uncased"))
        self.assertTrue("bert-base-uncased" in getattr(spaces[0], "models", []))

    def test_list_spaces_with_datasets(self):
        spaces = list(self._api.list_spaces(datasets="wikipedia"))
        self.assertTrue("wikipedia" in getattr(spaces[0], "datasets", []))

    def test_list_spaces_linked(self):
        spaces = list(self._api.list_spaces(linked=True))
        self.assertTrue(any((getattr(space, "models", None) is not None) for space in spaces))
        self.assertTrue(any((getattr(space, "datasets", None) is not None) for space in spaces))
        self.assertTrue(
            any((getattr(space, "models", None) is not None) and getattr(space, "datasets", None) is not None)
            for space in spaces
        )
        self.assertTrue(
            all((getattr(space, "models", None) is not None) or getattr(space, "datasets", None) is not None)
            for space in spaces
        )


class HfApiPrivateTest(HfApiCommonTest):
    @retry_endpoint
    def setUp(self) -> None:
        super().setUp()
        self.REPO_NAME = repo_name("private")
        self._api.create_repo(repo_id=self.REPO_NAME, private=True)
        self._api.create_repo(repo_id=self.REPO_NAME, private=True, repo_type="dataset")

    def tearDown(self) -> None:
        self._api.delete_repo(repo_id=self.REPO_NAME)
        self._api.delete_repo(repo_id=self.REPO_NAME, repo_type="dataset")

    def test_model_info(self):
        shutil.rmtree(os.path.dirname(HfFolder.path_token), ignore_errors=True)
        with patch.object(self._api, "token", None):  # no default token
            # Test we cannot access model info without a token
            with self.assertRaisesRegex(
                requests.exceptions.HTTPError,
                re.compile(
                    r"401 Client Error(.+)\(Request ID: .+\)(.*)Repository Not Found",
                    flags=re.DOTALL,
                ),
            ):
                _ = self._api.model_info(repo_id=f"{USER}/{self.REPO_NAME}")

            model_info = self._api.model_info(repo_id=f"{USER}/{self.REPO_NAME}", use_auth_token=self._token)
            self.assertIsInstance(model_info, ModelInfo)

    def test_dataset_info(self):
        shutil.rmtree(os.path.dirname(HfFolder.path_token), ignore_errors=True)
        with patch.object(self._api, "token", None):  # no default token
            # Test we cannot access model info without a token
            with self.assertRaisesRegex(
                requests.exceptions.HTTPError,
                re.compile(
                    r"401 Client Error(.+)\(Request ID: .+\)(.*)Repository Not Found",
                    flags=re.DOTALL,
                ),
            ):
                _ = self._api.dataset_info(repo_id=f"{USER}/{self.REPO_NAME}")

            dataset_info = self._api.dataset_info(repo_id=f"{USER}/{self.REPO_NAME}", use_auth_token=self._token)
            self.assertIsInstance(dataset_info, DatasetInfo)

    def test_list_private_datasets(self):
        orig = len(list(self._api.list_datasets(use_auth_token=False)))
        new = len(list(self._api.list_datasets(use_auth_token=self._token)))
        self.assertGreater(new, orig)

    def test_list_private_models(self):
        orig = len(list(self._api.list_models(use_auth_token=False)))
        new = len(list(self._api.list_models(use_auth_token=self._token)))
        self.assertGreater(new, orig)

    @with_production_testing
    def test_list_private_spaces(self):
        orig = len(list(self._api.list_spaces(use_auth_token=False)))
        new = len(list(self._api.list_spaces(use_auth_token=self._token)))
        self.assertGreaterEqual(new, orig)


@pytest.mark.usefixtures("fx_cache_dir")
class UploadFolderMockedTest(unittest.TestCase):
    api = HfApi()
    cache_dir: Path

    def setUp(self) -> None:
        (self.cache_dir / "file.txt").write_text("content")
        (self.cache_dir / "lfs.bin").write_text("content")

        (self.cache_dir / "sub").mkdir()
        (self.cache_dir / "sub" / "file.txt").write_text("content")
        (self.cache_dir / "sub" / "lfs_in_sub.bin").write_text("content")

        (self.cache_dir / "subdir").mkdir()
        (self.cache_dir / "subdir" / "file.txt").write_text("content")
        (self.cache_dir / "subdir" / "lfs_in_subdir.bin").write_text("content")

        self.all_local_files = {
            "lfs.bin",
            "file.txt",
            "sub/file.txt",
            "sub/lfs_in_sub.bin",
            "subdir/file.txt",
            "subdir/lfs_in_subdir.bin",
        }

        self.repo_files_mock = Mock()
        self.repo_files_mock.return_value = [  # all remote files
            ".gitattributes",
            "file.txt",
            "file1.txt",
            "sub/file.txt",
            "sub/file1.txt",
            "subdir/file.txt",
            "subdir/lfs_in_subdir.bin",
        ]
        self.api.list_repo_files = self.repo_files_mock

        self.create_commit_mock = Mock()
        self.create_commit_mock.return_value.pr_url = None
        self.api.create_commit = self.create_commit_mock

    def _upload_folder_alias(self, **kwargs) -> List[Union[CommitOperationAdd, CommitOperationDelete]]:
        """Alias to call `upload_folder` + retrieve the CommitOperation list passed to `create_commit`."""
        if "folder_path" not in kwargs:
            kwargs["folder_path"] = self.cache_dir
        self.api.upload_folder(repo_id="repo_id", **kwargs)
        return self.create_commit_mock.call_args_list[0][1]["operations"]

    def test_allow_everything(self):
        operations = self._upload_folder_alias()
        self.assertTrue(all(isinstance(op, CommitOperationAdd) for op in operations))
        self.assertEqual({op.path_in_repo for op in operations}, self.all_local_files)

    def test_allow_everything_in_subdir_no_trailing_slash(self):
        operations = self._upload_folder_alias(folder_path=self.cache_dir / "subdir", path_in_repo="subdir")
        self.assertTrue(all(isinstance(op, CommitOperationAdd) for op in operations))
        self.assertEqual(
            {op.path_in_repo for op in operations},
            {"subdir/file.txt", "subdir/lfs_in_subdir.bin"},  # correct `path_in_repo`
        )

    def test_allow_everything_in_subdir_with_trailing_slash(self):
        operations = self._upload_folder_alias(folder_path=self.cache_dir / "subdir", path_in_repo="subdir/")
        self.assertTrue(all(isinstance(op, CommitOperationAdd) for op in operations))
        self.assertEqual(
            {op.path_in_repo for op in operations},
            {"subdir/file.txt", "subdir/lfs_in_subdir.bin"},  # correct `path_in_repo`
        )

    def test_allow_txt_ignore_subdir(self):
        operations = self._upload_folder_alias(allow_patterns="*.txt", ignore_patterns="subdir/*")
        self.assertTrue(all(isinstance(op, CommitOperationAdd) for op in operations))
        self.assertEqual(
            {op.path_in_repo for op in operations},
            {"sub/file.txt", "file.txt"},  # only .txt files, not in subdir
        )

    def test_allow_txt_not_root_ignore_subdir(self):
        operations = self._upload_folder_alias(allow_patterns="**/*.txt", ignore_patterns="subdir/*")
        self.assertTrue(all(isinstance(op, CommitOperationAdd) for op in operations))
        self.assertEqual(
            {op.path_in_repo for op in operations},
            {"sub/file.txt"},  # only .txt files, not in subdir, not at root
        )

    def test_path_in_repo_dot(self):
        """Regression test for #1382 when using `path_in_repo="."`.

        Using `path_in_repo="."` or `path_in_repo=None` should be equivalent.
        See https://github.com/huggingface/huggingface_hub/pull/1382.
        """
        operation_with_dot = self._upload_folder_alias(path_in_repo=".", allow_patterns=["file.txt"])[0]
        operation_with_none = self._upload_folder_alias(path_in_repo=None, allow_patterns=["file.txt"])[0]
        self.assertEqual(operation_with_dot.path_in_repo, "file.txt")
        self.assertEqual(operation_with_none.path_in_repo, "file.txt")

    def test_delete_txt(self):
        operations = self._upload_folder_alias(delete_patterns="*.txt")
        added_files = {op.path_in_repo for op in operations if isinstance(op, CommitOperationAdd)}
        deleted_files = {op.path_in_repo for op in operations if isinstance(op, CommitOperationDelete)}

        self.assertEqual(added_files, self.all_local_files)
        self.assertEqual(deleted_files, {"file1.txt", "sub/file1.txt"})

        # since "file.txt" and "sub/file.txt" are overwritten, no need to delete them first
        self.assertIn("file.txt", added_files)
        self.assertIn("sub/file.txt", added_files)

    def test_delete_txt_in_sub(self):
        operations = self._upload_folder_alias(
            path_in_repo="sub/", folder_path=self.cache_dir / "sub", delete_patterns="*.txt"
        )
        added_files = {op.path_in_repo for op in operations if isinstance(op, CommitOperationAdd)}
        deleted_files = {op.path_in_repo for op in operations if isinstance(op, CommitOperationDelete)}

        self.assertEqual(added_files, {"sub/file.txt", "sub/lfs_in_sub.bin"})  # added only in sub/
        self.assertEqual(deleted_files, {"sub/file1.txt"})  # delete only in sub/

    def test_delete_txt_in_sub_ignore_sub_file_txt(self):
        operations = self._upload_folder_alias(
            path_in_repo="sub", folder_path=self.cache_dir / "sub", ignore_patterns="file.txt", delete_patterns="*.txt"
        )
        added_files = {op.path_in_repo for op in operations if isinstance(op, CommitOperationAdd)}
        deleted_files = {op.path_in_repo for op in operations if isinstance(op, CommitOperationDelete)}

        # since "sub/file.txt" should be deleted and is not overwritten (ignore_patterns), we delete it explicitly
        self.assertEqual(added_files, {"sub/lfs_in_sub.bin"})  # no "sub/file.txt"
        self.assertEqual(deleted_files, {"sub/file1.txt", "sub/file.txt"})


@pytest.mark.usefixtures("fx_cache_dir")
class HfLargefilesTest(HfApiCommonTest):
    cache_dir: Path

    def tearDown(self):
        self._api.delete_repo(repo_id=self.repo_id)

    def setup_local_clone(self) -> None:
        REMOTE_URL_AUTH = self.repo_url.replace(ENDPOINT_STAGING, ENDPOINT_STAGING_BASIC_AUTH)
        subprocess.run(
            ["git", "clone", REMOTE_URL_AUTH, str(self.cache_dir)],
            check=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )
        subprocess.run(["git", "lfs", "track", "*.pdf"], check=True, cwd=self.cache_dir)
        subprocess.run(["git", "lfs", "track", "*.epub"], check=True, cwd=self.cache_dir)

    @retry_endpoint
    @require_git_lfs
    def test_end_to_end_thresh_6M(self):
        # Little-hack: create repo with defined `_lfsmultipartthresh`. Only for tests purposes
        self._api._lfsmultipartthresh = 6 * 10**6
        self.repo_url = self._api.create_repo(repo_id=repo_name())
        self.repo_id = self.repo_url.repo_id
        self._api._lfsmultipartthresh = None
        self.setup_local_clone()

        subprocess.run(
            ["wget", LARGE_FILE_18MB], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.cache_dir
        )
        subprocess.run(["git", "add", "*"], check=True, cwd=self.cache_dir)
        subprocess.run(["git", "commit", "-m", "commit message"], check=True, cwd=self.cache_dir)

        # This will fail as we haven't set up our custom transfer agent yet.
        failed_process = subprocess.run(
            ["git", "push"],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=self.cache_dir,
        )
        self.assertEqual(failed_process.returncode, 1)
        self.assertIn("cli lfs-enable-largefiles", failed_process.stderr.decode())
        # ^ Instructions on how to fix this are included in the error message.
        subprocess.run(["huggingface-cli", "lfs-enable-largefiles", self.cache_dir], check=True)

        start_time = time.time()
        subprocess.run(["git", "push"], check=True, cwd=self.cache_dir)
        print("took", time.time() - start_time)

        # To be 100% sure, let's download the resolved file
        pdf_url = f"{self.repo_url}/resolve/main/progit.pdf"
        DEST_FILENAME = "uploaded.pdf"
        subprocess.run(
            ["wget", pdf_url, "-O", DEST_FILENAME],
            check=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=self.cache_dir,
        )
        dest_filesize = (self.cache_dir / DEST_FILENAME).stat().st_size
        self.assertEqual(dest_filesize, 18685041)

    @retry_endpoint
    @require_git_lfs
    def test_end_to_end_thresh_16M(self):
        # Here we'll push one multipart and one non-multipart file in the same commit, and see what happens
        # Little-hack: create repo with defined `_lfsmultipartthresh`. Only for tests purposes
        self._api._lfsmultipartthresh = 16 * 10**6
        self.repo_url = self._api.create_repo(repo_id=repo_name())
        self.repo_id = self.repo_url.repo_id
        self._api._lfsmultipartthresh = None
        self.setup_local_clone()

        subprocess.run(
            ["wget", LARGE_FILE_18MB], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.cache_dir
        )
        subprocess.run(
            ["wget", LARGE_FILE_14MB], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.cache_dir
        )
        subprocess.run(["git", "add", "*"], check=True, cwd=self.cache_dir)
        subprocess.run(["git", "commit", "-m", "both files in same commit"], check=True, cwd=self.cache_dir)
        subprocess.run(["huggingface-cli", "lfs-enable-largefiles", self.cache_dir], check=True)

        start_time = time.time()
        subprocess.run(["git", "push"], check=True, cwd=self.cache_dir)
        print("took", time.time() - start_time)

    def test_upload_lfs_file_multipart(self):
        """End to end test to check upload an LFS file using multipart upload works."""
        self._api._lfsmultipartthresh = 16 * 10**6
        self.repo_id = self._api.create_repo(repo_id=repo_name()).repo_id
        self._api._lfsmultipartthresh = None

        with patch.object(
            huggingface_hub.lfs,
            "_upload_parts_iteratively",
            wraps=huggingface_hub.lfs._upload_parts_iteratively,
        ) as mock:
            self._api.upload_file(repo_id=self.repo_id, path_or_fileobj=b"0" * 18 * 10**6, path_in_repo="lfs.bin")
            mock.assert_called_once()  # It used multipart upload


class ParseHFUrlTest(unittest.TestCase):
    def test_repo_type_and_id_from_hf_id_on_correct_values(self):
        possible_values = {
            "https://huggingface.co/id": [None, None, "id"],
            "https://huggingface.co/user/id": [None, "user", "id"],
            "https://huggingface.co/datasets/user/id": ["dataset", "user", "id"],
            "https://huggingface.co/spaces/user/id": ["space", "user", "id"],
            "user/id": [None, "user", "id"],
            "dataset/user/id": ["dataset", "user", "id"],
            "space/user/id": ["space", "user", "id"],
            "id": [None, None, "id"],
            "hf://id": [None, None, "id"],
            "hf://user/id": [None, "user", "id"],
            "hf://model/user/name": ["model", "user", "name"],  # 's' is optional
            "hf://models/user/name": ["model", "user", "name"],
        }

        for key, value in possible_values.items():
            self.assertEqual(
                repo_type_and_id_from_hf_id(key, hub_url="https://huggingface.co"),
                tuple(value),
            )

    def test_repo_type_and_id_from_hf_id_on_wrong_values(self):
        for hub_id in [
            "https://unknown-endpoint.co/id",
            "https://huggingface.co/datasets/user/id@revision",  # @ forbidden
            "datasets/user/id/subpath",
            "hffs://model/user/name",
            "spaeces/user/id",  # with typo in repo type
        ]:
            with self.assertRaises(ValueError):
                repo_type_and_id_from_hf_id(hub_id, hub_url="https://huggingface.co")


class HfApiDiscussionsTest(HfApiCommonTest):
    def setUp(self):
        self.repo_id = self._api.create_repo(repo_id=repo_name()).repo_id
        self.pull_request = self._api.create_discussion(
            repo_id=self.repo_id, pull_request=True, title="Test Pull Request"
        )
        self.discussion = self._api.create_discussion(
            repo_id=self.repo_id, pull_request=False, title="Test Discussion"
        )

    def tearDown(self):
        self._api.delete_repo(repo_id=self.repo_id)

    def test_create_discussion(self):
        discussion = self._api.create_discussion(repo_id=self.repo_id, title=" Test discussion !  ")
        self.assertEqual(discussion.num, 3)
        self.assertEqual(discussion.author, USER)
        self.assertEqual(discussion.is_pull_request, False)
        self.assertEqual(discussion.title, "Test discussion !")

    @use_tmp_repo("dataset")
    def test_create_discussion_space(self, repo_url: RepoUrl):
        """Regression test for #1463.

        Computed URL was malformed with `dataset` and `space` repo_types.
        See https://github.com/huggingface/huggingface_hub/issues/1463.
        """
        discussion = self._api.create_discussion(repo_id=repo_url.repo_id, repo_type="dataset", title="title")
        self.assertEqual(discussion.url, f"{repo_url}/discussions/1")

    def test_create_pull_request(self):
        discussion = self._api.create_discussion(repo_id=self.repo_id, title=" Test PR !  ", pull_request=True)
        self.assertEqual(discussion.num, 3)
        self.assertEqual(discussion.author, USER)
        self.assertEqual(discussion.is_pull_request, True)
        self.assertEqual(discussion.title, "Test PR !")

        model_info = self._api.repo_info(repo_id=self.repo_id, revision="refs/pr/1")
        self.assertIsInstance(model_info, ModelInfo)

    def test_get_repo_discussion(self):
        discussions_generator = self._api.get_repo_discussions(repo_id=self.repo_id)
        self.assertIsInstance(discussions_generator, types.GeneratorType)
        self.assertListEqual(
            list([d.num for d in discussions_generator]), [self.discussion.num, self.pull_request.num]
        )

    def test_get_discussion_details(self):
        retrieved = self._api.get_discussion_details(repo_id=self.repo_id, discussion_num=2)
        self.assertEqual(retrieved, self.discussion)

    def test_edit_discussion_comment(self):
        def get_first_comment(discussion: DiscussionWithDetails) -> DiscussionComment:
            return [evt for evt in discussion.events if evt.type == "comment"][0]

        edited_comment = self._api.edit_discussion_comment(
            repo_id=self.repo_id,
            discussion_num=self.pull_request.num,
            comment_id=get_first_comment(self.pull_request).id,
            new_content="**Edited** comment 🤗",
        )
        retrieved = self._api.get_discussion_details(repo_id=self.repo_id, discussion_num=self.pull_request.num)
        self.assertEqual(get_first_comment(retrieved).edited, True)
        self.assertEqual(get_first_comment(retrieved).id, get_first_comment(self.pull_request).id)
        self.assertEqual(get_first_comment(retrieved).content, "**Edited** comment 🤗")

        self.assertEqual(get_first_comment(retrieved), edited_comment)

    def test_comment_discussion(self):
        new_comment = self._api.comment_discussion(
            repo_id=self.repo_id,
            discussion_num=self.discussion.num,
            comment="""\
                # Multi-line comment

                **With formatting**, including *italic text* & ~strike through~
                And even [links](http://hf.co)! 💥🤯
            """,
        )
        retrieved = self._api.get_discussion_details(repo_id=self.repo_id, discussion_num=self.discussion.num)
        self.assertEqual(len(retrieved.events), 2)
        self.assertIn(new_comment.id, {event.id for event in retrieved.events})

    def test_rename_discussion(self):
        rename_event = self._api.rename_discussion(
            repo_id=self.repo_id, discussion_num=self.discussion.num, new_title="New title2"
        )
        retrieved = self._api.get_discussion_details(repo_id=self.repo_id, discussion_num=self.discussion.num)
        self.assertIn(rename_event.id, (event.id for event in retrieved.events))
        self.assertEqual(rename_event.old_title, self.discussion.title)
        self.assertEqual(rename_event.new_title, "New title2")

    def test_change_discussion_status(self):
        status_change_event = self._api.change_discussion_status(
            repo_id=self.repo_id, discussion_num=self.discussion.num, new_status="closed"
        )
        retrieved = self._api.get_discussion_details(repo_id=self.repo_id, discussion_num=self.discussion.num)
        self.assertIn(status_change_event.id, (event.id for event in retrieved.events))
        self.assertEqual(status_change_event.new_status, "closed")

        with self.assertRaises(ValueError):
            self._api.change_discussion_status(
                repo_id=self.repo_id, discussion_num=self.discussion.num, new_status="published"
            )

    def test_merge_pull_request(self):
        self._api.create_commit(
            repo_id=self.repo_id,
            commit_message="Commit some file",
            operations=[CommitOperationAdd(path_in_repo="file.test", path_or_fileobj=b"Content")],
            revision=self.pull_request.git_reference,
        )
        self._api.change_discussion_status(
            repo_id=self.repo_id, discussion_num=self.pull_request.num, new_status="open"
        )
        self._api.merge_pull_request(self.repo_id, self.pull_request.num)

        retrieved = self._api.get_discussion_details(repo_id=self.repo_id, discussion_num=self.pull_request.num)
        self.assertEqual(retrieved.status, "merged")
        self.assertIsNotNone(retrieved.merge_commit_oid)


class ActivityApiTest(unittest.TestCase):
    @classmethod
    def setUpClass(cls) -> None:
        cls.api = HfApi()  # no auth!

    def test_like_and_unlike_repo(self) -> None:
        # Create and like a private and a public repo
        repo_id_private = self.api.create_repo(repo_name(), token=TOKEN, private=True).repo_id
        self.api.like(repo_id_private, token=TOKEN)

        repo_id_public = self.api.create_repo(repo_name(), token=TOKEN, private=False).repo_id
        self.api.like(repo_id_public, token=TOKEN)

        # Get likes as public and authenticated
        likes = self.api.list_liked_repos(USER)
        likes_with_auth = self.api.list_liked_repos(USER, token=TOKEN)

        # Public repo is shown in liked repos
        self.assertIn(repo_id_public, likes.models)
        self.assertIn(repo_id_public, likes_with_auth.models)

        # Private repo is NOT shown in liked repos, even when authenticated
        # This is by design. See https://github.com/huggingface/moon-landing/pull/4879 (internal link)
        self.assertNotIn(repo_id_private, likes.models)
        self.assertNotIn(repo_id_private, likes_with_auth.models)

        # Unlike repo and check not in liked list
        self.api.unlike(repo_id_public, token=TOKEN)
        self.api.unlike(repo_id_private, token=TOKEN)
        likes_after_unlike = self.api.list_liked_repos(USER)
        self.assertNotIn(repo_id_public, likes_after_unlike.models)  # Unliked

        # Cleanup
        self.api.delete_repo(repo_id_public, token=TOKEN)
        self.api.delete_repo(repo_id_private, token=TOKEN)

    def test_like_missing_repo(self) -> None:
        with self.assertRaises(RepositoryNotFoundError):
            self.api.like("missing_repo_id", token=TOKEN)

        with self.assertRaises(RepositoryNotFoundError):
            self.api.unlike("missing_repo_id", token=TOKEN)

    def test_like_twice(self) -> None:
        # Create and like repo
        repo_id = self.api.create_repo(repo_name(), token=TOKEN, private=True).repo_id

        # Can like twice
        self.api.like(repo_id, token=TOKEN)
        self.api.like(repo_id, token=TOKEN)

        # Can unlike twice
        self.api.unlike(repo_id, token=TOKEN)
        self.api.unlike(repo_id, token=TOKEN)

        # Cleanup
        self.api.delete_repo(repo_id, token=TOKEN)

    def test_list_liked_repos_no_auth(self) -> None:
        # Create a list 1 liked repo
        liked_repo_name = "repo-that-is-liked-public"
        repo_url = self.api.create_repo(liked_repo_name, exist_ok=True, token=TOKEN)
        self.api.like(repo_url.repo_id, token=TOKEN)

        # Fetch liked repos without auth
        likes = self.api.list_liked_repos(USER)
        self.assertEqual(likes.user, USER)
        self.assertGreater(len(likes.models) + len(likes.datasets) + len(likes.spaces), 0)
        self.assertIn(repo_url.repo_id, likes.models)

    def test_list_likes_repos_auth_and_implicit_user(self) -> None:
        # User is implicit
        likes = self.api.list_liked_repos(token=TOKEN)
        self.assertEqual(likes.user, USER)

    def test_list_likes_repos_auth_and_explicit_user(self) -> None:
        # User is explicit even if auth
        likes = self.api.list_liked_repos(user="__DUMMY_DATASETS_SERVER_USER__", token=TOKEN)
        self.assertEqual(likes.user, "__DUMMY_DATASETS_SERVER_USER__")

    @with_production_testing
    def test_list_likes_on_production(self) -> None:
        # Test julien-c likes a lot of repos !
        likes = HfApi().list_liked_repos("julien-c")
        self.assertEqual(len(likes.models) + len(likes.datasets) + len(likes.spaces), likes.total)
        self.assertGreater(len(likes.models), 0)
        self.assertGreater(len(likes.datasets), 0)
        self.assertGreater(len(likes.spaces), 0)


@pytest.mark.usefixtures("fx_production_space")
class TestSpaceAPIProduction(unittest.TestCase):
    """
    Testing Space API is not possible on staging. Tests are run against production
    server using a token stored under `HUGGINGFACE_PRODUCTION_USER_TOKEN` environment
    variable. Tests requiring hardware are mocked to spare some resources.
    """

    repo_id: str
    api: HfApi

    def test_manage_secrets(self) -> None:
        # Add 3 secrets
        self.api.add_space_secret(self.repo_id, "foo", "123")
        self.api.add_space_secret(self.repo_id, "token", "hf_api_123456")
        self.api.add_space_secret(self.repo_id, "gh_api_key", "******")

        # Update secret
        self.api.add_space_secret(self.repo_id, "foo", "456")

        # Delete secret
        self.api.delete_space_secret(self.repo_id, "gh_api_key")

        # Doesn't fail on missing key
        self.api.delete_space_secret(self.repo_id, "missing_key")

    def test_space_runtime(self) -> None:
        runtime = self.api.get_space_runtime(self.repo_id)

        # Space has just been created: hardware might not be set yet.
        self.assertIn(runtime.hardware, (None, SpaceHardware.CPU_BASIC))
        self.assertIn(runtime.requested_hardware, (None, SpaceHardware.CPU_BASIC))

        # Space is either "BUILDING" (if not yet done) or "NO_APP_FILE" (if building failed)
        self.assertIn(runtime.stage, (SpaceStage.NO_APP_FILE, SpaceStage.BUILDING))
        self.assertIn(runtime.stage, ("NO_APP_FILE", "BUILDING"))  # str works as well

        # Raw response from Hub
        self.assertIsInstance(runtime.raw, dict)

    def test_pause_and_restart_space(self) -> None:
        # Upload a fake app.py file
        self.api.upload_file(path_or_fileobj=b"", path_in_repo="app.py", repo_id=self.repo_id, repo_type="space")

        # Wait for the Space to be "BUILDING"
        count = 0
        while True:
            if self.api.get_space_runtime(self.repo_id).stage == SpaceStage.BUILDING:
                break
            time.sleep(1.0)
            count += 1
            if count > 10:
                raise Exception("Space is not building after 10 seconds.")

        # Pause it
        runtime_after_pause = self.api.pause_space(self.repo_id)
        self.assertEqual(runtime_after_pause.stage, SpaceStage.PAUSED)

        # Restart
        self.api.restart_space(self.repo_id)
        time.sleep(1.0)
        runtime_after_restart = self.api.get_space_runtime(self.repo_id)
        self.assertIn(runtime_after_restart.stage, (SpaceStage.BUILDING, SpaceStage.RUNNING_BUILDING))


@pytest.mark.usefixtures("fx_cache_dir")
class TestCommitInBackground(HfApiCommonTest):
    cache_dir: Path

    @use_tmp_repo()
    def test_commit_to_repo_in_background(self, repo_url: RepoUrl) -> None:
        repo_id = repo_url.repo_id
        (self.cache_dir / "file.txt").write_text("content")
        (self.cache_dir / "lfs.bin").write_text("content")

        t0 = time.time()
        upload_future_1 = self._api.upload_file(
            path_or_fileobj=b"1", path_in_repo="1.txt", repo_id=repo_id, commit_message="Upload 1", run_as_future=True
        )
        upload_future_2 = self._api.upload_file(
            path_or_fileobj=b"2", path_in_repo="2.txt", repo_id=repo_id, commit_message="Upload 2", run_as_future=True
        )
        upload_future_3 = self._api.upload_folder(
            repo_id=repo_id, folder_path=self.cache_dir, commit_message="Upload folder", run_as_future=True
        )
        t1 = time.time()

        # all futures are queued instantly
        self.assertLessEqual(t1 - t0, 0.01)

        # wait for the last job to complete
        upload_future_3.result()

        # all of them are now complete (ran in order)
        self.assertTrue(upload_future_1.done())
        self.assertTrue(upload_future_2.done())
        self.assertTrue(upload_future_3.done())

        # 4 commits, sorted in reverse order of creation
        commits = self._api.list_repo_commits(repo_id=repo_id)
        self.assertEqual(len(commits), 4)
        self.assertEqual(commits[0].title, "Upload folder")
        self.assertEqual(commits[1].title, "Upload 2")
        self.assertEqual(commits[2].title, "Upload 1")
        self.assertEqual(commits[3].title, "initial commit")

    @use_tmp_repo()
    def test_run_as_future(self, repo_url: RepoUrl) -> None:
        repo_id = repo_url.repo_id
        self._api.run_as_future(self._api.like, repo_id)
        future_1 = self._api.run_as_future(self._api.model_info, repo_id=repo_id)
        self._api.run_as_future(self._api.unlike, repo_id)
        future_2 = self._api.run_as_future(self._api.model_info, repo_id=repo_id)

        self.assertIsInstance(future_1, Future)
        self.assertIsInstance(future_2, Future)

        # Wait for first info future
        info_1 = future_1.result()
        self.assertFalse(future_2.done())

        # Wait for second info future
        info_2 = future_2.result()
        self.assertTrue(future_2.done())

        # Like/unlike is correct
        self.assertEqual(info_1.likes, 1)
        self.assertEqual(info_2.likes, 0)


class TestSpaceAPIMocked(unittest.TestCase):
    """
    Testing Space hardware requests is resource intensive for the server (need to spawn
    GPUs). Tests are mocked to check the correct values are sent.
    """

    def setUp(self) -> None:
        self.api = HfApi(token="fake_token")
        self.repo_id = "fake_repo_id"

        get_session_mock = Mock()
        self.post_mock = get_session_mock().post
        self.post_mock.return_value.json.return_value = {
            "url": f"{self.api.endpoint}/spaces/user/repo_id",
            "stage": "RUNNING",
            "sdk": "gradio",
            "sdkVersion": "3.17.0",
            "hardware": {
                "current": "t4-medium",
                "requested": "t4-medium",
            },
            "gcTimeout": None,
        }
        self.patcher = patch("huggingface_hub.hf_api.get_session", get_session_mock)
        self.patcher.start()

    def tearDown(self) -> None:
        self.patcher.stop()

    def test_create_space_with_hardware(self) -> None:
        self.api.create_repo(
            self.repo_id,
            repo_type="space",
            space_sdk="gradio",
            space_hardware=SpaceHardware.T4_MEDIUM,
        )
        self.post_mock.assert_called_once_with(
            f"{self.api.endpoint}/api/repos/create",
            headers=self.api._build_hf_headers(),
            json={
                "name": self.repo_id,
                "organization": None,
                "private": False,
                "type": "space",
                "sdk": "gradio",
                "hardware": "t4-medium",
            },
        )

    def test_request_space_hardware_no_sleep_time(self) -> None:
        self.api.request_space_hardware(self.repo_id, SpaceHardware.T4_MEDIUM)
        self.post_mock.assert_called_once_with(
            f"{self.api.endpoint}/api/spaces/{self.repo_id}/hardware",
            headers=self.api._build_hf_headers(),
            json={"flavor": "t4-medium"},
        )

    def test_request_space_hardware_with_sleep_time(self) -> None:
        self.api.request_space_hardware(self.repo_id, SpaceHardware.T4_MEDIUM, sleep_time=123)
        self.post_mock.assert_called_once_with(
            f"{self.api.endpoint}/api/spaces/{self.repo_id}/hardware",
            headers=self.api._build_hf_headers(),
            json={"flavor": "t4-medium", "sleepTimeSeconds": 123},
        )

    def test_set_space_sleep_time_upgraded_hardware(self) -> None:
        self.api.set_space_sleep_time(self.repo_id, sleep_time=123)
        self.post_mock.assert_called_once_with(
            f"{self.api.endpoint}/api/spaces/{self.repo_id}/sleeptime",
            headers=self.api._build_hf_headers(),
            json={"seconds": 123},
        )

    def test_set_space_sleep_time_cpu_basic(self) -> None:
        self.post_mock.return_value.json.return_value["hardware"]["requested"] = "cpu-basic"
        with self.assertWarns(UserWarning):
            self.api.set_space_sleep_time(self.repo_id, sleep_time=123)


class ListGitRefsTest(unittest.TestCase):
    @classmethod
    @with_production_testing
    def setUpClass(cls) -> None:
        cls.api = HfApi()
        return super().setUpClass()

    def test_list_refs_gpt2(self) -> None:
        refs = self.api.list_repo_refs("gpt2")
        self.assertGreater(len(refs.branches), 0)
        main_branch = [branch for branch in refs.branches if branch.name == "main"][0]
        self.assertEqual(main_branch.ref, "refs/heads/main")
        # Can get info by revision
        self.api.repo_info("gpt2", revision=main_branch.target_commit)

    def test_list_refs_bigcode(self) -> None:
        refs = self.api.list_repo_refs("bigcode/admin", repo_type="dataset")
        self.assertGreater(len(refs.branches), 0)
        self.assertGreater(len(refs.converts), 0)
        main_branch = [branch for branch in refs.branches if branch.name == "main"][0]
        self.assertEqual(main_branch.ref, "refs/heads/main")

        convert_branch = [branch for branch in refs.converts if branch.name == "parquet"][0]
        self.assertEqual(convert_branch.ref, "refs/convert/parquet")

        # Can get info by convert revision
        self.api.repo_info(
            "bigcode/admin",
            repo_type="dataset",
            revision=convert_branch.target_commit,
        )


class ListGitCommitsTest(unittest.TestCase):
    @classmethod
    def setUpClass(cls) -> None:
        cls.api = HfApi(token=TOKEN)
        # Create repo (with initial commit)
        cls.repo_id = cls.api.create_repo(repo_name()).repo_id

        # Create a commit on `main` branch
        cls.api.upload_file(repo_id=cls.repo_id, path_or_fileobj=b"content", path_in_repo="content.txt")

        # Create a commit in a PR
        cls.api.upload_file(repo_id=cls.repo_id, path_or_fileobj=b"on_pr", path_in_repo="on_pr.txt", create_pr=True)

        # Create another commit on `main` branch
        cls.api.upload_file(repo_id=cls.repo_id, path_or_fileobj=b"on_main", path_in_repo="on_main.txt")
        return super().setUpClass()

    @classmethod
    def tearDownClass(cls) -> None:
        cls.api.delete_repo(cls.repo_id)
        return super().tearDownClass()

    def test_list_commits_on_main(self) -> None:
        commits = self.api.list_repo_commits(self.repo_id)

        # "on_pr" commit not returned
        self.assertEquals(len(commits), 3)
        self.assertTrue(all("on_pr" not in commit.title for commit in commits))

        # USER is always the author
        self.assertTrue(all(commit.authors == [USER] for commit in commits))

        # latest commit first
        self.assertEquals(commits[0].title, "Upload on_main.txt with huggingface_hub")

        # Formatted field not returned by default
        for commit in commits:
            self.assertIsNone(commit.formatted_title)
            self.assertIsNone(commit.formatted_message)

    def test_list_commits_on_pr(self) -> None:
        commits = self.api.list_repo_commits(self.repo_id, revision="refs/pr/1")

        # "on_pr" commit returned but not the "on_main" one
        self.assertEquals(len(commits), 3)
        self.assertTrue(all("on_main" not in commit.title for commit in commits))
        self.assertEquals(commits[0].title, "Upload on_pr.txt with huggingface_hub")

    def test_list_commits_include_formatted(self) -> None:
        for commit in self.api.list_repo_commits(self.repo_id, formatted=True):
            self.assertIsNotNone(commit.formatted_title)
            self.assertIsNotNone(commit.formatted_message)

    def test_list_commits_on_missing_repo(self) -> None:
        with self.assertRaises(RepositoryNotFoundError):
            self.api.list_repo_commits("missing_repo_id")

    def test_list_commits_on_missing_revision(self) -> None:
        with self.assertRaises(RevisionNotFoundError):
            self.api.list_repo_commits(self.repo_id, revision="missing_revision")


@patch("huggingface_hub.hf_api.build_hf_headers")
class HfApiTokenAttributeTest(unittest.TestCase):
    def test_token_passed(self, mock_build_hf_headers: Mock) -> None:
        HfApi(token="default token")._build_hf_headers(token="A token")
        self._assert_token_is(mock_build_hf_headers, "A token")

    def test_no_token_passed(self, mock_build_hf_headers: Mock) -> None:
        HfApi(token="default token")._build_hf_headers()
        self._assert_token_is(mock_build_hf_headers, "default token")

    def test_token_true_passed(self, mock_build_hf_headers: Mock) -> None:
        HfApi(token="default token")._build_hf_headers(token=True)
        self._assert_token_is(mock_build_hf_headers, True)

    def test_token_false_passed(self, mock_build_hf_headers: Mock) -> None:
        HfApi(token="default token")._build_hf_headers(token=False)
        self._assert_token_is(mock_build_hf_headers, False)

    def test_no_token_at_all(self, mock_build_hf_headers: Mock) -> None:
        HfApi()._build_hf_headers(token=None)
        self._assert_token_is(mock_build_hf_headers, None)

    def _assert_token_is(self, mock_build_hf_headers: Mock, expected_value: str) -> None:
        self.assertEqual(mock_build_hf_headers.call_args[1]["token"], expected_value)

    def test_library_name_and_version_are_set(self, mock_build_hf_headers: Mock) -> None:
        HfApi(library_name="a", library_version="b")._build_hf_headers()
        self.assertEqual(mock_build_hf_headers.call_args[1]["library_name"], "a")
        self.assertEqual(mock_build_hf_headers.call_args[1]["library_version"], "b")

    def test_library_name_and_version_are_overwritten(self, mock_build_hf_headers: Mock) -> None:
        api = HfApi(library_name="a", library_version="b")
        api._build_hf_headers(library_name="A", library_version="B")
        self.assertEqual(mock_build_hf_headers.call_args[1]["library_name"], "A")
        self.assertEqual(mock_build_hf_headers.call_args[1]["library_version"], "B")

    def test_user_agent_is_set(self, mock_build_hf_headers: Mock) -> None:
        HfApi(user_agent={"a": "b"})._build_hf_headers()
        self.assertEqual(mock_build_hf_headers.call_args[1]["user_agent"], {"a": "b"})

    def test_user_agent_is_overwritten(self, mock_build_hf_headers: Mock) -> None:
        HfApi(user_agent={"a": "b"})._build_hf_headers(user_agent={"A": "B"})
        self.assertEqual(mock_build_hf_headers.call_args[1]["user_agent"], {"A": "B"})


@patch("huggingface_hub.hf_api.ENDPOINT", "https://huggingface.co")
class RepoUrlTest(unittest.TestCase):
    def test_repo_url_class(self):
        url = RepoUrl("https://huggingface.co/gpt2")

        # RepoUrl Is a string
        self.assertIsInstance(url, str)
        self.assertEqual(url, "https://huggingface.co/gpt2")

        # Any str-method can be applied
        self.assertEqual(url.split("/"), "https://huggingface.co/gpt2".split("/"))

        # String formatting and concatenation work
        self.assertEqual(f"New repo: {url}", "New repo: https://huggingface.co/gpt2")
        self.assertEqual("New repo: " + url, "New repo: https://huggingface.co/gpt2")

        # __repr__ is modified for debugging purposes
        self.assertEqual(
            repr(url),
            (
                "RepoUrl('https://huggingface.co/gpt2',"
                " endpoint='https://huggingface.co', repo_type='model', repo_id='gpt2')"
            ),
        )

    def test_repo_url_endpoint(self):
        # Implicit endpoint
        url = RepoUrl("https://huggingface.co/gpt2")
        self.assertEqual(url.endpoint, "https://huggingface.co")

        # Explicit endpoint
        url = RepoUrl("https://example.com/gpt2", endpoint="https://example.com")
        self.assertEqual(url.endpoint, "https://example.com")

    def test_repo_url_repo_type(self):
        # Explicit repo type
        url = RepoUrl("https://huggingface.co/user/repo_name")
        self.assertEqual(url.repo_type, "model")

        url = RepoUrl("https://huggingface.co/datasets/user/repo_name")
        self.assertEqual(url.repo_type, "dataset")

        url = RepoUrl("https://huggingface.co/spaces/user/repo_name")
        self.assertEqual(url.repo_type, "space")

        # Implicit repo type (model)
        url = RepoUrl("https://huggingface.co/user/repo_name")
        self.assertEqual(url.repo_type, "model")

    def test_repo_url_namespace(self):
        # Canonical model (e.g. no username)
        url = RepoUrl("https://huggingface.co/gpt2")
        self.assertIsNone(url.namespace)
        self.assertEqual(url.repo_id, "gpt2")

        # "Normal" model
        url = RepoUrl("https://huggingface.co/dummy_user/dummy_model")
        self.assertEqual(url.namespace, "dummy_user")
        self.assertEqual(url.repo_id, "dummy_user/dummy_model")

    def test_repo_url_url_property(self):
        # RepoUrl.url returns a pure `str` value
        url = RepoUrl("https://huggingface.co/gpt2")
        self.assertEqual(url, "https://huggingface.co/gpt2")
        self.assertEqual(url.url, "https://huggingface.co/gpt2")
        self.assertIsInstance(url, RepoUrl)
        self.assertNotIsInstance(url.url, RepoUrl)

    def test_repo_url_canonical_model(self):
        for _id in ("gpt2", "hf://gpt2", "https://huggingface.co/gpt2"):
            with self.subTest(_id):
                url = RepoUrl(_id)
                self.assertEqual(url.repo_id, "gpt2")
                self.assertEqual(url.repo_type, "model")

    def test_repo_url_canonical_dataset(self):
        for _id in ("datasets/squad", "hf://datasets/squad", "https://huggingface.co/datasets/squad"):
            with self.subTest(_id):
                url = RepoUrl(_id)
                self.assertEqual(url.repo_id, "squad")
                self.assertEqual(url.repo_type, "dataset")


class HfApiDuplicateSpaceTest(HfApiCommonTest):
    @retry_endpoint
    @unittest.skip("HTTP 500 currently on staging")
    def test_duplicate_space_success(self) -> None:
        """Check `duplicate_space` works."""
        from_repo_name = space_repo_name("original_repo_name")
        from_repo_id = self._api.create_repo(
            repo_id=from_repo_name,
            repo_type="space",
            space_sdk="static",
            token=OTHER_TOKEN,
        ).repo_id
        self._api.upload_file(
            path_or_fileobj=b"data",
            path_in_repo="temp/new_file.md",
            repo_id=from_repo_id,
            repo_type="space",
            token=OTHER_TOKEN,
        )

        to_repo_id = self._api.duplicate_space(from_repo_id).repo_id

        self.assertEqual(to_repo_id, f"{USER}/{from_repo_name}")
        self.assertEqual(
            self._api.list_repo_files(repo_id=from_repo_id, repo_type="space"),
            [".gitattributes", "README.md", "index.html", "style.css", "temp/new_file.md"],
        )
        self.assertEqual(
            self._api.list_repo_files(repo_id=to_repo_id, repo_type="space"),
            self._api.list_repo_files(repo_id=from_repo_id, repo_type="space"),
        )

        self._api.delete_repo(repo_id=from_repo_id, repo_type="space", token=OTHER_TOKEN)
        self._api.delete_repo(repo_id=to_repo_id, repo_type="space")

    def test_duplicate_space_from_missing_repo(self) -> None:
        """Check `duplicate_space` fails when the from_repo doesn't exist."""

        with self.assertRaises(RepositoryNotFoundError):
            self._api.duplicate_space(f"{OTHER_USER}/repo_that_does_not_exist")


class ReprMixinTest(unittest.TestCase):
    def test_repr_mixin(self) -> None:
        class MyClass(ReprMixin):
            def __init__(self, **kwargs: Dict[str, Any]) -> None:
                self.__dict__.update(kwargs)

        self.assertEqual(
            repr(MyClass(foo="foo", bar="bar")),
            "MyClass: {'bar': 'bar', 'foo': 'foo'}",  # keys are sorted
        )


# tests/test_hf_file_system.py

import datetime
import unittest
from typing import Optional
from unittest.mock import patch

import fsspec
import pytest

from huggingface_hub.constants import REPO_TYPES_URL_PREFIXES
from huggingface_hub.hf_file_system import HfFileSystem
from huggingface_hub.utils import RepositoryNotFoundError, RevisionNotFoundError

from .testing_constants import ENDPOINT_STAGING, TOKEN, USER
from .testing_utils import repo_name, retry_endpoint


class HfFileSystemTests(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        """Register `HfFileSystem` as a `fsspec` filesystem if not already registered."""
        if HfFileSystem.protocol not in fsspec.available_protocols():
            fsspec.register_implementation(HfFileSystem.protocol, HfFileSystem)

    def setUp(self):
        self.repo_id = f"{USER}/{repo_name()}"
        self.repo_type = "dataset"
        self.hf_path = REPO_TYPES_URL_PREFIXES.get(self.repo_type, "") + self.repo_id
        self.hffs = HfFileSystem(endpoint=ENDPOINT_STAGING, token=TOKEN)
        self.api = self.hffs._api

        # Create dummy repo
        self.api.create_repo(self.repo_id, repo_type=self.repo_type)
        self.api.upload_file(
            path_or_fileobj=b"dummy binary data on pr",
            path_in_repo="data/binary_data_for_pr.bin",
            repo_id=self.repo_id,
            repo_type=self.repo_type,
            create_pr=True,
        )
        self.api.upload_file(
            path_or_fileobj="dummy text data".encode("utf-8"),
            path_in_repo="data/text_data.txt",
            repo_id=self.repo_id,
            repo_type=self.repo_type,
        )
        self.api.upload_file(
            path_or_fileobj=b"dummy binary data",
            path_in_repo="data/binary_data.bin",
            repo_id=self.repo_id,
            repo_type=self.repo_type,
        )

    def tearDown(self):
        self.api.delete_repo(self.repo_id, repo_type=self.repo_type)

    @retry_endpoint
    def test_glob(self):
        self.assertEqual(
            sorted(self.hffs.glob(self.hf_path + "/*")),
            sorted([self.hf_path + "/.gitattributes", self.hf_path + "/data"]),
        )

        self.assertEqual(
            sorted(self.hffs.glob(self.hf_path + "/*", revision="main")),
            sorted([self.hf_path + "/.gitattributes", self.hf_path + "/data"]),
        )
        self.assertEqual(
            sorted(self.hffs.glob(self.hf_path + "@main" + "/*")),
            sorted([self.hf_path + "@main" + "/.gitattributes", self.hf_path + "@main" + "/data"]),
        )

    @retry_endpoint
    def test_file_type(self):
        self.assertTrue(
            self.hffs.isdir(self.hf_path + "/data") and not self.hffs.isdir(self.hf_path + "/.gitattributes")
        )
        self.assertTrue(
            self.hffs.isfile(self.hf_path + "/data/text_data.txt") and not self.hffs.isfile(self.hf_path + "/data")
        )

    @retry_endpoint
    def test_remove_file(self):
        self.hffs.rm_file(self.hf_path + "/data/text_data.txt")
        self.assertEqual(self.hffs.glob(self.hf_path + "/data/*"), [self.hf_path + "/data/binary_data.bin"])

    @retry_endpoint
    def test_remove_directory(self):
        self.hffs.rm(self.hf_path + "/data", recursive=True)
        self.assertNotIn(self.hf_path + "/data", self.hffs.ls(self.hf_path))

    @retry_endpoint
    def test_read_file(self):
        with self.hffs.open(self.hf_path + "/data/text_data.txt", "r") as f:
            self.assertEqual(f.read(), "dummy text data")

    @retry_endpoint
    def test_write_file(self):
        data = "new text data"
        with self.hffs.open(self.hf_path + "/data/new_text_data.txt", "w") as f:
            f.write(data)
        self.assertIn(self.hf_path + "/data/new_text_data.txt", self.hffs.glob(self.hf_path + "/data/*"))
        with self.hffs.open(self.hf_path + "/data/new_text_data.txt", "r") as f:
            self.assertEqual(f.read(), data)

    @retry_endpoint
    def test_write_file_multiple_chunks(self):
        # TODO: try with files between 10 and 50MB (as of 16 March 2023 I was getting 504 errors on hub-ci)
        data = "a" * (4 << 20)  # 4MB
        with self.hffs.open(self.hf_path + "/data/new_text_data_big.txt", "w") as f:
            for _ in range(2):  # 8MB in total
                f.write(data)

        self.assertIn(self.hf_path + "/data/new_text_data_big.txt", self.hffs.glob(self.hf_path + "/data/*"))
        with self.hffs.open(self.hf_path + "/data/new_text_data_big.txt", "r") as f:
            for _ in range(2):
                self.assertEqual(f.read(len(data)), data)

    @unittest.skip("Not implemented yet")
    @retry_endpoint
    def test_append_file(self):
        with self.hffs.open(self.hf_path + "/data/text_data.txt", "a") as f:
            f.write(" appended text")

        with self.hffs.open(self.hf_path + "/data/text_data.txt", "r") as f:
            self.assertEqual(f.read(), "dummy text data appended text")

    @retry_endpoint
    def test_copy_file(self):
        # Non-LFS file
        self.assertIsNone(self.hffs.info(self.hf_path + "/data/text_data.txt")["lfs"])
        self.hffs.cp_file(self.hf_path + "/data/text_data.txt", self.hf_path + "/data/text_data_copy.txt")
        with self.hffs.open(self.hf_path + "/data/text_data_copy.txt", "r") as f:
            self.assertEqual(f.read(), "dummy text data")
        self.assertIsNone(self.hffs.info(self.hf_path + "/data/text_data_copy.txt")["lfs"])
        # LFS file
        self.assertIsNotNone(self.hffs.info(self.hf_path + "/data/binary_data.bin")["lfs"])
        self.hffs.cp_file(self.hf_path + "/data/binary_data.bin", self.hf_path + "/data/binary_data_copy.bin")
        with self.hffs.open(self.hf_path + "/data/binary_data_copy.bin", "rb") as f:
            self.assertEqual(f.read(), b"dummy binary data")
        self.assertIsNotNone(self.hffs.info(self.hf_path + "/data/binary_data_copy.bin")["lfs"])

    @retry_endpoint
    def test_modified_time(self):
        self.assertIsInstance(self.hffs.modified(self.hf_path + "/data/text_data.txt"), datetime.datetime)
        # should fail on a non-existing file
        with self.assertRaises(FileNotFoundError):
            self.hffs.modified(self.hf_path + "/data/not_existing_file.txt")
        # should fail on a directory
        with self.assertRaises(IsADirectoryError):
            self.hffs.modified(self.hf_path + "/data")

    @retry_endpoint
    def test_initialize_from_fsspec(self):
        fs, _, paths = fsspec.get_fs_token_paths(
            f"hf://{self.repo_type}s/{self.repo_id}/data/text_data.txt",
            storage_options={
                "endpoint": ENDPOINT_STAGING,
                "token": TOKEN,
            },
        )
        self.assertIsInstance(fs, HfFileSystem)
        self.assertEqual(fs._api.endpoint, ENDPOINT_STAGING)
        self.assertEqual(fs.token, TOKEN)
        self.assertEqual(paths, [self.hf_path + "/data/text_data.txt"])

        fs, _, paths = fsspec.get_fs_token_paths(f"hf://{self.repo_id}/data/text_data.txt")
        self.assertIsInstance(fs, HfFileSystem)
        self.assertEqual(paths, [f"{self.repo_id}/data/text_data.txt"])

    @retry_endpoint
    def test_list_root_directory_no_revision(self):
        files = self.hffs.ls(self.hf_path)
        self.assertEqual(len(files), 2)

        self.assertEqual(files[0]["type"], "directory")
        self.assertEqual(files[0]["size"], 0)
        self.assertTrue(files[0]["name"].endswith("/data"))

        self.assertEqual(files[1]["type"], "file")
        self.assertGreater(files[1]["size"], 0)  # not empty
        self.assertTrue(files[1]["name"].endswith("/.gitattributes"))

    @retry_endpoint
    def test_list_data_directory_no_revision(self):
        files = self.hffs.ls(self.hf_path + "/data")
        self.assertEqual(len(files), 2)

        self.assertEqual(files[0]["type"], "file")
        self.assertGreater(files[0]["size"], 0)  # not empty
        self.assertTrue(files[0]["name"].endswith("/data/binary_data.bin"))
        self.assertIsNotNone(files[0]["lfs"])
        self.assertIn("oid", files[0]["lfs"])
        self.assertIn("size", files[0]["lfs"])
        self.assertIn("pointerSize", files[0]["lfs"])

        self.assertEqual(files[1]["type"], "file")
        self.assertGreater(files[1]["size"], 0)  # not empty
        self.assertTrue(files[1]["name"].endswith("/data/text_data.txt"))
        self.assertIsNone(files[1]["lfs"])

    @retry_endpoint
    def test_list_data_directory_with_revision(self):
        files = self.hffs.ls(self.hf_path + "@refs%2Fpr%2F1" + "/data")

        for test_name, files in {
            "rev_in_path": self.hffs.ls(self.hf_path + "@refs%2Fpr%2F1" + "/data"),
            "rev_as_arg": self.hffs.ls(self.hf_path + "/data", revision="refs/pr/1"),
            "rev_in_path_and_as_arg": self.hffs.ls(self.hf_path + "@refs%2Fpr%2F1" + "/data", revision="refs/pr/1"),
        }.items():
            with self.subTest(test_name):
                self.assertEqual(len(files), 1)  # only one file in PR
                self.assertEqual(files[0]["type"], "file")
                self.assertTrue(files[0]["name"].endswith("/data/binary_data_for_pr.bin"))  # PR file


@pytest.mark.parametrize("path_in_repo", ["", "foo"])
@pytest.mark.parametrize(
    "root_path,revision,repo_type,repo_id,resolved_revision",
    [
        # Parse without namespace
        ("gpt2", None, "model", "gpt2", "main"),
        ("gpt2", "dev", "model", "gpt2", "dev"),
        ("gpt2@dev", None, "model", "gpt2", "dev"),
        ("datasets/squad", None, "dataset", "squad", "main"),
        ("datasets/squad", "dev", "dataset", "squad", "dev"),
        ("datasets/squad@dev", None, "dataset", "squad", "dev"),
        # Parse with namespace
        ("username/my_model", None, "model", "username/my_model", "main"),
        ("username/my_model", "dev", "model", "username/my_model", "dev"),
        ("username/my_model@dev", None, "model", "username/my_model", "dev"),
        ("datasets/username/my_dataset", None, "dataset", "username/my_dataset", "main"),
        ("datasets/username/my_dataset", "dev", "dataset", "username/my_dataset", "dev"),
        ("datasets/username/my_dataset@dev", None, "dataset", "username/my_dataset", "dev"),
        # Parse with hf:// protocol
        ("hf://gpt2", None, "model", "gpt2", "main"),
        ("hf://gpt2", "dev", "model", "gpt2", "dev"),
        ("hf://gpt2@dev", None, "model", "gpt2", "dev"),
        ("hf://datasets/squad", None, "dataset", "squad", "main"),
        ("hf://datasets/squad", "dev", "dataset", "squad", "dev"),
        ("hf://datasets/squad@dev", None, "dataset", "squad", "dev"),
    ],
)
def test_resolve_path(
    root_path: str,
    revision: Optional[str],
    repo_type: str,
    repo_id: str,
    resolved_revision: str,
    path_in_repo: str,
):
    fs = HfFileSystem()
    path = root_path + "/" + path_in_repo if path_in_repo else root_path

    def mock_repo_info(repo_id: str, *, revision: str, repo_type: str, **kwargs):
        if repo_id not in ["gpt2", "squad", "username/my_dataset", "username/my_model"]:
            raise RepositoryNotFoundError(repo_id)
        if revision is not None and revision not in ["main", "dev"]:
            raise RevisionNotFoundError(revision)

    with patch.object(fs._api, "repo_info", mock_repo_info):
        resolved_path = fs.resolve_path(path, revision=revision)
        assert (
            resolved_path.repo_type,
            resolved_path.repo_id,
            resolved_path.revision,
            resolved_path.path_in_repo,
        ) == (repo_type, repo_id, resolved_revision, path_in_repo)


def test_resolve_path_with_non_matching_revisions():
    fs = HfFileSystem()
    with pytest.raises(ValueError):
        fs.resolve_path("gpt2@dev", revision="main")


@pytest.mark.parametrize("not_supported_path", ["", "foo", "datasets", "datasets/foo"])
def test_access_repositories_lists(not_supported_path):
    fs = HfFileSystem()
    with pytest.raises(NotImplementedError):
        fs.ls(not_supported_path)
    with pytest.raises(NotImplementedError):
        fs.glob(not_supported_path + "/")
    with pytest.raises(NotImplementedError):
        fs.open(not_supported_path)


# tests/test_hubmixin.py

import json
import os
import unittest
from pathlib import Path
from typing import TypeVar
from unittest.mock import Mock, patch

import pytest

from huggingface_hub import HfApi, hf_hub_download
from huggingface_hub.hub_mixin import PyTorchModelHubMixin
from huggingface_hub.utils import SoftTemporaryDirectory, is_torch_available

from .testing_constants import ENDPOINT_STAGING, TOKEN, USER
from .testing_utils import repo_name


if is_torch_available():
    import torch.nn as nn


def require_torch(test_case):
    """
    Decorator marking a test that requires PyTorch.
    These tests are skipped when PyTorch isn't installed.
    """
    if not is_torch_available():
        return unittest.skip("test requires PyTorch")(test_case)
    else:
        return test_case


if is_torch_available():
    CONFIG = {"num": 10, "act": "gelu_fast"}

    class DummyModel(nn.Module, PyTorchModelHubMixin):
        def __init__(self, **kwargs):
            super().__init__()
            self.config = kwargs.pop("config", None)
            self.l1 = nn.Linear(2, 2)

        def forward(self, x):
            return self.l1(x)

else:
    DummyModel = None


@require_torch
@pytest.mark.usefixtures("fx_cache_dir")
class HubMixingTest(unittest.TestCase):
    cache_dir: Path

    @classmethod
    def setUpClass(cls):
        """
        Share this valid token in all tests below.
        """
        cls._api = HfApi(endpoint=ENDPOINT_STAGING, token=TOKEN)

    def test_save_pretrained_basic(self):
        DummyModel().save_pretrained(self.cache_dir)
        files = os.listdir(self.cache_dir)
        self.assertTrue("pytorch_model.bin" in files)
        self.assertEqual(len(files), 1)

    def test_save_pretrained_with_config(self):
        DummyModel().save_pretrained(self.cache_dir, config=CONFIG)
        files = os.listdir(self.cache_dir)
        self.assertTrue("config.json" in files)
        self.assertTrue("pytorch_model.bin" in files)
        self.assertEqual(len(files), 2)

    def test_save_pretrained_with_push_to_hub(self):
        repo_id = repo_name("save")
        save_directory = self.cache_dir / repo_id

        config = {"hello": "world"}
        mocked_model = DummyModel()
        mocked_model.push_to_hub = Mock()
        mocked_model._save_pretrained = Mock()  # disable _save_pretrained to speed-up

        # Not pushed to hub
        mocked_model.save_pretrained(save_directory)
        mocked_model.push_to_hub.assert_not_called()

        # Push to hub with repo_id
        mocked_model.save_pretrained(save_directory, push_to_hub=True, repo_id="CustomID", config=config)
        mocked_model.push_to_hub.assert_called_with(repo_id="CustomID", config=config)

        # Push to hub with default repo_id (based on dir name)
        mocked_model.save_pretrained(save_directory, push_to_hub=True, config=config)
        mocked_model.push_to_hub.assert_called_with(repo_id=repo_id, config=config)

    @patch.object(DummyModel, "_from_pretrained")
    def test_from_pretrained_model_id_only(self, from_pretrained_mock: Mock) -> None:
        model = DummyModel.from_pretrained("namespace/repo_name")
        from_pretrained_mock.assert_called_once()
        self.assertIs(model, from_pretrained_mock.return_value)

    @patch.object(DummyModel, "_from_pretrained")
    def test_from_pretrained_model_id_and_revision(self, from_pretrained_mock: Mock) -> None:
        """Regression test for #1313.
        See https://github.com/huggingface/huggingface_hub/issues/1313."""
        model = DummyModel.from_pretrained("namespace/repo_name", revision="123456789")
        from_pretrained_mock.assert_called_once_with(
            model_id="namespace/repo_name",
            revision="123456789",  # Revision is passed correctly!
            cache_dir=None,
            force_download=False,
            proxies=None,
            resume_download=False,
            local_files_only=False,
            token=None,
        )
        self.assertIs(model, from_pretrained_mock.return_value)

    def test_from_pretrained_to_relative_path(self):
        with SoftTemporaryDirectory(dir=Path(".")) as tmp_relative_dir:
            relative_save_directory = Path(tmp_relative_dir) / "model"
            DummyModel().save_pretrained(relative_save_directory, config=CONFIG)
            model = DummyModel.from_pretrained(relative_save_directory)
            self.assertDictEqual(model.config, CONFIG)

    def test_from_pretrained_to_absolute_path(self):
        save_directory = self.cache_dir / "subfolder"
        DummyModel().save_pretrained(save_directory, config=CONFIG)
        model = DummyModel.from_pretrained(save_directory)
        self.assertDictEqual(model.config, CONFIG)

    def test_from_pretrained_to_absolute_string_path(self):
        save_directory = str(self.cache_dir / "subfolder")
        DummyModel().save_pretrained(save_directory, config=CONFIG)
        model = DummyModel.from_pretrained(save_directory)
        self.assertDictEqual(model.config, CONFIG)

    def test_return_type_hint_from_pretrained(self):
        self.assertIn(
            "return",
            DummyModel.from_pretrained.__annotations__,
            "`PyTorchModelHubMixin.from_pretrained` does not set a return type annotation.",
        )
        self.assertIsInstance(
            DummyModel.from_pretrained.__annotations__["return"],
            TypeVar,
            "`PyTorchModelHubMixin.from_pretrained` return type annotation is not a TypeVar.",
        )
        self.assertEqual(
            DummyModel.from_pretrained.__annotations__["return"].__bound__.__forward_arg__,
            "ModelHubMixin",
            "`PyTorchModelHubMixin.from_pretrained` return type annotation is not a TypeVar bound by `ModelHubMixin`.",
        )

    def test_push_to_hub(self):
        repo_id = f"{USER}/{repo_name('push_to_hub')}"
        DummyModel().push_to_hub(repo_id=repo_id, api_endpoint=ENDPOINT_STAGING, token=TOKEN, config=CONFIG)

        # Test model id exists
        model_info = self._api.model_info(repo_id)
        self.assertEqual(model_info.modelId, repo_id)

        # Test config has been pushed to hub
        tmp_config_path = hf_hub_download(
            repo_id=repo_id,
            filename="config.json",
            use_auth_token=TOKEN,
            cache_dir=self.cache_dir,
        )
        with open(tmp_config_path) as f:
            self.assertDictEqual(json.load(f), CONFIG)

        # Delete repo
        self._api.delete_repo(repo_id=repo_id)


# tests/test_inference_api.py

# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from pathlib import Path
from unittest.mock import patch

from PIL import Image

from huggingface_hub import hf_hub_download
from huggingface_hub.inference_api import InferenceApi

from .testing_utils import with_production_testing


@with_production_testing
class InferenceApiTest(unittest.TestCase):
    def read(self, filename: str) -> bytes:
        return Path(filename).read_bytes()

    @classmethod
    @with_production_testing
    def setUpClass(cls) -> None:
        cls.image_file = hf_hub_download(repo_id="Narsil/image_dummy", repo_type="dataset", filename="lena.png")
        return super().setUpClass()

    def test_simple_inference(self):
        api = InferenceApi("bert-base-uncased")
        inputs = "Hi, I think [MASK] is cool"
        results = api(inputs)
        self.assertIsInstance(results, list)

        result = results[0]
        self.assertIsInstance(result, dict)
        self.assertTrue("sequence" in result)
        self.assertTrue("score" in result)

    def test_inference_with_params(self):
        api = InferenceApi("typeform/distilbert-base-uncased-mnli")
        inputs = "I bought a device but it is not working and I would like to get reimbursed!"
        params = {"candidate_labels": ["refund", "legal", "faq"]}
        result = api(inputs, params)
        self.assertIsInstance(result, dict)
        self.assertTrue("sequence" in result)
        self.assertTrue("scores" in result)

    def test_inference_with_dict_inputs(self):
        api = InferenceApi("distilbert-base-cased-distilled-squad")
        inputs = {
            "question": "What's my name?",
            "context": "My name is Clara and I live in Berkeley.",
        }
        result = api(inputs)
        self.assertIsInstance(result, dict)
        self.assertTrue("score" in result)
        self.assertTrue("answer" in result)

    def test_inference_with_audio(self):
        api = InferenceApi("facebook/wav2vec2-base-960h")
        file = hf_hub_download(
            repo_id="hf-internal-testing/dummy-flac-single-example",
            repo_type="dataset",
            filename="example.flac",
        )
        data = self.read(file)
        result = api(data=data)
        self.assertIsInstance(result, dict)
        self.assertTrue("text" in result, f"We received {result} instead")

    def test_inference_with_image(self):
        api = InferenceApi("google/vit-base-patch16-224")
        data = self.read(self.image_file)
        result = api(data=data)
        self.assertIsInstance(result, list)
        for classification in result:
            self.assertIsInstance(classification, dict)
            self.assertTrue("score" in classification)
            self.assertTrue("label" in classification)

    def test_text_to_image(self):
        api = InferenceApi("stabilityai/stable-diffusion-2-1")
        with patch("huggingface_hub.inference_api.get_session") as mock:
            mock().post.return_value.headers = {"Content-Type": "image/jpeg"}
            mock().post.return_value.content = self.read(self.image_file)
            output = api("cat")
        self.assertIsInstance(output, Image.Image)

    def test_text_to_image_raw_response(self):
        api = InferenceApi("stabilityai/stable-diffusion-2-1")
        with patch("huggingface_hub.inference_api.get_session") as mock:
            mock().post.return_value.headers = {"Content-Type": "image/jpeg"}
            mock().post.return_value.content = self.read(self.image_file)
            output = api("cat", raw_response=True)
        # Raw response is returned
        self.assertEqual(output, mock().post.return_value)

    def test_inference_overriding_task(self):
        api = InferenceApi(
            "sentence-transformers/paraphrase-albert-small-v2",
            task="feature-extraction",
        )
        inputs = "This is an example again"
        result = api(inputs)
        self.assertIsInstance(result, list)

    def test_inference_overriding_invalid_task(self):
        with self.assertRaises(ValueError, msg="Invalid task invalid-task. Make sure it's valid."):
            InferenceApi("bert-base-uncased", task="invalid-task")

    def test_inference_missing_input(self):
        api = InferenceApi("deepset/roberta-base-squad2")
        result = api({"question": "What's my name?"})
        self.assertIsInstance(result, dict)
        self.assertTrue("error" in result)


# tests/test_inference_client.py

# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import io
import unittest
from pathlib import Path
from unittest.mock import MagicMock, patch

import numpy as np
import pytest
from PIL import Image

from huggingface_hub import InferenceClient, hf_hub_download
from huggingface_hub.inference._client import _open_as_binary
from huggingface_hub.utils import build_hf_headers

from .testing_utils import with_production_testing


# Avoid call to hf.co/api/models in VCRed tests
_RECOMMENDED_MODELS_FOR_VCR = {
    "audio-classification": "speechbrain/google_speech_command_xvector",
    "audio-to-audio": "speechbrain/sepformer-wham",
    "automatic-speech-recognition": "facebook/wav2vec2-base-960h",
    "conversational": "facebook/blenderbot-400M-distill",
    "feature-extraction": "facebook/bart-base",
    "image-classification": "google/vit-base-patch16-224",
    "image-segmentation": "facebook/detr-resnet-50-panoptic",
    "sentence-similarity": "sentence-transformers/all-MiniLM-L6-v2",
    "summarization": "sshleifer/distilbart-cnn-12-6",
    "text-classification": "distilbert-base-uncased-finetuned-sst-2-english",
    "text-to-image": "CompVis/stable-diffusion-v1-4",
    "text-to-speech": "espnet/kan-bayashi_ljspeech_vits",
    "zero-shot-image-classification": "openai/clip-vit-base-patch32",
}


class InferenceClientTest(unittest.TestCase):
    @classmethod
    @with_production_testing
    def setUpClass(cls) -> None:
        super().setUpClass()
        cls.image_file = hf_hub_download(repo_id="Narsil/image_dummy", repo_type="dataset", filename="lena.png")
        cls.audio_file = hf_hub_download(repo_id="Narsil/image_dummy", repo_type="dataset", filename="sample1.flac")


@pytest.mark.vcr
@with_production_testing
@patch("huggingface_hub.inference._client._fetch_recommended_models", lambda: _RECOMMENDED_MODELS_FOR_VCR)
class InferenceClientVCRTest(InferenceClientTest):
    """
    Test for the main tasks implemented in InferenceClient. Since Inference API can be flaky, we use VCR.py and
    pytest-vcr to record and replay the inference calls (see https://pytest-vcr.readthedocs.io/en/latest/).

    Tips when adding new tasks:
    - Most of the time, we only test that the return values are correct. We don't always test the actual output of the model.
    - In the CI, VRC replay is always on. If you want to test locally against the server, you can use the `--vcr-mode`
      and `--disable-vcr` command line options. See https://pytest-vcr.readthedocs.io/en/latest/configuration/.
    - If you get rate-limited locally, you can use your own token when initializing InferenceClient.
      /!\\ WARNING: if you do so, you must delete the token from the cassette before committing!
    - If the model is not loaded on the server, you will save a lot of HTTP 503 responses in the cassette. We don't
      want those to be committed. Either delete them manually or rerun the test once the model is loaded on the server.
    """

    def setUp(self) -> None:
        super().setUp()
        self.client = InferenceClient()

    def test_audio_classification(self) -> None:
        output = self.client.audio_classification(self.audio_file)
        self.assertIsInstance(output, list)
        self.assertGreater(len(output), 0)
        for item in output:
            self.assertIsInstance(item["score"], float)
            self.assertIsInstance(item["label"], str)

    def test_automatic_speech_recognition(self) -> None:
        output = self.client.automatic_speech_recognition(self.audio_file)
        self.assertEqual(output, "A MAN SAID TO THE UNIVERSE SIR I EXIST")

    def test_conversational(self) -> None:
        output = self.client.conversational("Hi, who are you?")
        self.assertEqual(
            output,
            {
                "generated_text": "I am the one who knocks.",
                "conversation": {
                    "generated_responses": ["I am the one who knocks."],
                    "past_user_inputs": ["Hi, who are you?"],
                },
                "warnings": ["Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation."],
            },
        )

        output2 = self.client.conversational(
            "Wow, that's scary!",
            generated_responses=output["conversation"]["generated_responses"],
            past_user_inputs=output["conversation"]["past_user_inputs"],
        )
        self.assertEqual(
            output2,
            {
                "generated_text": "I am the one who knocks.",
                "conversation": {
                    "generated_responses": ["I am the one who knocks.", "I am the one who knocks."],
                    "past_user_inputs": ["Hi, who are you?", "Wow, that's scary!"],
                },
                "warnings": ["Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation."],
            },
        )

    def test_feature_extraction(self) -> None:
        embedding = self.client.feature_extraction("Hi, who are you?")
        self.assertIsInstance(embedding, np.ndarray)
        self.assertEqual(embedding.shape, (8, 768))

    def test_image_classification(self) -> None:
        output = self.client.image_classification(self.image_file)
        self.assertIsInstance(output, list)
        self.assertGreater(len(output), 0)
        for item in output:
            self.assertIsInstance(item["score"], float)
            self.assertIsInstance(item["label"], str)

    def test_image_segmentation(self) -> None:
        output = self.client.image_segmentation(self.image_file)
        self.assertIsInstance(output, list)
        self.assertGreater(len(output), 0)
        for item in output:
            self.assertIsInstance(item["score"], float)
            self.assertIsInstance(item["label"], str)
            self.assertIsInstance(item["mask"], Image.Image)
            self.assertEqual(item["mask"].height, 512)
            self.assertEqual(item["mask"].width, 512)

    # ERROR 500 from server
    # Only during tests, not when running locally. Has to be investigated.
    # def test_image_to_image(self) -> None:
    #     image = self.client.image_to_image(self.image_file, prompt="turn the woman into a man")
    #     self.assertIsInstance(image, Image.Image)
    #     self.assertEqual(image.height, 512)
    #     self.assertEqual(image.width, 512)

    # ERROR 500 from server
    # Only during tests, not when running locally. Has to be investigated.
    # def test_image_to_text(self) -> None:
    #     caption = self.client.image_to_text(self.image_file)
    #     self.assertEqual(caption, "")

    def test_sentence_similarity(self) -> None:
        scores = self.client.sentence_similarity(
            "Machine learning is so easy.",
            other_sentences=[
                "Deep learning is so straightforward.",
                "This is so difficult, like rocket science.",
                "I can't believe how much I struggled with this.",
            ],
        )
        self.assertEqual(scores, [0.7785726189613342, 0.45876261591911316, 0.2906220555305481])

    def test_summarization(self) -> None:
        summary = self.client.summarization(
            "The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, and the tallest"
            " structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. During its"
            " construction, the Eiffel Tower surpassed the Washington Monument to become the tallest man-made"
            " structure in the world, a title it held for 41 years until the Chrysler Building in New York City was"
            " finished in 1930. It was the first structure to reach a height of 300 metres. Due to the addition of a"
            " broadcasting aerial at the top of the tower in 1957, it is now taller than the Chrysler Building by 5.2"
            " metres (17 ft). Excluding transmitters, the Eiffel Tower is the second tallest free-standing structure"
            " in France after the Millau Viaduct."
        )
        self.assertEqual(
            summary,
            (
                "The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building. Its base is"
                " square, measuring 125 metres (410 ft) on each side. During its construction, the Eiffel Tower"
                " surpassed the Washington Monument to become the tallest man-made structure in the world."
            ),
        )

    def test_text_to_image(self) -> None:
        image = self.client.text_to_image("An astronaut riding a horse on the moon.")
        self.assertIsInstance(image, Image.Image)
        self.assertEqual(image.height, 768)
        self.assertEqual(image.width, 768)

    def test_text_to_speech(self) -> None:
        audio = self.client.text_to_speech("Hello world")
        self.assertIsInstance(audio, bytes)

    def test_zero_shot_image_classification(self) -> None:
        output = self.client.zero_shot_image_classification(self.image_file, ["tree", "woman", "cat"])
        self.assertIsInstance(output, list)
        self.assertGreater(len(output), 0)
        for item in output:
            self.assertIsInstance(item["label"], str)
            self.assertIsInstance(item["score"], float)


class TestOpenAsBinary(InferenceClientTest):
    def test_open_as_binary_with_none(self) -> None:
        with _open_as_binary(None) as content:
            self.assertIsNone(content)

    def test_open_as_binary_from_str_path(self) -> None:
        with _open_as_binary(self.image_file) as content:
            self.assertIsInstance(content, io.BufferedReader)

    def test_open_as_binary_from_pathlib_path(self) -> None:
        with _open_as_binary(Path(self.image_file)) as content:
            self.assertIsInstance(content, io.BufferedReader)

    def test_open_as_binary_from_url(self) -> None:
        with _open_as_binary("https://huggingface.co/datasets/Narsil/image_dummy/resolve/main/tree.png") as content:
            self.assertIsInstance(content, bytes)

    def test_open_as_binary_opened_file(self) -> None:
        with Path(self.image_file).open("rb") as f:
            with _open_as_binary(f) as content:
                self.assertEqual(content, f)
                self.assertIsInstance(content, io.BufferedReader)

    def test_open_as_binary_from_bytes(self) -> None:
        content_bytes = Path(self.image_file).read_bytes()
        with _open_as_binary(content_bytes) as content:
            self.assertEqual(content, content_bytes)


class TestResolveURL(InferenceClientTest):
    FAKE_ENDPOINT = "https://my-endpoint.hf.co"

    def test_model_as_url(self):
        self.assertEqual(InferenceClient()._resolve_url(model=self.FAKE_ENDPOINT), self.FAKE_ENDPOINT)

    def test_model_as_id_no_task(self):
        self.assertEqual(
            InferenceClient()._resolve_url(model="username/repo_name"),
            "https://api-inference.huggingface.co/models/username/repo_name",
        )

    def test_model_as_id_and_task_ignored(self):
        self.assertEqual(
            InferenceClient()._resolve_url(model="username/repo_name", task="text-to-image"),
            # /models endpoint
            "https://api-inference.huggingface.co/models/username/repo_name",
        )

    def test_model_as_id_and_task_not_ignored(self):
        # Special case for feature-extraction and sentence-similarity
        self.assertEqual(
            InferenceClient()._resolve_url(model="username/repo_name", task="feature-extraction"),
            # /pipeline/{task} endpoint
            "https://api-inference.huggingface.co/pipeline/feature-extraction/username/repo_name",
        )

    def test_method_level_has_priority(self) -> None:
        # Priority to method-level
        self.assertEqual(
            InferenceClient(model=self.FAKE_ENDPOINT + "_instance")._resolve_url(model=self.FAKE_ENDPOINT + "_method"),
            self.FAKE_ENDPOINT + "_method",
        )

    def test_recommended_model_from_supported_task(self) -> None:
        # Get recommended model
        self.assertEqual(
            InferenceClient()._resolve_url(task="text-to-image"),
            "https://api-inference.huggingface.co/models/CompVis/stable-diffusion-v1-4",
        )

    def test_unsupported_task(self) -> None:
        with self.assertRaises(ValueError):
            InferenceClient()._resolve_url(task="unknown-task")


class TestHeadersAndCookies(unittest.TestCase):
    def test_headers_and_cookies(self) -> None:
        client = InferenceClient(headers={"X-My-Header": "foo"}, cookies={"my-cookie": "bar"})
        self.assertEqual(client.headers["X-My-Header"], "foo")
        self.assertEqual(client.cookies["my-cookie"], "bar")

    def test_headers_overwrite(self) -> None:
        # Default user agent
        self.assertTrue(InferenceClient().headers["user-agent"].startswith("unknown/None;"))

        # Overwritten user-agent
        self.assertEqual(InferenceClient(headers={"user-agent": "bar"}).headers["user-agent"], "bar")

        # Case-insensitive overwrite
        self.assertEqual(InferenceClient(headers={"USER-agent": "bar"}).headers["user-agent"], "bar")

    @patch("huggingface_hub.inference._client.get_session")
    def test_mocked_post(self, get_session_mock: MagicMock) -> None:
        """Test that headers and cookies are correctly passed to the request."""
        client = InferenceClient(headers={"X-My-Header": "foo"}, cookies={"my-cookie": "bar"})
        response = client.post(data=b"content", model="username/repo_name")
        self.assertEqual(response, get_session_mock().post.return_value)

        expected_user_agent = build_hf_headers()["user-agent"]
        get_session_mock().post.assert_called_once_with(
            "https://api-inference.huggingface.co/models/username/repo_name",
            json=None,
            data=b"content",
            headers={"user-agent": expected_user_agent, "X-My-Header": "foo"},
            cookies={"my-cookie": "bar"},
            timeout=None,
            stream=False,
        )


# tests/test_inference_text_generation.py

# Original implementation taken from the `text-generation` Python client (see https://pypi.org/project/text-generation/
# and https://github.com/huggingface/text-generation-inference/tree/main/clients/python)
#
# See './src/huggingface_hub/inference/_text_generation.py' for details.
import unittest
from typing import Dict
from unittest.mock import MagicMock, patch

import pytest
from pydantic import ValidationError
from requests import HTTPError

from huggingface_hub import InferenceClient
from huggingface_hub.inference._client import _NON_TGI_SERVERS
from huggingface_hub.inference._text_generation import (
    FinishReason,
    GenerationError,
    IncompleteGenerationError,
    InputToken,
    OverloadedError,
    TextGenerationParameters,
    TextGenerationRequest,
    raise_text_generation_error,
)
from huggingface_hub.inference._text_generation import (
    ValidationError as TextGenerationValidationError,
)


class TestTextGenerationTypes(unittest.TestCase):
    def test_parameters_validation(self):
        # Test best_of
        TextGenerationParameters(best_of=1)
        with self.assertRaises(ValidationError):
            TextGenerationParameters(best_of=0)
        with self.assertRaises(ValidationError):
            TextGenerationParameters(best_of=-1)
        TextGenerationParameters(best_of=2, do_sample=True)
        with self.assertRaises(ValidationError):
            TextGenerationParameters(best_of=2)
        with self.assertRaises(ValidationError):
            TextGenerationParameters(best_of=2, seed=1)

        # Test repetition_penalty
        TextGenerationParameters(repetition_penalty=1)
        with self.assertRaises(ValidationError):
            TextGenerationParameters(repetition_penalty=0)
        with self.assertRaises(ValidationError):
            TextGenerationParameters(repetition_penalty=-1)

        # Test seed
        TextGenerationParameters(seed=1)
        with self.assertRaises(ValidationError):
            TextGenerationParameters(seed=-1)

        # Test temperature
        TextGenerationParameters(temperature=1)
        with self.assertRaises(ValidationError):
            TextGenerationParameters(temperature=0)
        with self.assertRaises(ValidationError):
            TextGenerationParameters(temperature=-1)

        # Test top_k
        TextGenerationParameters(top_k=1)
        with self.assertRaises(ValidationError):
            TextGenerationParameters(top_k=0)
        with self.assertRaises(ValidationError):
            TextGenerationParameters(top_k=-1)

        # Test top_p
        TextGenerationParameters(top_p=0.5)
        with self.assertRaises(ValidationError):
            TextGenerationParameters(top_p=0)
        with self.assertRaises(ValidationError):
            TextGenerationParameters(top_p=-1)
        with self.assertRaises(ValidationError):
            TextGenerationParameters(top_p=1)

        # Test truncate
        TextGenerationParameters(truncate=1)
        with self.assertRaises(ValidationError):
            TextGenerationParameters(truncate=0)
        with self.assertRaises(ValidationError):
            TextGenerationParameters(truncate=-1)

        # Test typical_p
        TextGenerationParameters(typical_p=0.5)
        with self.assertRaises(ValidationError):
            TextGenerationParameters(typical_p=0)
        with self.assertRaises(ValidationError):
            TextGenerationParameters(typical_p=-1)
        with self.assertRaises(ValidationError):
            TextGenerationParameters(typical_p=1)

    def test_request_validation(self):
        TextGenerationRequest(inputs="test")

        with self.assertRaises(ValidationError):
            TextGenerationRequest(inputs="")

        TextGenerationRequest(inputs="test", stream=True)
        TextGenerationRequest(inputs="test", parameters=TextGenerationParameters(best_of=2, do_sample=True))

        with self.assertRaises(ValidationError):
            TextGenerationRequest(
                inputs="test", parameters=TextGenerationParameters(best_of=2, do_sample=True), stream=True
            )


class TestTextGenerationErrors(unittest.TestCase):
    def test_generation_error(self):
        error = _mocked_error({"error_type": "generation", "error": "test"})
        with self.assertRaises(GenerationError):
            raise_text_generation_error(error)

    def test_incomplete_generation_error(self):
        error = _mocked_error({"error_type": "incomplete_generation", "error": "test"})
        with self.assertRaises(IncompleteGenerationError):
            raise_text_generation_error(error)

    def test_overloaded_error(self):
        error = _mocked_error({"error_type": "overloaded", "error": "test"})
        with self.assertRaises(OverloadedError):
            raise_text_generation_error(error)

    def test_validation_error(self):
        error = _mocked_error({"error_type": "validation", "error": "test"})
        with self.assertRaises(TextGenerationValidationError):
            raise_text_generation_error(error)


def _mocked_error(payload: Dict) -> MagicMock:
    error = HTTPError(response=MagicMock())
    error.response.json.return_value = payload
    return error


@pytest.mark.vcr
@patch.dict("huggingface_hub.inference._client._NON_TGI_SERVERS", {})
class TestTextGenerationClientVCR(unittest.TestCase):
    """Use VCR test to avoid making requests to the prod infra."""

    def setUp(self) -> None:
        self.client = InferenceClient(model="google/flan-t5-xxl")
        return super().setUp()

    def test_generate_no_details(self):
        response = self.client.text_generation("test", details=False, max_new_tokens=1)

        assert response == ""

    def test_generate_with_details(self):
        response = self.client.text_generation("test", details=True, max_new_tokens=1, decoder_input_details=True)

        assert response.generated_text == ""
        assert response.details.finish_reason == FinishReason.Length
        assert response.details.generated_tokens == 1
        assert response.details.seed is None
        assert len(response.details.prefill) == 1
        assert response.details.prefill[0] == InputToken(id=0, text="<pad>", logprob=None)
        assert len(response.details.tokens) == 1
        assert response.details.tokens[0].id == 3
        assert response.details.tokens[0].text == " "
        assert not response.details.tokens[0].special

    def test_generate_best_of(self):
        response = self.client.text_generation(
            "test", max_new_tokens=1, best_of=2, do_sample=True, decoder_input_details=True, details=True
        )

        assert response.details.seed is not None
        assert response.details.best_of_sequences is not None
        assert len(response.details.best_of_sequences) == 1
        assert response.details.best_of_sequences[0].seed is not None

    def test_generate_validation_error(self):
        with self.assertRaises(TextGenerationValidationError):
            self.client.text_generation("test", max_new_tokens=10_000)

    def test_generate_stream_no_details(self):
        responses = [
            response for response in self.client.text_generation("test", max_new_tokens=1, stream=True, details=True)
        ]

        assert len(responses) == 1
        response = responses[0]

        assert response.generated_text == ""
        assert response.details.finish_reason == FinishReason.Length
        assert response.details.generated_tokens == 1
        assert response.details.seed is None

    def test_generate_stream_with_details(self):
        responses = [
            response for response in self.client.text_generation("test", max_new_tokens=1, stream=True, details=True)
        ]

        assert len(responses) == 1
        response = responses[0]

        assert response.generated_text == ""
        assert response.details.finish_reason == FinishReason.Length
        assert response.details.generated_tokens == 1
        assert response.details.seed is None

    def test_generate_non_tgi_endpoint(self):
        text = self.client.text_generation("0 1 2", model="gpt2", max_new_tokens=10)
        self.assertEqual(text, " 3 4 5 6 7 8 9 10 11 12")
        self.assertIn("gpt2", _NON_TGI_SERVERS)

        # Watermark is ignored (+ warning)
        with self.assertWarns(UserWarning):
            self.client.text_generation("4 5 6", model="gpt2", max_new_tokens=10, watermark=True)

        # Return as detail even if details=True (+ warning)
        with self.assertWarns(UserWarning):
            text = self.client.text_generation("0 1 2", model="gpt2", max_new_tokens=10, details=True)
            self.assertIsInstance(text, str)

        # Return as stream raises error
        with self.assertRaises(ValueError):
            self.client.text_generation("0 1 2", model="gpt2", max_new_tokens=10, stream=True)


# tests/test_init_lazy_loading.py

import unittest

import jedi


class TestHuggingfaceHubInit(unittest.TestCase):
    def test_autocomplete_on_root_imports(self) -> None:
        """Test autocomplete with `huggingface_hub` works with Jedi.

        Not all autocomplete systems are based on Jedi but if this one works we can
        assume others do as well.
        """
        source = """from huggingface_hub import c"""
        script = jedi.Script(source, path="example.py")
        completions = script.complete(1, len(source))

        for completion in completions:
            if completion.name == "create_commit":
                # Assert `create_commit` is suggestion from `huggingface_hub` lib
                self.assertEqual(completion.module_name, "huggingface_hub")

                # Assert autocomplete knows where `create_commit` lives
                # It would not be the case with a dynamic import.
                goto_list = completion.goto()
                self.assertEqual(len(goto_list), 1)

                # Assert docstring is find. This means autocomplete can also provide
                # the help section.
                signature_list = goto_list[0].get_signatures()
                self.assertEqual(len(signature_list), 2)  # create_commit has 2 signatures (normal and `run_as_future`)
                self.assertTrue(signature_list[0].docstring().startswith("create_commit(repo_id: str,"))
                break
        else:
            self.fail(
                "Jedi autocomplete did not suggest `create_commit` to complete the"
                f" line `{source}`. It is most probable that static imports are not"
                " correct in `./src/huggingface_hub/__init__.py`. Please run `make"
                " style` to fix this."
            )


# tests/test_keras_integration.py

import json
import os
import re
import unittest
from pathlib import Path

import pytest

from huggingface_hub import HfApi, hf_hub_download, snapshot_download
from huggingface_hub.keras_mixin import (
    KerasModelHubMixin,
    from_pretrained_keras,
    push_to_hub_keras,
    save_pretrained_keras,
)
from huggingface_hub.utils import (
    is_graphviz_available,
    is_pydot_available,
    is_tf_available,
    logging,
)

from .testing_constants import ENDPOINT_STAGING, TOKEN, USER
from .testing_utils import (
    repo_name,
    retry_endpoint,
)


logger = logging.get_logger(__name__)

WORKING_REPO_SUBDIR = f"fixtures/working_repo_{__name__.split('.')[-1]}"
WORKING_REPO_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), WORKING_REPO_SUBDIR)

PUSH_TO_HUB_KERAS_WARNING_REGEX = re.escape("Deprecated argument(s) used in 'push_to_hub_keras':")

if is_tf_available():
    import tensorflow as tf


def require_tf(test_case):
    """
    Decorator marking a test that requires TensorFlow, graphviz and pydot.

    These tests are skipped when TensorFlow, graphviz and pydot are installed.

    """
    if not is_tf_available() or not is_pydot_available() or not is_graphviz_available():
        return unittest.skip("test requires Tensorflow, graphviz and pydot.")(test_case)
    else:
        return test_case


if is_tf_available():
    # Define dummy mixin model...
    class DummyModel(tf.keras.Model, KerasModelHubMixin):
        def __init__(self, **kwargs):
            super().__init__()
            self.l1 = tf.keras.layers.Dense(2, activation="relu")
            dummy_batch_size = input_dim = 2
            self.dummy_inputs = tf.ones([dummy_batch_size, input_dim])

        def call(self, x):
            return self.l1(x)

else:
    DummyModel = None


@require_tf
@pytest.mark.usefixtures("fx_cache_dir")
class CommonKerasTest(unittest.TestCase):
    cache_dir: Path

    @classmethod
    def setUpClass(cls):
        """
        Share this valid token in all tests below.
        """
        cls._api = HfApi(endpoint=ENDPOINT_STAGING, token=TOKEN)


class HubMixingTestKeras(CommonKerasTest):
    def test_save_pretrained(self):
        model = DummyModel()
        model(model.dummy_inputs)

        model.save_pretrained(self.cache_dir)
        files = os.listdir(self.cache_dir)
        self.assertTrue("saved_model.pb" in files)
        self.assertTrue("keras_metadata.pb" in files)
        self.assertTrue("README.md" in files)
        self.assertTrue("model.png" in files)
        self.assertEqual(len(files), 7)

        model.save_pretrained(self.cache_dir, config={"num": 12, "act": "gelu"})
        files = os.listdir(self.cache_dir)
        self.assertTrue("config.json" in files)
        self.assertTrue("saved_model.pb" in files)
        self.assertEqual(len(files), 8)

    def test_keras_from_pretrained_weights(self):
        model = DummyModel()
        model(model.dummy_inputs)

        model.save_pretrained(self.cache_dir)
        new_model = DummyModel.from_pretrained(self.cache_dir)

        # Check the reloaded model's weights match the original model's weights
        self.assertTrue(tf.reduce_all(tf.equal(new_model.weights[0], model.weights[0])))

        # Check a new model's weights are not the same as the reloaded model's weights
        another_model = DummyModel()
        another_model(tf.ones([2, 2]))
        self.assertFalse(tf.reduce_all(tf.equal(new_model.weights[0], another_model.weights[0])).numpy().item())

    def test_abs_path_from_pretrained(self):
        model = DummyModel()
        model(model.dummy_inputs)
        model.save_pretrained(self.cache_dir, config={"num": 10, "act": "gelu_fast"})
        model = DummyModel.from_pretrained(self.cache_dir)
        self.assertTrue(model.config == {"num": 10, "act": "gelu_fast"})

    @retry_endpoint
    def test_push_to_hub_keras_mixin_via_http_basic(self):
        repo_id = f"{USER}/{repo_name()}"

        model = DummyModel()
        model(model.dummy_inputs)

        model.push_to_hub(
            repo_id=repo_id, api_endpoint=ENDPOINT_STAGING, token=TOKEN, config={"num": 7, "act": "gelu_fast"}
        )

        # Test model id exists
        model_info = self._api.model_info(repo_id)
        self.assertEqual(model_info.modelId, repo_id)

        # Test config has been pushed to hub
        config_path = hf_hub_download(
            repo_id=repo_id, filename="config.json", use_auth_token=TOKEN, cache_dir=self.cache_dir
        )
        with open(config_path) as f:
            self.assertEqual(json.load(f), {"num": 7, "act": "gelu_fast"})

        # Delete tmp file and repo
        self._api.delete_repo(repo_id=repo_id)


@require_tf
class HubKerasSequentialTest(CommonKerasTest):
    def model_init(self):
        model = tf.keras.models.Sequential()
        model.add(tf.keras.layers.Dense(2, activation="relu"))
        model.compile(optimizer="adam", loss="mse")
        return model

    def model_fit(self, model):
        x = tf.constant([[0.44, 0.90], [0.65, 0.39]])
        y = tf.constant([[1, 1], [0, 0]])
        model.fit(x, y)
        return model

    def test_save_pretrained(self):
        model = self.model_init()
        with pytest.raises(ValueError, match="Model should be built*"):
            save_pretrained_keras(model, save_directory=self.cache_dir)
        model.build((None, 2))

        save_pretrained_keras(model, save_directory=self.cache_dir)
        files = os.listdir(self.cache_dir)
        self.assertIn("saved_model.pb", files)
        self.assertIn("keras_metadata.pb", files)
        self.assertIn("model.png", files)
        self.assertIn("README.md", files)
        self.assertEqual(len(files), 7)

        loaded_model = from_pretrained_keras(self.cache_dir)
        self.assertIsNone(loaded_model.optimizer)

    def test_save_pretrained_model_card_fit(self):
        model = self.model_init()
        model = self.model_fit(model)

        save_pretrained_keras(model, save_directory=self.cache_dir)
        files = os.listdir(self.cache_dir)
        history = json.loads((self.cache_dir / "history.json").read_text())

        self.assertIn("saved_model.pb", files)
        self.assertIn("keras_metadata.pb", files)
        self.assertIn("model.png", files)
        self.assertIn("README.md", files)
        self.assertIn("history.json", files)
        self.assertEqual(history, model.history.history)
        self.assertEqual(len(files), 8)

    def test_save_model_card_history_removal(self):
        model = self.model_init()
        model = self.model_fit(model)

        history_path = self.cache_dir / "history.json"
        history_path.write_text("Keras FTW")

        with pytest.warns(UserWarning, match="`history.json` file already exists, *"):
            save_pretrained_keras(model, save_directory=self.cache_dir)
            # assert that it's not the same as old history file and it's overridden
            self.assertNotEqual("Keras FTW", history_path.read_text())

            # Check the history is saved as a json in the repository.
            files = os.listdir(self.cache_dir)
            self.assertIn("history.json", files)

            # Check that there is no "Training Metrics" section in the model card.
            # This was done in an older version.
            self.assertNotIn("Training Metrics", (self.cache_dir / "README.md").read_text())

    def test_save_pretrained_optimizer_state(self):
        model = self.model_init()
        model.build((None, 2))
        save_pretrained_keras(model, self.cache_dir, include_optimizer=True)
        loaded_model = from_pretrained_keras(self.cache_dir)
        self.assertIsNotNone(loaded_model.optimizer)

    def test_from_pretrained_weights(self):
        model = self.model_init()
        model.build((None, 2))

        save_pretrained_keras(model, self.cache_dir)
        new_model = from_pretrained_keras(self.cache_dir)

        # Check a new model's weights are not the same as the reloaded model's weights
        another_model = DummyModel()
        another_model(tf.ones([2, 2]))
        self.assertFalse(tf.reduce_all(tf.equal(new_model.weights[0], another_model.weights[0])).numpy().item())

    def test_save_pretrained_task_name_deprecation(self):
        model = self.model_init()
        model.build((None, 2))

        with pytest.warns(
            FutureWarning,
            match="`task_name` input argument is deprecated. Pass `tags` instead.",
        ):
            save_pretrained_keras(model, self.cache_dir, tags=["test"], task_name="test", save_traces=True)

    def test_abs_path_from_pretrained(self):
        model = self.model_init()
        model.build((None, 2))
        save_pretrained_keras(
            model, self.cache_dir, config={"num": 10, "act": "gelu_fast"}, plot_model=True, tags=None
        )
        new_model = from_pretrained_keras(self.cache_dir)
        self.assertTrue(tf.reduce_all(tf.equal(new_model.weights[0], model.weights[0])))
        self.assertTrue(new_model.config == {"num": 10, "act": "gelu_fast"})

    @retry_endpoint
    def test_push_to_hub_keras_sequential_via_http_basic(self):
        repo_id = f"{USER}/{repo_name()}"
        model = self.model_init()
        model = self.model_fit(model)

        push_to_hub_keras(model, repo_id=repo_id, token=TOKEN, api_endpoint=ENDPOINT_STAGING)
        model_info = self._api.model_info(repo_id)
        self.assertEqual(model_info.modelId, repo_id)
        self.assertTrue("README.md" in [f.rfilename for f in model_info.siblings])
        self.assertTrue("model.png" in [f.rfilename for f in model_info.siblings])
        self._api.delete_repo(repo_id=repo_id)

    @retry_endpoint
    def test_push_to_hub_keras_sequential_via_http_plot_false(self):
        repo_id = f"{USER}/{repo_name()}"
        model = self.model_init()
        model = self.model_fit(model)

        push_to_hub_keras(model, repo_id=repo_id, token=TOKEN, api_endpoint=ENDPOINT_STAGING, plot_model=False)
        model_info = self._api.model_info(repo_id)
        self.assertFalse("model.png" in [f.rfilename for f in model_info.siblings])
        self._api.delete_repo(repo_id=repo_id)

    @retry_endpoint
    def test_push_to_hub_keras_via_http_override_tensorboard(self):
        """Test log directory is overwritten when pushing a keras model a 2nd time."""
        repo_id = f"{USER}/{repo_name()}"

        log_dir = self.cache_dir / "tb_log_dir"
        log_dir.mkdir(parents=True, exist_ok=True)
        (log_dir / "tensorboard.txt").write_text("Keras FTW")

        model = self.model_init()
        model.build((None, 2))
        push_to_hub_keras(model, repo_id=repo_id, log_dir=log_dir, api_endpoint=ENDPOINT_STAGING, token=TOKEN)

        log_dir2 = self.cache_dir / "tb_log_dir2"
        log_dir2.mkdir(parents=True, exist_ok=True)
        (log_dir2 / "override.txt").write_text("Keras FTW")
        push_to_hub_keras(model, repo_id=repo_id, log_dir=log_dir2, api_endpoint=ENDPOINT_STAGING, token=TOKEN)

        files = self._api.list_repo_files(repo_id)
        self.assertIn("logs/override.txt", files)
        self.assertNotIn("logs/tensorboard.txt", files)

        self._api.delete_repo(repo_id=repo_id)

    @retry_endpoint
    def test_push_to_hub_keras_via_http_with_model_kwargs(self):
        repo_id = f"{USER}/{repo_name()}"

        model = self.model_init()
        model = self.model_fit(model)
        push_to_hub_keras(
            model,
            repo_id=repo_id,
            api_endpoint=ENDPOINT_STAGING,
            token=TOKEN,
            include_optimizer=True,
            save_traces=False,
        )

        model_info = self._api.model_info(repo_id)
        self.assertEqual(model_info.modelId, repo_id)

        snapshot_path = snapshot_download(repo_id=repo_id, cache_dir=self.cache_dir)
        from_pretrained_keras(snapshot_path)

        self._api.delete_repo(repo_id)


@require_tf
class HubKerasFunctionalTest(CommonKerasTest):
    def model_init(self):
        inputs = tf.keras.layers.Input(shape=(2,))
        outputs = tf.keras.layers.Dense(2, activation="relu")(inputs)
        model = tf.keras.models.Model(inputs=inputs, outputs=outputs)
        model.compile(optimizer="adam", loss="mse")
        return model

    def model_fit(self, model):
        x = tf.constant([[0.44, 0.90], [0.65, 0.39]])
        y = tf.constant([[1, 1], [0, 0]])
        model.fit(x, y)
        return model

    def test_save_pretrained(self):
        model = self.model_init()
        model.build((None, 2))
        self.assertTrue(model.built)

        save_pretrained_keras(model, self.cache_dir)
        files = os.listdir(self.cache_dir)

        self.assertIn("saved_model.pb", files)
        self.assertIn("keras_metadata.pb", files)
        self.assertEqual(len(files), 7)

    def test_save_pretrained_fit(self):
        model = self.model_init()
        model = self.model_fit(model)

        save_pretrained_keras(model, self.cache_dir)
        files = os.listdir(self.cache_dir)

        self.assertIn("saved_model.pb", files)
        self.assertIn("keras_metadata.pb", files)
        self.assertEqual(len(files), 8)


# tests/test_lfs.py

import os
import unittest
from hashlib import sha256
from io import BytesIO

from huggingface_hub.lfs import SliceFileObj, UploadInfo
from huggingface_hub.utils import SoftTemporaryDirectory


class TestUploadInfo(unittest.TestCase):
    def setUp(self) -> None:
        self.content = b"RandOm ConTEnT" * 1024
        self.size = len(self.content)
        self.sha = sha256(self.content).digest()
        self.sample = self.content[:512]

    def test_upload_info_from_path(self):
        with SoftTemporaryDirectory() as tmpdir:
            filepath = os.path.join(tmpdir, "file.bin")
            with open(filepath, "wb+") as file:
                file.write(self.content)
            upload_info = UploadInfo.from_path(filepath)

        self.assertEqual(upload_info.sample, self.sample)
        self.assertEqual(upload_info.size, self.size)
        self.assertEqual(upload_info.sha256, self.sha)

    def test_upload_info_from_bytes(self):
        upload_info = UploadInfo.from_bytes(self.content)

        self.assertEqual(upload_info.sample, self.sample)
        self.assertEqual(upload_info.size, self.size)
        self.assertEqual(upload_info.sha256, self.sha)

    def test_upload_info_from_bytes_io(self):
        upload_info = UploadInfo.from_fileobj(BytesIO(self.content))

        self.assertEqual(upload_info.sample, self.sample)
        self.assertEqual(upload_info.size, self.size)
        self.assertEqual(upload_info.sha256, self.sha)


class TestSliceFileObj(unittest.TestCase):
    def setUp(self) -> None:
        self.content = b"RANDOM self.content uauabciabeubahveb" * 1024

    def test_slice_fileobj_BytesIO(self):
        fileobj = BytesIO(self.content)
        prev_pos = fileobj.tell()

        # Test read
        with SliceFileObj(fileobj, seek_from=24, read_limit=18) as fileobj_slice:
            self.assertEqual(fileobj_slice.tell(), 0)
            self.assertEqual(fileobj_slice.read(), self.content[24:42])
            self.assertEqual(fileobj_slice.tell(), 18)
            self.assertEqual(fileobj_slice.read(), b"")
            self.assertEqual(fileobj_slice.tell(), 18)

        self.assertEqual(fileobj.tell(), prev_pos)

        with SliceFileObj(fileobj, seek_from=0, read_limit=990) as fileobj_slice:
            self.assertEqual(fileobj_slice.tell(), 0)
            self.assertEqual(fileobj_slice.read(200), self.content[0:200])
            self.assertEqual(fileobj_slice.read(500), self.content[200:700])
            self.assertEqual(fileobj_slice.read(200), self.content[700:900])
            self.assertEqual(fileobj_slice.read(200), self.content[900:990])
            self.assertEqual(fileobj_slice.read(200), b"")

        # Test seek with whence = os.SEEK_SET
        with SliceFileObj(fileobj, seek_from=100, read_limit=100) as fileobj_slice:
            self.assertEqual(fileobj_slice.tell(), 0)
            fileobj_slice.seek(2, os.SEEK_SET)
            self.assertEqual(fileobj_slice.tell(), 2)
            self.assertEqual(fileobj_slice.fileobj.tell(), 102)
            fileobj_slice.seek(-4, os.SEEK_SET)
            self.assertEqual(fileobj_slice.tell(), 0)
            self.assertEqual(fileobj_slice.fileobj.tell(), 100)
            fileobj_slice.seek(100 + 4, os.SEEK_SET)
            self.assertEqual(fileobj_slice.tell(), 100)
            self.assertEqual(fileobj_slice.fileobj.tell(), 200)

        # Test seek with whence = os.SEEK_CUR
        with SliceFileObj(fileobj, seek_from=100, read_limit=100) as fileobj_slice:
            self.assertEqual(fileobj_slice.tell(), 0)
            fileobj_slice.seek(-5, os.SEEK_CUR)
            self.assertEqual(fileobj_slice.tell(), 0)
            self.assertEqual(fileobj_slice.fileobj.tell(), 100)
            fileobj_slice.seek(50, os.SEEK_CUR)
            self.assertEqual(fileobj_slice.tell(), 50)
            self.assertEqual(fileobj_slice.fileobj.tell(), 150)
            fileobj_slice.seek(100, os.SEEK_CUR)
            self.assertEqual(fileobj_slice.tell(), 100)
            self.assertEqual(fileobj_slice.fileobj.tell(), 200)
            fileobj_slice.seek(-300, os.SEEK_CUR)
            self.assertEqual(fileobj_slice.tell(), 0)
            self.assertEqual(fileobj_slice.fileobj.tell(), 100)

        # Test seek with whence = os.SEEK_END
        with SliceFileObj(fileobj, seek_from=100, read_limit=100) as fileobj_slice:
            self.assertEqual(fileobj_slice.tell(), 0)
            fileobj_slice.seek(-5, os.SEEK_END)
            self.assertEqual(fileobj_slice.tell(), 95)
            self.assertEqual(fileobj_slice.fileobj.tell(), 195)
            fileobj_slice.seek(50, os.SEEK_END)
            self.assertEqual(fileobj_slice.tell(), 100)
            self.assertEqual(fileobj_slice.fileobj.tell(), 200)
            fileobj_slice.seek(-200, os.SEEK_END)
            self.assertEqual(fileobj_slice.tell(), 0)
            self.assertEqual(fileobj_slice.fileobj.tell(), 100)

    def test_slice_fileobj_file(self):
        self.content = b"RANDOM self.content uauabciabeubahveb" * 1024

        with SoftTemporaryDirectory() as tmpdir:
            filepath = os.path.join(tmpdir, "file.bin")
            with open(filepath, "wb+") as f:
                f.write(self.content)
            with open(filepath, "rb") as fileobj:
                prev_pos = fileobj.tell()
                # Test read
                with SliceFileObj(fileobj, seek_from=24, read_limit=18) as fileobj_slice:
                    self.assertEqual(fileobj_slice.tell(), 0)
                    self.assertEqual(fileobj_slice.read(), self.content[24:42])
                    self.assertEqual(fileobj_slice.tell(), 18)
                    self.assertEqual(fileobj_slice.read(), b"")
                    self.assertEqual(fileobj_slice.tell(), 18)

                self.assertEqual(fileobj.tell(), prev_pos)

                with SliceFileObj(fileobj, seek_from=0, read_limit=990) as fileobj_slice:
                    self.assertEqual(fileobj_slice.tell(), 0)
                    self.assertEqual(fileobj_slice.read(200), self.content[0:200])
                    self.assertEqual(fileobj_slice.read(500), self.content[200:700])
                    self.assertEqual(fileobj_slice.read(200), self.content[700:900])
                    self.assertEqual(fileobj_slice.read(200), self.content[900:990])
                    self.assertEqual(fileobj_slice.read(200), b"")

                # Test seek with whence = os.SEEK_SET
                with SliceFileObj(fileobj, seek_from=100, read_limit=100) as fileobj_slice:
                    self.assertEqual(fileobj_slice.tell(), 0)
                    fileobj_slice.seek(2, os.SEEK_SET)
                    self.assertEqual(fileobj_slice.tell(), 2)
                    self.assertEqual(fileobj_slice.fileobj.tell(), 102)
                    fileobj_slice.seek(-4, os.SEEK_SET)
                    self.assertEqual(fileobj_slice.tell(), 0)
                    self.assertEqual(fileobj_slice.fileobj.tell(), 100)
                    fileobj_slice.seek(100 + 4, os.SEEK_SET)
                    self.assertEqual(fileobj_slice.tell(), 100)
                    self.assertEqual(fileobj_slice.fileobj.tell(), 200)

                # Test seek with whence = os.SEEK_CUR
                with SliceFileObj(fileobj, seek_from=100, read_limit=100) as fileobj_slice:
                    self.assertEqual(fileobj_slice.tell(), 0)
                    fileobj_slice.seek(-5, os.SEEK_CUR)
                    self.assertEqual(fileobj_slice.tell(), 0)
                    self.assertEqual(fileobj_slice.fileobj.tell(), 100)
                    fileobj_slice.seek(50, os.SEEK_CUR)
                    self.assertEqual(fileobj_slice.tell(), 50)
                    self.assertEqual(fileobj_slice.fileobj.tell(), 150)
                    fileobj_slice.seek(100, os.SEEK_CUR)
                    self.assertEqual(fileobj_slice.tell(), 100)
                    self.assertEqual(fileobj_slice.fileobj.tell(), 200)
                    fileobj_slice.seek(-300, os.SEEK_CUR)
                    self.assertEqual(fileobj_slice.tell(), 0)
                    self.assertEqual(fileobj_slice.fileobj.tell(), 100)

                # Test seek with whence = os.SEEK_END
                with SliceFileObj(fileobj, seek_from=100, read_limit=100) as fileobj_slice:
                    self.assertEqual(fileobj_slice.tell(), 0)
                    fileobj_slice.seek(-5, os.SEEK_END)
                    self.assertEqual(fileobj_slice.tell(), 95)
                    self.assertEqual(fileobj_slice.fileobj.tell(), 195)
                    fileobj_slice.seek(50, os.SEEK_END)
                    self.assertEqual(fileobj_slice.tell(), 100)
                    self.assertEqual(fileobj_slice.fileobj.tell(), 200)
                    fileobj_slice.seek(-200, os.SEEK_END)
                    self.assertEqual(fileobj_slice.tell(), 0)
                    self.assertEqual(fileobj_slice.fileobj.tell(), 100)


# tests/test_login_utils.py

import subprocess
import unittest
from typing import Optional

from huggingface_hub._login import _set_store_as_git_credential_helper_globally
from huggingface_hub.utils import run_subprocess


class TestSetGlobalStore(unittest.TestCase):
    previous_config: Optional[str]

    def setUp(self) -> None:
        """Get current global config value."""
        try:
            self.previous_config = run_subprocess("git config --global credential.helper").stdout
        except subprocess.CalledProcessError:
            self.previous_config = None  # Means global credential.helper value not set

        run_subprocess("git config --global credential.helper store")

    def tearDown(self) -> None:
        """Reset global config value."""
        if self.previous_config is None:
            run_subprocess("git config --global --unset credential.helper")
        else:
            run_subprocess(f"git config --global credential.helper {self.previous_config}")

    def test_set_store_as_git_credential_helper_globally(self) -> None:
        """Test `_set_store_as_git_credential_helper_globally` works as expected.

        Previous value from the machine is restored after the test.
        """
        _set_store_as_git_credential_helper_globally()
        new_config = run_subprocess("git config --global credential.helper").stdout
        self.assertEqual(new_config, "store\n")


# tests/test_offline_utils.py

from io import BytesIO

import pytest
import requests

from huggingface_hub.file_download import http_get

from .testing_utils import (
    OfflineSimulationMode,
    RequestWouldHangIndefinitelyError,
    offline,
)


def test_offline_with_timeout():
    with offline(OfflineSimulationMode.CONNECTION_TIMES_OUT):
        with pytest.raises(RequestWouldHangIndefinitelyError):
            requests.request("GET", "https://huggingface.co")
        with pytest.raises(requests.exceptions.ConnectTimeout):
            requests.request("GET", "https://huggingface.co", timeout=1.0)
        with pytest.raises(requests.exceptions.ConnectTimeout):
            http_get("https://huggingface.co", BytesIO())


def test_offline_with_connection_error():
    with offline(OfflineSimulationMode.CONNECTION_FAILS):
        with pytest.raises(requests.exceptions.ConnectionError):
            requests.request("GET", "https://huggingface.co")
        with pytest.raises(requests.exceptions.ConnectionError):
            http_get("https://huggingface.co", BytesIO())


def test_offline_with_datasets_offline_mode_enabled():
    with offline(OfflineSimulationMode.HF_HUB_OFFLINE_SET_TO_1):
        with pytest.raises(ConnectionError):
            http_get("https://huggingface.co", BytesIO())


# tests/test_repocard.py

# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import os
import re
import unittest
from pathlib import Path

import pytest
import requests
import yaml

from huggingface_hub import (
    DatasetCard,
    DatasetCardData,
    EvalResult,
    ModelCard,
    ModelCardData,
    RepoCard,
    SpaceCard,
    SpaceCardData,
    metadata_eval_result,
    metadata_load,
    metadata_save,
    metadata_update,
)
from huggingface_hub.constants import REPOCARD_NAME
from huggingface_hub.file_download import hf_hub_download
from huggingface_hub.hf_api import HfApi
from huggingface_hub.repocard import REGEX_YAML_BLOCK
from huggingface_hub.repocard_data import CardData
from huggingface_hub.utils import SoftTemporaryDirectory, is_jinja_available, logging

from .testing_constants import (
    ENDPOINT_STAGING,
    ENDPOINT_STAGING_BASIC_AUTH,
    TOKEN,
    USER,
)
from .testing_utils import (
    repo_name,
    retry_endpoint,
    with_production_testing,
)


SAMPLE_CARDS_DIR = Path(__file__).parent / "fixtures/cards"

ROUND_TRIP_MODELCARD_CASE = """
---
language: no
datasets: CLUECorpusSmall
widget:
- text: 北京是[MASK]国的首都。
---

# Title
"""

DUMMY_MODELCARD = """
---
license: mit
datasets:
- foo
- bar
---

Hello
"""

DUMMY_MODELCARD_TARGET = """---
meaning_of_life: 42
---

Hello
"""

DUMMY_MODELCARD_TARGET_WITH_EMOJI = """---
emoji: 🎁
---

Hello
"""

DUMMY_MODELCARD_TARGET_NO_YAML = """---
meaning_of_life: 42
---
Hello
"""

DUMMY_NEW_MODELCARD_TARGET = """---
meaning_of_life: 42
---
"""

DUMMY_MODELCARD_TARGET_NO_TAGS = """
Hello
"""

DUMMY_MODELCARD_EVAL_RESULT = """---
model-index:
- name: RoBERTa fine-tuned on ReactionGIF
  results:
  - task:
      type: text-classification
      name: Text Classification
    dataset:
      name: ReactionGIF
      type: julien-c/reactiongif
      config: default
      split: test
    metrics:
    - type: accuracy
      value: 0.2662102282047272
      name: Accuracy
      config: default
      verified: true
---
"""

DUMMY_MODELCARD_NO_TEXT_CONTENT = """---
license: cc-by-sa-4.0
---
"""

DUMMY_MODELCARD_EVAL_RESULT_BOTH_VERIFIED_AND_UNVERIFIED = """---
model-index:
- name: RoBERTa fine-tuned on ReactionGIF
  results:
  - task:
      type: text-classification
      name: Text Classification
    dataset:
      name: ReactionGIF
      type: julien-c/reactiongif
      config: default
      split: test
    metrics:
    - type: accuracy
      value: 0.2662102282047272
      name: Accuracy
      config: default
      verified: false
  - task:
      type: text-classification
      name: Text Classification
    dataset:
      name: ReactionGIF
      type: julien-c/reactiongif
      config: default
      split: test
    metrics:
    - type: accuracy
      value: 0.6666666666666666
      name: Accuracy
      config: default
      verified: true
---

This is a test model card.
"""


def require_jinja(test_case):
    """
    Decorator marking a test that requires Jinja2.

    These tests are skipped when Jinja2 is not installed.
    """
    if not is_jinja_available():
        return unittest.skip("test requires Jinja2.")(test_case)
    else:
        return test_case


@pytest.mark.usefixtures("fx_cache_dir")
class RepocardMetadataTest(unittest.TestCase):
    cache_dir: Path

    def setUp(self) -> None:
        self.filepath = self.cache_dir / REPOCARD_NAME

    def test_metadata_load(self):
        self.filepath.write_text(DUMMY_MODELCARD)
        data = metadata_load(self.filepath)
        self.assertDictEqual(data, {"license": "mit", "datasets": ["foo", "bar"]})

    def test_metadata_save(self):
        self.filepath.write_text(DUMMY_MODELCARD)
        metadata_save(self.filepath, {"meaning_of_life": 42})
        content = self.filepath.read_text()
        self.assertEqual(content, DUMMY_MODELCARD_TARGET)

    def test_metadata_save_with_emoji_character(self):
        self.filepath.write_text(DUMMY_MODELCARD)
        metadata_save(self.filepath, {"emoji": "🎁"})
        content = self.filepath.read_text(encoding="utf-8")
        self.assertEqual(content, DUMMY_MODELCARD_TARGET_WITH_EMOJI)

    def test_metadata_save_from_file_no_yaml(self):
        self.filepath.write_text("Hello\n")
        metadata_save(self.filepath, {"meaning_of_life": 42})
        content = self.filepath.read_text()
        self.assertEqual(content, DUMMY_MODELCARD_TARGET_NO_YAML)

    def test_metadata_save_new_file(self):
        metadata_save(self.filepath, {"meaning_of_life": 42})
        content = self.filepath.read_text()
        self.assertEqual(content, DUMMY_NEW_MODELCARD_TARGET)

    def test_no_metadata_returns_none(self):
        self.filepath.write_text(DUMMY_MODELCARD_TARGET_NO_TAGS)
        data = metadata_load(self.filepath)
        self.assertEqual(data, None)

    def test_metadata_eval_result(self):
        data = metadata_eval_result(
            model_pretty_name="RoBERTa fine-tuned on ReactionGIF",
            task_pretty_name="Text Classification",
            task_id="text-classification",
            metrics_pretty_name="Accuracy",
            metrics_id="accuracy",
            metrics_value=0.2662102282047272,
            metrics_config="default",
            metrics_verified=True,
            dataset_pretty_name="ReactionGIF",
            dataset_id="julien-c/reactiongif",
            dataset_config="default",
            dataset_split="test",
        )
        metadata_save(self.filepath, data)
        content = self.filepath.read_text().splitlines()
        self.assertEqual(content, DUMMY_MODELCARD_EVAL_RESULT.splitlines())


class RepocardMetadataUpdateTest(unittest.TestCase):
    @retry_endpoint
    def setUp(self) -> None:
        self.token = TOKEN
        self.api = HfApi(token=TOKEN)

        self.repo_id = self.api.create_repo(repo_name()).repo_id
        self.api.upload_file(
            path_or_fileobj=DUMMY_MODELCARD_EVAL_RESULT.encode(), repo_id=self.repo_id, path_in_repo=REPOCARD_NAME
        )
        self.existing_metadata = yaml.safe_load(DUMMY_MODELCARD_EVAL_RESULT.strip().strip("-"))

    def tearDown(self) -> None:
        self.api.delete_repo(repo_id=self.repo_id)

    def _get_remote_card(self) -> str:
        return hf_hub_download(repo_id=self.repo_id, filename=REPOCARD_NAME)

    def test_update_dataset_name(self):
        new_datasets_data = {"datasets": ["test/test_dataset"]}
        metadata_update(self.repo_id, new_datasets_data, token=self.token)

        hf_hub_download(repo_id=self.repo_id, filename=REPOCARD_NAME)
        updated_metadata = metadata_load(self._get_remote_card())
        expected_metadata = copy.deepcopy(self.existing_metadata)
        expected_metadata.update(new_datasets_data)
        self.assertDictEqual(updated_metadata, expected_metadata)

    def test_update_existing_result_with_overwrite(self):
        new_metadata = copy.deepcopy(self.existing_metadata)
        new_metadata["model-index"][0]["results"][0]["metrics"][0]["value"] = 0.2862102282047272
        metadata_update(self.repo_id, new_metadata, token=self.token, overwrite=True)

        updated_metadata = metadata_load(self._get_remote_card())
        self.assertDictEqual(updated_metadata, new_metadata)

    def test_update_verify_token(self):
        """Tests whether updating the verification token updates in-place.

        Regression test for https://github.com/huggingface/huggingface_hub/issues/1210
        """
        new_metadata = copy.deepcopy(self.existing_metadata)
        new_metadata["model-index"][0]["results"][0]["metrics"][0]["verifyToken"] = "1234"
        metadata_update(self.repo_id, new_metadata, token=self.token, overwrite=True)

        updated_metadata = metadata_load(self._get_remote_card())
        self.assertDictEqual(updated_metadata, new_metadata)

    def test_metadata_update_upstream(self):
        new_metadata = copy.deepcopy(self.existing_metadata)
        new_metadata["model-index"][0]["results"][0]["metrics"][0]["value"] = 0.1

        # download first, then update
        path = self._get_remote_card()
        metadata_update(self.repo_id, new_metadata, token=self.token, overwrite=True)

        self.assertNotEqual(metadata_load(path), new_metadata)
        self.assertEqual(metadata_load(path), self.existing_metadata)

    def test_update_existing_result_without_overwrite(self):
        new_metadata = copy.deepcopy(self.existing_metadata)
        new_metadata["model-index"][0]["results"][0]["metrics"][0]["value"] = 0.2862102282047272

        with pytest.raises(
            ValueError,
            match=(
                "You passed a new value for the existing metric 'name: Accuracy, type:"
                " accuracy'. Set `overwrite=True` to overwrite existing metrics."
            ),
        ):
            metadata_update(self.repo_id, new_metadata, token=self.token, overwrite=False)

    def test_update_existing_field_without_overwrite(self):
        new_datasets_data = {"datasets": "['test/test_dataset']"}
        metadata_update(self.repo_id, new_datasets_data, token=self.token)

        with pytest.raises(
            ValueError,
            match=(
                "You passed a new value for the existing meta data field 'datasets'."
                " Set `overwrite=True` to overwrite existing metadata."
            ),
        ):
            new_datasets_data = {"datasets": "['test/test_dataset_2']"}
            metadata_update(self.repo_id, new_datasets_data, token=self.token, overwrite=False)

    def test_update_new_result_existing_dataset(self):
        new_result = metadata_eval_result(
            model_pretty_name="RoBERTa fine-tuned on ReactionGIF",
            task_pretty_name="Text Classification",
            task_id="text-classification",
            metrics_pretty_name="Recall",
            metrics_id="recall",
            metrics_value=0.7762102282047272,
            metrics_config="default",
            metrics_verified=False,
            dataset_pretty_name="ReactionGIF",
            dataset_id="julien-c/reactiongif",
            dataset_config="default",
            dataset_split="test",
        )

        metadata_update(self.repo_id, new_result, token=self.token, overwrite=False)

        expected_metadata = copy.deepcopy(self.existing_metadata)
        expected_metadata["model-index"][0]["results"][0]["metrics"].append(
            new_result["model-index"][0]["results"][0]["metrics"][0]
        )

        updated_metadata = metadata_load(self._get_remote_card())
        self.assertDictEqual(updated_metadata, expected_metadata)

    def test_update_new_result_new_dataset(self):
        new_result = metadata_eval_result(
            model_pretty_name="RoBERTa fine-tuned on ReactionGIF",
            task_pretty_name="Text Classification",
            task_id="text-classification",
            metrics_pretty_name="Accuracy",
            metrics_id="accuracy",
            metrics_value=0.2662102282047272,
            metrics_config="default",
            metrics_verified=False,
            dataset_pretty_name="ReactionJPEG",
            dataset_id="julien-c/reactionjpeg",
            dataset_config="default",
            dataset_split="test",
        )

        metadata_update(self.repo_id, new_result, token=self.token, overwrite=False)

        expected_metadata = copy.deepcopy(self.existing_metadata)
        expected_metadata["model-index"][0]["results"].append(new_result["model-index"][0]["results"][0])

        updated_metadata = metadata_load(self._get_remote_card())
        self.assertDictEqual(updated_metadata, expected_metadata)

    def test_update_metadata_on_empty_text_content(self) -> None:
        """Test `update_metadata` on a model card that has metadata but no text content

        Regression test for https://github.com/huggingface/huggingface_hub/issues/1010
        """
        # Create modelcard with metadata but empty text content
        self.api.upload_file(
            path_or_fileobj=DUMMY_MODELCARD_NO_TEXT_CONTENT.encode(), path_in_repo=REPOCARD_NAME, repo_id=self.repo_id
        )
        metadata_update(self.repo_id, {"tag": "test"}, token=self.token)

        # Check update went fine
        updated_metadata = metadata_load(self._get_remote_card())
        expected_metadata = {"license": "cc-by-sa-4.0", "tag": "test"}
        self.assertDictEqual(updated_metadata, expected_metadata)

    def test_update_with_existing_name(self):
        new_metadata = copy.deepcopy(self.existing_metadata)
        new_metadata["model-index"][0].pop("name")
        new_metadata["model-index"][0]["results"][0]["metrics"][0]["value"] = 0.2862102282047272
        metadata_update(self.repo_id, new_metadata, token=self.token, overwrite=True)

        card_data = ModelCard.load(self.repo_id)
        self.assertEqual(card_data.data.model_name, self.existing_metadata["model-index"][0]["name"])

    def test_update_without_existing_name(self):
        # delete existing metadata
        self.api.upload_file(path_or_fileobj="# Test".encode(), repo_id=self.repo_id, path_in_repo="README.md")

        new_metadata = copy.deepcopy(self.existing_metadata)
        new_metadata["model-index"][0].pop("name")

        metadata_update(self.repo_id, new_metadata, token=self.token, overwrite=True)

        card_data = ModelCard.load(self.repo_id)
        self.assertEqual(card_data.data.model_name, self.repo_id)

    def test_update_with_both_verified_and_unverified_metric(self):
        """Regression test for #1185.

        See https://github.com/huggingface/huggingface_hub/issues/1185.
        """
        self.api.upload_file(
            path_or_fileobj=DUMMY_MODELCARD_EVAL_RESULT_BOTH_VERIFIED_AND_UNVERIFIED.encode(),
            repo_id=self.repo_id,
            path_in_repo="README.md",
        )
        card = ModelCard.load(self.repo_id)
        metadata = card.data.to_dict()
        metadata_update(self.repo_id, metadata=metadata, overwrite=True, token=self.token)

        new_card = ModelCard.load(self.repo_id)
        self.assertEqual(len(new_card.data.eval_results), 2)
        first_result = new_card.data.eval_results[0]
        second_result = new_card.data.eval_results[1]

        # One is verified, the other not
        self.assertFalse(first_result.verified)
        self.assertTrue(second_result.verified)

        # Result values are different
        self.assertEqual(first_result.metric_value, 0.2662102282047272)
        self.assertEqual(second_result.metric_value, 0.6666666666666666)


class TestMetadataUpdateOnMissingCard(unittest.TestCase):
    def setUp(self) -> None:
        """
        Share this valid token in all tests below.
        """
        self._token = TOKEN
        self._api = HfApi(endpoint=ENDPOINT_STAGING, token=TOKEN)
        self._repo_id = f"{USER}/{repo_name()}"

    def test_metadata_update_missing_readme_on_model(self) -> None:
        self._api.create_repo(self._repo_id)
        metadata_update(self._repo_id, {"tag": "this_is_a_test"}, token=self._token)
        model_card = ModelCard.load(self._repo_id, token=self._token)

        # Created a card with default template + metadata
        self.assertIn("# Model Card for Model ID", str(model_card))
        self.assertEqual(model_card.data.to_dict(), {"tag": "this_is_a_test"})

        self._api.delete_repo(self._repo_id)

    def test_metadata_update_missing_readme_on_dataset(self) -> None:
        self._api.create_repo(self._repo_id, repo_type="dataset")
        metadata_update(
            self._repo_id,
            {"tag": "this is a dataset test"},
            token=self._token,
            repo_type="dataset",
        )
        dataset_card = DatasetCard.load(self._repo_id, token=self._token)

        # Created a card with default template + metadata
        self.assertIn("# Dataset Card for Dataset Name", str(dataset_card))
        self.assertEqual(dataset_card.data.to_dict(), {"tag": "this is a dataset test"})

        self._api.delete_repo(self._repo_id, repo_type="dataset")

    def test_metadata_update_missing_readme_on_space(self) -> None:
        self._api.create_repo(self._repo_id, repo_type="space", space_sdk="static")
        self._api.delete_file("README.md", self._repo_id, repo_type="space")
        with self.assertRaises(ValueError):
            # Cannot create a default readme on a space repo (should be automatically
            # created on the Hub).
            metadata_update(
                self._repo_id,
                {"tag": "this is a space test"},
                token=self._token,
                repo_type="space",
            )
        self._api.delete_repo(self._repo_id, repo_type="space")


class TestCaseWithCapLog(unittest.TestCase):
    _api = HfApi(endpoint=ENDPOINT_STAGING, token=TOKEN)

    @pytest.fixture(autouse=True)
    def inject_fixtures(self, caplog):
        """Assign pytest caplog as attribute so we can use captured log messages in tests below."""
        self.caplog = caplog


class RepoCardTest(TestCaseWithCapLog):
    def test_load_repocard_from_file(self):
        sample_path = SAMPLE_CARDS_DIR / "sample_simple.md"
        card = RepoCard.load(sample_path)
        self.assertEqual(
            card.data.to_dict(),
            {
                "language": ["en"],
                "license": "mit",
                "library_name": "pytorch-lightning",
                "tags": ["pytorch", "image-classification"],
                "datasets": ["beans"],
                "metrics": ["acc"],
            },
        )
        self.assertTrue(
            card.text.strip().startswith("# my-cool-model"),
            "Card text not loaded properly",
        )

    def test_change_repocard_data(self):
        sample_path = SAMPLE_CARDS_DIR / "sample_simple.md"
        card = RepoCard.load(sample_path)
        card.data.language = ["fr"]

        with SoftTemporaryDirectory() as tempdir:
            updated_card_path = Path(tempdir) / "updated.md"
            card.save(updated_card_path)

            updated_card = RepoCard.load(updated_card_path)
            self.assertEqual(updated_card.data.language, ["fr"], "Card data not updated properly")

    @require_jinja
    def test_repo_card_from_default_template(self):
        card = RepoCard.from_template(
            card_data=CardData(
                language="en",
                license="mit",
                library_name="pytorch",
                tags=["image-classification", "resnet"],
                datasets="imagenet",
                metrics=["acc", "f1"],
            ),
            model_id=None,
        )
        self.assertIsInstance(card, RepoCard)
        self.assertTrue(
            card.text.strip().startswith("# Model Card for Model ID"),
            "Default model name not set correctly",
        )

    @require_jinja
    def test_repo_card_from_default_template_with_model_id(self):
        card = RepoCard.from_template(
            card_data=CardData(
                language="en",
                license="mit",
                library_name="pytorch",
                tags=["image-classification", "resnet"],
                datasets="imagenet",
                metrics=["acc", "f1"],
            ),
            model_id="my-cool-model",
        )
        self.assertTrue(
            card.text.strip().startswith("# Model Card for my-cool-model"),
            "model_id not properly set in card template",
        )

    @require_jinja
    def test_repo_card_from_custom_template(self):
        template_path = SAMPLE_CARDS_DIR / "sample_template.md"
        card = RepoCard.from_template(
            card_data=CardData(
                language="en",
                license="mit",
                library_name="pytorch",
                tags="text-classification",
                datasets="glue",
                metrics="acc",
            ),
            template_path=template_path,
            some_data="asdf",
        )
        self.assertTrue(
            card.text.endswith("asdf"),
            "Custom template didn't set jinja variable correctly",
        )

    def test_repo_card_data_must_be_dict(self):
        sample_path = SAMPLE_CARDS_DIR / "sample_invalid_card_data.md"
        with pytest.raises(ValueError, match="repo card metadata block should be a dict"):
            RepoCard(sample_path.read_text())

    def test_repo_card_without_metadata(self):
        sample_path = SAMPLE_CARDS_DIR / "sample_no_metadata.md"

        with self.caplog.at_level(logging.WARNING):
            card = RepoCard(sample_path.read_text())
        self.assertIn(
            "Repo card metadata block was not found. Setting CardData to empty.",
            self.caplog.text,
        )
        self.assertEqual(card.data, CardData())

    def test_validate_repocard(self):
        sample_path = SAMPLE_CARDS_DIR / "sample_simple.md"
        card = RepoCard.load(sample_path)
        card.validate()

        card.data.license = "asdf"
        with pytest.raises(ValueError, match='- Error: "license" must be one of'):
            card.validate()

    def test_push_to_hub(self):
        repo_id = f"{USER}/{repo_name('push-card')}"
        self._api.create_repo(repo_id)

        card_data = CardData(
            language="en",
            license="mit",
            library_name="pytorch",
            tags=["text-classification"],
            datasets="glue",
            metrics="acc",
        )
        # Mock what RepoCard.from_template does so we can test w/o Jinja2
        content = f"---\n{card_data.to_yaml()}\n---\n\n# MyModel\n\nHello, world!"
        card = RepoCard(content)

        url = f"{ENDPOINT_STAGING_BASIC_AUTH}/{repo_id}/resolve/main/README.md"

        # Check this file doesn't exist (sanity check)
        with pytest.raises(requests.exceptions.HTTPError):
            r = requests.get(url)
            r.raise_for_status()

        # Push the card up to README.md in the repo
        card.push_to_hub(repo_id, token=TOKEN)

        # No error should occur now, as README.md should exist
        r = requests.get(url)
        r.raise_for_status()

        self._api.delete_repo(repo_id=repo_id)

    def test_push_and_create_pr(self):
        repo_id = f"{USER}/{repo_name('pr-card')}"
        self._api.create_repo(repo_id)
        card_data = CardData(
            language="en",
            license="mit",
            library_name="pytorch",
            tags=["text-classification"],
            datasets="glue",
            metrics="acc",
        )
        # Mock what RepoCard.from_template does so we can test w/o Jinja2
        content = f"---\n{card_data.to_yaml()}\n---\n\n# MyModel\n\nHello, world!"
        card = RepoCard(content)

        url = f"{ENDPOINT_STAGING_BASIC_AUTH}/api/models/{repo_id}/discussions"
        r = requests.get(url)
        data = r.json()
        self.assertEqual(data["count"], 0)
        card.push_to_hub(repo_id, token=TOKEN, create_pr=True)
        r = requests.get(url)
        data = r.json()
        self.assertEqual(data["count"], 1)

        self._api.delete_repo(repo_id=repo_id)

    def test_preserve_windows_linebreaks(self):
        card_path = SAMPLE_CARDS_DIR / "sample_windows_line_breaks.md"
        card = RepoCard.load(card_path)
        self.assertIn("\r\n", str(card))

    def test_preserve_linebreaks_when_saving(self):
        card_path = SAMPLE_CARDS_DIR / "sample_simple.md"
        card = RepoCard.load(card_path)
        with SoftTemporaryDirectory() as tmpdir:
            tmpfile = os.path.join(tmpdir, "readme.md")
            card.save(tmpfile)
            card2 = RepoCard.load(tmpfile)
        self.assertEqual(str(card), str(card2))

    def test_updating_text_updates_content(self):
        sample_path = SAMPLE_CARDS_DIR / "sample_simple.md"
        card = RepoCard.load(sample_path)
        card.text = "Hello, world!"
        line_break = "\r\n" if os.name == "nt" else "\n"
        self.assertEqual(
            card.content,
            # line_break depends on platform. Correctly set when using RepoCard.save(...) to avoid diffs
            f"---\n{card.data.to_yaml()}\n---\nHello, world!".replace("\n", line_break),
        )


class TestRegexYamlBlock(unittest.TestCase):
    def test_match_with_leading_whitespace(self):
        self.assertIsNotNone(REGEX_YAML_BLOCK.search("   \n---\nmetadata: 1\n---"))

    def test_match_without_leading_whitespace(self):
        self.assertIsNotNone(REGEX_YAML_BLOCK.search("---\nmetadata: 1\n---"))

    def test_does_not_match_with_leading_text(self):
        self.assertIsNone(REGEX_YAML_BLOCK.search("something\n---\nmetadata: 1\n---"))


class ModelCardTest(TestCaseWithCapLog):
    def test_model_card_with_invalid_model_index(self):
        """Test raise an error when loading a card that has invalid model-index."""
        sample_path = SAMPLE_CARDS_DIR / "sample_invalid_model_index.md"
        with self.assertRaises(ValueError):
            ModelCard.load(sample_path)

    def test_model_card_with_invalid_model_index_and_ignore_error(self):
        """Test trigger a warning when loading a card that has invalid model-index and `ignore_metadata_errors=True`

        Some information is lost.
        """
        sample_path = SAMPLE_CARDS_DIR / "sample_invalid_model_index.md"
        with self.caplog.at_level(logging.WARNING):
            card = ModelCard.load(sample_path, ignore_metadata_errors=True)
        self.assertIn(
            "Invalid model-index. Not loading eval results into CardData.",
            self.caplog.text,
        )
        self.assertIsNone(card.data.eval_results)

    def test_model_card_with_model_index(self):
        """Test that loading a model card with multiple evaluations is consistent with `metadata_load`.

        Regression test for https://github.com/huggingface/huggingface_hub/issues/1208
        """
        sample_path = SAMPLE_CARDS_DIR / "sample_simple_model_index.md"
        card = ModelCard.load(sample_path)
        metadata = metadata_load(sample_path)
        self.assertDictEqual(card.data.to_dict(), metadata)

    def test_load_model_card_from_file(self):
        sample_path = SAMPLE_CARDS_DIR / "sample_simple.md"
        card = ModelCard.load(sample_path)
        self.assertIsInstance(card, ModelCard)
        self.assertEqual(
            card.data.to_dict(),
            {
                "language": ["en"],
                "license": "mit",
                "library_name": "pytorch-lightning",
                "tags": ["pytorch", "image-classification"],
                "datasets": ["beans"],
                "metrics": ["acc"],
            },
        )
        self.assertTrue(
            card.text.strip().startswith("# my-cool-model"),
            "Card text not loaded properly",
        )

    @require_jinja
    def test_model_card_from_custom_template(self):
        template_path = SAMPLE_CARDS_DIR / "sample_template.md"
        card = ModelCard.from_template(
            card_data=ModelCardData(
                language="en",
                license="mit",
                library_name="pytorch",
                tags="text-classification",
                datasets="glue",
                metrics="acc",
            ),
            template_path=template_path,
            some_data="asdf",
        )
        self.assertIsInstance(card, ModelCard)
        self.assertTrue(
            card.text.endswith("asdf"),
            "Custom template didn't set jinja variable correctly",
        )

    @require_jinja
    def test_model_card_from_template_eval_results(self):
        template_path = SAMPLE_CARDS_DIR / "sample_template.md"
        card = ModelCard.from_template(
            card_data=ModelCardData(
                eval_results=[
                    EvalResult(
                        task_type="text-classification",
                        task_name="Text Classification",
                        dataset_type="julien-c/reactiongif",
                        dataset_name="ReactionGIF",
                        dataset_config="default",
                        dataset_split="test",
                        metric_type="accuracy",
                        metric_value=0.2662102282047272,
                        metric_name="Accuracy",
                        metric_config="default",
                        verified=True,
                    ),
                ],
                model_name="RoBERTa fine-tuned on ReactionGIF",
            ),
            template_path=template_path,
            some_data="asdf",
        )
        self.assertIsInstance(card, ModelCard)
        self.assertTrue(card.text.endswith("asdf"))
        self.assertTrue(card.data.to_dict().get("eval_results") is None)
        self.assertEqual(str(card)[: len(DUMMY_MODELCARD_EVAL_RESULT)], DUMMY_MODELCARD_EVAL_RESULT)


class DatasetCardTest(TestCaseWithCapLog):
    def test_load_datasetcard_from_file(self):
        sample_path = SAMPLE_CARDS_DIR / "sample_datasetcard_simple.md"
        card = DatasetCard.load(sample_path)
        self.assertEqual(
            card.data.to_dict(),
            {
                "annotations_creators": ["crowdsourced", "expert-generated"],
                "language_creators": ["found"],
                "language": ["en"],
                "license": ["bsd-3-clause"],
                "multilinguality": ["monolingual"],
                "size_categories": ["n<1K"],
                "task_categories": ["image-segmentation"],
                "task_ids": ["semantic-segmentation"],
                "pretty_name": "Sample Segmentation",
            },
        )
        self.assertIsInstance(card, DatasetCard)
        self.assertIsInstance(card.data, DatasetCardData)
        self.assertTrue(card.text.strip().startswith("# Dataset Card for"))

    @require_jinja
    def test_dataset_card_from_default_template(self):
        card_data = DatasetCardData(
            language="en",
            license="mit",
        )

        # Here we check default title when pretty_name not provided.
        card = DatasetCard.from_template(card_data)
        self.assertTrue(card.text.strip().startswith("# Dataset Card for Dataset Name"))

        card_data = DatasetCardData(
            language="en",
            license="mit",
            pretty_name="My Cool Dataset",
        )

        # Here we pass the card data as kwargs as well so template picks up pretty_name.
        card = DatasetCard.from_template(card_data, **card_data.to_dict())
        self.assertTrue(card.text.strip().startswith("# Dataset Card for My Cool Dataset"))

        self.assertIsInstance(card, DatasetCard)

    @require_jinja
    def test_dataset_card_from_default_template_with_template_variables(self):
        card_data = DatasetCardData(
            language="en",
            license="mit",
            pretty_name="My Cool Dataset",
        )

        # Here we pass the card data as kwargs as well so template picks up pretty_name.
        card = DatasetCard.from_template(
            card_data,
            homepage_url="https://huggingface.co",
            repo_url="https://github.com/huggingface/huggingface_hub",
            paper_url="https://arxiv.org/pdf/1910.03771.pdf",
            point_of_contact="https://huggingface.co/nateraw",
            dataset_summary=(
                "This is a test dataset card to check if the template variables "
                "in the dataset card template are working."
            ),
        )
        self.assertTrue(card.text.strip().startswith("# Dataset Card for My Cool Dataset"))
        self.assertIsInstance(card, DatasetCard)

        matches = re.findall(r"Homepage:\*\* https:\/\/huggingface\.co", str(card))
        self.assertEqual(matches[0], "Homepage:** https://huggingface.co")

    @require_jinja
    def test_dataset_card_from_custom_template(self):
        card = DatasetCard.from_template(
            card_data=DatasetCardData(
                language="en",
                license="mit",
                pretty_name="My Cool Dataset",
            ),
            template_path=SAMPLE_CARDS_DIR / "sample_datasetcard_template.md",
            pretty_name="My Cool Dataset",
            some_data="asdf",
        )
        self.assertIsInstance(card, DatasetCard)

        # Title this time is just # {{ pretty_name }}
        self.assertTrue(card.text.strip().startswith("# My Cool Dataset"))

        # some_data is at the bottom of the template, so should end with whatever we passed to it
        self.assertTrue(card.text.strip().endswith("asdf"))


@with_production_testing
class SpaceCardTest(TestCaseWithCapLog):
    def test_load_spacecard_from_hub(self) -> None:
        card = SpaceCard.load("multimodalart/dreambooth-training")
        self.assertIsInstance(card, SpaceCard)
        self.assertIsInstance(card.data, SpaceCardData)
        self.assertEqual(card.data.title, "Dreambooth Training")
        self.assertIsNone(card.data.app_port)


# tests/test_repocard_data.py

import unittest

import pytest
import yaml

from huggingface_hub import SpaceCardData
from huggingface_hub.repocard_data import (
    CardData,
    DatasetCardData,
    EvalResult,
    ModelCardData,
    eval_results_to_model_index,
    model_index_to_eval_results,
)


DUMMY_METADATA_WITH_MODEL_INDEX = """
language: en
license: mit
library_name: timm
tags:
- pytorch
- image-classification
datasets:
- beans
metrics:
- acc
model-index:
- name: my-cool-model
  results:
  - task:
      type: image-classification
    dataset:
      type: beans
      name: Beans
    metrics:
    - type: acc
      value: 0.9
"""


class BaseCardDataTest(unittest.TestCase):
    def test_metadata_behave_as_dict(self):
        metadata = CardData(foo="bar")

        # .get and __getitem__
        self.assertEqual(metadata.get("foo"), "bar")
        self.assertEqual(metadata.get("FOO"), None)  # case sensitive
        self.assertEqual(metadata["foo"], "bar")
        with self.assertRaises(KeyError):  # case sensitive
            _ = metadata["FOO"]

        # __setitem__
        metadata["foo"] = "BAR"
        self.assertEqual(metadata.get("foo"), "BAR")
        self.assertEqual(metadata["foo"], "BAR")

        # __contains__
        self.assertTrue("foo" in metadata)
        self.assertFalse("FOO" in metadata)

        # export
        self.assertEqual(str(metadata), "foo: BAR")

        # .pop
        self.assertEqual(metadata.pop("foo"), "BAR")


class ModelCardDataTest(unittest.TestCase):
    def test_eval_results_to_model_index(self):
        expected_results = yaml.safe_load(DUMMY_METADATA_WITH_MODEL_INDEX)

        eval_results = [
            EvalResult(
                task_type="image-classification",
                dataset_type="beans",
                dataset_name="Beans",
                metric_type="acc",
                metric_value=0.9,
            ),
        ]

        model_index = eval_results_to_model_index("my-cool-model", eval_results)

        self.assertEqual(model_index, expected_results["model-index"])

    def test_model_index_to_eval_results(self):
        model_index = [
            {
                "name": "my-cool-model",
                "results": [
                    {
                        "task": {
                            "type": "image-classification",
                        },
                        "dataset": {
                            "type": "cats_vs_dogs",
                            "name": "Cats vs. Dogs",
                        },
                        "metrics": [
                            {
                                "type": "acc",
                                "value": 0.85,
                            },
                            {
                                "type": "f1",
                                "value": 0.9,
                            },
                        ],
                    },
                    {
                        "task": {
                            "type": "image-classification",
                        },
                        "dataset": {
                            "type": "beans",
                            "name": "Beans",
                        },
                        "metrics": [
                            {
                                "type": "acc",
                                "value": 0.9,
                                "verified": True,
                                "verifyToken": 1234,
                            }
                        ],
                    },
                ],
            }
        ]
        model_name, eval_results = model_index_to_eval_results(model_index)

        self.assertEqual(len(eval_results), 3)
        self.assertEqual(model_name, "my-cool-model")
        self.assertEqual(eval_results[0].dataset_type, "cats_vs_dogs")
        self.assertEqual(eval_results[1].metric_type, "f1")
        self.assertEqual(eval_results[1].metric_value, 0.9)
        self.assertEqual(eval_results[2].task_type, "image-classification")
        self.assertEqual(eval_results[2].dataset_type, "beans")
        self.assertEqual(eval_results[2].verified, True)
        self.assertEqual(eval_results[2].verify_token, 1234)

    def test_card_data_requires_model_name_for_eval_results(self):
        with pytest.raises(ValueError, match="`eval_results` requires `model_name` to be set."):
            ModelCardData(
                eval_results=[
                    EvalResult(
                        task_type="image-classification",
                        dataset_type="beans",
                        dataset_name="Beans",
                        metric_type="acc",
                        metric_value=0.9,
                    ),
                ],
            )

        data = ModelCardData(
            model_name="my-cool-model",
            eval_results=[
                EvalResult(
                    task_type="image-classification",
                    dataset_type="beans",
                    dataset_name="Beans",
                    metric_type="acc",
                    metric_value=0.9,
                ),
            ],
        )

        model_index = eval_results_to_model_index(data.model_name, data.eval_results)

        self.assertEqual(model_index[0]["name"], "my-cool-model")
        self.assertEqual(model_index[0]["results"][0]["task"]["type"], "image-classification")

    def test_arbitrary_incoming_card_data(self):
        data = ModelCardData(
            model_name="my-cool-model",
            eval_results=[
                EvalResult(
                    task_type="image-classification",
                    dataset_type="beans",
                    dataset_name="Beans",
                    metric_type="acc",
                    metric_value=0.9,
                ),
            ],
            some_arbitrary_kwarg="some_value",
        )

        self.assertEqual(data.some_arbitrary_kwarg, "some_value")

        data_dict = data.to_dict()
        self.assertEqual(data_dict["some_arbitrary_kwarg"], "some_value")


class DatasetCardDataTest(unittest.TestCase):
    def test_train_eval_index_keys_updated(self):
        train_eval_index = [
            {
                "config": "plain_text",
                "task": "text-classification",
                "task_id": "binary_classification",
                "splits": {"train_split": "train", "eval_split": "test"},
                "col_mapping": {"text": "text", "label": "target"},
                "metrics": [
                    {
                        "type": "accuracy",
                        "name": "Accuracy",
                    },
                    {"type": "f1", "name": "F1 macro", "args": {"average": "macro"}},
                ],
            }
        ]
        card_data = DatasetCardData(
            language="en",
            license="mit",
            pretty_name="My Cool Dataset",
            train_eval_index=train_eval_index,
        )
        # The init should have popped this out of kwargs and into train_eval_index attr
        self.assertEqual(card_data.train_eval_index, train_eval_index)
        # Underlying train_eval_index gets converted to train-eval-index in DatasetCardData._to_dict.
        # So train_eval_index should be None in the dict
        self.assertTrue(card_data.to_dict().get("train_eval_index") is None)
        # And train-eval-index should be in the dict
        self.assertEqual(card_data.to_dict()["train-eval-index"], train_eval_index)


class SpaceCardDataTest(unittest.TestCase):
    def test_space_card_data(self) -> None:
        card_data = SpaceCardData(
            title="Dreambooth Training",
            license="mit",
            sdk="gradio",
            duplicated_from="multimodalart/dreambooth-training",
        )
        self.assertEqual(
            card_data.to_dict(),
            {
                "title": "Dreambooth Training",
                "sdk": "gradio",
                "license": "mit",
                "duplicated_from": "multimodalart/dreambooth-training",
            },
        )
        self.assertIsNone(card_data.tags)  # SpaceCardData has some default attributes


# tests/test_repository.py

# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
import time
import unittest
from pathlib import Path

import pytest
import requests

from huggingface_hub import RepoUrl
from huggingface_hub.hf_api import HfApi
from huggingface_hub.repository import (
    Repository,
    is_tracked_upstream,
    is_tracked_with_lfs,
)
from huggingface_hub.utils import SoftTemporaryDirectory, logging, run_subprocess

from .testing_constants import ENDPOINT_STAGING, TOKEN
from .testing_utils import (
    repo_name,
    retry_endpoint,
    use_tmp_repo,
    with_production_testing,
)


logger = logging.get_logger(__name__)


@pytest.mark.usefixtures("fx_cache_dir")
class RepositoryTestAbstract(unittest.TestCase):
    cache_dir: Path
    repo_path: Path

    # This content is 5MB (under 10MB)
    small_content = json.dumps([100] * int(1e6))

    # This content is 20MB (over 10MB)
    large_content = json.dumps([100] * int(4e6))

    # This content is binary (contains the null character)
    binary_content = "\x00\x00\x00\x00"

    _api = HfApi(endpoint=ENDPOINT_STAGING, token=TOKEN)

    @classmethod
    def setUp(self) -> None:
        self.repo_path = self.cache_dir / "working_dir"
        self.repo_path.mkdir()

    def _create_dummy_files(self):
        # Create dummy files
        # one is lfs-tracked, the other is not.
        small_file = self.repo_path / "dummy.txt"
        small_file.write_text(self.small_content)

        binary_file = self.repo_path / "model.bin"
        binary_file.write_text(self.binary_content)


class TestRepositoryShared(RepositoryTestAbstract):
    """Tests in this class shares a single repo on the Hub (common to all tests).

    These tests must not push data to it.
    """

    @classmethod
    def setUpClass(cls):
        """
        Share this valid token in all tests below.
        """
        super().setUpClass()
        cls.repo_url = cls._api.create_repo(repo_id=repo_name())
        cls.repo_id = cls.repo_url.repo_id
        cls._api.upload_file(
            path_or_fileobj=cls.binary_content.encode(),
            path_in_repo="random_file.txt",
            repo_id=cls.repo_id,
        )

    @classmethod
    def tearDownClass(cls):
        cls._api.delete_repo(repo_id=cls.repo_id)

    def test_clone_from_repo_url(self):
        Repository(self.repo_path, clone_from=self.repo_url)

    @retry_endpoint
    def test_clone_from_repo_id(self):
        Repository(self.repo_path, clone_from=self.repo_id)

    @retry_endpoint
    def test_clone_from_repo_name_no_namespace_fails(self):
        with self.assertRaises(EnvironmentError):
            Repository(self.repo_path, clone_from=self.repo_id.split("/")[1], token=TOKEN)

    @retry_endpoint
    def test_clone_from_not_hf_url(self):
        # Should not error out
        Repository(self.repo_path, clone_from="https://hf.co/hf-internal-testing/huggingface-hub-dummy-repository")

    def test_clone_from_missing_repo(self):
        """If the repo does not exist an EnvironmentError is raised."""
        with self.assertRaises(EnvironmentError):
            Repository(self.repo_path, clone_from="missing_repo")

    @with_production_testing
    @retry_endpoint
    def test_clone_from_prod_canonical_repo_id(self):
        Repository(self.repo_path, clone_from="bert-base-cased", skip_lfs_files=True)

    @with_production_testing
    @retry_endpoint
    def test_clone_from_prod_canonical_repo_url(self):
        Repository(self.repo_path, clone_from="https://huggingface.co/bert-base-cased", skip_lfs_files=True)

    def test_init_from_existing_local_clone(self):
        run_subprocess(["git", "clone", self.repo_url, str(self.repo_path)])

        repo = Repository(self.repo_path)
        repo.lfs_track(["*.pdf"])
        repo.lfs_enable_largefiles()
        repo.git_pull()

    def test_init_failure(self):
        with self.assertRaises(ValueError):
            Repository(self.repo_path)

    @retry_endpoint
    def test_init_clone_in_empty_folder(self):
        repo = Repository(self.repo_path, clone_from=self.repo_url)
        repo.lfs_track(["*.pdf"])
        repo.lfs_enable_largefiles()
        repo.git_pull()
        self.assertIn("random_file.txt", os.listdir(self.repo_path))

    def test_git_lfs_filename(self):
        run_subprocess("git init", folder=self.repo_path)

        repo = Repository(self.repo_path)
        large_file = self.repo_path / "large_file[].txt"
        large_file.write_text(self.large_content)

        repo.git_add()

        repo.lfs_track([large_file.name])
        self.assertFalse(is_tracked_with_lfs(large_file))

        repo.lfs_track([large_file.name], filename=True)
        self.assertTrue(is_tracked_with_lfs(large_file))

    def test_init_clone_in_nonempty_folder(self):
        self._create_dummy_files()
        with self.assertRaises(EnvironmentError):
            Repository(self.repo_path, clone_from=self.repo_url)

    @retry_endpoint
    def test_init_clone_in_nonempty_linked_git_repo_with_token(self):
        Repository(self.repo_path, clone_from=self.repo_url, token=TOKEN)
        Repository(self.repo_path, clone_from=self.repo_url, token=TOKEN)

    @retry_endpoint
    def test_is_tracked_upstream(self):
        Repository(self.repo_path, clone_from=self.repo_id)
        self.assertTrue(is_tracked_upstream(self.repo_path))

    @retry_endpoint
    def test_push_errors_on_wrong_checkout(self):
        repo = Repository(self.repo_path, clone_from=self.repo_id)

        head_commit_ref = run_subprocess("git show --oneline -s", folder=self.repo_path).stdout.split()[0]

        repo.git_checkout(head_commit_ref)

        with self.assertRaises(OSError):
            with repo.commit("New commit"):
                with open("new_file", "w+") as f:
                    f.write("Ok")


class TestRepositoryUniqueRepos(RepositoryTestAbstract):
    """Tests in this class use separated repos on the Hub (i.e. 1 test = 1 repo).

    These tests can push data to it.
    """

    @retry_endpoint
    def setUp(self):
        super().setUp()
        self.repo_url = self._api.create_repo(repo_id=repo_name())
        self.repo_id = self.repo_url.repo_id
        self._api.upload_file(
            path_or_fileobj=self.binary_content.encode(), path_in_repo="random_file.txt", repo_id=self.repo_id
        )

    def tearDown(self):
        self._api.delete_repo(repo_id=self.repo_id)

    def clone_repo(self, **kwargs) -> Repository:
        if "local_dir" not in kwargs:
            kwargs["local_dir"] = self.repo_path
        if "clone_from" not in kwargs:
            kwargs["clone_from"] = self.repo_url
        if "token" not in kwargs:
            kwargs["token"] = TOKEN
        if "git_user" not in kwargs:
            kwargs["git_user"] = "ci"
        if "git_email" not in kwargs:
            kwargs["git_email"] = "ci@dummy.com"
        return Repository(**kwargs)

    @retry_endpoint
    @use_tmp_repo()
    def test_init_clone_in_nonempty_non_linked_git_repo(self, repo_url: RepoUrl):
        self.clone_repo()

        # Try and clone another repository within the same directory.
        # Should error out due to mismatched remotes.
        with self.assertRaises(EnvironmentError):
            Repository(self.repo_path, clone_from=repo_url)

    @retry_endpoint
    def test_init_clone_in_nonempty_linked_git_repo(self):
        # Clone the repository to disk
        self.clone_repo()

        # Add to the remote repository without doing anything to the local repository.
        self._api.upload_file(
            path_or_fileobj=self.binary_content.encode(), path_in_repo="random_file_3.txt", repo_id=self.repo_id
        )

        # Cloning the repository in the same directory should not result in a git pull.
        self.clone_repo(clone_from=self.repo_url)
        self.assertNotIn("random_file_3.txt", os.listdir(self.repo_path))

    @retry_endpoint
    def test_init_clone_in_nonempty_linked_git_repo_unrelated_histories(self):
        # Clone the repository to disk
        repo = self.clone_repo()

        # Create and commit file locally
        (self.repo_path / "random_file_3.txt").write_text("hello world")
        repo.git_add()
        repo.git_commit("Unrelated commit")

        # Add to the remote repository without doing anything to the local repository.
        self._api.upload_file(
            path_or_fileobj=self.binary_content.encode(),
            path_in_repo="random_file_3.txt",
            repo_id=self.repo_url.repo_id,
        )

        # The repo should initialize correctly as the remote is the same, even with unrelated historied
        self.clone_repo()

    @retry_endpoint
    def test_add_commit_push(self):
        repo = self.clone_repo()
        self._create_dummy_files()
        repo.git_add()
        repo.git_commit()
        url = repo.git_push()

        # Check that the returned commit url
        # actually exists.
        r = requests.head(url)
        r.raise_for_status()

    @retry_endpoint
    def test_add_commit_push_non_blocking(self):
        repo = self.clone_repo()
        self._create_dummy_files()
        repo.git_add()
        repo.git_commit()
        url, result = repo.git_push(blocking=False)

        # Check background process
        if result._process.poll() is None:
            self.assertEqual(result.status, -1)

        while not result.is_done:
            time.sleep(0.5)

        self.assertTrue(result.is_done)
        self.assertEqual(result.status, 0)

        # Check that the returned commit url
        # actually exists.
        r = requests.head(url)
        r.raise_for_status()

    @retry_endpoint
    def test_context_manager_non_blocking(self):
        repo = self.clone_repo()

        with repo.commit("New commit", blocking=False):
            (self.repo_path / "dummy.txt").write_text("hello world")

        while repo.commands_in_progress:
            time.sleep(1)

        self.assertEqual(len(repo.commands_in_progress), 0)
        self.assertEqual(len(repo.command_queue), 1)
        self.assertEqual(repo.command_queue[-1].status, 0)
        self.assertEqual(repo.command_queue[-1].is_done, True)
        self.assertEqual(repo.command_queue[-1].title, "push")

    @unittest.skipIf(os.name == "nt", "Killing a process on Windows works differently.")
    def test_add_commit_push_non_blocking_process_killed(self):
        repo = self.clone_repo()

        # Far too big file: will take forever
        (self.repo_path / "dummy.txt").write_text(str([[[1] * 10000] * 1000] * 10))
        repo.git_add(auto_lfs_track=True)
        repo.git_commit()
        _, result = repo.git_push(blocking=False)

        result._process.kill()

        while result._process.poll() is None:
            time.sleep(0.5)

        self.assertTrue(result.is_done)
        self.assertEqual(result.status, -9)

    @retry_endpoint
    def test_commit_context_manager(self):
        # Clone and commit from a first folder
        folder_1 = self.repo_path / "folder_1"
        clone = self.clone_repo(local_dir=folder_1)
        with clone.commit("Commit"):
            with open("dummy.txt", "w") as f:
                f.write("hello")
            with open("model.bin", "w") as f:
                f.write("hello")

        # Clone in second folder. Check existence of committed files
        folder_2 = self.repo_path / "folder_2"
        self.clone_repo(local_dir=folder_2)
        files = os.listdir(folder_2)
        self.assertTrue("dummy.txt" in files)
        self.assertTrue("model.bin" in files)

    @retry_endpoint
    def test_clone_skip_lfs_files(self):
        # Upload LFS file
        self._api.upload_file(path_or_fileobj=b"Bin file", path_in_repo="file.bin", repo_id=self.repo_id)

        repo = self.clone_repo(skip_lfs_files=True)
        file_bin = self.repo_path / "file.bin"

        self.assertTrue(file_bin.read_text().startswith("version"))

        repo.git_pull(lfs=True)

        self.assertEqual(file_bin.read_text(), "Bin file")

    @retry_endpoint
    def test_commits_on_correct_branch(self):
        repo = self.clone_repo()
        branch = repo.current_branch
        repo.git_checkout("new-branch", create_branch_ok=True)
        repo.git_checkout(branch)

        with repo.commit("New commit"):
            with open("file.txt", "w+") as f:
                f.write("Ok")

        repo.git_checkout("new-branch")

        with repo.commit("New commit"):
            with open("new_file.txt", "w+") as f:
                f.write("Ok")

        with SoftTemporaryDirectory() as tmp:
            clone = self.clone_repo(local_dir=tmp)
            files = os.listdir(clone.local_dir)
            self.assertTrue("file.txt" in files)
            self.assertFalse("new_file.txt" in files)

            clone.git_checkout("new-branch")
            files = os.listdir(clone.local_dir)
            self.assertFalse("file.txt" in files)
            self.assertTrue("new_file.txt" in files)

    @retry_endpoint
    def test_repo_checkout_push(self):
        repo = self.clone_repo()

        repo.git_checkout("new-branch", create_branch_ok=True)
        repo.git_checkout("main")

        (self.repo_path / "file.txt").write_text("OK")

        repo.push_to_hub("Commit #1")
        repo.git_checkout("new-branch", create_branch_ok=True)

        (self.repo_path / "new_file.txt").write_text("OK")

        repo.push_to_hub("Commit #2")

        with SoftTemporaryDirectory() as tmp:
            clone = self.clone_repo(local_dir=tmp)
            files = os.listdir(clone.local_dir)
            self.assertTrue("file.txt" in files)
            self.assertFalse("new_file.txt" in files)

            clone.git_checkout("new-branch")
            files = os.listdir(clone.local_dir)
            self.assertFalse("file.txt" in files)
            self.assertTrue("new_file.txt" in files)

    @retry_endpoint
    def test_repo_checkout_commit_context_manager(self):
        repo = self.clone_repo()

        with repo.commit("Commit #1", branch="new-branch"):
            with open(os.path.join(repo.local_dir, "file.txt"), "w+") as f:
                f.write("Ok")

        with repo.commit("Commit #2", branch="main"):
            with open(os.path.join(repo.local_dir, "new_file.txt"), "w+") as f:
                f.write("Ok")

        # Maintains lastly used branch
        with repo.commit("Commit #3"):
            with open(os.path.join(repo.local_dir, "new_file-2.txt"), "w+") as f:
                f.write("Ok")

        with SoftTemporaryDirectory() as tmp:
            clone = self.clone_repo(local_dir=tmp)
            files = os.listdir(clone.local_dir)
            self.assertFalse("file.txt" in files)
            self.assertTrue("new_file-2.txt" in files)
            self.assertTrue("new_file.txt" in files)

            clone.git_checkout("new-branch")
            files = os.listdir(clone.local_dir)
            self.assertTrue("file.txt" in files)
            self.assertFalse("new_file.txt" in files)
            self.assertFalse("new_file-2.txt" in files)

    @retry_endpoint
    def test_add_tag(self):
        repo = self.clone_repo()
        repo.add_tag("v4.6.0", remote="origin")
        self.assertTrue(repo.tag_exists("v4.6.0", remote="origin"))

    @retry_endpoint
    def test_add_annotated_tag(self):
        repo = self.clone_repo()
        repo.add_tag("v4.5.0", message="This is an annotated tag", remote="origin")

        # Unfortunately git offers no built-in way to check the annotated
        # message of a remote tag.
        # In order to check that the remote tag was correctly annotated,
        # we delete the local tag before pulling the remote tag (which
        # should be the same). We then check that this tag is correctly
        # annotated.
        repo.delete_tag("v4.5.0")

        self.assertTrue(repo.tag_exists("v4.5.0", remote="origin"))
        self.assertFalse(repo.tag_exists("v4.5.0"))

        # Tag still exists on remote
        run_subprocess("git pull --tags", folder=self.repo_path)
        self.assertTrue(repo.tag_exists("v4.5.0"))

        # Tag is annotated
        result = run_subprocess("git tag -n9", folder=self.repo_path).stdout.strip()
        self.assertIn("This is an annotated tag", result)

    @retry_endpoint
    def test_delete_tag(self):
        repo = self.clone_repo()

        repo.add_tag("v4.6.0", message="This is an annotated tag", remote="origin")
        self.assertTrue(repo.tag_exists("v4.6.0", remote="origin"))

        repo.delete_tag("v4.6.0")
        self.assertFalse(repo.tag_exists("v4.6.0"))
        self.assertTrue(repo.tag_exists("v4.6.0", remote="origin"))

        repo.delete_tag("v4.6.0", remote="origin")
        self.assertFalse(repo.tag_exists("v4.6.0", remote="origin"))

    @retry_endpoint
    def test_lfs_prune(self):
        repo = self.clone_repo()

        with repo.commit("Committing LFS file"):
            with open("file.bin", "w+") as f:
                f.write("Random string 1")

        with repo.commit("Committing LFS file"):
            with open("file.bin", "w+") as f:
                f.write("Random string 2")

        root_directory = self.repo_path / ".git" / "lfs"
        git_lfs_files_size = sum(f.stat().st_size for f in root_directory.glob("**/*") if f.is_file())
        repo.lfs_prune()
        post_prune_git_lfs_files_size = sum(f.stat().st_size for f in root_directory.glob("**/*") if f.is_file())

        # Size of the directory holding LFS files was reduced
        self.assertLess(post_prune_git_lfs_files_size, git_lfs_files_size)

    @retry_endpoint
    def test_lfs_prune_git_push(self):
        repo = self.clone_repo()
        with repo.commit("Committing LFS file"):
            with open("file.bin", "w+") as f:
                f.write("Random string 1")

        root_directory = self.repo_path / ".git" / "lfs"
        git_lfs_files_size = sum(f.stat().st_size for f in root_directory.glob("**/*") if f.is_file())

        with open(os.path.join(repo.local_dir, "file.bin"), "w+") as f:
            f.write("Random string 2")

        repo.git_add()
        repo.git_commit("New commit")
        repo.git_push(auto_lfs_prune=True)

        post_prune_git_lfs_files_size = sum(f.stat().st_size for f in root_directory.glob("**/*") if f.is_file())

        # Size of the directory holding LFS files is the exact same
        self.assertEqual(post_prune_git_lfs_files_size, git_lfs_files_size)


class TestRepositoryOffline(RepositoryTestAbstract):
    """Class to test `Repository` object on local folders only (no cloning from Hub)."""

    repo: Repository

    @classmethod
    def setUp(self) -> None:
        super().setUp()

        run_subprocess("git init", folder=self.repo_path)

        self.repo = Repository(self.repo_path, git_user="ci", git_email="ci@dummy.ci")

        git_attributes_path = self.repo_path / ".gitattributes"
        git_attributes_path.write_text("*.pt filter=lfs diff=lfs merge=lfs -text")

        self.repo.git_add(".gitattributes")
        self.repo.git_commit("Add .gitattributes")

    def test_is_tracked_with_lfs(self):
        txt_1 = self.repo_path / "small_file_1.txt"
        txt_2 = self.repo_path / "small_file_2.txt"
        pt_1 = self.repo_path / "model.pt"

        txt_1.write_text(self.small_content)
        txt_2.write_text(self.small_content)
        pt_1.write_text(self.small_content)

        self.repo.lfs_track("small_file_1.txt")

        self.assertTrue(is_tracked_with_lfs(txt_1))
        self.assertFalse(is_tracked_with_lfs(txt_2))
        self.assertTrue(pt_1)

    def test_is_tracked_with_lfs_with_pattern(self):
        txt_small_file = self.repo_path / "small_file.txt"
        txt_small_file.write_text(self.small_content)

        txt_large_file = self.repo_path / "large_file.txt"
        txt_large_file.write_text(self.large_content)

        (self.repo_path / "dir").mkdir()
        txt_small_file_in_dir = self.repo_path / "dir" / "small_file.txt"
        txt_small_file_in_dir.write_text(self.small_content)

        txt_large_file_in_dir = self.repo_path / "dir" / "large_file.txt"
        txt_large_file_in_dir.write_text(self.large_content)

        self.repo.auto_track_large_files("dir")

        self.assertFalse(is_tracked_with_lfs(txt_large_file))
        self.assertFalse(is_tracked_with_lfs(txt_small_file))
        self.assertTrue(is_tracked_with_lfs(txt_large_file_in_dir))
        self.assertFalse(is_tracked_with_lfs(txt_small_file_in_dir))

    def test_auto_track_large_files(self):
        txt_small_file = self.repo_path / "small_file.txt"
        txt_small_file.write_text(self.small_content)

        txt_large_file = self.repo_path / "large_file.txt"
        txt_large_file.write_text(self.large_content)

        self.repo.auto_track_large_files()

        self.assertTrue(is_tracked_with_lfs(txt_large_file))
        self.assertFalse(is_tracked_with_lfs(txt_small_file))

    def test_auto_track_binary_files(self):
        non_binary_file = self.repo_path / "non_binary_file.txt"
        non_binary_file.write_text(self.small_content)

        binary_file = self.repo_path / "binary_file.txt"
        binary_file.write_text(self.binary_content)

        self.repo.auto_track_binary_files()

        self.assertFalse(is_tracked_with_lfs(non_binary_file))
        self.assertTrue(is_tracked_with_lfs(binary_file))

    def test_auto_track_large_files_ignored_with_gitignore(self):
        (self.repo_path / "dir").mkdir()

        # Test nested gitignores
        gitignore_file = self.repo_path / ".gitignore"
        gitignore_file.write_text("large_file.txt")

        gitignore_file_in_dir = self.repo_path / "dir" / ".gitignore"
        gitignore_file_in_dir.write_text("large_file_3.txt")

        large_file = self.repo_path / "large_file.txt"
        large_file.write_text(self.large_content)

        large_file_2 = self.repo_path / "large_file_2.txt"
        large_file_2.write_text(self.large_content)

        large_file_3 = self.repo_path / "dir" / "large_file_3.txt"
        large_file_3.write_text(self.large_content)

        large_file_4 = self.repo_path / "dir" / "large_file_4.txt"
        large_file_4.write_text(self.large_content)

        self.repo.auto_track_large_files()

        # Large files
        self.assertFalse(is_tracked_with_lfs(large_file))
        self.assertTrue(is_tracked_with_lfs(large_file_2))

        self.assertFalse(is_tracked_with_lfs(large_file_3))
        self.assertTrue(is_tracked_with_lfs(large_file_4))

    def test_auto_track_binary_files_ignored_with_gitignore(self):
        (self.repo_path / "dir").mkdir()

        # Test nested gitignores
        gitignore_file = self.repo_path / ".gitignore"
        gitignore_file.write_text("binary_file.txt")

        gitignore_file_in_dir = self.repo_path / "dir" / ".gitignore"
        gitignore_file_in_dir.write_text("binary_file_3.txt")

        binary_file = self.repo_path / "binary_file.txt"
        binary_file.write_text(self.binary_content)

        binary_file_2 = self.repo_path / "binary_file_2.txt"
        binary_file_2.write_text(self.binary_content)

        binary_file_3 = self.repo_path / "dir" / "binary_file_3.txt"
        binary_file_3.write_text(self.binary_content)

        binary_file_4 = self.repo_path / "dir" / "binary_file_4.txt"
        binary_file_4.write_text(self.binary_content)

        self.repo.auto_track_binary_files()

        # Binary files
        self.assertFalse(is_tracked_with_lfs(binary_file))
        self.assertTrue(is_tracked_with_lfs(binary_file_2))
        self.assertFalse(is_tracked_with_lfs(binary_file_3))
        self.assertTrue(is_tracked_with_lfs(binary_file_4))

    def test_auto_track_large_files_through_git_add(self):
        txt_small_file = self.repo_path / "small_file.txt"
        txt_small_file.write_text(self.small_content)

        txt_large_file = self.repo_path / "large_file.txt"
        txt_large_file.write_text(self.large_content)

        self.repo.git_add(auto_lfs_track=True)

        self.assertTrue(is_tracked_with_lfs(txt_large_file))
        self.assertFalse(is_tracked_with_lfs(txt_small_file))

    def test_auto_track_binary_files_through_git_add(self):
        non_binary_file = self.repo_path / "small_file.txt"
        non_binary_file.write_text(self.small_content)

        binary_file = self.repo_path / "binary.txt"
        binary_file.write_text(self.binary_content)

        self.repo.git_add(auto_lfs_track=True)

        self.assertTrue(is_tracked_with_lfs(binary_file))
        self.assertFalse(is_tracked_with_lfs(non_binary_file))

    def test_auto_no_track_large_files_through_git_add(self):
        txt_small_file = self.repo_path / "small_file.txt"
        txt_small_file.write_text(self.small_content)

        txt_large_file = self.repo_path / "large_file.txt"
        txt_large_file.write_text(self.large_content)

        self.repo.git_add(auto_lfs_track=False)

        self.assertFalse(is_tracked_with_lfs(txt_large_file))
        self.assertFalse(is_tracked_with_lfs(txt_small_file))

    def test_auto_no_track_binary_files_through_git_add(self):
        non_binary_file = self.repo_path / "small_file.txt"
        non_binary_file.write_text(self.small_content)

        binary_file = self.repo_path / "binary.txt"
        binary_file.write_text(self.binary_content)

        self.repo.git_add(auto_lfs_track=False)

        self.assertFalse(is_tracked_with_lfs(binary_file))
        self.assertFalse(is_tracked_with_lfs(non_binary_file))

    def test_auto_track_updates_removed_gitattributes(self):
        txt_small_file = self.repo_path / "small_file.txt"
        txt_small_file.write_text(self.small_content)

        txt_large_file = self.repo_path / "large_file.txt"
        txt_large_file.write_text(self.large_content)

        self.repo.git_add(auto_lfs_track=True)

        self.assertTrue(is_tracked_with_lfs(txt_large_file))
        self.assertFalse(is_tracked_with_lfs(txt_small_file))

        # Remove large file
        txt_large_file.unlink()

        # Auto track should remove the entry from .gitattributes
        self.repo.auto_track_large_files()

        # Recreate the large file with smaller contents
        txt_large_file.write_text(self.small_content)

        # Ensure the file is not LFS tracked anymore
        self.repo.auto_track_large_files()
        self.assertFalse(is_tracked_with_lfs(txt_large_file))

    def test_checkout_non_existing_branch(self):
        self.assertRaises(EnvironmentError, self.repo.git_checkout, "brand-new-branch")

    def test_checkout_new_branch(self):
        self.repo.git_checkout("new-branch", create_branch_ok=True)
        self.assertEqual(self.repo.current_branch, "new-branch")

    def test_is_not_tracked_upstream(self):
        self.repo.git_checkout("new-branch", create_branch_ok=True)
        self.assertFalse(is_tracked_upstream(self.repo.local_dir))

    def test_no_branch_checked_out_raises(self):
        head_commit_ref = run_subprocess("git show --oneline -s", folder=self.repo_path).stdout.split()[0]

        self.repo.git_checkout(head_commit_ref)
        self.assertRaises(OSError, is_tracked_upstream, self.repo.local_dir)

    def test_repo_init_checkout_default_revision(self):
        # Instantiate repository on a given revision
        repo = Repository(self.repo_path, revision="new-branch")
        self.assertEqual(repo.current_branch, "new-branch")

        # The revision should be kept when re-initializing the repo
        repo_2 = Repository(self.repo_path)
        self.assertEqual(repo_2.current_branch, "new-branch")

    def test_repo_init_checkout_revision(self):
        current_head_hash = self.repo.git_head_hash()

        (self.repo_path / "file.txt").write_text("hello world")

        self.repo.git_add()
        self.repo.git_commit("Add file.txt")

        new_head_hash = self.repo.git_head_hash()

        self.assertNotEqual(current_head_hash, new_head_hash)

        previous_head_repo = Repository(self.repo_path, revision=current_head_hash)
        files = os.listdir(previous_head_repo.local_dir)
        self.assertNotIn("file.txt", files)

        current_head_repo = Repository(self.repo_path, revision=new_head_hash)
        files = os.listdir(current_head_repo.local_dir)
        self.assertIn("file.txt", files)

    def test_repo_user(self):
        _ = Repository(self.repo_path, token=TOKEN)
        username = run_subprocess("git config user.name", folder=self.repo_path).stdout
        email = run_subprocess("git config user.email", folder=self.repo_path).stdout

        # hardcode values to avoid another api call to whoami
        self.assertEqual(username.strip(), "Dummy User")
        self.assertEqual(email.strip(), "julien@huggingface.co")

    def test_repo_passed_user(self):
        _ = Repository(self.repo_path, token=TOKEN, git_user="RANDOM_USER", git_email="EMAIL@EMAIL.EMAIL")
        username = run_subprocess("git config user.name", folder=self.repo_path).stdout
        email = run_subprocess("git config user.email", folder=self.repo_path).stdout

        self.assertEqual(username.strip(), "RANDOM_USER")
        self.assertEqual(email.strip(), "EMAIL@EMAIL.EMAIL")

    def test_add_tag(self):
        self.repo.add_tag("v4.6.0")
        self.assertTrue(self.repo.tag_exists("v4.6.0"))

    def test_add_annotated_tag(self):
        self.repo.add_tag("v4.6.0", message="This is an annotated tag")
        self.assertTrue(self.repo.tag_exists("v4.6.0"))

        result = run_subprocess("git tag -n9", folder=self.repo_path).stdout.strip()
        self.assertIn("This is an annotated tag", result)

    def test_delete_tag(self):
        self.repo.add_tag("v4.6.0", message="This is an annotated tag")
        self.assertTrue(self.repo.tag_exists("v4.6.0"))

        self.repo.delete_tag("v4.6.0")
        self.assertFalse(self.repo.tag_exists("v4.6.0"))

    def test_repo_clean(self):
        self.assertTrue(self.repo.is_repo_clean())
        (self.repo_path / "file.txt").write_text("hello world")
        self.assertFalse(self.repo.is_repo_clean())


class TestRepositoryDataset(RepositoryTestAbstract):
    """Class to test that cloning from a different repo_type works fine."""

    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.repo_url = cls._api.create_repo(repo_id=repo_name(), repo_type="dataset")
        cls.repo_id = cls.repo_url.repo_id
        cls._api.upload_file(
            path_or_fileobj=cls.binary_content.encode(),
            path_in_repo="file.txt",
            repo_id=cls.repo_id,
            repo_type="dataset",
        )

    @classmethod
    def tearDownClass(cls):
        super().tearDownClass()
        cls._api.delete_repo(repo_id=cls.repo_id, repo_type="dataset")

    @retry_endpoint
    def test_clone_dataset_with_endpoint_explicit_repo_type(self):
        Repository(
            self.repo_path, clone_from=self.repo_url, repo_type="dataset", git_user="ci", git_email="ci@dummy.com"
        )
        self.assertTrue((self.repo_path / "file.txt").exists())

    @retry_endpoint
    def test_clone_dataset_with_endpoint_implicit_repo_type(self):
        self.assertIn("dataset", self.repo_url)  # Implicit
        Repository(self.repo_path, clone_from=self.repo_url, git_user="ci", git_email="ci@dummy.com")
        self.assertTrue((self.repo_path / "file.txt").exists())

    @retry_endpoint
    def test_clone_dataset_with_repo_id_and_repo_type(self):
        Repository(
            self.repo_path, clone_from=self.repo_id, repo_type="dataset", git_user="ci", git_email="ci@dummy.com"
        )
        self.assertTrue((self.repo_path / "file.txt").exists())

    @retry_endpoint
    def test_clone_dataset_no_ci_user_and_email(self):
        Repository(self.repo_path, clone_from=self.repo_id, repo_type="dataset")
        self.assertTrue((self.repo_path / "file.txt").exists())

    @retry_endpoint
    def test_clone_dataset_with_repo_name_and_repo_type_fails(self):
        with self.assertRaises(EnvironmentError):
            Repository(
                self.repo_path,
                clone_from=self.repo_id.split("/")[1],
                repo_type="dataset",
                token=TOKEN,
                git_user="ci",
                git_email="ci@dummy.com",
            )


# tests/test_snapshot_download.py

import os
import unittest
from pathlib import Path
from unittest.mock import patch

import requests

from huggingface_hub import CommitOperationAdd, HfApi, snapshot_download
from huggingface_hub.utils import HfFolder, SoftTemporaryDirectory

from .testing_constants import TOKEN
from .testing_utils import repo_name


class SnapshotDownloadTests(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        """
        Share this valid token in all tests below.
        """
        cls.api = HfApi(token=TOKEN)
        cls.repo_id = cls.api.create_repo(repo_name("snapshot-download")).repo_id

        # First commit on `main`
        cls.first_commit_hash = cls.api.create_commit(
            repo_id=cls.repo_id,
            operations=[
                CommitOperationAdd(path_in_repo="dummy_file.txt", path_or_fileobj=b"v1"),
                CommitOperationAdd(path_in_repo="subpath/file.txt", path_or_fileobj=b"content in subpath"),
            ],
            commit_message="Add file to main branch",
        ).oid

        # Second commit on `main`
        cls.second_commit_hash = cls.api.create_commit(
            repo_id=cls.repo_id,
            operations=[
                CommitOperationAdd(path_in_repo="dummy_file.txt", path_or_fileobj=b"v2"),
                CommitOperationAdd(path_in_repo="dummy_file_2.txt", path_or_fileobj=b"v3"),
            ],
            commit_message="Add file to main branch",
        ).oid

        # Third commit on `other`
        cls.api.create_branch(repo_id=cls.repo_id, branch="other")
        cls.third_commit_hash = cls.api.create_commit(
            repo_id=cls.repo_id,
            operations=[
                CommitOperationAdd(path_in_repo="dummy_file_2.txt", path_or_fileobj=b"v4"),
            ],
            commit_message="Add file to other branch",
            revision="other",
        ).oid

    @classmethod
    def tearDownClass(cls) -> None:
        cls.api.delete_repo(repo_id=cls.repo_id)

    def test_download_model(self):
        # Test `main` branch
        with SoftTemporaryDirectory() as tmpdir:
            storage_folder = snapshot_download(self.repo_id, revision="main", cache_dir=tmpdir)

            # folder contains the two files contributed and the .gitattributes
            folder_contents = os.listdir(storage_folder)
            self.assertEqual(len(folder_contents), 4)
            self.assertTrue("dummy_file.txt" in folder_contents)
            self.assertTrue("dummy_file_2.txt" in folder_contents)
            self.assertTrue(".gitattributes" in folder_contents)

            with open(os.path.join(storage_folder, "dummy_file.txt"), "r") as f:
                contents = f.read()
                self.assertEqual(contents, "v2")

            # folder name contains the revision's commit sha.
            self.assertTrue(self.second_commit_hash in storage_folder)

        # Test with specific revision
        with SoftTemporaryDirectory() as tmpdir:
            storage_folder = snapshot_download(
                self.repo_id,
                revision=self.first_commit_hash,
                cache_dir=tmpdir,
            )

            # folder contains the two files contributed and the .gitattributes
            folder_contents = os.listdir(storage_folder)
            self.assertEqual(len(folder_contents), 3)
            self.assertTrue("dummy_file.txt" in folder_contents)
            self.assertTrue(".gitattributes" in folder_contents)

            with open(os.path.join(storage_folder, "dummy_file.txt"), "r") as f:
                contents = f.read()
                self.assertEqual(contents, "v1")

            # folder name contains the revision's commit sha.
            self.assertTrue(self.first_commit_hash in storage_folder)

    def test_download_private_model(self):
        self.api.update_repo_visibility(repo_id=self.repo_id, private=True)

        # Test download fails without token
        with SoftTemporaryDirectory() as tmpdir:
            with self.assertRaisesRegex(requests.exceptions.HTTPError, "401 Client Error"):
                _ = snapshot_download(self.repo_id, revision="main", cache_dir=tmpdir)

        # Test we can download with token from cache
        with SoftTemporaryDirectory() as tmpdir:
            HfFolder.save_token(TOKEN)
            storage_folder = snapshot_download(self.repo_id, revision="main", cache_dir=tmpdir)
            self.assertTrue(self.second_commit_hash in storage_folder)

        # Test we can download with explicit token
        with SoftTemporaryDirectory() as tmpdir:
            storage_folder = snapshot_download(self.repo_id, revision="main", cache_dir=tmpdir, token=TOKEN)
            self.assertTrue(self.second_commit_hash in storage_folder)

        self.api.update_repo_visibility(repo_id=self.repo_id, private=False)

    def test_download_model_local_only(self):
        # Test no branch specified
        with SoftTemporaryDirectory() as tmpdir:
            # first download folder to cache it
            snapshot_download(self.repo_id, cache_dir=tmpdir)
            # now load from cache
            storage_folder = snapshot_download(self.repo_id, cache_dir=tmpdir, local_files_only=True)
            self.assertTrue(self.second_commit_hash in storage_folder)  # has expected revision

        # Test with specific revision branch
        with SoftTemporaryDirectory() as tmpdir:
            # first download folder to cache it
            snapshot_download(self.repo_id, revision="other", cache_dir=tmpdir)
            # now load from cache
            storage_folder = snapshot_download(self.repo_id, revision="other", cache_dir=tmpdir, local_files_only=True)
            self.assertTrue(self.third_commit_hash in storage_folder)  # has expected revision

        # Test with specific revision hash
        with SoftTemporaryDirectory() as tmpdir:
            # first download folder to cache it
            snapshot_download(self.repo_id, revision=self.first_commit_hash, cache_dir=tmpdir)
            # now load from cache
            storage_folder = snapshot_download(
                self.repo_id, revision=self.first_commit_hash, cache_dir=tmpdir, local_files_only=True
            )
            self.assertTrue(self.first_commit_hash in storage_folder)  # has expected revision

    def test_download_model_local_only_multiple(self):
        # cache multiple commits and make sure correct commit is taken
        with SoftTemporaryDirectory() as tmpdir:
            # download folder from main and other to cache it
            snapshot_download(self.repo_id, cache_dir=tmpdir)
            snapshot_download(self.repo_id, revision="other", cache_dir=tmpdir)

            # now make sure that loading "main" branch gives correct branch
            # folder name contains the 2nd commit sha and not the 3rd
            storage_folder = snapshot_download(self.repo_id, cache_dir=tmpdir, local_files_only=True)
            self.assertTrue(self.second_commit_hash in storage_folder)

    def check_download_model_with_pattern(self, pattern, allow=True):
        # Test `main` branch
        allow_patterns = pattern if allow else None
        ignore_patterns = pattern if not allow else None

        with SoftTemporaryDirectory() as tmpdir:
            storage_folder = snapshot_download(
                self.repo_id,
                revision="main",
                cache_dir=tmpdir,
                allow_patterns=allow_patterns,
                ignore_patterns=ignore_patterns,
            )

            # folder contains the three text files but not the .gitattributes
            folder_contents = os.listdir(storage_folder)
            self.assertEqual(len(folder_contents), 3)
            self.assertTrue("dummy_file.txt" in folder_contents)
            self.assertTrue("dummy_file_2.txt" in folder_contents)
            self.assertTrue(".gitattributes" not in folder_contents)

    def test_download_model_with_allow_pattern(self):
        self.check_download_model_with_pattern("*.txt")

    def test_download_model_with_allow_pattern_list(self):
        self.check_download_model_with_pattern(["dummy_file.txt", "dummy_file_2.txt", "subpath/*"])

    def test_download_model_with_ignore_pattern(self):
        self.check_download_model_with_pattern(".gitattributes", allow=False)

    def test_download_model_with_ignore_pattern_list(self):
        self.check_download_model_with_pattern(["*.git*", "*.pt"], allow=False)

    @patch("huggingface_hub.constants.HF_HUB_LOCAL_DIR_AUTO_SYMLINK_THRESHOLD", 10)  # >10b => "big file"
    def test_download_to_local_dir(self) -> None:
        """Download a repository to local dir.

        Cache dir is used and symlinks are adding to local dir. This test is here to check once the normal behavior
        with snapshot_download. More combinations of cache_dir/local_dir/use_symlinks are tested separately in
        `test_file_download.py`.
        """
        with SoftTemporaryDirectory() as cache_dir:
            with SoftTemporaryDirectory() as local_dir:
                returned_path = snapshot_download(self.repo_id, cache_dir=cache_dir, local_dir=local_dir)

                # Files have been downloaded
                self.assertTrue((Path(local_dir) / "dummy_file.txt").is_file())
                self.assertTrue((Path(local_dir) / "dummy_file_2.txt").is_file())

                # Files are small so duplicated from cache (no symlinks)
                self.assertFalse((Path(local_dir) / "dummy_file.txt").is_symlink())  # smaller than 10b => duplicated
                self.assertFalse((Path(local_dir) / "dummy_file_2.txt").is_symlink())  # smaller than 10b => duplicated

                # File structure is preserved (+check content)
                subpath_file = Path(local_dir) / "subpath" / "file.txt"
                self.assertTrue(subpath_file.is_file())
                self.assertEqual(subpath_file.read_text(), "content in subpath")
                if os.name != "nt":
                    self.assertTrue(subpath_file.is_symlink())  # bigger than 10b => symlinked

                # Check returns local dir and not cache dir
                self.assertEqual(returned_path, local_dir)


# tests/test_tf_import.py

import sys
import unittest

from huggingface_hub.utils import is_tf_available


def require_tf(test_case):
    """
    Decorator marking a test that requires TensorFlow.

    These tests are skipped when TensorFlow is not installed.

    """
    if not is_tf_available():
        return unittest.skip("test requires Tensorflow")(test_case)
    else:
        return test_case


@require_tf
def test_import_huggingface_hub_does_not_import_tensorflow():
    # `import huggingface_hub` is not necessary since huggingface_hub is already imported at the top of this file,
    # but let's keep it here anyway just in case
    import huggingface_hub  # noqa

    assert "tensorflow" not in sys.modules


# tests/test_utils_assets.py

import unittest
from pathlib import Path
from unittest.mock import patch

import pytest

from huggingface_hub import cached_assets_path


@pytest.mark.usefixtures("fx_cache_dir")
class CacheAssetsTest(unittest.TestCase):
    cache_dir: Path

    def test_cached_assets_path_with_namespace_and_subfolder(self) -> None:
        expected_path = self.cache_dir / "datasets" / "SQuAD" / "download"
        self.assertFalse(expected_path.is_dir())

        path = cached_assets_path(
            library_name="datasets",
            namespace="SQuAD",
            subfolder="download",
            assets_dir=self.cache_dir,
        )

        self.assertEqual(path, expected_path)  # Path is generated
        self.assertTrue(path.is_dir())  # And dir is created

    def test_cached_assets_path_without_subfolder(self) -> None:
        path = cached_assets_path(library_name="datasets", namespace="SQuAD", assets_dir=self.cache_dir)
        self.assertEqual(path, self.cache_dir / "datasets" / "SQuAD" / "default")
        self.assertTrue(path.is_dir())

    def test_cached_assets_path_without_namespace(self) -> None:
        path = cached_assets_path(library_name="datasets", subfolder="download", assets_dir=self.cache_dir)
        self.assertEqual(path, self.cache_dir / "datasets" / "default" / "download")
        self.assertTrue(path.is_dir())

    def test_cached_assets_path_without_namespace_and_subfolder(self) -> None:
        path = cached_assets_path(library_name="datasets", assets_dir=self.cache_dir)
        self.assertEqual(path, self.cache_dir / "datasets" / "default" / "default")
        self.assertTrue(path.is_dir())

    def test_cached_assets_path_forbidden_symbols(self) -> None:
        path = cached_assets_path(
            library_name="ReAlLy dumb",
            namespace="user/repo_name",
            subfolder="this is/not\\clever",
            assets_dir=self.cache_dir,
        )
        self.assertEqual(
            path,
            self.cache_dir / "ReAlLy--dumb" / "user--repo_name" / "this--is--not--clever",
        )
        self.assertTrue(path.is_dir())

    def test_cached_assets_path_default_assets_dir(self) -> None:
        with patch(
            "huggingface_hub.utils._cache_assets.HUGGINGFACE_ASSETS_CACHE",
            self.cache_dir,
        ):  # Uses environment variable from HUGGINGFACE_ASSETS_CACHE
            self.assertEqual(
                cached_assets_path(library_name="datasets"),
                self.cache_dir / "datasets" / "default" / "default",
            )

    def test_cached_assets_path_is_a_file(self) -> None:
        expected_path = self.cache_dir / "datasets" / "default" / "default"
        expected_path.parent.mkdir(parents=True)
        expected_path.touch()  # this should be the generated folder but is a file !

        with self.assertRaises(ValueError):
            cached_assets_path(library_name="datasets", assets_dir=self.cache_dir)

    def test_cached_assets_path_parent_is_a_file(self) -> None:
        expected_path = self.cache_dir / "datasets" / "default" / "default"
        expected_path.parent.parent.mkdir(parents=True)
        expected_path.parent.touch()  # cannot create folder as a parent is a file !

        with self.assertRaises(ValueError):
            cached_assets_path(library_name="datasets", assets_dir=self.cache_dir)


# tests/test_utils_cache.py

import os
import tempfile
import time
import unittest
from pathlib import Path
from unittest.mock import Mock

import pytest

from huggingface_hub._snapshot_download import snapshot_download
from huggingface_hub.commands.scan_cache import ScanCacheCommand
from huggingface_hub.utils import DeleteCacheStrategy, HFCacheInfo, capture_output, scan_cache_dir
from huggingface_hub.utils._cache_manager import (
    CacheNotFound,
    _format_size,
    _format_timesince,
    _try_delete_path,
)

from .testing_constants import TOKEN
from .testing_utils import (
    rmtree_with_retry,
    with_production_testing,
    xfail_on_windows,
)


# On production server to avoid recreating them all the time
MODEL_ID = "hf-internal-testing/hfh_ci_scan_repo_a"
MODEL_PATH = "models--hf-internal-testing--hfh_ci_scan_repo_a"

DATASET_ID = "hf-internal-testing/hfh_ci_scan_dataset_b"
DATASET_PATH = "datasets--hf-internal-testing--hfh_ci_scan_dataset_b"

REPO_A_MAIN_HASH = "c0d57e03d9f128062eadb6665618982db612b2e3"
REPO_A_PR_1_HASH = "1a665a9d28a66b1d0f8edd9359fc824aacc63234"
REPO_A_OTHER_HASH = "f95875cd910793299a545417cc4b3c9055202883"
REPO_A_MAIN_README_BLOB_HASH = "fffc22b462ba2368b09b4d38527760051c9090a9"
REPO_B_MAIN_HASH = "f1cdcd4641b3ea2dfa8d4333dba1ea3b532735e1"

REF_1_NAME = "refs/pr/1"


@pytest.mark.usefixtures("fx_cache_dir")
class TestMissingCacheUtils(unittest.TestCase):
    cache_dir: Path

    def test_cache_dir_is_missing(self) -> None:
        """Directory to scan does not exist raises CacheNotFound."""
        self.assertRaises(CacheNotFound, scan_cache_dir, self.cache_dir / "does_not_exist")

    def test_cache_dir_is_a_file(self) -> None:
        """Directory to scan is a file raises ValueError."""
        file_path = self.cache_dir / "file.txt"
        file_path.touch()
        self.assertRaises(ValueError, scan_cache_dir, file_path)


@pytest.mark.usefixtures("fx_cache_dir")
class TestValidCacheUtils(unittest.TestCase):
    cache_dir: Path

    @with_production_testing
    def setUp(self) -> None:
        """Setup a clean cache for tests that will remain valid in all tests."""
        # Download latest main
        snapshot_download(
            repo_id=MODEL_ID,
            repo_type="model",
            cache_dir=self.cache_dir,
            use_auth_token=TOKEN,
        )

        # Download latest commit which is same as `main`
        snapshot_download(
            repo_id=MODEL_ID,
            revision=REPO_A_MAIN_HASH,
            repo_type="model",
            cache_dir=self.cache_dir,
            use_auth_token=TOKEN,
        )

        # Download the first commit
        snapshot_download(
            repo_id=MODEL_ID,
            revision=REPO_A_OTHER_HASH,
            repo_type="model",
            cache_dir=self.cache_dir,
            use_auth_token=TOKEN,
        )

        # Download from a PR
        snapshot_download(
            repo_id=MODEL_ID,
            revision="refs/pr/1",
            repo_type="model",
            cache_dir=self.cache_dir,
            use_auth_token=TOKEN,
        )

        # Download a Dataset repo from "main"
        snapshot_download(
            repo_id=DATASET_ID,
            revision="main",
            repo_type="dataset",
            cache_dir=self.cache_dir,
            use_auth_token=TOKEN,
        )

    @unittest.skipIf(os.name == "nt", "Windows cache is tested separately")
    def test_scan_cache_on_valid_cache_unix(self) -> None:
        """Scan the cache dir without warnings (on unix-based platform).

        This test is duplicated and adapted for Windows in `test_scan_cache_on_valid_cache_windows`.
        Note: Please make sure to updated both if any change is made.
        """
        report = scan_cache_dir(self.cache_dir)

        # Check general information about downloaded snapshots
        self.assertEqual(report.size_on_disk, 3766)
        self.assertEqual(len(report.repos), 2)  # Model and dataset
        self.assertEqual(len(report.warnings), 0)  # Repos are valid

        repo_a = [repo for repo in report.repos if repo.repo_id == MODEL_ID][0]

        # Check repo A general information
        repo_a_path = self.cache_dir / MODEL_PATH
        self.assertEqual(repo_a.repo_id, MODEL_ID)
        self.assertEqual(repo_a.repo_type, "model")
        self.assertEqual(repo_a.repo_path, repo_a_path)

        # 4 downloads but 3 revisions because "main" and REPO_A_MAIN_HASH are the same
        self.assertEqual(len(repo_a.revisions), 3)
        self.assertEqual(
            {rev.commit_hash for rev in repo_a.revisions},
            {REPO_A_MAIN_HASH, REPO_A_PR_1_HASH, REPO_A_OTHER_HASH},
        )

        # Repo size on disk is less than sum of revisions !
        self.assertEqual(repo_a.size_on_disk, 1501)
        self.assertEqual(sum(rev.size_on_disk for rev in repo_a.revisions), 4463)

        # Repo nb files is less than sum of revisions !
        self.assertEqual(repo_a.nb_files, 3)
        self.assertEqual(sum(rev.nb_files for rev in repo_a.revisions), 6)

        # 2 REFS in the repo: "main" and "refs/pr/1"
        # We could have add a tag as well
        self.assertEqual(set(repo_a.refs.keys()), {"main", REF_1_NAME})
        self.assertEqual(repo_a.refs["main"].commit_hash, REPO_A_MAIN_HASH)
        self.assertEqual(repo_a.refs[REF_1_NAME].commit_hash, REPO_A_PR_1_HASH)

        # Check "main" revision information
        main_revision = repo_a.refs["main"]
        main_revision_path = repo_a_path / "snapshots" / REPO_A_MAIN_HASH

        self.assertEqual(main_revision.commit_hash, REPO_A_MAIN_HASH)
        self.assertEqual(main_revision.snapshot_path, main_revision_path)
        self.assertEqual(main_revision.refs, {"main"})

        # Same nb of files and size on disk that the sum
        self.assertEqual(main_revision.nb_files, len(main_revision.files))
        self.assertEqual(
            main_revision.size_on_disk,
            sum(file.size_on_disk for file in main_revision.files),
        )

        # Check readme file from "main" revision
        main_readme_file = [file for file in main_revision.files if file.file_name == "README.md"][0]
        main_readme_file_path = main_revision_path / "README.md"
        main_readme_blob_path = repo_a_path / "blobs" / REPO_A_MAIN_README_BLOB_HASH

        self.assertEqual(main_readme_file.file_name, "README.md")
        self.assertEqual(main_readme_file.file_path, main_readme_file_path)
        self.assertEqual(main_readme_file.blob_path, main_readme_blob_path)

        # Check readme file from "refs/pr/1" revision
        pr_1_revision = repo_a.refs[REF_1_NAME]
        pr_1_revision_path = repo_a_path / "snapshots" / REPO_A_PR_1_HASH
        pr_1_readme_file = [file for file in pr_1_revision.files if file.file_name == "README.md"][0]
        pr_1_readme_file_path = pr_1_revision_path / "README.md"

        # file_path in "refs/pr/1" revision is different than "main" but same blob path
        self.assertEqual(pr_1_readme_file.file_path, pr_1_readme_file_path)  # different
        self.assertEqual(pr_1_readme_file.blob_path, main_readme_blob_path)  # same

    @unittest.skipIf(os.name != "nt", "Windows cache is tested separately")
    def test_scan_cache_on_valid_cache_windows(self) -> None:
        """Scan the cache dir without warnings (on Windows).

        Windows tests do not use symlinks which leads to duplication in the cache.
        This test is duplicated from `test_scan_cache_on_valid_cache_unix` with a few
        tweaks specific to windows.
        Note: Please make sure to updated both if any change is made.
        """
        report = scan_cache_dir(self.cache_dir)

        # Check general information about downloaded snapshots
        self.assertEqual(report.size_on_disk, 6728)
        self.assertEqual(len(report.repos), 2)  # Model and dataset
        self.assertEqual(len(report.warnings), 0)  # Repos are valid

        repo_a = [repo for repo in report.repos if repo.repo_id == MODEL_ID][0]

        # Check repo A general information
        repo_a_path = self.cache_dir / MODEL_PATH
        self.assertEqual(repo_a.repo_id, MODEL_ID)
        self.assertEqual(repo_a.repo_type, "model")
        self.assertEqual(repo_a.repo_path, repo_a_path)

        # 4 downloads but 3 revisions because "main" and REPO_A_MAIN_HASH are the same
        self.assertEqual(len(repo_a.revisions), 3)
        self.assertEqual(
            {rev.commit_hash for rev in repo_a.revisions},
            {REPO_A_MAIN_HASH, REPO_A_PR_1_HASH, REPO_A_OTHER_HASH},
        )

        # Repo size on disk is equal to the sum of revisions (no symlinks)
        self.assertEqual(repo_a.size_on_disk, 4463)  # Windows-specific
        self.assertEqual(sum(rev.size_on_disk for rev in repo_a.revisions), 4463)

        # Repo nb files is equal to the sum of revisions !
        self.assertEqual(repo_a.nb_files, 6)  # Windows-specific
        self.assertEqual(sum(rev.nb_files for rev in repo_a.revisions), 6)

        # 2 REFS in the repo: "main" and "refs/pr/1"
        # We could have add a tag as well
        REF_1_NAME = "refs\\pr\\1"  # Windows-specific
        self.assertEqual(set(repo_a.refs.keys()), {"main", REF_1_NAME})
        self.assertEqual(repo_a.refs["main"].commit_hash, REPO_A_MAIN_HASH)
        self.assertEqual(repo_a.refs[REF_1_NAME].commit_hash, REPO_A_PR_1_HASH)

        # Check "main" revision information
        main_revision = repo_a.refs["main"]
        main_revision_path = repo_a_path / "snapshots" / REPO_A_MAIN_HASH

        self.assertEqual(main_revision.commit_hash, REPO_A_MAIN_HASH)
        self.assertEqual(main_revision.snapshot_path, main_revision_path)
        self.assertEqual(main_revision.refs, {"main"})

        # Same nb of files and size on disk that the sum
        self.assertEqual(main_revision.nb_files, len(main_revision.files))
        self.assertEqual(
            main_revision.size_on_disk,
            sum(file.size_on_disk for file in main_revision.files),
        )

        # Check readme file from "main" revision
        main_readme_file = [file for file in main_revision.files if file.file_name == "README.md"][0]
        main_readme_file_path = main_revision_path / "README.md"
        main_readme_blob_path = repo_a_path / "blobs" / REPO_A_MAIN_README_BLOB_HASH

        self.assertEqual(main_readme_file.file_name, "README.md")
        self.assertEqual(main_readme_file.file_path, main_readme_file_path)
        self.assertEqual(main_readme_file.blob_path, main_readme_file_path)  # Windows-specific: no blob file
        self.assertFalse(main_readme_blob_path.exists())  # Windows-specific

        # Check readme file from "refs/pr/1" revision
        pr_1_revision = repo_a.refs[REF_1_NAME]
        pr_1_revision_path = repo_a_path / "snapshots" / REPO_A_PR_1_HASH
        pr_1_readme_file = [file for file in pr_1_revision.files if file.file_name == "README.md"][0]
        pr_1_readme_file_path = pr_1_revision_path / "README.md"

        # file_path in "refs/pr/1" revision is different than "main"
        # Windows-specific: even blob path is different
        self.assertEqual(pr_1_readme_file.file_path, pr_1_readme_file_path)
        self.assertNotEqual(  # Windows-specific: different as well
            pr_1_readme_file.blob_path, main_readme_file.blob_path
        )

    @xfail_on_windows("Size on disk and paths differ on Windows. Not useful to test.")
    def test_cli_scan_cache_quiet(self) -> None:
        """Test output from CLI scan cache with non verbose output.

        End-to-end test just to see if output is in expected format.
        """
        args = Mock()
        args.verbose = 0
        args.dir = self.cache_dir

        with capture_output() as output:
            ScanCacheCommand(args).run()

        expected_output = f"""
        REPO ID                       REPO TYPE SIZE ON DISK NB FILES LAST_ACCESSED     LAST_MODIFIED     REFS            LOCAL PATH
        ----------------------------- --------- ------------ -------- ----------------- ----------------- --------------- ---------------------------------------------------------
        {DATASET_ID} dataset           2.3K        1 a few seconds ago a few seconds ago main            {self.cache_dir}/{DATASET_PATH}
        {MODEL_ID}   model             1.5K        3 a few seconds ago a few seconds ago main, refs/pr/1 {self.cache_dir}/{MODEL_PATH}

        Done in 0.0s. Scanned 2 repo(s) for a total of \x1b[1m\x1b[31m3.8K\x1b[0m.
        """

        self.assertListEqual(
            output.getvalue().replace("-", "").split(),
            expected_output.replace("-", "").split(),
        )

    @xfail_on_windows("Size on disk and paths differ on Windows. Not useful to test.")
    def test_cli_scan_cache_verbose(self) -> None:
        """Test output from CLI scan cache with verbose output.

        End-to-end test just to see if output is in expected format.
        """
        args = Mock()
        args.verbose = 1
        args.dir = self.cache_dir

        with capture_output() as output:
            ScanCacheCommand(args).run()

        expected_output = f"""
        REPO ID                       REPO TYPE REVISION                                 SIZE ON DISK NB FILES LAST_MODIFIED     REFS      LOCAL PATH
        ----------------------------- --------- ---------------------------------------- ------------ -------- ----------------- --------- ------------------------------------------------------------------------------------------------------------
        {DATASET_ID} dataset   {REPO_B_MAIN_HASH}          2.3K        1 a few seconds ago main      {self.cache_dir}/{DATASET_PATH}/snapshots/{REPO_B_MAIN_HASH}
        {MODEL_ID}   model     {REPO_A_PR_1_HASH}          1.5K        3 a few seconds ago refs/pr/1 {self.cache_dir}/{MODEL_PATH}/snapshots/{REPO_A_PR_1_HASH}
        {MODEL_ID}   model     {REPO_A_MAIN_HASH}          1.5K        2 a few seconds ago main      {self.cache_dir}/{MODEL_PATH}/snapshots/{REPO_A_MAIN_HASH}
        {MODEL_ID}   model     {REPO_A_OTHER_HASH}         1.5K        1 a few seconds ago           {self.cache_dir}/{MODEL_PATH}/snapshots/{REPO_A_OTHER_HASH}

        Done in 0.0s. Scanned 2 repo(s) for a total of \x1b[1m\x1b[31m3.8K\x1b[0m.
        """

        self.assertListEqual(
            output.getvalue().replace("-", "").split(),
            expected_output.replace("-", "").split(),
        )

    def test_cli_scan_missing_cache(self) -> None:
        """Test output from CLI scan cache when cache does not exist.

        End-to-end test just to see if output is in expected format.
        """
        tmp_dir = tempfile.mkdtemp()
        os.rmdir(tmp_dir)

        args = Mock()
        args.verbose = 0
        args.dir = tmp_dir

        with capture_output() as output:
            ScanCacheCommand(args).run()

        expected_output = f"""
        Cache directory not found: {Path(tmp_dir).resolve()}
        """

        self.assertListEqual(output.getvalue().split(), expected_output.split())


@pytest.mark.usefixtures("fx_cache_dir")
class TestCorruptedCacheUtils(unittest.TestCase):
    cache_dir: Path
    repo_path: Path
    refs_path: Path
    snapshots_path: Path

    @with_production_testing
    def setUp(self) -> None:
        """Setup a clean cache for tests that will get corrupted/modified in tests."""
        # Download latest main
        snapshot_download(
            repo_id=MODEL_ID,
            repo_type="model",
            cache_dir=self.cache_dir,
            use_auth_token=TOKEN,
        )

        self.repo_path = self.cache_dir / MODEL_PATH
        self.refs_path = self.repo_path / "refs"
        self.snapshots_path = self.repo_path / "snapshots"

    def test_repo_path_not_valid_dir(self) -> None:
        """Test if found a not valid path in cache dir."""
        # Case 1: a file
        repo_path = self.cache_dir / "a_file_that_should_not_be_there.txt"
        repo_path.touch()

        report = scan_cache_dir(self.cache_dir)
        self.assertEqual(len(report.repos), 1)  # Scan still worked !

        self.assertEqual(len(report.warnings), 1)
        self.assertEqual(str(report.warnings[0]), f"Repo path is not a directory: {repo_path}")

        # Case 2: a folder with wrong naming
        os.remove(repo_path)
        repo_path = self.cache_dir / "a_folder_that_should_not_be_there"
        repo_path.mkdir()

        report = scan_cache_dir(self.cache_dir)
        self.assertEqual(len(report.repos), 1)  # Scan still worked !

        self.assertEqual(len(report.warnings), 1)
        self.assertEqual(
            str(report.warnings[0]),
            f"Repo path is not a valid HuggingFace cache directory: {repo_path}",
        )

        # Case 3: good naming but not a dataset/model/space
        rmtree_with_retry(repo_path)
        repo_path = self.cache_dir / "not-models--t5-small"
        repo_path.mkdir()

        report = scan_cache_dir(self.cache_dir)
        self.assertEqual(len(report.repos), 1)  # Scan still worked !

        self.assertEqual(len(report.warnings), 1)
        self.assertEqual(
            str(report.warnings[0]),
            f"Repo type must be `dataset`, `model` or `space`, found `not-model` ({repo_path}).",
        )

    def test_snapshots_path_not_found(self) -> None:
        """Test if snapshots directory is missing in cached repo."""
        rmtree_with_retry(self.snapshots_path)

        report = scan_cache_dir(self.cache_dir)
        self.assertEqual(len(report.repos), 0)  # Failed

        self.assertEqual(len(report.warnings), 1)
        self.assertEqual(
            str(report.warnings[0]),
            f"Snapshots dir doesn't exist in cached repo: {self.snapshots_path}",
        )

    def test_file_in_snapshots_dir(self) -> None:
        """Test if snapshots directory contains a file."""
        wrong_file_path = self.snapshots_path / "should_not_be_there"
        wrong_file_path.touch()

        report = scan_cache_dir(self.cache_dir)
        self.assertEqual(len(report.repos), 0)  # Failed

        self.assertEqual(len(report.warnings), 1)
        self.assertEqual(
            str(report.warnings[0]),
            f"Snapshots folder corrupted. Found a file: {wrong_file_path}",
        )

    def test_snapshot_with_no_blob_files(self) -> None:
        """Test if a snapshot directory (e.g. a cached revision) is empty."""
        for revision_path in self.snapshots_path.glob("*"):
            # Delete content of the revision
            rmtree_with_retry(revision_path)
            revision_path.mkdir()

        # Scan
        report = scan_cache_dir(self.cache_dir)

        # Get single repo
        self.assertEqual(len(report.warnings), 0)  # Did not fail
        self.assertEqual(len(report.repos), 1)
        repo_report = list(report.repos)[0]

        # Repo report is empty
        self.assertEqual(repo_report.size_on_disk, 0)
        self.assertEqual(len(repo_report.revisions), 1)
        revision_report = list(repo_report.revisions)[0]

        # No files in revision so last_modified is the one from the revision folder
        self.assertEqual(revision_report.nb_files, 0)
        self.assertEqual(revision_report.last_modified, revision_path.stat().st_mtime)

    def test_repo_with_no_snapshots(self) -> None:
        """Test if the snapshot directory exists but is empty."""
        rmtree_with_retry(self.refs_path)
        rmtree_with_retry(self.snapshots_path)
        self.snapshots_path.mkdir()

        # Scan
        report = scan_cache_dir(self.cache_dir)

        # Get single repo
        self.assertEqual(len(report.warnings), 0)  # Did not fail
        self.assertEqual(len(report.repos), 1)
        repo_report = list(report.repos)[0]

        # No revisions in repos so last_modified is the one from the repo folder
        self.assertEqual(repo_report.size_on_disk, 0)
        self.assertEqual(len(repo_report.revisions), 0)
        self.assertEqual(repo_report.last_modified, self.repo_path.stat().st_mtime)
        self.assertEqual(repo_report.last_accessed, self.repo_path.stat().st_atime)

    def test_ref_to_missing_revision(self) -> None:
        """Test if a `refs` points to a missing revision."""
        new_ref = self.repo_path / "refs" / "not_main"
        with new_ref.open("w") as f:
            f.write("revision_hash_that_does_not_exist")

        report = scan_cache_dir(self.cache_dir)
        self.assertEqual(len(report.repos), 0)  # Failed

        self.assertEqual(len(report.warnings), 1)
        self.assertEqual(
            str(report.warnings[0]),
            "Reference(s) refer to missing commit hashes: {'revision_hash_that_does_not_exist': {'not_main'}} "
            + f"({self.repo_path }).",
        )

    @xfail_on_windows("Last modified/last accessed work a bit differently on Windows.")
    def test_scan_cache_last_modified_and_last_accessed(self) -> None:
        """Scan the last_modified and last_accessed properties when scanning."""
        TIME_GAP = 0.1

        # Make a first scan
        report_1 = scan_cache_dir(self.cache_dir)

        # Values from first report
        repo_1 = list(report_1.repos)[0]
        revision_1 = list(repo_1.revisions)[0]
        readme_file_1 = [file for file in revision_1.files if file.file_name == "README.md"][0]
        another_file_1 = [file for file in revision_1.files if file.file_name == ".gitattributes"][0]

        # Comparison of last_accessed/last_modified between file and repo
        self.assertLessEqual(readme_file_1.blob_last_accessed, repo_1.last_accessed)
        self.assertLessEqual(readme_file_1.blob_last_modified, repo_1.last_modified)
        self.assertEqual(revision_1.last_modified, repo_1.last_modified)

        # Sleep and write new readme
        time.sleep(TIME_GAP)
        readme_file_1.file_path.write_text("modified readme")

        # Sleep and read content from readme
        time.sleep(TIME_GAP)
        with readme_file_1.file_path.open("r") as f:
            _ = f.read()

        # Sleep and re-scan
        time.sleep(TIME_GAP)
        report_2 = scan_cache_dir(self.cache_dir)

        # Values from second report
        repo_2 = list(report_2.repos)[0]
        revision_2 = list(repo_2.revisions)[0]
        readme_file_2 = [file for file in revision_2.files if file.file_name == "README.md"][0]
        another_file_2 = [file for file in revision_1.files if file.file_name == ".gitattributes"][0]

        # Report 1 is not updated when cache changes
        self.assertLess(repo_1.last_accessed, repo_2.last_accessed)
        self.assertLess(repo_1.last_modified, repo_2.last_modified)

        # "Another_file.md" did not change
        self.assertEqual(another_file_1, another_file_2)

        # Readme.md has been modified and then accessed more recently
        self.assertGreaterEqual(
            readme_file_2.blob_last_modified - readme_file_1.blob_last_modified,
            TIME_GAP * 0.9,  # 0.9 factor because not exactly precise
        )
        self.assertGreaterEqual(
            readme_file_2.blob_last_accessed - readme_file_1.blob_last_accessed,
            2 * TIME_GAP * 0.9,  # 0.9 factor because not exactly precise
        )
        self.assertGreaterEqual(
            readme_file_2.blob_last_accessed - readme_file_2.blob_last_modified,
            TIME_GAP * 0.9,  # 0.9 factor because not exactly precise
        )

        # Comparison of last_accessed/last_modified between file and repo
        self.assertEqual(readme_file_2.blob_last_accessed, repo_2.last_accessed)
        self.assertEqual(readme_file_2.blob_last_modified, repo_2.last_modified)
        self.assertEqual(revision_2.last_modified, repo_2.last_modified)


class TestDeleteRevisionsDryRun(unittest.TestCase):
    cache_info: Mock  # Mocked HFCacheInfo

    def setUp(self) -> None:
        """Set up fake cache scan report."""
        repo_A_path = Path("repo_A")
        blobs_path = repo_A_path / "blobs"
        snapshots_path = repo_A_path / "snapshots_path"

        # Define blob files
        main_only_file = Mock()
        main_only_file.blob_path = blobs_path / "main_only_hash"
        main_only_file.size_on_disk = 1

        detached_only_file = Mock()
        detached_only_file.blob_path = blobs_path / "detached_only_hash"
        detached_only_file.size_on_disk = 10

        pr_1_only_file = Mock()
        pr_1_only_file.blob_path = blobs_path / "pr_1_only_hash"
        pr_1_only_file.size_on_disk = 100

        detached_and_pr_1_only_file = Mock()
        detached_and_pr_1_only_file.blob_path = blobs_path / "detached_and_pr_1_only_hash"
        detached_and_pr_1_only_file.size_on_disk = 1000

        shared_file = Mock()
        shared_file.blob_path = blobs_path / "shared_file_hash"
        shared_file.size_on_disk = 10000

        # Define revisions
        repo_A_rev_main = Mock()
        repo_A_rev_main.commit_hash = "repo_A_rev_main"
        repo_A_rev_main.snapshot_path = snapshots_path / "repo_A_rev_main"
        repo_A_rev_main.files = {main_only_file, shared_file}
        repo_A_rev_main.refs = {"main"}

        repo_A_rev_detached = Mock()
        repo_A_rev_detached.commit_hash = "repo_A_rev_detached"
        repo_A_rev_detached.snapshot_path = snapshots_path / "repo_A_rev_detached"
        repo_A_rev_detached.files = {
            detached_only_file,
            detached_and_pr_1_only_file,
            shared_file,
        }
        repo_A_rev_detached.refs = {}

        repo_A_rev_pr_1 = Mock()
        repo_A_rev_pr_1.commit_hash = "repo_A_rev_pr_1"
        repo_A_rev_pr_1.snapshot_path = snapshots_path / "repo_A_rev_pr_1"
        repo_A_rev_pr_1.files = {
            pr_1_only_file,
            detached_and_pr_1_only_file,
            shared_file,
        }
        repo_A_rev_pr_1.refs = {"refs/pr/1"}

        # Define repo
        repo_A = Mock()
        repo_A.repo_path = Path("repo_A")
        repo_A.size_on_disk = 4444
        repo_A.revisions = {repo_A_rev_main, repo_A_rev_detached, repo_A_rev_pr_1}

        # Define cache
        cache_info = Mock()
        cache_info.repos = [repo_A]
        self.cache_info = cache_info

    def test_delete_detached_revision(self) -> None:
        strategy = HFCacheInfo.delete_revisions(self.cache_info, "repo_A_rev_detached")
        expected = DeleteCacheStrategy(
            expected_freed_size=10,
            blobs={
                # "shared_file_hash" and "detached_and_pr_1_only_hash" are not deleted
                Path("repo_A/blobs/detached_only_hash"),
            },
            refs=set(),  # No ref deleted since detached
            repos=set(),  # No repo deleted as other revisions exist
            snapshots={Path("repo_A/snapshots_path/repo_A_rev_detached")},
        )
        self.assertEqual(strategy, expected)

    def test_delete_pr_1_revision(self) -> None:
        strategy = HFCacheInfo.delete_revisions(self.cache_info, "repo_A_rev_pr_1")
        expected = DeleteCacheStrategy(
            expected_freed_size=100,
            blobs={
                # "shared_file_hash" and "detached_and_pr_1_only_hash" are not deleted
                Path("repo_A/blobs/pr_1_only_hash")
            },
            refs={Path("repo_A/refs/refs/pr/1")},  # Ref is deleted !
            repos=set(),  # No repo deleted as other revisions exist
            snapshots={Path("repo_A/snapshots_path/repo_A_rev_pr_1")},
        )
        self.assertEqual(strategy, expected)

    def test_delete_pr_1_and_detached(self) -> None:
        strategy = HFCacheInfo.delete_revisions(self.cache_info, "repo_A_rev_detached", "repo_A_rev_pr_1")
        expected = DeleteCacheStrategy(
            expected_freed_size=1110,
            blobs={
                Path("repo_A/blobs/detached_only_hash"),
                Path("repo_A/blobs/pr_1_only_hash"),
                # blob shared in both revisions and only those two
                Path("repo_A/blobs/detached_and_pr_1_only_hash"),
            },
            refs={Path("repo_A/refs/refs/pr/1")},
            repos=set(),
            snapshots={
                Path("repo_A/snapshots_path/repo_A_rev_detached"),
                Path("repo_A/snapshots_path/repo_A_rev_pr_1"),
            },
        )
        self.assertEqual(strategy, expected)

    def test_delete_all_revisions(self) -> None:
        strategy = HFCacheInfo.delete_revisions(
            self.cache_info, "repo_A_rev_detached", "repo_A_rev_pr_1", "repo_A_rev_main"
        )
        expected = DeleteCacheStrategy(
            expected_freed_size=4444,
            blobs=set(),
            refs=set(),
            repos={Path("repo_A")},  # No remaining revisions: full repo is deleted
            snapshots=set(),
        )
        self.assertEqual(strategy, expected)

    def test_delete_unknown_revision(self) -> None:
        with self.assertLogs() as captured:
            strategy = HFCacheInfo.delete_revisions(self.cache_info, "repo_A_rev_detached", "abcdef123456789")

        # Expected is same strategy as without "abcdef123456789"
        expected = HFCacheInfo.delete_revisions(self.cache_info, "repo_A_rev_detached")
        self.assertEqual(strategy, expected)

        # Expect a warning message
        self.assertEqual(len(captured.records), 1)
        self.assertEqual(captured.records[0].levelname, "WARNING")
        self.assertEqual(
            captured.records[0].message,
            "Revision(s) not found - cannot delete them: abcdef123456789",
        )


@pytest.mark.usefixtures("fx_cache_dir")
class TestDeleteStrategyExecute(unittest.TestCase):
    cache_dir: Path

    def test_execute(self) -> None:
        # Repo folders
        repo_A_path = self.cache_dir / "repo_A"
        repo_A_path.mkdir()
        repo_B_path = self.cache_dir / "repo_B"
        repo_B_path.mkdir()

        # Refs files in repo_B
        refs_main_path = repo_B_path / "refs" / "main"
        refs_main_path.parent.mkdir(parents=True)
        refs_main_path.touch()
        refs_pr_1_path = repo_B_path / "refs" / "refs" / "pr" / "1"
        refs_pr_1_path.parent.mkdir(parents=True)
        refs_pr_1_path.touch()

        # Blobs files in repo_B
        (repo_B_path / "blobs").mkdir()
        blob_1 = repo_B_path / "blobs" / "blob_1"
        blob_2 = repo_B_path / "blobs" / "blob_2"
        blob_3 = repo_B_path / "blobs" / "blob_3"
        blob_1.touch()
        blob_2.touch()
        blob_3.touch()

        # Snapshot folders in repo_B
        snapshot_1 = repo_B_path / "snapshots" / "snapshot_1"
        snapshot_2 = repo_B_path / "snapshots" / "snapshot_2"

        snapshot_1.mkdir(parents=True)
        snapshot_2.mkdir()

        # Execute deletion
        # Delete repo_A + keep only blob_1, main ref and snapshot_1 in repo_B.
        DeleteCacheStrategy(
            expected_freed_size=123456,
            blobs={blob_2, blob_3},
            refs={refs_pr_1_path},
            repos={repo_A_path},
            snapshots={snapshot_2},
        ).execute()

        # Repo A deleted
        self.assertFalse(repo_A_path.exists())
        self.assertTrue(repo_B_path.exists())

        # Only `blob` 1 remains
        self.assertTrue(blob_1.exists())
        self.assertFalse(blob_2.exists())
        self.assertFalse(blob_3.exists())

        # Only ref `main` remains
        self.assertTrue(refs_main_path.exists())
        self.assertFalse(refs_pr_1_path.exists())

        # Only `snapshot_1` remains
        self.assertTrue(snapshot_1.exists())
        self.assertFalse(snapshot_2.exists())


@pytest.mark.usefixtures("fx_cache_dir")
class TestTryDeletePath(unittest.TestCase):
    cache_dir: Path

    def test_delete_path_on_file_success(self) -> None:
        """Successfully delete a local file."""
        file_path = self.cache_dir / "file.txt"
        file_path.touch()
        _try_delete_path(file_path, path_type="TYPE")
        self.assertFalse(file_path.exists())

    def test_delete_path_on_folder_success(self) -> None:
        """Successfully delete a local folder."""
        dir_path = self.cache_dir / "something"
        subdir_path = dir_path / "bar"
        subdir_path.mkdir(parents=True)  # subfolder

        file_path_1 = dir_path / "file.txt"  # file at root
        file_path_1.touch()

        file_path_2 = subdir_path / "config.json"  # file in subfolder
        file_path_2.touch()

        _try_delete_path(dir_path, path_type="TYPE")

        self.assertFalse(dir_path.exists())
        self.assertFalse(subdir_path.exists())
        self.assertFalse(file_path_1.exists())
        self.assertFalse(file_path_2.exists())

    def test_delete_path_on_missing_file(self) -> None:
        """Try delete a missing file."""
        file_path = self.cache_dir / "file.txt"

        with self.assertLogs() as captured:
            _try_delete_path(file_path, path_type="TYPE")

        # Assert warning message with traceback for debug purposes
        self.assertEqual(len(captured.output), 1)
        self.assertTrue(
            captured.output[0].startswith(
                "WARNING:huggingface_hub.utils._cache_manager:Couldn't delete TYPE:"
                f" file not found ({file_path})\nTraceback (most recent call last):"
            )
        )

    def test_delete_path_on_missing_folder(self) -> None:
        """Try delete a missing folder."""
        dir_path = self.cache_dir / "folder"

        with self.assertLogs() as captured:
            _try_delete_path(dir_path, path_type="TYPE")

        # Assert warning message with traceback for debug purposes
        self.assertEqual(len(captured.output), 1)
        self.assertTrue(
            captured.output[0].startswith(
                "WARNING:huggingface_hub.utils._cache_manager:Couldn't delete TYPE:"
                f" file not found ({dir_path})\nTraceback (most recent call last):"
            )
        )

    @xfail_on_windows(reason="Permissions are handled differently on Windows.")
    def test_delete_path_on_local_folder_with_wrong_permission(self) -> None:
        """Try delete a local folder that is protected."""
        dir_path = self.cache_dir / "something"
        dir_path.mkdir()
        file_path_1 = dir_path / "file.txt"  # file at root
        file_path_1.touch()
        dir_path.chmod(444)  # Read-only folder

        with self.assertLogs() as captured:
            _try_delete_path(dir_path, path_type="TYPE")

        # Folder still exists (couldn't be deleted)
        self.assertTrue(dir_path.is_dir())

        # Assert warning message with traceback for debug purposes
        self.assertEqual(len(captured.output), 1)
        self.assertTrue(
            captured.output[0].startswith(
                "WARNING:huggingface_hub.utils._cache_manager:Couldn't delete TYPE:"
                f" permission denied ({dir_path})\nTraceback (most recent call last):"
            )
        )

        # For proper cleanup
        dir_path.chmod(509)


class TestStringFormatters(unittest.TestCase):
    SIZES = {
        16.0: "16.0",
        1000.0: "1.0K",
        1024 * 1024 * 1024: "1.1G",  # not 1.0GiB
    }

    SINCE = {
        1: "a few seconds ago",
        15: "a few seconds ago",
        25: "25 seconds ago",
        80: "1 minute ago",
        1000: "17 minutes ago",
        4000: "1 hour ago",
        8000: "2 hours ago",
        3600 * 24 * 13: "2 weeks ago",
        3600 * 24 * 30 * 8.2: "8 months ago",
        3600 * 24 * 365: "1 year ago",
        3600 * 24 * 365 * 9.6: "10 years ago",
    }

    def test_format_size(self) -> None:
        """Test `_format_size` formatter."""
        for size, expected in self.SIZES.items():
            self.assertEqual(
                _format_size(size),
                expected,
                msg=f"Wrong formatting for {size} == '{expected}'",
            )

    def test_format_timesince(self) -> None:
        """Test `_format_timesince` formatter."""
        for ts, expected in self.SINCE.items():
            self.assertEqual(
                _format_timesince(time.time() - ts),
                expected,
                msg=f"Wrong formatting for {ts} == '{expected}'",
            )


# tests/test_utils_chunks.py

import unittest

from huggingface_hub.utils._chunk_utils import chunk_iterable


class TestUtilsCommon(unittest.TestCase):
    def test_chunk_iterable_non_truncated(self):
        # Can iterable over any iterable (iterator, list, tuple,...)
        for iterable in (range(12), list(range(12)), tuple(range(12))):
            # 12 is a multiple of 4 -> last chunk is not truncated
            for chunk, expected_chunk in zip(
                chunk_iterable(iterable, chunk_size=4),
                [[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]],
            ):
                self.assertListEqual(list(chunk), expected_chunk)

    def test_chunk_iterable_last_chunk_truncated(self):
        # Can iterable over any iterable (iterator, list, tuple,...)
        for iterable in (range(12), list(range(12)), tuple(range(12))):
            # 12 is NOT a multiple of 5 -> last chunk is truncated
            for chunk, expected_chunk in zip(
                chunk_iterable(iterable, chunk_size=5),
                [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9], [10, 11]],
            ):
                self.assertListEqual(list(chunk), expected_chunk)

    def test_chunk_iterable_validation(self):
        with self.assertRaises(ValueError):
            next(chunk_iterable(range(128), 0))

        with self.assertRaises(ValueError):
            next(chunk_iterable(range(128), -1))


# tests/test_utils_cli.py

import os
import unittest
from unittest import mock

from huggingface_hub.commands._cli_utils import ANSI, tabulate


class TestCLIUtils(unittest.TestCase):
    @mock.patch.dict(os.environ, {}, clear=True)
    def test_ansi_utils(self) -> None:
        """Test `ANSI` works as expected."""
        self.assertEqual(
            ANSI.bold("this is bold"),
            "\x1b[1mthis is bold\x1b[0m",
        )

        self.assertEqual(
            ANSI.gray("this is gray"),
            "\x1b[90mthis is gray\x1b[0m",
        )

        self.assertEqual(
            ANSI.red("this is red"),
            "\x1b[1m\x1b[31mthis is red\x1b[0m",
        )

        self.assertEqual(
            ANSI.gray(ANSI.bold("this is bold and grey")),
            "\x1b[90m\x1b[1mthis is bold and grey\x1b[0m\x1b[0m",
        )

    @mock.patch.dict(os.environ, {"NO_COLOR": "1"}, clear=True)
    def test_ansi_no_color(self) -> None:
        """Test `ANSI` respects `NO_COLOR` env var."""

        self.assertEqual(
            ANSI.bold("this is bold"),
            "this is bold",
        )

        self.assertEqual(
            ANSI.gray("this is gray"),
            "this is gray",
        )

        self.assertEqual(
            ANSI.red("this is red"),
            "this is red",
        )

        self.assertEqual(
            ANSI.gray(ANSI.bold("this is bold and grey")),
            "this is bold and grey",
        )

    def test_tabulate_utility(self) -> None:
        """Test `tabulate` works as expected."""
        rows = [[1, 2, 3], ["a very long value", "foo", "bar"], ["", 123, 456]]
        headers = ["Header 1", "something else", "a third column"]
        self.assertEqual(
            tabulate(rows=rows, headers=headers),
            (
                "Header 1          something else a third column \n"
                "----------------- -------------- -------------- \n"
                "                1              2              3 \n"
                "a very long value foo            bar            \n"
                "                             123            456 "
            ),
        )

    def test_tabulate_utility_with_too_short_row(self) -> None:
        """
        Test `tabulate` throw IndexError when a row has less values than the header
        list.
        """
        self.assertRaises(
            IndexError,
            tabulate,
            rows=[[1]],
            headers=["Header 1", "Header 2"],
        )


# tests/test_utils_datetime.py

import unittest
from datetime import datetime, timezone

import pytest

from huggingface_hub.utils import parse_datetime


class TestDatetimeUtils(unittest.TestCase):
    def test_parse_datetime(self):
        """Test `parse_datetime` works correctly on datetimes returned by server."""
        self.assertEqual(
            parse_datetime("2022-08-19T07:19:38.123Z"),
            datetime(2022, 8, 19, 7, 19, 38, 123000, tzinfo=timezone.utc),
        )

        with pytest.raises(ValueError, match=r".*Cannot parse '2022-08-19T07:19:38' as a datetime.*"):
            parse_datetime("2022-08-19T07:19:38")

        with pytest.raises(
            ValueError,
            match=r".*Cannot parse '2022-08-19T07:19:38.123' as a datetime.*",
        ):
            parse_datetime("2022-08-19T07:19:38.123")

        with pytest.raises(
            ValueError,
            match=r".*Cannot parse '2022-08-19 07:19:38.123Z\+6:00' as a datetime.*",
        ):
            parse_datetime("2022-08-19 07:19:38.123Z+6:00")


# tests/test_utils_deprecation.py

import unittest
import warnings

import pytest

from huggingface_hub.utils._deprecation import (
    _deprecate_arguments,
    _deprecate_list_output,
    _deprecate_method,
    _deprecate_positional_args,
)


class TestDeprecationUtils(unittest.TestCase):
    def test_deprecate_positional_args(self):
        """Test warnings are triggered when using deprecated positional args."""

        @_deprecate_positional_args(version="xxx")
        def dummy_position_deprecated(a, *, b="b", c="c"):
            pass

        with warnings.catch_warnings():
            # Assert no warnings when used correctly.
            # Taken from https://docs.pytest.org/en/latest/how-to/capture-warnings.html#additional-use-cases-of-warnings-in-tests
            warnings.simplefilter("error")
            dummy_position_deprecated(a="A", b="B", c="C")
            dummy_position_deprecated("A", b="B", c="C")

        with pytest.warns(FutureWarning):
            dummy_position_deprecated("A", "B", c="C")

        with pytest.warns(FutureWarning):
            dummy_position_deprecated("A", "B", "C")

    def test_deprecate_arguments(self):
        """Test warnings are triggered when using deprecated arguments."""

        @_deprecate_arguments(version="xxx", deprecated_args={"c"})
        def dummy_c_deprecated(a, b="b", c="c"):
            pass

        @_deprecate_arguments(version="xxx", deprecated_args={"b", "c"})
        def dummy_b_c_deprecated(a, b="b", c="c"):
            pass

        with warnings.catch_warnings():
            # Assert no warnings when used correctly.
            # Taken from https://docs.pytest.org/en/latest/how-to/capture-warnings.html#additional-use-cases-of-warnings-in-tests
            warnings.simplefilter("error")
            dummy_c_deprecated("A")
            dummy_c_deprecated("A", "B")
            dummy_c_deprecated("A", b="B")

            dummy_b_c_deprecated("A")

            dummy_b_c_deprecated("A", b="b")
            dummy_b_c_deprecated("A", b="b", c="c")

        with pytest.warns(FutureWarning):
            dummy_c_deprecated("A", "B", "C")

        with pytest.warns(FutureWarning):
            dummy_c_deprecated("A", c="C")

        with pytest.warns(FutureWarning):
            dummy_c_deprecated("A", b="B", c="C")

        with pytest.warns(FutureWarning):
            dummy_b_c_deprecated("A", b="B")

        with pytest.warns(FutureWarning):
            dummy_b_c_deprecated("A", c="C")

        with pytest.warns(FutureWarning):
            dummy_b_c_deprecated("A", b="B", c="C")

    def test_deprecate_arguments_with_default_warning_message(self) -> None:
        """Test default warning message when deprecating arguments."""

        @_deprecate_arguments(version="xxx", deprecated_args={"a"})
        def dummy_deprecated_default_message(a: str = "a") -> None:
            pass

        # Default message
        with pytest.warns(FutureWarning) as record:
            dummy_deprecated_default_message(a="A")
        self.assertEqual(len(record), 1)
        self.assertEqual(
            record[0].message.args[0],
            (
                "Deprecated argument(s) used in 'dummy_deprecated_default_message': a."
                " Will not be supported from version 'xxx'."
            ),
        )

    def test_deprecate_arguments_with_custom_warning_message(self) -> None:
        """Test custom warning message when deprecating arguments."""

        @_deprecate_arguments(
            version="xxx",
            deprecated_args={"a"},
            custom_message="This is a custom message.",
        )
        def dummy_deprecated_custom_message(a: str = "a") -> None:
            pass

        # Custom message
        with pytest.warns(FutureWarning) as record:
            dummy_deprecated_custom_message(a="A")
        self.assertEqual(len(record), 1)
        self.assertEqual(
            record[0].message.args[0],
            (
                "Deprecated argument(s) used in 'dummy_deprecated_custom_message': a."
                " Will not be supported from version 'xxx'.\n\nThis is a custom"
                " message."
            ),
        )

    def test_deprecated_method(self) -> None:
        """Test deprecate method throw warning."""

        @_deprecate_method(version="xxx", message="This is a custom message.")
        def dummy_deprecated() -> None:
            pass

        # Custom message
        with pytest.warns(FutureWarning) as record:
            dummy_deprecated()
        self.assertEqual(len(record), 1)
        self.assertEqual(
            record[0].message.args[0],
            (
                "'dummy_deprecated' (from 'tests.test_utils_deprecation') is deprecated"
                " and will be removed from version 'xxx'. This is a custom message."
            ),
        )

    def test_deprecate_list_output(self) -> None:
        """Test test_deprecate_list_output throw warnings."""

        @_deprecate_list_output(version="xxx")
        def dummy_deprecated() -> None:
            return [1, 2, 3]

        output = dummy_deprecated()

        # Still a list !
        self.assertIsInstance(output, list)

        # __getitem__
        with pytest.warns(FutureWarning) as record:
            self.assertEqual(output[0], 1)

        # (check real message once)
        self.assertEqual(
            record[0].message.args[0],
            (
                "'dummy_deprecated' currently returns a list of objects but is planned to be a generator starting from"
                " version xxx in order to implement pagination. Please avoid to use"
                " `dummy_deprecated(...).__getitem__` or explicitly convert the output to a list first with `[item for"
                " item in dummy_deprecated(...)]`."
            ),
        )

        # __setitem__
        with pytest.warns(FutureWarning):
            output[0] = 10

        # __delitem__
        with pytest.warns(FutureWarning):
            del output[1]

        # `.append` deprecated (as .index, .pop, .insert, .remove, .sort,...)
        with pytest.warns(FutureWarning):
            output.append(5)

        # Magic __len__ deprecated (as __add__, __mul__, __contains__,...)
        with pytest.warns(FutureWarning):
            self.assertEqual(len(output), 3)

        with pytest.warns(FutureWarning):
            self.assertTrue(3 in output)

        # List has been modified
        # Iterating over items is NOT deprecated !
        for item, expected in zip(output, [10, 3, 5]):
            self.assertEqual(item, expected)

        # If output is converted to list, no warning is raised.
        output_as_list = list(iter(dummy_deprecated()))
        self.assertEqual(output_as_list[0], 1)
        del output_as_list[1]
        output_as_list.append(5)
        self.assertEqual(len(output_as_list), 3)
        self.assertTrue(3 in output_as_list)


# tests/test_utils_errors.py

import unittest

from requests.models import Response

from huggingface_hub.utils._errors import (
    BadRequestError,
    EntryNotFoundError,
    HfHubHTTPError,
    RepositoryNotFoundError,
    RevisionNotFoundError,
    hf_raise_for_status,
)


class TestErrorUtils(unittest.TestCase):
    def test_hf_raise_for_status_repo_not_found(self) -> None:
        response = Response()
        response.headers = {"X-Error-Code": "RepoNotFound", "X-Request-Id": 123}
        response.status_code = 404
        with self.assertRaisesRegex(RepositoryNotFoundError, "Repository Not Found") as context:
            hf_raise_for_status(response)

        self.assertEqual(context.exception.response.status_code, 404)
        self.assertIn("Request ID: 123", str(context.exception))

    def test_hf_raise_for_status_repo_not_found_without_error_code(self) -> None:
        response = Response()
        response.headers = {"X-Request-Id": 123}
        response.status_code = 401
        with self.assertRaisesRegex(RepositoryNotFoundError, "Repository Not Found") as context:
            hf_raise_for_status(response)

        self.assertEqual(context.exception.response.status_code, 401)
        self.assertIn("Request ID: 123", str(context.exception))

    def test_hf_raise_for_status_revision_not_found(self) -> None:
        response = Response()
        response.headers = {"X-Error-Code": "RevisionNotFound", "X-Request-Id": 123}
        response.status_code = 404
        with self.assertRaisesRegex(RevisionNotFoundError, "Revision Not Found") as context:
            hf_raise_for_status(response)

        self.assertEqual(context.exception.response.status_code, 404)
        self.assertIn("Request ID: 123", str(context.exception))

    def test_hf_raise_for_status_entry_not_found(self) -> None:
        response = Response()
        response.headers = {"X-Error-Code": "EntryNotFound", "X-Request-Id": 123}
        response.status_code = 404
        with self.assertRaisesRegex(EntryNotFoundError, "Entry Not Found") as context:
            hf_raise_for_status(response)

        self.assertEqual(context.exception.response.status_code, 404)
        self.assertIn("Request ID: 123", str(context.exception))

    def test_hf_raise_for_status_bad_request_no_endpoint_name(self) -> None:
        """Test HTTPError converted to BadRequestError if error 400."""
        response = Response()
        response.status_code = 400
        with self.assertRaisesRegex(BadRequestError, "Bad request:") as context:
            hf_raise_for_status(response)
        self.assertEqual(context.exception.response.status_code, 400)

    def test_hf_raise_for_status_bad_request_with_endpoint_name(self) -> None:
        """Test endpoint name is added to BadRequestError message."""
        response = Response()
        response.status_code = 400
        with self.assertRaisesRegex(BadRequestError, "Bad request for preupload endpoint:") as context:
            hf_raise_for_status(response, endpoint_name="preupload")
        self.assertEqual(context.exception.response.status_code, 400)

    def test_hf_raise_for_status_fallback(self) -> None:
        """Test HTTPError is converted to HfHubHTTPError."""
        response = Response()
        response.status_code = 404
        response.headers = {
            "X-Request-Id": "test-id",
        }
        response.url = "test_URL"
        with self.assertRaisesRegex(HfHubHTTPError, "Request ID: test-id") as context:
            hf_raise_for_status(response)

        self.assertEqual(context.exception.response.status_code, 404)
        self.assertEqual(context.exception.response.url, "test_URL")


class TestHfHubHTTPError(unittest.TestCase):
    response: Response

    def setUp(self) -> None:
        """Setup with a default response."""
        self.response = Response()
        self.response.status_code = 404
        self.response.url = "test_URL"

    def test_hf_hub_http_error_initialization(self) -> None:
        """Test HfHubHTTPError is initialized properly."""
        error = HfHubHTTPError("this is a message", response=self.response)
        self.assertEqual(str(error), "this is a message")
        self.assertEqual(error.response, self.response)
        self.assertIsNone(error.request_id)
        self.assertIsNone(error.server_message)

    def test_hf_hub_http_error_init_with_request_id(self) -> None:
        """Test request id is added to the message."""
        self.response.headers = {"X-Request-Id": "test-id"}
        error = HfHubHTTPError("this is a message", response=self.response)
        self.assertEqual(str(error), "this is a message (Request ID: test-id)")
        self.assertEqual(error.request_id, "test-id")

    def test_hf_hub_http_error_init_with_request_id_and_multiline_message(self) -> None:
        """Test request id is added to the end of the first line."""
        self.response.headers = {"X-Request-Id": "test-id"}
        error = HfHubHTTPError("this is a message\nthis is more details", response=self.response)
        self.assertEqual(str(error), "this is a message (Request ID: test-id)\nthis is more details")

        error = HfHubHTTPError("this is a message\n\nthis is more details", response=self.response)
        self.assertEqual(
            str(error),
            "this is a message (Request ID: test-id)\n\nthis is more details",
        )

    def test_hf_hub_http_error_init_with_request_id_already_in_message(self) -> None:
        """Test request id is not duplicated in error message (case insensitive)"""
        self.response.headers = {"X-Request-Id": "test-id"}
        error = HfHubHTTPError("this is a message on request TEST-ID", response=self.response)
        self.assertEqual(str(error), "this is a message on request TEST-ID")
        self.assertEqual(error.request_id, "test-id")

    def test_hf_hub_http_error_init_with_server_error(self) -> None:
        """Test server error is added to the error message."""
        self.response._content = b'{"error": "This is a message returned by the server"}'
        error = HfHubHTTPError("this is a message", response=self.response)
        self.assertEqual(str(error), "this is a message\n\nThis is a message returned by the server")
        self.assertEqual(error.server_message, "This is a message returned by the server")

    def test_hf_hub_http_error_init_with_server_error_and_multiline_message(
        self,
    ) -> None:
        """Test server error is added to the error message after the details."""
        self.response._content = b'{"error": "This is a message returned by the server"}'
        error = HfHubHTTPError("this is a message\n\nSome details.", response=self.response)
        self.assertEqual(
            str(error),
            "this is a message\n\nSome details.\nThis is a message returned by the server",
        )

    def test_hf_hub_http_error_init_with_multiple_server_errors(
        self,
    ) -> None:
        """Test server errors are added to the error message after the details.

        Regression test for https://github.com/huggingface/huggingface_hub/issues/1114.
        """
        self.response._content = (
            b'{"httpStatusCode": 400, "errors": [{"message": "this is error 1", "type":'
            b' "error"}, {"message": "this is error 2", "type": "error"}]}'
        )
        error = HfHubHTTPError("this is a message\n\nSome details.", response=self.response)
        self.assertEqual(
            str(error),
            "this is a message\n\nSome details.\nthis is error 1\nthis is error 2",
        )

    def test_hf_hub_http_error_init_with_server_error_already_in_message(
        self,
    ) -> None:
        """Test server error is not duplicated if already in details.

        Case insensitive.
        """
        self.response._content = b'{"error": "repo NOT found"}'
        error = HfHubHTTPError(
            "this is a message\n\nRepo Not Found. and more\nand more",
            response=self.response,
        )
        self.assertEqual(
            str(error),
            "this is a message\n\nRepo Not Found. and more\nand more",
        )

    def test_hf_hub_http_error_init_with_unparsable_server_error(
        self,
    ) -> None:
        """Test error message is unchanged and exception is not raised.."""
        self.response._content = b"this is not a json-formatted string"
        error = HfHubHTTPError("this is a message", response=self.response)
        self.assertEqual(str(error), "this is a message")
        self.assertIsNone(error.server_message)  # still None since not parsed

    def test_hf_hub_http_error_append_to_message(self) -> None:
        """Test add extra information to existing HfHubHTTPError."""
        error = HfHubHTTPError("this is a message", response=self.response)
        error.args = error.args + (1, 2, 3)  # faking some extra args

        error.append_to_message("\nthis is an additional message")
        self.assertEqual(
            error.args,
            ("this is a message\nthis is an additional message", 1, 2, 3),
        )
        self.assertIsNone(error.server_message)  # added message is not from server

    def test_hf_hub_http_error_init_with_error_message_in_header(self) -> None:
        """Test server error from header is added to the error message."""
        self.response.headers = {"X-Error-Message": "Error message from headers."}
        error = HfHubHTTPError("this is a message", response=self.response)
        self.assertEqual(str(error), "this is a message\n\nError message from headers.")
        self.assertEqual(error.server_message, "Error message from headers.")

    def test_hf_hub_http_error_init_with_error_message_from_header_and_body(
        self,
    ) -> None:
        """Test server error from header and from body are added to the error message."""
        self.response._content = b'{"error": "Error message from body."}'
        self.response.headers = {"X-Error-Message": "Error message from headers."}
        error = HfHubHTTPError("this is a message", response=self.response)
        self.assertEqual(
            str(error),
            "this is a message\n\nError message from headers.\nError message from body.",
        )
        self.assertEqual(
            error.server_message,
            "Error message from headers.\nError message from body.",
        )

    def test_hf_hub_http_error_init_with_error_message_duplicated_in_header_and_body(
        self,
    ) -> None:
        """Test server error from header and from body are the same.

        Should not duplicate it in the raised `HfHubHTTPError`.
        """
        self.response._content = b'{"error": "Error message duplicated in headers and body."}'
        self.response.headers = {"X-Error-Message": "Error message duplicated in headers and body."}
        error = HfHubHTTPError("this is a message", response=self.response)
        self.assertEqual(
            str(error),
            "this is a message\n\nError message duplicated in headers and body.",
        )
        self.assertEqual(error.server_message, "Error message duplicated in headers and body.")


# tests/test_utils_experimental.py

import unittest
import warnings
from unittest.mock import patch

from huggingface_hub.utils import experimental


@experimental
def dummy_function():
    return "success"


class TestExperimentalFlag(unittest.TestCase):
    def test_experimental_warning(self):
        with patch("huggingface_hub.constants.HF_HUB_DISABLE_EXPERIMENTAL_WARNING", False):
            with warnings.catch_warnings(record=True) as w:
                warnings.simplefilter("always")
                self.assertEqual(dummy_function(), "success")
            self.assertEqual(len(w), 1)

    def test_experimental_no_warning(self):
        with patch("huggingface_hub.constants.HF_HUB_DISABLE_EXPERIMENTAL_WARNING", True):
            with warnings.catch_warnings(record=True) as w:
                warnings.simplefilter("always")
                self.assertEqual(dummy_function(), "success")
            self.assertEqual(len(w), 0)


# tests/test_utils_fixes.py

import unittest
from pathlib import Path

from huggingface_hub.utils import SoftTemporaryDirectory, yaml_dump


class TestYamlDump(unittest.TestCase):
    def test_yaml_dump_emoji(self) -> None:
        self.assertEqual(yaml_dump({"emoji": "👀"}), "emoji: 👀\n")

    def test_yaml_dump_japanese_characters(self) -> None:
        self.assertEqual(yaml_dump({"some unicode": "日本か"}), "some unicode: 日本か\n")

    def test_yaml_dump_explicit_no_unicode(self) -> None:
        self.assertEqual(yaml_dump({"emoji": "👀"}, allow_unicode=False), 'emoji: "\\U0001F440"\n')


class TestTemporaryDirectory(unittest.TestCase):
    def test_temporary_directory(self) -> None:
        with SoftTemporaryDirectory(prefix="prefix", suffix="suffix") as tmpdir:
            self.assertIsInstance(tmpdir, str)
            path = Path(tmpdir)
            self.assertTrue(path.name.startswith("prefix"))
            self.assertTrue(path.name.endswith("suffix"))
            self.assertTrue(path.is_dir())
        # Tmpdir is deleted
        self.assertFalse(path.is_dir())


# tests/test_utils_git_credentials.py

import time
import unittest
from pathlib import Path

import pytest

from huggingface_hub.constants import ENDPOINT
from huggingface_hub.utils import run_interactive_subprocess, run_subprocess
from huggingface_hub.utils._git_credential import (
    list_credential_helpers,
    set_git_credential,
    unset_git_credential,
)


STORE_AND_CACHE_HELPERS_CONFIG = """
[credential]
    helper = store
    helper = cache --timeout 30000
"""


@pytest.mark.usefixtures("fx_cache_dir")
class TestGitCredentials(unittest.TestCase):
    cache_dir: Path

    def setUp(self):
        """Initialize and configure a local repo.

        Avoid to configure git helpers globally on a contributor's machine.
        """
        run_subprocess("git init", folder=self.cache_dir)
        with (self.cache_dir / ".git" / "config").open("w") as f:
            f.write(STORE_AND_CACHE_HELPERS_CONFIG)

    def test_list_credential_helpers(self) -> None:
        helpers = list_credential_helpers(folder=self.cache_dir)
        self.assertIn("cache", helpers)
        self.assertIn("store", helpers)

    def test_set_and_unset_git_credential(self) -> None:
        username = "hf_test_user_" + str(round(time.time()))  # make username unique

        # Set credentials
        set_git_credential(token="hf_test_token", username=username, folder=self.cache_dir)

        # Check credentials are stored
        with run_interactive_subprocess("git credential fill", folder=self.cache_dir) as (stdin, stdout):
            stdin.write(f"url={ENDPOINT}\nusername={username}\n\n")
            stdin.flush()
            output = stdout.read()
        self.assertIn("password=hf_test_token", output)

        # Unset credentials
        unset_git_credential(username=username, folder=self.cache_dir)

        # Check credentials are NOT stored
        # Cannot check with `git credential fill` as it would hang forever: only
        # checking `store` helper instead.
        with run_interactive_subprocess("git credential-store get", folder=self.cache_dir) as (stdin, stdout):
            stdin.write(f"url={ENDPOINT}\nusername={username}\n\n")
            stdin.flush()
            output = stdout.read()
        self.assertEqual("", output)


# tests/test_utils_headers.py

import unittest
from unittest.mock import Mock, patch

from huggingface_hub.utils import get_hf_hub_version, get_python_version
from huggingface_hub.utils._headers import _deduplicate_user_agent, _http_user_agent, build_hf_headers

from .testing_utils import handle_injection, handle_injection_in_test


# Only for tests that are not related to user agent
DEFAULT_USER_AGENT = _http_user_agent()

FAKE_TOKEN = "123456789"
FAKE_TOKEN_ORG = "api_org_123456789"
FAKE_TOKEN_HEADER = {
    "authorization": f"Bearer {FAKE_TOKEN}",
    "user-agent": DEFAULT_USER_AGENT,
}
NO_AUTH_HEADER = {"user-agent": DEFAULT_USER_AGENT}


@patch("huggingface_hub.utils._headers.HfFolder")
@handle_injection
class TestAuthHeadersUtil(unittest.TestCase):
    def test_use_auth_token_str(self) -> None:
        self.assertEqual(build_hf_headers(use_auth_token=FAKE_TOKEN), FAKE_TOKEN_HEADER)

    def test_use_auth_token_true_no_cached_token(self, mock_HfFolder: Mock) -> None:
        mock_HfFolder().get_token.return_value = None
        with self.assertRaises(EnvironmentError):
            build_hf_headers(use_auth_token=True)

    def test_use_auth_token_true_has_cached_token(self, mock_HfFolder: Mock) -> None:
        mock_HfFolder().get_token.return_value = FAKE_TOKEN
        self.assertEqual(build_hf_headers(use_auth_token=True), FAKE_TOKEN_HEADER)

    def test_use_auth_token_false(self, mock_HfFolder: Mock) -> None:
        mock_HfFolder().get_token.return_value = FAKE_TOKEN
        self.assertEqual(build_hf_headers(use_auth_token=False), NO_AUTH_HEADER)

    def test_use_auth_token_none_no_cached_token(self, mock_HfFolder: Mock) -> None:
        mock_HfFolder().get_token.return_value = None
        self.assertEqual(build_hf_headers(), NO_AUTH_HEADER)

    def test_use_auth_token_none_has_cached_token(self, mock_HfFolder: Mock) -> None:
        mock_HfFolder().get_token.return_value = FAKE_TOKEN
        self.assertEqual(build_hf_headers(), FAKE_TOKEN_HEADER)

    def test_write_action_org_token(self) -> None:
        with self.assertRaises(ValueError):
            build_hf_headers(use_auth_token=FAKE_TOKEN_ORG, is_write_action=True)

    def test_write_action_none_token(self, mock_HfFolder: Mock) -> None:
        mock_HfFolder().get_token.return_value = None
        with self.assertRaises(ValueError):
            build_hf_headers(is_write_action=True)

    def test_write_action_use_auth_token_false(self) -> None:
        with self.assertRaises(ValueError):
            build_hf_headers(use_auth_token=False, is_write_action=True)

    def test_implicit_use_disabled(self, mock_HfFolder: Mock) -> None:
        with patch(  # not as decorator to avoid friction with @handle_injection
            "huggingface_hub.constants.HF_HUB_DISABLE_IMPLICIT_TOKEN", True
        ):
            mock_HfFolder().get_token.return_value = FAKE_TOKEN
            self.assertEqual(build_hf_headers(), NO_AUTH_HEADER)  # token is not sent

    def test_implicit_use_disabled_but_explicit_use(self, mock_HfFolder: Mock) -> None:
        with patch(  # not as decorator to avoid friction with @handle_injection
            "huggingface_hub.constants.HF_HUB_DISABLE_IMPLICIT_TOKEN", True
        ):
            mock_HfFolder().get_token.return_value = FAKE_TOKEN

            # This is not an implicit use so we still send it
            self.assertEqual(build_hf_headers(use_auth_token=True), FAKE_TOKEN_HEADER)


class TestUserAgentHeadersUtil(unittest.TestCase):
    def _get_user_agent(self, **kwargs) -> str:
        return build_hf_headers(**kwargs)["user-agent"]

    @patch("huggingface_hub.utils._headers.get_fastai_version")
    @patch("huggingface_hub.utils._headers.get_fastcore_version")
    @patch("huggingface_hub.utils._headers.get_tf_version")
    @patch("huggingface_hub.utils._headers.get_torch_version")
    @patch("huggingface_hub.utils._headers.is_fastai_available")
    @patch("huggingface_hub.utils._headers.is_fastcore_available")
    @patch("huggingface_hub.utils._headers.is_tf_available")
    @patch("huggingface_hub.utils._headers.is_torch_available")
    @handle_injection_in_test
    def test_default_user_agent(
        self,
        mock_get_fastai_version: Mock,
        mock_get_fastcore_version: Mock,
        mock_get_tf_version: Mock,
        mock_get_torch_version: Mock,
        mock_is_fastai_available: Mock,
        mock_is_fastcore_available: Mock,
        mock_is_tf_available: Mock,
        mock_is_torch_available: Mock,
    ) -> None:
        mock_get_fastai_version.return_value = "fastai_version"
        mock_get_fastcore_version.return_value = "fastcore_version"
        mock_get_tf_version.return_value = "tf_version"
        mock_get_torch_version.return_value = "torch_version"
        mock_is_fastai_available.return_value = True
        mock_is_fastcore_available.return_value = True
        mock_is_tf_available.return_value = True
        mock_is_torch_available.return_value = True
        self.assertEqual(
            self._get_user_agent(),
            (
                f"unknown/None; hf_hub/{get_hf_hub_version()};"
                f" python/{get_python_version()}; torch/torch_version;"
                " tensorflow/tf_version; fastai/fastai_version;"
                " fastcore/fastcore_version"
            ),
        )

    @patch("huggingface_hub.utils._headers.is_torch_available")
    @patch("huggingface_hub.utils._headers.is_tf_available")
    @handle_injection_in_test
    def test_user_agent_with_library_name_multiple_missing(
        self, mock_is_torch_available: Mock, mock_is_tf_available: Mock
    ) -> None:
        mock_is_torch_available.return_value = False
        mock_is_tf_available.return_value = False
        self.assertNotIn("torch", self._get_user_agent())
        self.assertNotIn("tensorflow", self._get_user_agent())

    def test_user_agent_with_library_name_and_version(self) -> None:
        self.assertTrue(
            self._get_user_agent(
                library_name="foo",
                library_version="bar",
            ).startswith("foo/bar;")
        )

    def test_user_agent_with_library_name_no_version(self) -> None:
        self.assertTrue(self._get_user_agent(library_name="foo").startswith("foo/None;"))

    def test_user_agent_with_custom_agent_string(self) -> None:
        self.assertTrue(self._get_user_agent(user_agent="this is a custom agent").endswith("this is a custom agent"))

    def test_user_agent_with_custom_agent_dict(self) -> None:
        self.assertTrue(self._get_user_agent(user_agent={"a": "b", "c": "d"}).endswith("a/b; c/d"))

    def test_user_agent_deduplicate(self) -> None:
        self.assertEqual(
            _deduplicate_user_agent(
                "python/3.7; python/3.8; hf_hub/0.12; transformers/None; hf_hub/0.12; python/3.7; diffusers/0.12.1"
            ),
            # 1. "python" is kept twice with different values
            # 2. "python/3.7" second occurrence is removed
            # 3. "hf_hub" second occurrence is removed
            # 4. order is preserved
            "python/3.7; python/3.8; hf_hub/0.12; transformers/None; diffusers/0.12.1",
        )


# tests/test_utils_hf_folder.py

# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contain tests for `HfFolder` utility."""
import os
import unittest
from pathlib import Path
from unittest.mock import patch
from uuid import uuid4

from huggingface_hub.utils import HfFolder, SoftTemporaryDirectory


def _generate_token() -> str:
    return f"token-{uuid4()}"


class HfFolderTest(unittest.TestCase):
    def test_token_workflow(self):
        """
        Test the whole token save/get/delete workflow,
        with the desired behavior with respect to non-existent tokens.
        """
        token = _generate_token()
        HfFolder.save_token(token)
        self.assertEqual(HfFolder.get_token(), token)
        HfFolder.delete_token()
        HfFolder.delete_token()
        # ^^ not an error, we test that the
        # second call does not fail.
        self.assertEqual(HfFolder.get_token(), None)
        # test TOKEN in env
        self.assertEqual(HfFolder.get_token(), None)
        with unittest.mock.patch.dict(os.environ, {"HUGGING_FACE_HUB_TOKEN": token}):
            self.assertEqual(HfFolder.get_token(), token)

    def test_token_in_old_path(self):
        token = _generate_token()
        token2 = _generate_token()
        with SoftTemporaryDirectory() as tmpdir:
            path_token = Path(tmpdir) / "new_token_path"
            old_path_token = Path(tmpdir) / "old_path_token"

            # Use dummy paths
            new_patcher = patch.object(HfFolder, "path_token", path_token)
            old_patcher = patch.object(HfFolder, "_old_path_token", old_path_token)
            new_patcher.start()
            old_patcher.start()

            # Reads from old path -> works but warn
            old_path_token.write_text(token)
            with self.assertWarns(UserWarning):
                self.assertEqual(HfFolder.get_token(), token)
            # Old path still exists
            self.assertEqual(old_path_token.read_text(), token)
            # New path is created
            self.assertEqual(path_token.read_text(), token)

            # Delete -> works, doesn't warn, delete both paths
            HfFolder.delete_token()
            self.assertFalse(old_path_token.exists())
            self.assertFalse(path_token.exists())

            # Write -> only to new path
            HfFolder.save_token(token)
            self.assertFalse(old_path_token.exists())
            self.assertEqual(path_token.read_text(), token)

            # Read -> new path has priority. No warning message.
            old_path_token.write_text(token2)
            self.assertEqual(HfFolder.get_token(), token)

            # Un-patch
            new_patcher.stop()
            old_patcher.stop()


# tests/test_utils_http.py

import threading
import time
import unittest
from typing import Generator
from unittest.mock import Mock, call, patch
from uuid import UUID

import requests
from requests import ConnectTimeout, HTTPError

from huggingface_hub.constants import ENDPOINT
from huggingface_hub.utils._http import configure_http_backend, get_session, http_backoff


URL = "https://www.google.com"


class TestHttpBackoff(unittest.TestCase):
    def setUp(self) -> None:
        get_session_mock = Mock()
        self.mock_request = get_session_mock().request

        self.patcher = patch("huggingface_hub.utils._http.get_session", get_session_mock)
        self.patcher.start()

    def tearDown(self) -> None:
        self.patcher.stop()

    def test_backoff_no_errors(self) -> None:
        """Test normal usage of `http_backoff`."""
        data_mock = Mock()
        response = http_backoff("GET", URL, data=data_mock)
        self.mock_request.assert_called_once_with(method="GET", url=URL, data=data_mock)
        self.assertIs(response, self.mock_request())

    def test_backoff_3_calls(self) -> None:
        """Test `http_backoff` with 2 fails."""
        response_mock = Mock()
        self.mock_request.side_effect = (ValueError(), ValueError(), response_mock)
        response = http_backoff(  # retry on ValueError, instant retry
            "GET", URL, retry_on_exceptions=ValueError, base_wait_time=0.0
        )
        self.assertEqual(self.mock_request.call_count, 3)
        self.mock_request.assert_has_calls(
            calls=[
                call(method="GET", url=URL),
                call(method="GET", url=URL),
                call(method="GET", url=URL),
            ]
        )
        self.assertIs(response, response_mock)

    def test_backoff_on_exception_until_max(self) -> None:
        """Test `http_backoff` until max limit is reached with exceptions."""
        self.mock_request.side_effect = ConnectTimeout()

        with self.assertRaises(ConnectTimeout):
            http_backoff("GET", URL, base_wait_time=0.0, max_retries=3)

        self.assertEqual(self.mock_request.call_count, 4)

    def test_backoff_on_status_code_until_max(self) -> None:
        """Test `http_backoff` until max limit is reached with status codes."""
        mock_503 = Mock()
        mock_503.status_code = 503
        mock_504 = Mock()
        mock_504.status_code = 504
        mock_504.raise_for_status.side_effect = HTTPError()
        self.mock_request.side_effect = (mock_503, mock_504, mock_503, mock_504)

        with self.assertRaises(HTTPError):
            http_backoff(
                "GET",
                URL,
                base_wait_time=0.0,
                max_retries=3,
                retry_on_status_codes=(503, 504),
            )

        self.assertEqual(self.mock_request.call_count, 4)

    def test_backoff_on_exceptions_and_status_codes(self) -> None:
        """Test `http_backoff` until max limit with status codes and exceptions."""
        mock_503 = Mock()
        mock_503.status_code = 503
        self.mock_request.side_effect = (mock_503, ConnectTimeout())

        with self.assertRaises(ConnectTimeout):
            http_backoff("GET", URL, base_wait_time=0.0, max_retries=1)

        self.assertEqual(self.mock_request.call_count, 2)

    def test_backoff_on_valid_status_code(self) -> None:
        """Test `http_backoff` until max limit with a valid status code.

        Quite a corner case: the user wants to retry is status code is 200. Requests are
        retried but in the end, the HTTP 200 response is returned if the server returned
        only 200 responses.
        """
        mock_200 = Mock()
        mock_200.status_code = 200
        self.mock_request.side_effect = (mock_200, mock_200, mock_200, mock_200)

        response = http_backoff("GET", URL, base_wait_time=0.0, max_retries=3, retry_on_status_codes=200)

        self.assertEqual(self.mock_request.call_count, 4)
        self.assertIs(response, mock_200)

    def test_backoff_sleep_time(self) -> None:
        """Test `http_backoff` sleep time goes exponential until max limit.

        Since timing between 2 requests is sleep duration + some other stuff, this test
        can be unstable. However, sleep durations between 10ms and 50ms should be enough
        to make the approximation that measured durations are the "sleep time" waited by
        `http_backoff`. If this is not the case, just increase `base_wait_time`,
        `max_wait_time` and `expected_sleep_times` with bigger values.
        """
        sleep_times = []

        def _side_effect_timer() -> Generator[ConnectTimeout, None, None]:
            t0 = time.time()
            while True:
                yield ConnectTimeout()
                t1 = time.time()
                sleep_times.append(round(t1 - t0, 1))
                t0 = t1

        self.mock_request.side_effect = _side_effect_timer()

        with self.assertRaises(ConnectTimeout):
            http_backoff("GET", URL, base_wait_time=0.1, max_wait_time=0.5, max_retries=5)

        self.assertEqual(self.mock_request.call_count, 6)

        # Assert sleep times are exponential until plateau
        expected_sleep_times = [0.1, 0.2, 0.4, 0.5, 0.5]
        self.assertListEqual(sleep_times, expected_sleep_times)


class TestConfigureSession(unittest.TestCase):
    def setUp(self) -> None:
        # Reconfigure + clear session cache between each test
        configure_http_backend()

    @classmethod
    def tearDownClass(cls) -> None:
        # Clear all sessions after tests
        configure_http_backend()

    @staticmethod
    def _factory() -> requests.Session:
        session = requests.Session()
        session.headers.update({"x-test-header": 4})
        return session

    def test_default_configuration(self) -> None:
        session = get_session()
        self.assertEqual(session.headers["connection"], "keep-alive")  # keep connection alive by default
        self.assertIsNone(session.auth)
        self.assertEqual(session.proxies, {})
        self.assertEqual(session.verify, True)
        self.assertIsNone(session.cert)
        self.assertEqual(session.max_redirects, 30)
        self.assertEqual(session.trust_env, True)
        self.assertEqual(session.hooks, {"response": []})

    def test_set_configuration(self) -> None:
        configure_http_backend(backend_factory=self._factory)

        # Check headers have been set correctly
        session = get_session()
        self.assertNotEqual(session.headers, {"x-test-header": 4})
        self.assertEqual(session.headers["x-test-header"], 4)

    def test_get_session_twice(self):
        session_1 = get_session()
        session_2 = get_session()
        self.assertIs(session_1, session_2)  # exact same instance

    def test_get_session_twice_but_reconfigure_in_between(self):
        """Reconfiguring the session clears the cache."""
        session_1 = get_session()
        configure_http_backend(backend_factory=self._factory)

        session_2 = get_session()
        self.assertIsNot(session_1, session_2)
        self.assertIsNone(session_1.headers.get("x-test-header"))
        self.assertEqual(session_2.headers["x-test-header"], 4)

    def test_get_session_multiple_threads(self):
        N = 3
        sessions = [None] * N

        def _get_session_in_thread(index: int) -> None:
            time.sleep(0.01)
            sessions[index] = get_session()

        # Get main thread session
        main_session = get_session()

        # Start 3 threads and get sessions in each of them
        threads = [threading.Thread(target=_get_session_in_thread, args=(index,)) for index in range(N)]
        for th in threads:
            th.start()
            print(th)
        for th in threads:
            th.join()

        # Check all sessions are different
        for i in range(N):
            self.assertIsNot(main_session, sessions[i])
            for j in range(N):
                if i != j:
                    self.assertIsNot(sessions[i], sessions[j])


class TestUniqueRequestId(unittest.TestCase):
    api_endpoint = ENDPOINT + "/api/tasks"  # any endpoint is fine

    def test_request_id_is_used_by_server(self):
        response = get_session().get(self.api_endpoint)

        request_id = response.request.headers.get("x-request-id")
        response_id = response.headers.get("x-request-id")
        self.assertEqual(request_id, response_id)
        self.assertTrue(_is_uuid(response_id))

    def test_request_id_is_unique(self):
        response_1 = get_session().get(self.api_endpoint)
        response_2 = get_session().get(self.api_endpoint)

        response_id_1 = response_1.headers["x-request-id"]
        response_id_2 = response_2.headers["x-request-id"]
        self.assertNotEqual(response_id_1, response_id_2)

        self.assertTrue(_is_uuid(response_id_1))
        self.assertTrue(_is_uuid(response_id_2))

    def test_request_id_not_overwritten(self):
        response = get_session().get(self.api_endpoint, headers={"x-request-id": "custom-id"})

        request_id = response.request.headers["x-request-id"]
        self.assertEqual(request_id, "custom-id")

        response_id = response.headers["x-request-id"]
        self.assertEqual(response_id, "custom-id")


def _is_uuid(string: str) -> bool:
    # Taken from https://stackoverflow.com/a/33245493
    try:
        uuid_obj = UUID(string)
    except ValueError:
        return False
    return str(uuid_obj) == string


# tests/test_utils_pagination.py

import unittest
from unittest.mock import Mock, call, patch

from huggingface_hub.utils._pagination import paginate

from .testing_utils import handle_injection_in_test


class TestPagination(unittest.TestCase):
    @patch("huggingface_hub.utils._pagination.get_session")
    @patch("huggingface_hub.utils._pagination.hf_raise_for_status")
    @handle_injection_in_test
    def test_mocked_paginate(self, mock_get_session: Mock, mock_hf_raise_for_status: Mock) -> None:
        mock_get = mock_get_session().get
        mock_params = Mock()
        mock_headers = Mock()

        # Simulate page 1
        mock_response_page_1 = Mock()
        mock_response_page_1.json.return_value = [1, 2, 3]
        mock_response_page_1.links = {"next": {"url": "url_p2"}}

        # Simulate page 2
        mock_response_page_2 = Mock()
        mock_response_page_2.json.return_value = [4, 5, 6]
        mock_response_page_2.links = {"next": {"url": "url_p3"}}

        # Simulate page 3
        mock_response_page_3 = Mock()
        mock_response_page_3.json.return_value = [7, 8]
        mock_response_page_3.links = {}

        # Mock response
        mock_get.side_effect = [
            mock_response_page_1,
            mock_response_page_2,
            mock_response_page_3,
        ]

        results = paginate("url", params=mock_params, headers=mock_headers)

        # Requests are made only when generator is yielded
        self.assertEqual(mock_get.call_count, 0)

        # Results after concatenating pages
        self.assertListEqual(list(results), [1, 2, 3, 4, 5, 6, 7, 8])

        # All pages requested: 3 requests, 3 raise for status
        self.assertEqual(mock_get.call_count, 3)
        self.assertEqual(mock_hf_raise_for_status.call_count, 3)

        # Params not passed to next pages
        self.assertListEqual(
            mock_get.call_args_list,
            [
                call("url", params=mock_params, headers=mock_headers),
                call("url_p2", headers=mock_headers),
                call("url_p3", headers=mock_headers),
            ],
        )

    def test_paginate_github_api(self) -> None:
        # Real test: paginate over huggingface repos on Github
        # Use enumerate and stop after first page to avoid loading all repos
        for num, _ in enumerate(
            paginate("https://api.github.com/orgs/huggingface/repos?limit=4", params={}, headers={})
        ):
            if num == 6:
                break
        else:
            self.fail("Did not get more than 6 repos")


# tests/test_utils_paths.py

import unittest
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable, List, Optional, Union

from huggingface_hub.utils import IGNORE_GIT_FOLDER_PATTERNS, filter_repo_objects


@dataclass
class DummyObject:
    path: Path


DUMMY_FILES = ["not_hidden.pdf", "profile.jpg", ".hidden.pdf", ".hidden_picture.png"]
DUMMY_PATHS = [Path(path) for path in DUMMY_FILES]
DUMMY_OBJECTS = [DummyObject(path=path) for path in DUMMY_FILES]


class TestPathsUtils(unittest.TestCase):
    def test_get_all_pdfs(self) -> None:
        """Get all PDFs even hidden ones."""
        self._check(
            items=DUMMY_FILES,
            expected_items=["not_hidden.pdf", ".hidden.pdf"],
            allow_patterns=["*.pdf"],
        )

    def test_get_all_pdfs_except_hidden(self) -> None:
        """Get all PDFs except hidden ones."""
        self._check(
            items=DUMMY_FILES,
            expected_items=["not_hidden.pdf"],
            allow_patterns=["*.pdf"],
            ignore_patterns=[".*"],
        )

    def test_get_all_pdfs_except_hidden_using_single_pattern(self) -> None:
        """Get all PDFs except hidden ones, using single pattern."""
        self._check(
            items=DUMMY_FILES,
            expected_items=["not_hidden.pdf"],
            allow_patterns="*.pdf",  # not a list
            ignore_patterns=".*",  # not a list
        )

    def test_get_all_images(self) -> None:
        """Get all images."""
        self._check(
            items=DUMMY_FILES,
            expected_items=["profile.jpg", ".hidden_picture.png"],
            allow_patterns=["*.png", "*.jpg"],
        )

    def test_get_all_images_except_hidden_from_paths(self) -> None:
        """Get all images except hidden ones, from Path list."""
        self._check(
            items=DUMMY_PATHS,
            expected_items=[Path("profile.jpg")],
            allow_patterns=["*.png", "*.jpg"],
            ignore_patterns=".*",
        )

    def test_get_all_images_except_hidden_from_objects(self) -> None:
        """Get all images except hidden ones, from object list."""
        self._check(
            items=DUMMY_OBJECTS,
            expected_items=[DummyObject(path="profile.jpg")],
            allow_patterns=["*.png", "*.jpg"],
            ignore_patterns=".*",
            key=lambda x: x.path,
        )

    def test_filter_objects_key_not_provided(self) -> None:
        """Test ValueError is raised if filtering non-string objects."""
        with self.assertRaisesRegex(ValueError, "Please provide `key` argument"):
            list(
                filter_repo_objects(
                    items=DUMMY_OBJECTS,
                    allow_patterns=["*.png", "*.jpg"],
                    ignore_patterns=".*",
                )
            )

    def _check(
        self,
        items: List[Any],
        expected_items: List[Any],
        allow_patterns: Optional[Union[List[str], str]] = None,
        ignore_patterns: Optional[Union[List[str], str]] = None,
        key: Optional[Callable[[Any], str]] = None,
    ) -> None:
        """Run `filter_repo_objects` and check output against expected result."""
        self.assertListEqual(
            list(
                filter_repo_objects(
                    items=items,
                    allow_patterns=allow_patterns,
                    ignore_patterns=ignore_patterns,
                    key=key,
                )
            ),
            expected_items,
        )


class TestGitFolderExclusion(unittest.TestCase):
    GIT_FOLDER_PATHS = [
        ".git",
        ".git/file.txt",
        ".git/folder/file.txt",
        "path/to/folder/.git",
        "path/to/folder/.git/file.txt",
        "path/to/.git/folder/file.txt",
    ]

    NOT_GIT_FOLDER_PATHS = [
        ".gitignore",
        "path/foo.git/file.txt",
        "path/.git_bar/file.txt",
        "path/to/file.git",
    ]

    def test_exclude_git_folder(self):
        filtered_paths = filter_repo_objects(
            items=self.GIT_FOLDER_PATHS + self.NOT_GIT_FOLDER_PATHS, ignore_patterns=IGNORE_GIT_FOLDER_PATTERNS
        )
        self.assertListEqual(list(filtered_paths), self.NOT_GIT_FOLDER_PATHS)


# tests/test_utils_runtime.py

import unittest

from huggingface_hub.utils._runtime import is_google_colab, is_notebook


class TestRuntimeUtils(unittest.TestCase):
    def test_is_notebook(self) -> None:
        """Test `is_notebook`."""
        self.assertFalse(is_notebook())

    def test_is_google_colab(self) -> None:
        """Test `is_google_colab`."""
        self.assertFalse(is_google_colab())


# tests/test_utils_sha.py

import os
import unittest
from hashlib import sha256
from io import BytesIO

from huggingface_hub.utils import SoftTemporaryDirectory
from huggingface_hub.utils.sha import sha_fileobj


class TestShaUtils(unittest.TestCase):
    def test_sha_fileobj(self):
        with SoftTemporaryDirectory() as tmpdir:
            content = b"Random content" * 1000
            sha = sha256(content).digest()

            # Test with file object
            filepath = os.path.join(tmpdir, "file.bin")
            with open(filepath, "wb+") as file:
                file.write(content)

            with open(filepath, "rb") as fileobj:
                self.assertEqual(sha_fileobj(fileobj, None), sha)
            with open(filepath, "rb") as fileobj:
                self.assertEqual(sha_fileobj(fileobj, 50), sha)
            with open(filepath, "rb") as fileobj:
                self.assertEqual(sha_fileobj(fileobj, 50_000), sha)

            # Test with in-memory file object
            self.assertEqual(sha_fileobj(BytesIO(content), None), sha)
            self.assertEqual(sha_fileobj(BytesIO(content), 50), sha)
            self.assertEqual(sha_fileobj(BytesIO(content), 50_000), sha)


# tests/test_utils_telemetry.py

import unittest
from queue import Queue
from unittest.mock import Mock, patch

from huggingface_hub.utils._telemetry import send_telemetry

from .testing_constants import ENDPOINT_STAGING


@patch("huggingface_hub.utils._telemetry._TELEMETRY_QUEUE", new_callable=Queue)
@patch("huggingface_hub.utils._telemetry._TELEMETRY_THREAD", None)
class TestSendTelemetry(unittest.TestCase):
    def setUp(self) -> None:
        get_session_mock = Mock()
        self.mock_head = get_session_mock().head

        self.patcher = patch("huggingface_hub.utils._telemetry.get_session", get_session_mock)
        self.patcher.start()

    def tearDown(self) -> None:
        self.patcher.stop()

    def test_topic_normal(self, queue: Queue) -> None:
        send_telemetry(topic="examples")
        queue.join()  # Wait for the telemetry tasks to be completed
        self.mock_head.assert_called_once()
        self.assertEqual(self.mock_head.call_args[0][0], f"{ENDPOINT_STAGING}/api/telemetry/examples")

    def test_topic_multiple(self, queue: Queue) -> None:
        send_telemetry(topic="example1")
        send_telemetry(topic="example2")
        send_telemetry(topic="example3")
        queue.join()  # Wait for the telemetry tasks to be completed

        self.assertEqual(self.mock_head.call_count, 3)  # 3 calls and order is preserved
        self.assertEqual(self.mock_head.call_args_list[0][0][0], f"{ENDPOINT_STAGING}/api/telemetry/example1")
        self.assertEqual(self.mock_head.call_args_list[1][0][0], f"{ENDPOINT_STAGING}/api/telemetry/example2")
        self.assertEqual(self.mock_head.call_args_list[2][0][0], f"{ENDPOINT_STAGING}/api/telemetry/example3")

    def test_topic_with_subtopic(self, queue: Queue) -> None:
        send_telemetry(topic="gradio/image/this_one")
        queue.join()  # Wait for the telemetry tasks to be completed
        self.mock_head.assert_called_once()
        self.assertEqual(self.mock_head.call_args[0][0], f"{ENDPOINT_STAGING}/api/telemetry/gradio/image/this_one")

    def test_topic_quoted(self, queue: Queue) -> None:
        send_telemetry(topic="foo bar")
        queue.join()  # Wait for the telemetry tasks to be completed
        self.mock_head.assert_called_once()
        self.assertEqual(self.mock_head.call_args[0][0], f"{ENDPOINT_STAGING}/api/telemetry/foo%20bar")

    @patch("huggingface_hub.utils._telemetry.constants.HF_HUB_OFFLINE", True)
    def test_hub_offline(self, queue: Queue) -> None:
        send_telemetry(topic="topic")
        self.assertTrue(queue.empty())  # no tasks
        self.mock_head.assert_not_called()

    @patch("huggingface_hub.utils._telemetry.constants.HF_HUB_DISABLE_TELEMETRY", True)
    def test_telemetry_disabled(self, queue: Queue) -> None:
        send_telemetry(topic="topic")
        self.assertTrue(queue.empty())  # no tasks
        self.mock_head.assert_not_called()

    @patch("huggingface_hub.utils._telemetry.build_hf_headers")
    def test_telemetry_use_build_hf_headers(self, mock_headers: Mock, queue: Queue) -> None:
        send_telemetry(topic="topic")
        queue.join()  # Wait for the telemetry tasks to be completed
        self.mock_head.assert_called_once()
        mock_headers.assert_called_once()
        self.assertEqual(self.mock_head.call_args[1]["headers"], mock_headers.return_value)


@patch("huggingface_hub.utils._telemetry._TELEMETRY_QUEUE", new_callable=Queue)
@patch("huggingface_hub.utils._telemetry._TELEMETRY_THREAD", None)
class TestSendTelemetryConnectionError(unittest.TestCase):
    def setUp(self) -> None:
        get_session_mock = Mock()
        get_session_mock().head.side_effect = Exception("whatever")

        self.patcher = patch("huggingface_hub.utils._telemetry.get_session", get_session_mock)
        self.patcher.start()

    def tearDown(self) -> None:
        self.patcher.stop()

    def test_telemetry_exception_silenced(self, queue: Queue) -> None:
        with self.assertLogs(logger="huggingface_hub.utils._telemetry", level="DEBUG") as captured:
            send_telemetry(topic="topic")
            queue.join()

        # Assert debug message with traceback for debug purposes
        self.assertEqual(len(captured.output), 1)
        self.assertEqual(
            captured.output[0],
            "DEBUG:huggingface_hub.utils._telemetry:Error while sending telemetry: whatever",
        )


# tests/test_utils_tqdm.py

import time
import unittest
from pathlib import Path
from unittest.mock import patch

import pytest
from pytest import CaptureFixture

from huggingface_hub.utils import (
    SoftTemporaryDirectory,
    are_progress_bars_disabled,
    disable_progress_bars,
    enable_progress_bars,
    tqdm,
    tqdm_stream_file,
)


class TestTqdmUtils(unittest.TestCase):
    @pytest.fixture(autouse=True)
    def capsys(self, capsys: CaptureFixture) -> None:
        """Workaround to make capsys work in unittest framework.

        Capsys is a convenient pytest fixture to capture stdout.
        See https://waylonwalker.com/pytest-capsys/.

        Taken from https://github.com/pytest-dev/pytest/issues/2504#issuecomment-309475790.
        """
        self.capsys = capsys

    def setUp(self) -> None:
        """Get verbosity to set it back after the tests."""
        self._previous_are_progress_bars_disabled = are_progress_bars_disabled()
        return super().setUp()

    def tearDown(self) -> None:
        """Set back progress bars verbosity as before testing."""
        if self._previous_are_progress_bars_disabled:
            disable_progress_bars()
        else:
            enable_progress_bars()

    @patch("huggingface_hub.utils._tqdm.HF_HUB_DISABLE_PROGRESS_BARS", None)
    def test_tqdm_helpers(self) -> None:
        """Test helpers to enable/disable progress bars."""
        disable_progress_bars()
        self.assertTrue(are_progress_bars_disabled())

        enable_progress_bars()
        self.assertFalse(are_progress_bars_disabled())

    @patch("huggingface_hub.utils._tqdm.HF_HUB_DISABLE_PROGRESS_BARS", True)
    def test_cannot_enable_tqdm_when_env_variable_is_set(self) -> None:
        """
        Test helpers cannot enable/disable progress bars when
        `HF_HUB_DISABLE_PROGRESS_BARS` is set.
        """
        disable_progress_bars()
        self.assertTrue(are_progress_bars_disabled())

        with self.assertWarns(UserWarning):
            enable_progress_bars()
        self.assertTrue(are_progress_bars_disabled())  # Still disabled !

    @patch("huggingface_hub.utils._tqdm.HF_HUB_DISABLE_PROGRESS_BARS", False)
    def test_cannot_disable_tqdm_when_env_variable_is_set(self) -> None:
        """
        Test helpers cannot enable/disable progress bars when
        `HF_HUB_DISABLE_PROGRESS_BARS` is set.
        """
        enable_progress_bars()
        self.assertFalse(are_progress_bars_disabled())

        with self.assertWarns(UserWarning):
            disable_progress_bars()
        self.assertFalse(are_progress_bars_disabled())  # Still enabled !

    @patch("huggingface_hub.utils._tqdm.HF_HUB_DISABLE_PROGRESS_BARS", None)
    def test_tqdm_disabled(self) -> None:
        """Test TQDM not outputting anything when globally disabled."""
        disable_progress_bars()
        for _ in tqdm(range(10)):
            pass

        captured = self.capsys.readouterr()
        self.assertEqual(captured.out, "")
        self.assertEqual(captured.err, "")

    @patch("huggingface_hub.utils._tqdm.HF_HUB_DISABLE_PROGRESS_BARS", None)
    def test_tqdm_disabled_cannot_be_forced(self) -> None:
        """Test TQDM cannot be forced when globally disabled."""
        disable_progress_bars()
        for _ in tqdm(range(10), disable=False):
            pass

        captured = self.capsys.readouterr()
        self.assertEqual(captured.out, "")
        self.assertEqual(captured.err, "")

    @patch("huggingface_hub.utils._tqdm.HF_HUB_DISABLE_PROGRESS_BARS", None)
    def test_tqdm_can_be_disabled_when_globally_enabled(self) -> None:
        """Test TQDM can still be locally disabled even when globally enabled."""
        enable_progress_bars()
        for _ in tqdm(range(10), disable=True):
            pass

        captured = self.capsys.readouterr()
        self.assertEqual(captured.out, "")
        self.assertEqual(captured.err, "")

    @patch("huggingface_hub.utils._tqdm.HF_HUB_DISABLE_PROGRESS_BARS", None)
    def test_tqdm_enabled(self) -> None:
        """Test TQDM work normally when globally enabled."""
        enable_progress_bars()
        for _ in tqdm(range(10)):
            pass

        captured = self.capsys.readouterr()
        self.assertEqual(captured.out, "")
        self.assertIn("10/10", captured.err)  # tqdm log

    def test_tqdm_stream_file(self) -> None:
        with SoftTemporaryDirectory() as tmpdir:
            filepath = Path(tmpdir) / "config.json"
            with filepath.open("w") as f:
                f.write("#" * 1000)

            with tqdm_stream_file(filepath) as f:
                while True:
                    data = f.read(100)
                    if not data:
                        break
                    time.sleep(0.001)  # Simulate a delay between each chunk

            captured = self.capsys.readouterr()
            self.assertEqual(captured.out, "")
            self.assertIn("config.json: 100%", captured.err)  # log file name
            self.assertIn("|█████████", captured.err)  # tqdm bar
            self.assertIn("1.00k/1.00k", captured.err)  # size in B


# tests/test_utils_validators.py

import unittest
from pathlib import Path
from unittest.mock import Mock, patch

from huggingface_hub.utils import (
    HFValidationError,
    smoothly_deprecate_use_auth_token,
    validate_hf_hub_args,
    validate_repo_id,
)


@patch("huggingface_hub.utils._validators.validate_repo_id")
class TestHfHubValidator(unittest.TestCase):
    """Test `validate_hf_hub_args` decorator calls all default validators."""

    def test_validate_repo_id_as_arg(self, validate_repo_id_mock: Mock) -> None:
        """Test `validate_repo_id` is called when `repo_id` is passed as arg."""
        self.dummy_function(123)
        validate_repo_id_mock.assert_called_once_with(123)

    def test_validate_repo_id_as_kwarg(self, validate_repo_id_mock: Mock) -> None:
        """Test `validate_repo_id` is called when `repo_id` is passed as kwarg."""
        self.dummy_function(repo_id=123)
        validate_repo_id_mock.assert_called_once_with(123)

    @staticmethod
    @validate_hf_hub_args
    def dummy_function(repo_id: str) -> None:
        pass


class TestRepoIdValidator(unittest.TestCase):
    VALID_VALUES = (
        "123",
        "foo",
        "foo/bar",
        "Foo-BAR_foo.bar123",
    )
    NOT_VALID_VALUES = (
        Path("foo/bar"),  # Must be a string
        "a" * 100,  # Too long
        "datasets/foo/bar",  # Repo_type forbidden in repo_id
        ".repo_id",  # Cannot start with .
        "repo_id.",  # Cannot end with .
        "foo--bar",  # Cannot contain "--"
        "foo..bar",  # Cannot contain "."
        "foo.git",  # Cannot end with ".git"
    )

    def test_valid_repo_ids(self) -> None:
        """Test `repo_id` validation on valid values."""
        for repo_id in self.VALID_VALUES:
            validate_repo_id(repo_id)

    def test_not_valid_repo_ids(self) -> None:
        """Test `repo_id` validation on not valid values."""
        for repo_id in self.NOT_VALID_VALUES:
            with self.assertRaises(HFValidationError, msg=f"'{repo_id}' must not be valid"):
                validate_repo_id(repo_id)


class TestSmoothlyDeprecateUseAuthToken(unittest.TestCase):
    def test_token_normal_usage_as_arg(self) -> None:
        self.assertEqual(
            self.dummy_token_function("this_is_a_token"),
            ("this_is_a_token", {}),
        )

    def test_token_normal_usage_as_kwarg(self) -> None:
        self.assertEqual(
            self.dummy_token_function(token="this_is_a_token"),
            ("this_is_a_token", {}),
        )

    def test_token_normal_usage_with_more_kwargs(self) -> None:
        self.assertEqual(
            self.dummy_token_function(token="this_is_a_token", foo="bar"),
            ("this_is_a_token", {"foo": "bar"}),
        )

    def test_token_with_smoothly_deprecated_use_auth_token(self) -> None:
        self.assertEqual(
            self.dummy_token_function(use_auth_token="this_is_a_use_auth_token"),
            ("this_is_a_use_auth_token", {}),
        )

    def test_input_kwargs_not_mutated_by_smooth_deprecation(self) -> None:
        initial_kwargs = {"a": "b", "use_auth_token": "token"}
        kwargs = smoothly_deprecate_use_auth_token(fn_name="name", has_token=False, kwargs=initial_kwargs)
        self.assertEqual(kwargs, {"a": "b", "token": "token"})
        self.assertEqual(initial_kwargs, {"a": "b", "use_auth_token": "token"})  # not mutated!

    def test_with_both_token_and_use_auth_token(self) -> None:
        with self.assertWarns(UserWarning):
            # `use_auth_token` is ignored !
            self.assertEqual(
                self.dummy_token_function(token="this_is_a_token", use_auth_token="this_is_a_use_auth_token"),
                ("this_is_a_token", {}),
            )

    def test_not_deprecated_use_auth_token(self) -> None:
        # `use_auth_token` is accepted by `dummy_use_auth_token_function`
        # => `smoothly_deprecate_use_auth_token` is not called
        self.assertEqual(
            self.dummy_use_auth_token_function(use_auth_token="this_is_a_use_auth_token"),
            ("this_is_a_use_auth_token", {}),
        )

    @staticmethod
    @validate_hf_hub_args
    def dummy_token_function(token: str, **kwargs) -> None:
        return token, kwargs

    @staticmethod
    @validate_hf_hub_args
    def dummy_use_auth_token_function(use_auth_token: str, **kwargs) -> None:
        return use_auth_token, kwargs


# tests/test_webhooks_server.py

import unittest
from unittest.mock import patch

from fastapi import Request

from huggingface_hub.utils import capture_output, is_gradio_available

from .testing_utils import require_webhooks


if is_gradio_available():
    import gradio as gr
    from fastapi.testclient import TestClient

    import huggingface_hub._webhooks_server
    from huggingface_hub import WebhookPayload, WebhooksServer


# Taken from https://huggingface.co/docs/hub/webhooks#event
WEBHOOK_PAYLOAD_EXAMPLE = {
    "event": {"action": "create", "scope": "discussion"},
    "repo": {
        "type": "model",
        "name": "gpt2",
        "id": "621ffdc036468d709f17434d",
        "private": False,
        "url": {"web": "https://huggingface.co/gpt2", "api": "https://huggingface.co/api/models/gpt2"},
        "owner": {"id": "628b753283ef59b5be89e937"},
    },
    "discussion": {
        "id": "6399f58518721fdd27fc9ca9",
        "title": "Update co2 emissions",
        "url": {
            "web": "https://huggingface.co/gpt2/discussions/19",
            "api": "https://huggingface.co/api/models/gpt2/discussions/19",
        },
        "status": "open",
        "author": {"id": "61d2f90c3c2083e1c08af22d"},
        "num": 19,
        "isPullRequest": True,
        "changes": {"base": "refs/heads/main"},
    },
    "comment": {
        "id": "6399f58518721fdd27fc9caa",
        "author": {"id": "61d2f90c3c2083e1c08af22d"},
        "content": "Add co2 emissions information to the model card",
        "hidden": False,
        "url": {"web": "https://huggingface.co/gpt2/discussions/19#6399f58518721fdd27fc9caa"},
    },
    "webhook": {"id": "6390e855e30d9209411de93b", "version": 3},
}


@require_webhooks
class TestWebhooksServerDontRun(unittest.TestCase):
    def test_add_webhook_implicit_path(self):
        # Test adding a webhook
        app = WebhooksServer()

        @app.add_webhook
        async def handler():
            pass

        self.assertIn("/webhooks/handler", app.registered_webhooks)

    def test_add_webhook_explicit_path(self):
        # Test adding a webhook
        app = WebhooksServer()

        @app.add_webhook(path="/test_webhook")
        async def handler():
            pass

        self.assertIn("/webhooks/test_webhook", app.registered_webhooks)  # still registered under /webhooks

    def test_add_webhook_twice_should_fail(self):
        # Test adding a webhook
        app = WebhooksServer()

        @app.add_webhook("my_webhook")
        async def test_webhook():
            pass

        # Registering twice the same webhook should raise an error
        with self.assertRaises(ValueError):

            @app.add_webhook("my_webhook")
            async def test_webhook_2():
                pass


@require_webhooks
class TestWebhooksServerRun(unittest.TestCase):
    HEADERS_VALID_SECRET = {"x-webhook-secret": "my_webhook_secret"}
    HEADERS_WRONG_SECRET = {"x-webhook-secret": "wrong_webhook_secret"}

    def setUp(self) -> None:
        with gr.Blocks() as ui:
            gr.Markdown("Hello World!")
        app = WebhooksServer(ui=ui, webhook_secret="my_webhook_secret")

        # Route to check payload parsing
        @app.add_webhook
        async def test_webhook(payload: WebhookPayload) -> None:
            return {"scope": payload.event.scope}

        # Routes to check secret validation
        # Checks all 4 cases (async/sync, with/without request parameter)
        @app.add_webhook
        async def async_with_request(request: Request) -> None:
            return {"success": True}

        @app.add_webhook
        def sync_with_request(request: Request) -> None:
            return {"success": True}

        @app.add_webhook
        async def async_no_request() -> None:
            return {"success": True}

        @app.add_webhook
        def sync_no_request() -> None:
            return {"success": True}

        # Route to check explicit path
        @app.add_webhook(path="/explicit_path")
        async def with_explicit_path() -> None:
            return {"success": True}

        self.ui = ui
        self.app = app
        self.client = self.mocked_run_app()

    def tearDown(self) -> None:
        self.ui.server.close()

    def mocked_run_app(self) -> "TestClient":
        with patch.object(self.ui, "block_thread"):
            # Run without blocking
            with patch.object(huggingface_hub._webhooks_server, "_is_local", False):
                # Run without tunnel
                self.app.run()
                return TestClient(self.app.fastapi_app)

    def test_run_print_instructions(self):
        """Test that the instructions are printed when running the app."""
        # Test running the app
        with capture_output() as output:
            self.mocked_run_app()

        instructions = output.getvalue()
        self.assertIn("Webhooks are correctly setup and ready to use:", instructions)
        self.assertIn("- POST http://127.0.0.1:7860/webhooks/test_webhook", instructions)

    def test_run_parse_payload(self):
        """Test that the payload is correctly parsed when running the app."""
        response = self.client.post(
            "/webhooks/test_webhook", headers=self.HEADERS_VALID_SECRET, json=WEBHOOK_PAYLOAD_EXAMPLE
        )
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.json(), {"scope": "discussion"})

    def test_with_webhook_secret_should_succeed(self):
        """Test success if valid secret is sent."""
        for path in ["async_with_request", "sync_with_request", "async_no_request", "sync_no_request"]:
            with self.subTest(path):
                response = self.client.post(f"/webhooks/{path}", headers=self.HEADERS_VALID_SECRET)
                self.assertEqual(response.status_code, 200)
                self.assertEqual(response.json(), {"success": True})

    def test_no_webhook_secret_should_be_unauthorized(self):
        """Test failure if valid secret is sent."""
        for path in ["async_with_request", "sync_with_request", "async_no_request", "sync_no_request"]:
            with self.subTest(path):
                response = self.client.post(f"/webhooks/{path}")
                self.assertEqual(response.status_code, 401)

    def test_wrong_webhook_secret_should_be_forbidden(self):
        """Test failure if valid secret is sent."""
        for path in ["async_with_request", "sync_with_request", "async_no_request", "sync_no_request"]:
            with self.subTest(path):
                response = self.client.post(f"/webhooks/{path}", headers=self.HEADERS_WRONG_SECRET)
                self.assertEqual(response.status_code, 403)

    def test_route_with_explicit_path(self):
        """Test that the route with an explicit path is correctly registered."""
        response = self.client.post("/webhooks/explicit_path", headers=self.HEADERS_VALID_SECRET)
        self.assertEqual(response.status_code, 200)


# tests/test_windows.py

"""Contains tests that are specific to windows machines."""
import os
import unittest

from huggingface_hub.file_download import are_symlinks_supported


def require_windows(test_case):
    if os.name != "nt":
        return unittest.skip("test of git lfs workflow")(test_case)
    else:
        return test_case


@require_windows
class WindowsTests(unittest.TestCase):
    def test_are_symlink_supported(self) -> None:
        self.assertFalse(are_symlinks_supported())


# tests/testing_constants.py

import os


USER = "__DUMMY_TRANSFORMERS_USER__"
FULL_NAME = "Dummy User"
PASS = "__DUMMY_TRANSFORMERS_PASS__"

# Not critical, only usable on the sandboxed CI instance.
TOKEN = "hf_94wBhPGp6KrrTH3KDchhKpRxZwd6dmHWLL"

# Used to create repos that we don't own (example: for gated repo)
# Token is not critical. Also public in https://github.com/huggingface/datasets-server
OTHER_USER = "__DUMMY_DATASETS_SERVER_USER__"
OTHER_TOKEN = "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"

ENDPOINT_PRODUCTION = "https://huggingface.co"
ENDPOINT_STAGING = "https://hub-ci.huggingface.co"
ENDPOINT_STAGING_BASIC_AUTH = f"https://{USER}:{PASS}@hub-ci.huggingface.co"

ENDPOINT_PRODUCTION_URL_SCHEME = ENDPOINT_PRODUCTION + "/{repo_id}/resolve/{revision}/{filename}"

# Token to be set as environment variable.
# Almost all features are tested on staging environment. However, Spaces are not supported
# there which makes it impossible to test secrets/hardware requests.
#
# Value is set as a secret in Github actions. To make this test work locally, set
# `HUGGINGFACE_HUB_PRODUCTION_TEST` environment variable on your machine with a personal
# token (see https://huggingface.co/settings/tokens). The test pipeline will only create
# private spaces (and delete them afterwards).
#
# See `fx_production_space` fixture. Goal is to limit its usage as much as possible.
PRODUCTION_TOKEN = os.environ.get("HUGGINGFACE_PRODUCTION_USER_TOKEN")


# tests/testing_utils.py

import inspect
import os
import shutil
import stat
import time
import unittest
import uuid
from contextlib import contextmanager
from enum import Enum
from functools import wraps
from pathlib import Path
from typing import Callable, Optional, Type, TypeVar, Union
from unittest.mock import Mock, patch

import pytest
import requests
from requests.exceptions import HTTPError

from huggingface_hub.utils import is_gradio_available, logging
from tests.testing_constants import ENDPOINT_PRODUCTION, ENDPOINT_PRODUCTION_URL_SCHEME


logger = logging.get_logger(__name__)

SMALL_MODEL_IDENTIFIER = "julien-c/bert-xsmall-dummy"
DUMMY_DIFF_TOKENIZER_IDENTIFIER = "julien-c/dummy-diff-tokenizer"
# Example model ids

# An actual model hosted on huggingface.co,
# w/ more details.
DUMMY_MODEL_ID = "julien-c/dummy-unknown"
DUMMY_MODEL_ID_REVISION_ONE_SPECIFIC_COMMIT = "f2c752cfc5c0ab6f4bdec59acea69eefbee381c2"
# One particular commit (not the top of `main`)
DUMMY_MODEL_ID_REVISION_INVALID = "aaaaaaa"
# This commit does not exist, so we should 404.
DUMMY_MODEL_ID_PINNED_SHA1 = "d9e9f15bc825e4b2c9249e9578f884bbcb5e3684"
# Sha-1 of config.json on the top of `main`, for checking purposes
DUMMY_MODEL_ID_PINNED_SHA256 = "4b243c475af8d0a7754e87d7d096c92e5199ec2fe168a2ee7998e3b8e9bcb1d3"
# Sha-256 of pytorch_model.bin on the top of `main`, for checking purposes

# "hf-internal-testing/dummy-will-be-renamed" has been renamed to "hf-internal-testing/dummy-renamed"
DUMMY_RENAMED_OLD_MODEL_ID = "hf-internal-testing/dummy-will-be-renamed"
DUMMY_RENAMED_NEW_MODEL_ID = "hf-internal-testing/dummy-renamed"

SAMPLE_DATASET_IDENTIFIER = "lhoestq/custom_squad"
# Example dataset ids
DUMMY_DATASET_ID = "lhoestq/test"
DUMMY_DATASET_ID_REVISION_ONE_SPECIFIC_COMMIT = "81d06f998585f8ee10e6e3a2ea47203dc75f2a16"  # on branch "test-branch"

YES = ("y", "yes", "t", "true", "on", "1")
NO = ("n", "no", "f", "false", "off", "0")


def repo_name(id: Optional[str] = None, prefix: str = "repo") -> str:
    """
    Return a readable pseudo-unique repository name for tests.

    Example:
    ```py
    >>> repo_name()
    repo-2fe93f-16599646671840

    >>> repo_name("my-space", prefix='space')
    space-my-space-16599481979701
    """
    if id is None:
        id = uuid.uuid4().hex[:6]
    ts = int(time.time() * 10e3)
    return f"{prefix}-{id}-{ts}"


def parse_flag_from_env(key: str, default: bool = False) -> bool:
    try:
        value = os.environ[key]
    except KeyError:
        # KEY isn't set, default to `default`.
        return default

    # KEY is set, convert it to True or False.
    if value.lower() in YES:
        return True
    elif value.lower() in NO:
        return False
    else:
        # More values are supported, but let's keep the message simple.
        raise ValueError(f"If set, '{key}' must be one of {YES+NO}. Got '{value}'.")


def parse_int_from_env(key, default=None):
    try:
        value = os.environ[key]
    except KeyError:
        _value = default
    else:
        try:
            _value = int(value)
        except ValueError:
            raise ValueError("If set, {} must be a int.".format(key))
    return _value


_run_git_lfs_tests = parse_flag_from_env("RUN_GIT_LFS_TESTS", default=False)


def require_git_lfs(test_case):
    """
    Decorator to mark tests that requires git-lfs.

    git-lfs requires additional dependencies, and tests are skipped by default. Set the RUN_GIT_LFS_TESTS environment
    variable to a truthy value to run them.
    """
    if not _run_git_lfs_tests:
        return unittest.skip("test of git lfs workflow")(test_case)
    else:
        return test_case


def require_webhooks(test_case):
    """
    Decorator to mark tests that requires `webhooks` extra (i.e. gradio, fastapi, pydantic).

    git-lfs requires additional dependencies, and tests are skipped by default. Set the RUN_GIT_LFS_TESTS environment
    variable to a truthy value to run them.
    """
    if not is_gradio_available():
        return unittest.skip("Skip webhook test")(test_case)
    else:
        return test_case


class RequestWouldHangIndefinitelyError(Exception):
    pass


class OfflineSimulationMode(Enum):
    CONNECTION_FAILS = 0
    CONNECTION_TIMES_OUT = 1
    HF_HUB_OFFLINE_SET_TO_1 = 2


@contextmanager
def offline(mode=OfflineSimulationMode.CONNECTION_FAILS, timeout=1e-16):
    """
    Simulate offline mode.

    There are three offline simulation modes:

    CONNECTION_FAILS (default mode): a ConnectionError is raised for each network call.
        Connection errors are created by mocking socket.socket
    CONNECTION_TIMES_OUT: the connection hangs until it times out.
        The default timeout value is low (1e-16) to speed up the tests.
        Timeout errors are created by mocking requests.request
    HF_HUB_OFFLINE_SET_TO_1: the HF_HUB_OFFLINE_SET_TO_1 environment variable is set to 1.
        This makes the http/ftp calls of the library instantly fail and raise an OfflineModeEnabled error.
    """
    import socket

    from requests import request as online_request

    def timeout_request(method, url, **kwargs):
        # Change the url to an invalid url so that the connection hangs
        invalid_url = "https://10.255.255.1"
        if kwargs.get("timeout") is None:
            raise RequestWouldHangIndefinitelyError(
                f"Tried a call to {url} in offline mode with no timeout set. Please set a timeout."
            )
        kwargs["timeout"] = timeout
        try:
            return online_request(method, invalid_url, **kwargs)
        except Exception as e:
            # The following changes in the error are just here to make the offline timeout error prettier
            e.request.url = url
            max_retry_error = e.args[0]
            max_retry_error.args = (max_retry_error.args[0].replace("10.255.255.1", f"OfflineMock[{url}]"),)
            e.args = (max_retry_error,)
            raise

    def offline_socket(*args, **kwargs):
        raise socket.error("Offline mode is enabled.")

    if mode is OfflineSimulationMode.CONNECTION_FAILS:
        # inspired from https://stackoverflow.com/a/18601897
        with patch("socket.socket", offline_socket):
            with patch("huggingface_hub.utils._http.get_session") as get_session_mock:
                get_session_mock.return_value = requests.Session()  # not an existing one
                yield
    elif mode is OfflineSimulationMode.CONNECTION_TIMES_OUT:
        # inspired from https://stackoverflow.com/a/904609
        with patch("requests.request", timeout_request):
            with patch("huggingface_hub.utils._http.get_session") as get_session_mock:
                get_session_mock().request = timeout_request
                yield
    elif mode is OfflineSimulationMode.HF_HUB_OFFLINE_SET_TO_1:
        with patch("huggingface_hub.constants.HF_HUB_OFFLINE", True):
            yield
    else:
        raise ValueError("Please use a value from the OfflineSimulationMode enum.")


def set_write_permission_and_retry(func, path, excinfo):
    os.chmod(path, stat.S_IWRITE)
    func(path)


def rmtree_with_retry(path: Union[str, Path]) -> None:
    shutil.rmtree(path, onerror=set_write_permission_and_retry)


def with_production_testing(func):
    file_download = patch(
        "huggingface_hub.file_download.HUGGINGFACE_CO_URL_TEMPLATE",
        ENDPOINT_PRODUCTION_URL_SCHEME,
    )

    hf_api = patch(
        "huggingface_hub.hf_api.ENDPOINT",
        ENDPOINT_PRODUCTION,
    )

    return hf_api(file_download(func))


def retry_endpoint(function, number_of_tries: int = 3, wait_time: int = 5):
    """
    Retries test if failure, waiting `wait_time`.
    Should be added to any test hitting the `moon-staging` endpoint that is
    downloading Repositories or uploading data

    Args:
        number_of_tries: Number of tries to attempt a passing test
        wait_time: Time to wait in-between attempts in seconds
    """

    @wraps(function)
    def decorator(*args, **kwargs):
        retry_count = 1
        while retry_count < number_of_tries:
            try:
                return function(*args, **kwargs)
            except HTTPError as e:
                if e.response.status_code == 504:
                    logger.info(
                        f"Attempt {retry_count} failed with a 504 error. Retrying new"
                        f" execution in {wait_time} second(s)..."
                    )
                    time.sleep(5)
                    retry_count += 1
            except OSError:
                logger.info(
                    "Race condition met where we tried to `clone` before fully"
                    " deleting a repository. Retrying new execution in"
                    f" {wait_time} second(s)..."
                )
                retry_count += 1
            # Preserve original traceback
            return function(*args, **kwargs)

    return decorator


def expect_deprecation(function_name: str):
    """
    Decorator to flag tests that we expect to use deprecated arguments.

    Args:
        function_name (`str`):
            Name of the function that we expect to use in a deprecated way.

    NOTE: if a test is expected to warns FutureWarnings but is not, the test will fail.

    Context: over time, some arguments/methods become deprecated. In order to track
             deprecation in tests, we run pytest with flag `-Werror::FutureWarning`.
             In order to keep old tests during the deprecation phase (before removing
             the feature completely) without changing them internally, we can flag
             them with this decorator.
    See full discussion in https://github.com/huggingface/huggingface_hub/pull/952.

    This decorator works hand-in-hand with the `_deprecate_arguments` and
    `_deprecate_positional_args` decorators.

    Example
    ```py
    # in src/hub_mixins.py
    from .utils._deprecation import _deprecate_arguments

    @_deprecate_arguments(version="0.12", deprecated_args={"repo_url"})
    def push_to_hub(...):
        (...)

    # in tests/test_something.py
    from .testing_utils import expect_deprecation

    class SomethingTest(unittest.TestCase):
        (...)

        @expect_deprecation("push_to_hub"):
        def test_push_to_hub_git_version(self):
            (...)
            push_to_hub(repo_url="something") <- Should warn with FutureWarnings
            (...)
    ```
    """

    def _inner_decorator(test_function: Callable) -> Callable:
        @wraps(test_function)
        def _inner_test_function(*args, **kwargs):
            with pytest.warns(FutureWarning, match=f".*'{function_name}'.*"):
                return test_function(*args, **kwargs)

        return _inner_test_function

    return _inner_decorator


def xfail_on_windows(reason: str, raises: Optional[Type[Exception]] = None):
    """
    Decorator to flag tests that we expect to fail on Windows.

    Will not raise an error if the expected error happens while running on Windows machine.
    If error is expected but does not happen, the test fails as well.

    Args:
        reason (`str`):
            Reason why it should fail.
        raises (`Type[Exception]`):
            The error type we except to happen.
    """

    def _inner_decorator(test_function: Callable) -> Callable:
        return pytest.mark.xfail(os.name == "nt", reason=reason, raises=raises, strict=True, run=True)(test_function)

    return _inner_decorator


T = TypeVar("T")


def handle_injection(cls: T) -> T:
    """Handle mock injection for each test of a test class.

    When patching variables on a class level, only relevant mocks will be injected to
    the tests. This has 2 advantages:
    1. There is no need to expect all mocks in test arguments when they are not needed.
    2. Default mock injection append all mocks 1 by 1 to the test args. If the order of
       the patch calls or test argument is changed, it can lead to unexpected behavior.

    NOTE: `@handle_injection` has to be defined after the `@patch` calls.

    Example:
    ```py
    @patch("something.foo")
    @patch("something_else.foo.bar") # order doesn't matter
    @handle_injection # after @patch calls
    def TestHelloWorld(unittest.TestCase):

        def test_hello_foo(self, mock_foo: Mock) -> None:
            (...)

        def test_hello_bar(self, mock_bar: Mock) -> None
            (...)

        def test_hello_both(self, mock_foo: Mock, mock_bar: Mock) -> None:
            (...)
    ```

    There are limitations with the current implementation:
    1. All patched variables must have different names.
       Named injection will not work with both `@patch("something.foo")` and
       `@patch("something_else.foo")` patches.
    2. Tests are expected to take only `self` and mock arguments. If it's not the case,
       this helper will fail.
    3. Tests arguments must follow the `mock_{variable_name}` naming.
       Example: `@patch("something._foo")` -> `"mock__foo"`.
    4. Tests arguments must be typed as `Mock`.

    If required, we can improve the current implementation in the future to mitigate
    those limitations.

    Based on:
    - https://stackoverflow.com/a/3467879
    - https://stackoverflow.com/a/30764825
    - https://stackoverflow.com/a/57115876

    NOTE: this decorator is inspired from the fixture system from pytest.
    """
    # Iterate over class functions and decorate tests
    # Taken from https://stackoverflow.com/a/3467879
    #        and https://stackoverflow.com/a/30764825
    for name, fn in inspect.getmembers(cls):
        if name.startswith("test_"):
            setattr(cls, name, handle_injection_in_test(fn))

    # Return decorated class
    return cls


def handle_injection_in_test(fn: Callable) -> Callable:
    """
    Handle injections at a test level. See `handle_injection` for more details.

    Example:
    ```py
    def TestHelloWorld(unittest.TestCase):

        @patch("something.foo")
        @patch("something_else.foo.bar") # order doesn't matter
        @handle_injection_in_test # after @patch calls
        def test_hello_foo(self, mock_foo: Mock) -> None:
            (...)
    ```
    """
    signature = inspect.signature(fn)
    parameters = signature.parameters

    @wraps(fn)
    def _inner(*args, **kwargs):
        assert kwargs == {}

        # Initialize new dict at least with `self`.
        assert len(args) > 0
        assert len(parameters) > 0
        new_kwargs = {"self": args[0]}

        # Check which mocks have been injected
        mocks = {}
        for value in args[1:]:
            assert isinstance(value, Mock)
            mock_name = "mock_" + value._extract_mock_name()
            mocks[mock_name] = value

        # Check which mocks are expected
        for name, parameter in parameters.items():
            if name == "self":
                continue
            assert parameter.annotation is Mock
            assert (
                name in mocks
            ), f"Mock `{name}` not found for test `{fn.__name__}`. Available: {', '.join(sorted(mocks.keys()))}"
            new_kwargs[name] = mocks[name]

        # Run test only with a subset of mocks
        return fn(**new_kwargs)

    return _inner


def use_tmp_repo(repo_type: str = "model") -> Callable[[T], T]:
    """
    Test decorator to create a repo for the test and properly delete it afterward.

    TODO: could we make `_api`, `_user` and `_token` cleaner ?

    Example:
    ```py
    from huggingface_hub import RepoUrl
    from .testing_utils import use_tmp_repo

    class HfApiCommonTest(unittest.TestCase):
        _api = HfApi(endpoint=ENDPOINT_STAGING, token=TOKEN)

        @use_tmp_repo()
        def test_create_tag_on_model(self, repo_url: RepoUrl) -> None:
            (...)

        @use_tmp_repo("dataset")
        def test_create_tag_on_dataset(self, repo_url: RepoUrl) -> None:
            (...)
    ```
    """

    def _inner_use_tmp_repo(test_fn: T) -> T:
        @wraps(test_fn)
        def _inner(*args, **kwargs):
            self = args[0]
            assert isinstance(self, unittest.TestCase)

            repo_url = self._api.create_repo(repo_id=repo_name(prefix=repo_type), repo_type=repo_type)
            try:
                return test_fn(*args, **kwargs, repo_url=repo_url)
            finally:
                self._api.delete_repo(repo_id=repo_url.repo_id, repo_type=repo_type)

        return _inner

    return _inner_use_tmp_repo


# setup.py

from setuptools import find_packages, setup


def get_version() -> str:
    rel_path = "src/huggingface_hub/__init__.py"
    with open(rel_path, "r") as fp:
        for line in fp.read().splitlines():
            if line.startswith("__version__"):
                delim = '"' if '"' in line else "'"
                return line.split(delim)[1]
    raise RuntimeError("Unable to find version string.")


install_requires = [
    "filelock",
    "fsspec",
    "requests",
    "tqdm>=4.42.1",
    "pyyaml>=5.1",
    "typing-extensions>=3.7.4.3",  # to be able to import TypeAlias
    "importlib_metadata;python_version<'3.8'",
    "packaging>=20.9",
]

extras = {}

extras["cli"] = [
    "InquirerPy==0.3.4",
    # Note: installs `prompt-toolkit` in the background
]

extras["torch"] = [
    "torch",
]

extras["fastai"] = [
    "toml",
    "fastai>=2.4",
    "fastcore>=1.3.27",
]

extras["tensorflow"] = ["tensorflow", "pydot", "graphviz"]


extras["testing"] = extras["cli"] + [
    "jedi",
    "Jinja2",
    "pytest",
    "pytest-cov",
    "pytest-env",
    "pytest-xdist",
    "pytest-vcr",  # to mock Inference
    "urllib3<2.0",  # VCR.py broken with urllib3 2.0 (see https://urllib3.readthedocs.io/en/stable/v2-migration-guide.html)
    "soundfile",
    "Pillow",
    "gradio",  # to test webhooks
    "numpy",  # for embeddings
    "pydantic",  # for text-generation-inference
]

# Typing extra dependencies list is duplicated in `.pre-commit-config.yaml`
# Please make sure to update the list there when adding a new typing dependency.
extras["typing"] = [
    "types-PyYAML",
    "types-requests",
    "types-simplejson",
    "types-toml",
    "types-tqdm",
    "types-urllib3",
    "pydantic",  # for text-generation dataclasses
]

extras["quality"] = [
    "black~=23.1",
    "ruff>=0.0.241",
    "mypy==0.982",
]

extras["all"] = extras["testing"] + extras["quality"] + extras["typing"]

extras["dev"] = extras["all"]


setup(
    name="huggingface_hub",
    version=get_version(),
    author="Hugging Face, Inc.",
    author_email="julien@huggingface.co",
    description="Client library to download and publish models, datasets and other repos on the huggingface.co hub",
    long_description=open("README.md", "r", encoding="utf-8").read(),
    long_description_content_type="text/markdown",
    keywords="model-hub machine-learning models natural-language-processing deep-learning pytorch pretrained-models",
    license="Apache",
    url="https://github.com/huggingface/huggingface_hub",
    package_dir={"": "src"},
    packages=find_packages("src"),
    extras_require=extras,
    entry_points={
        "console_scripts": ["huggingface-cli=huggingface_hub.commands.huggingface_cli:main"],
        "fsspec.specs": "hf=huggingface_hub.HfFileSystem",
    },
    python_requires=">=3.7.0",
    install_requires=install_requires,
    classifiers=[
        "Intended Audience :: Developers",
        "Intended Audience :: Education",
        "Intended Audience :: Science/Research",
        "License :: OSI Approved :: Apache Software License",
        "Operating System :: OS Independent",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3 :: Only",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: 3.8",
        "Programming Language :: Python :: 3.9",
        "Programming Language :: Python :: 3.10",
        "Programming Language :: Python :: 3.11",
        "Topic :: Scientific/Engineering :: Artificial Intelligence",
    ],
    include_package_data=True,
)


# utils/_legacy_check_future_compatible_signatures.py

# coding=utf-8
# Copyright 2022-present, the HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains a tool to add/check the definition of "async" methods of `HfApi` in `huggingface_hub.hf_api.py`.

WARNING: this is a script kept to help with `@future_compatible` methods of `HfApi` but it is not 100% correct.
Keeping it here for reference but it is not used in the CI/Makefile.

What is done correctly:
1. Add "as_future" as argument to the method signature
2. Set Union[T, Future[T]] as return type to the method signature
3. Document "as_future" argument in the docstring of the method

What is NOT done correctly:
1. Generated stubs are grouped at the top of the `HfApi` class. They must be copy-pasted (overload definition must be
just before the method implementation)
2. `#type: ignore` must be adjusted in the first stub (if multiline definition)
"""
import argparse
import inspect
import os
import re
import tempfile
from pathlib import Path
from typing import Callable, NoReturn

import black
from ruff.__main__ import find_ruff_bin

from huggingface_hub.hf_api import HfApi


STUBS_SECTION_TEMPLATE = """
    ### Stubs section start ###

    # This section contains stubs for the methods that are marked as `@future_compatible`. Those methods have a
    # different return type depending on the `as_future: bool` value. For better integrations with IDEs, we provide
    # stubs for both return types. The actual implementation of those methods is written below.

    # WARNING: this section have been generated automatically. Do not modify it manually. If you modify it manually, your
    # changes will be overwritten. To re-generate this section, run `make style` (or `python utils/check_future_compatible_signatures.py`
    # directly).

    # FAQ:
    # 1. Why should we have these? For better type annotation which helps with IDE features like autocompletion.
    # 2. Why not a separate `hf_api.pyi` file? Would require to re-defined all the existing annotations from `hf_api.py`.
    # 3. Why not at the end of the module? Because `@overload` methods must be defined first.
    # 4. Why not another solution? I'd be glad, but this is the "less worse" I could find.
    # For more details, see https://github.com/huggingface/huggingface_hub/pull/1458


    {stubs}

    # WARNING: this section have been generated automatically. Do not modify it manually. If you modify it manually, your
    # changes will be overwritten. To re-generate this section, run `make style` (or `python utils/check_future_compatible_signatures.py`
    # directly).

    ### Stubs section end ###
"""

STUBS_SECTION_TEMPLATE_REGEX = re.compile(r"### Stubs section start ###.*### Stubs section end ###", re.DOTALL)

AS_FUTURE_SIGNATURE_TEMPLATE = "as_future: bool = False"

AS_FUTURE_DOCSTRING_TEMPLATE = """
            as_future (`bool`, *optional*):
                Whether or not to run this method in the background. Background jobs are run sequentially without
                blocking the main thread. Passing `as_future=True` will return a [Future](https://docs.python.org/3/library/concurrent.futures.html#future-objects)
                object. Defaults to `False`."""

ARGS_DOCSTRING_REGEX = re.compile(
    """
^[ ]{8}Args:   # Match args section ...
(.*?)          # ... everything ...
^[ ]{8}\\S      # ... until next section or end of docstring
""",
    re.MULTILINE | re.IGNORECASE | re.VERBOSE | re.DOTALL,
)

SIGNATURE_REGEX_FULL = re.compile(r"^\s*def.*?-> (.*?):", re.DOTALL | re.MULTILINE)
SIGNATURE_REGEX_RETURN_TYPE = re.compile(r"-> (.*?):")
SIGNATURE_REGEX_RETURN_TYPE_WITH_FUTURE = re.compile(r"-> Union\[(.*?), (.*?)\]:")


HF_API_FILE_PATH = Path(__file__).parents[1] / "src" / "huggingface_hub" / "hf_api.py"
HF_API_FILE_CONTENT = HF_API_FILE_PATH.read_text()


def generate_future_compatible_method(method: Callable, method_source: str) -> str:
    # 1. Document `as_future` parameter
    if AS_FUTURE_DOCSTRING_TEMPLATE not in method_source:
        match = ARGS_DOCSTRING_REGEX.search(method_source)
        if match is None:
            raise ValueError(f"Could not find `Args` section in docstring of {method}.")
        args_docs = match.group(1).strip()
        method_source = method_source.replace(args_docs, args_docs + AS_FUTURE_DOCSTRING_TEMPLATE)

    # 2. Update signature
    # 2.a. Add `as_future` parameter
    if AS_FUTURE_SIGNATURE_TEMPLATE not in method_source:
        match = SIGNATURE_REGEX_FULL.search(method_source)
        if match is None:
            raise ValueError(f"Could not find signature of {method} in source.")
        method_source = method_source.replace(
            match.group(), match.group().replace(") ->", f" {AS_FUTURE_SIGNATURE_TEMPLATE}) ->"), 1
        )

    # 2.b. Update return value
    if "Future[" not in method_source:
        match = SIGNATURE_REGEX_RETURN_TYPE.search(method_source)
        if match is None:
            raise ValueError(f"Could not find return type of {method} in source.")
        base_type = match.group(1).strip()
        return_type = f"Union[{base_type}, Future[{base_type}]]"
        return_value_replaced = match.group().replace(match.group(1), return_type)
        method_source = method_source.replace(match.group(), return_value_replaced)

    # 3. Generate @overload stubs
    match = SIGNATURE_REGEX_FULL.search(method_source)
    if match is None:
        raise ValueError(f"Could not find signature of {method} in source.")
    method_sig = match.group()

    match = SIGNATURE_REGEX_RETURN_TYPE_WITH_FUTURE.search(method_sig)
    if match is None:
        raise ValueError(f"Could not find return type (with Future) of {method} in source.")
    no_future_return_type = match.group(1).strip()
    with_future_return_type = match.group(2).strip()

    # 3.a. Stub when `as_future=False`
    no_future_stub = "    @overload\n" + method_sig
    no_future_stub = no_future_stub.replace(AS_FUTURE_SIGNATURE_TEMPLATE, "as_future: Literal[False] = ...")
    no_future_stub = SIGNATURE_REGEX_RETURN_TYPE.sub(rf"-> {no_future_return_type}:", no_future_stub)
    no_future_stub += "  # type: ignore\n        ..."  # only the first stub requires "type: ignore"

    # 3.b. Stub when `as_future=True`
    with_future_stub = "    @overload\n" + method_sig
    with_future_stub = with_future_stub.replace(AS_FUTURE_SIGNATURE_TEMPLATE, "as_future: Literal[True] = ...")
    with_future_stub = SIGNATURE_REGEX_RETURN_TYPE.sub(rf"-> {with_future_return_type}:", with_future_stub)
    with_future_stub += "\n        ..."

    stubs_source = no_future_stub + "\n\n" + with_future_stub + "\n\n"

    # 4. All good!
    return method_source, stubs_source


def generate_hf_api_module() -> str:
    raw_code = HF_API_FILE_CONTENT

    # Process all Future-compatible methods
    all_stubs_source = ""
    for _, method in inspect.getmembers(HfApi, predicate=inspect.isfunction):
        if not getattr(method, "is_future_compatible", False):
            continue
        source = inspect.getsource(method)
        method_source, stubs_source = generate_future_compatible_method(method, source)

        raw_code = raw_code.replace(source, method_source)
        all_stubs_source += "\n\n" + stubs_source

    # Generate code with stubs
    generated_code = STUBS_SECTION_TEMPLATE_REGEX.sub(STUBS_SECTION_TEMPLATE.format(stubs=all_stubs_source), raw_code)

    # Format (black+ruff)
    return format_generated_code(generated_code)


def format_generated_code(code: str) -> str:
    """
    Format some code with black+ruff. Cannot be done "on the fly" so we first save the code in a temporary file.
    """
    # Format with black
    code = black.format_file_contents(code, fast=False, mode=black.FileMode(line_length=119))

    # Format with ruff
    with tempfile.TemporaryDirectory() as tmpdir:
        filepath = Path(tmpdir) / "__init__.py"
        filepath.write_text(code)
        ruff_bin = find_ruff_bin()
        os.spawnv(os.P_WAIT, ruff_bin, ["ruff", str(filepath), "--fix", "--quiet"])
        return filepath.read_text()


def check_future_compatible_hf_api(update: bool) -> NoReturn:
    """Check that the code defining the threaded version of HfApi is up-to-date."""
    # If expected `__init__.py` content is different, test fails. If '--update-init-file'
    # is used, `__init__.py` file is updated before the test fails.
    expected_content = generate_hf_api_module()
    if expected_content != HF_API_FILE_CONTENT:
        if update:
            with HF_API_FILE_PATH.open("w") as f:
                f.write(expected_content)

            print(
                "✅ Signature/docstring/annotations for Future-compatible methods have been updated in"
                " `./src/huggingface_hub/hf_api.py`.\n   Please make sure the changes are accurate and commit them."
            )
            exit(0)
        else:
            print(
                "❌ Expected content mismatch for Future compatible methods in `./src/huggingface_hub/hf_api.py`.\n  "
                " Please run `make style` or `python utils/check_future_compatible_signatures.py --update`."
            )
            exit(1)

    print("✅ All good! (Future-compatible methods)")
    exit(0)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--update",
        action="store_true",
        help="Whether to override `./src/huggingface_hub/hf_api.py` if a change is detected.",
    )
    args = parser.parse_args()

    check_future_compatible_hf_api(update=args.update)


# utils/check_contrib_list.py

# coding=utf-8
# Copyright 2022-present, the HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains a tool to list contrib test suites automatically."""
import argparse
import re
from pathlib import Path
from typing import NoReturn


ROOT_DIR = Path(__file__).parent.parent
CONTRIB_PATH = ROOT_DIR / "contrib"
MAKEFILE_PATH = ROOT_DIR / "Makefile"
WORKFLOW_PATH = ROOT_DIR / ".github" / "workflows" / "contrib-tests.yml"

MAKEFILE_REGEX = re.compile(r"^CONTRIB_LIBS := .*$", flags=re.MULTILINE)
WORKFLOW_REGEX = re.compile(
    r"""
    # First: match "contrib: ["
    (?P<before>^\s{8}contrib:\s\[\n)
    # Match list of libs
    (\s{10}\".*\",\n)*
    # Finally: match trailing "]"
    (?P<after>^\s{8}\])
    """,
    flags=re.MULTILINE | re.VERBOSE,
)


def check_contrib_list(update: bool) -> NoReturn:
    """List `contrib` test suites.

    Make sure `Makefile` and `.github/workflows/contrib-tests.yml` are consistent with
    the list."""
    # List contrib test suites
    contrib_list = sorted(
        path.name for path in CONTRIB_PATH.glob("*") if path.is_dir() and not path.name.startswith("_")
    )

    # Check Makefile is consistent with list
    makefile_content = MAKEFILE_PATH.read_text()
    makefile_expected_content = MAKEFILE_REGEX.sub(f"CONTRIB_LIBS := {' '.join(contrib_list)}", makefile_content)

    # Check workflow is consistent with list
    workflow_content = WORKFLOW_PATH.read_text()
    _substitute = "\n".join(f'{" "*10}"{lib}",' for lib in contrib_list)
    workflow_content_expected = WORKFLOW_REGEX.sub(rf"\g<before>{_substitute}\n\g<after>", workflow_content)

    #
    failed = False
    if makefile_content != makefile_expected_content:
        if update:
            print(
                "✅ Contrib libs have been updated in `Makefile`."
                "\n   Please make sure the changes are accurate and commit them."
            )
            MAKEFILE_PATH.write_text(makefile_expected_content)
        else:
            print(
                "❌ Expected content mismatch in `Makefile`.\n   It is most likely that"
                " you added a contrib test and did not update the Makefile.\n   Please"
                " run `make style` or `python utils/check_contrib_list.py --update`."
            )
            failed = True

    if workflow_content != workflow_content_expected:
        if update:
            print(
                f"✅ Contrib libs have been updated in `{WORKFLOW_PATH}`."
                "\n   Please make sure the changes are accurate and commit them."
            )
            WORKFLOW_PATH.write_text(workflow_content_expected)
        else:
            print(
                f"❌ Expected content mismatch in `{WORKFLOW_PATH}`.\n   It is most"
                " likely that you added a contrib test and did not update the github"
                " workflow file.\n   Please run `make style` or `python"
                " utils/check_contrib_list.py --update`."
            )
            failed = True

    if failed:
        exit(1)
    print("✅ All good! (contrib list)")
    exit(0)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--update",
        action="store_true",
        help="Whether to fix Makefile and github workflow if a new lib is detected.",
    )
    args = parser.parse_args()

    check_contrib_list(update=args.update)


# utils/check_static_imports.py

# coding=utf-8
# Copyright 2022-present, the HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains a tool to reformat static imports in `huggingface_hub.__init__.py`."""
import argparse
import os
import re
import tempfile
from pathlib import Path
from typing import NoReturn

from ruff.__main__ import find_ruff_bin

from huggingface_hub import _SUBMOD_ATTRS


INIT_FILE_PATH = Path(__file__).parents[1] / "src" / "huggingface_hub" / "__init__.py"
SETUP_CFG_PATH = Path(__file__).parents[1] / "setup.cfg"

IF_TYPE_CHECKING_LINE = "\nif TYPE_CHECKING:  # pragma: no cover\n"
SUBMOD_ATTRS_PATTERN = re.compile("_SUBMOD_ATTRS = {[^}]+}")  # match the all dict


def check_static_imports(update: bool) -> NoReturn:
    """Check all imports are made twice (1 in lazy-loading and 1 in static checks).

    For more explanations, see `./src/huggingface_hub/__init__.py`.
    This script is used in the `make style` and `make quality` checks.
    """
    with INIT_FILE_PATH.open() as f:
        init_content = f.read()

    # Get first half of the `__init__.py` file.
    # WARNING: Content after this part will be entirely re-generated which means
    # human-edited changes will be lost !
    init_content_before_static_checks = init_content.split(IF_TYPE_CHECKING_LINE)[0]

    # Search and replace `_SUBMOD_ATTRS` dictionary definition. This ensures modules
    # and functions that can be lazy-loaded are alphabetically ordered for readability.
    if SUBMOD_ATTRS_PATTERN.search(init_content_before_static_checks) is None:
        print("Error: _SUBMOD_ATTRS dictionary definition not found in `./src/huggingface_hub/__init__.py`.")
        exit(1)

    _submod_attrs_definition = (
        "_SUBMOD_ATTRS = {\n"
        + "\n".join(
            f'    "{module}": [\n'
            + "\n".join(f'        "{attr}",' for attr in sorted(_SUBMOD_ATTRS[module]))
            + "\n    ],"
            for module in sorted(_SUBMOD_ATTRS.keys())
        )
        + "\n}"
    )
    reordered_content_before_static_checks = SUBMOD_ATTRS_PATTERN.sub(
        _submod_attrs_definition, init_content_before_static_checks
    )

    # Generate the static imports given the `_SUBMOD_ATTRS` dictionary.
    static_imports = [
        f"    from .{module} import {attr} # noqa: F401"
        for module, attributes in _SUBMOD_ATTRS.items()
        for attr in attributes
    ]

    # Generate the expected `__init__.py` file content and apply formatter on it.
    with tempfile.TemporaryDirectory() as tmpdir:
        filepath = Path(tmpdir) / "__init__.py"
        filepath.write_text(
            reordered_content_before_static_checks + IF_TYPE_CHECKING_LINE + "\n".join(static_imports) + "\n"
        )
        ruff_bin = find_ruff_bin()
        os.spawnv(os.P_WAIT, ruff_bin, ["ruff", str(filepath), "--fix", "--quiet"])
        expected_init_content = filepath.read_text()

    # If expected `__init__.py` content is different, test fails. If '--update-init-file'
    # is used, `__init__.py` file is updated before the test fails.
    if init_content != expected_init_content:
        if update:
            with INIT_FILE_PATH.open("w") as f:
                f.write(expected_init_content)

            print(
                "✅ Imports have been updated in `./src/huggingface_hub/__init__.py`."
                "\n   Please make sure the changes are accurate and commit them."
            )
            exit(0)
        else:
            print(
                "❌ Expected content mismatch in"
                " `./src/huggingface_hub/__init__.py`.\n   It is most likely that you"
                " added a module/function to `_SUBMOD_ATTRS` and did not update the"
                " 'static import'-part.\n   Please run `make style` or `python"
                " utils/check_static_imports.py --update`."
            )
            exit(1)

    print("✅ All good! (static imports)")
    exit(0)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--update",
        action="store_true",
        help="Whether to fix `./src/huggingface_hub/__init__.py` if a change is detected.",
    )
    args = parser.parse_args()

    check_static_imports(update=args.update)


# utils/push_repocard_examples.py

# coding=utf-8
# Copyright 2022-present, the HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Generate and push an empty ModelCard and DatasetCard to the Hub as examples."""
import argparse
from pathlib import Path

import jinja2

from huggingface_hub import DatasetCard, ModelCard, hf_hub_download, upload_file, whoami
from huggingface_hub.constants import REPOCARD_NAME


ORG_NAME = "templates"
MODEL_CARD_REPO_ID = "templates/model-card-example"
DATASET_CARD_REPO_ID = "templates/dataset-card-example"


def check_can_push():
    """Check the user can push to the `templates/` folder with its credentials."""
    try:
        me = whoami()
    except EnvironmentError:
        print("You must be logged in to push repo card examples.")

    if all(org["name"] != ORG_NAME for org in me.get("orgs", [])):
        print(f"❌ You must have access to organization '{ORG_NAME}' to push repo card examples.")
        exit(1)


def push_model_card_example(overwrite: bool) -> None:
    """Generate an empty model card from template for documentation purposes.

    Do not push if content has not changed. Script is triggered in CI on main branch.
    Card is pushed to https://huggingface.co/templates/model-card-example.
    """
    # Not using ModelCard directly to preserve comments in metadata part
    template = jinja2.Template(ModelCard.default_template_path.read_text())
    content = template.render(
        card_data="{}",
        model_summary=(
            "This modelcard aims to be a base template for new models. "
            "It has been generated using [this raw template]"
            "(https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/templates/modelcard_template.md?plain=1)."
        ),
    )
    if not overwrite:
        existing_content = Path(hf_hub_download(MODEL_CARD_REPO_ID, REPOCARD_NAME, repo_type="model")).read_text()
        if content == existing_content:
            print("Model Card not pushed: did not change.")
            return
    print(f"Pushing empty Model Card to Hub: {MODEL_CARD_REPO_ID}")
    upload_file(
        path_or_fileobj=content.encode(),
        path_in_repo=REPOCARD_NAME,
        repo_id=MODEL_CARD_REPO_ID,
        repo_type="model",
    )


def push_dataset_card_example(overwrite: bool) -> None:
    """Generate an empty dataset card from template for documentation purposes.

    Do not push if content has not changed. Script is triggered in CI on main branch.
    Card is pushed to https://huggingface.co/datasets/templates/dataset-card-example.
    """
    # Not using DatasetCard directly to preserve comments in metadata part
    template = jinja2.Template(DatasetCard.default_template_path.read_text())
    content = template.render(
        card_data="{}",
        dataset_summary=(
            "This dataset card aims to be a base template for new datasets. "
            "It has been generated using [this raw template]"
            "(https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/templates/datasetcard_template.md?plain=1)."
        ),
    )
    if not overwrite:
        existing_content = Path(hf_hub_download(DATASET_CARD_REPO_ID, REPOCARD_NAME, repo_type="dataset")).read_text()
        if content == existing_content:
            print("Dataset Card not pushed: did not change.")
            return
    print(f"Pushing empty Dataset Card to Hub: {DATASET_CARD_REPO_ID}")
    upload_file(
        path_or_fileobj=content.encode(),
        path_in_repo=REPOCARD_NAME,
        repo_id=DATASET_CARD_REPO_ID,
        repo_type="dataset",
    )


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--overwrite",
        action="store_true",
        help="Whether to force updating examples. By default, push to hub only if card is updated.",
    )
    args = parser.parse_args()

    check_can_push()
    push_model_card_example(args.overwrite)
    push_dataset_card_example(args.overwrite)


# utils/generate_async_inference_client.py
# coding=utf-8
# Copyright 2023-present, the HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains a tool to generate `src/huggingface_hub/inference/_generated/_async_client.py`."""
import argparse
import os
import re
import tempfile
from pathlib import Path
from typing import NoReturn

from ruff.__main__ import find_ruff_bin


ASYNC_CLIENT_FILE_PATH = (
    Path(__file__).parents[1] / "src" / "huggingface_hub" / "inference" / "_generated" / "_async_client.py"
)
SYNC_CLIENT_FILE_PATH = Path(__file__).parents[1] / "src" / "huggingface_hub" / "inference" / "_client.py"


def generate_async_client_code(code: str) -> str:
    """Generate AsyncInferenceClient source code."""
    # Warning message "this is an automatically generated file"
    code = _add_warning_to_file_header(code)

    # Imports specific to asyncio
    code = _add_imports(code)

    # Define `AsyncInferenceClient`
    code = _rename_to_AsyncInferenceClient(code)

    # Refactor `.post` method to be async + adapt calls
    code = _make_post_async(code)
    code = _await_post_method_call(code)
    code = _use_async_streaming_util(code)

    # Make all tasks-method async
    code = _make_tasks_methods_async(code)

    # Adapt text_generation error catching
    code = _adapt_text_generation_to_async(code)

    # Update some docstrings
    code = _rename_HTTPError_to_ClientResponseError_in_docstring(code)
    code = _update_examples_in_public_methods(code)
    return code


def format_source_code(code: str) -> str:
    """Apply formatter on a generated source code."""
    with tempfile.TemporaryDirectory() as tmpdir:
        filepath = Path(tmpdir) / "async_client.py"
        filepath.write_text(code)
        ruff_bin = find_ruff_bin()
        os.spawnv(os.P_WAIT, ruff_bin, ["ruff", str(filepath), "--fix", "--quiet"])
        return filepath.read_text()


def check_async_client(update: bool) -> NoReturn:
    """Check AsyncInferenceClient is correctly defined and consistent with InferenceClient.

    This script is used in the `make style` and `make quality` checks.
    """
    sync_client_code = SYNC_CLIENT_FILE_PATH.read_text()
    current_async_client_code = ASYNC_CLIENT_FILE_PATH.read_text()

    raw_async_client_code = generate_async_client_code(sync_client_code)

    formatted_async_client_code = format_source_code(raw_async_client_code)

    # If expected `__init__.py` content is different, test fails. If '--update-init-file'
    # is used, `__init__.py` file is updated before the test fails.
    if current_async_client_code != formatted_async_client_code:
        if update:
            ASYNC_CLIENT_FILE_PATH.write_text(formatted_async_client_code)

            print(
                "✅ AsyncInferenceClient source code has been updated in"
                " `./src/huggingface_hub/inference/_generated/_async_client.py`.\n   Please make sure the changes are"
                " accurate and commit them."
            )
            exit(0)
        else:
            print(
                "❌ Expected content mismatch in `./src/huggingface_hub/inference/_generated/_async_client.py`.\n   It"
                " is most likely that you modified some InferenceClient code and did not update the the"
                " AsyncInferenceClient one.\n   Please run `make style` or `python"
                " utils/generate_async_inference_client.py --update`."
            )
            exit(1)

    print("✅ All good! (AsyncInferenceClient)")
    exit(0)


def _add_warning_to_file_header(code: str) -> str:
    warning_message = (
        "#\n# WARNING\n# This entire file has been adapted from the sync-client code in"
        " `src/huggingface_hub/inference/_client.py`.\n# Any change in InferenceClient will be automatically reflected"
        " in AsyncInferenceClient.\n# To re-generate the code, run `make style` or `python"
        " ./utils/generate_async_inference_client.py --update`.\n# WARNING"
    )
    return re.sub(
        r"""
        ( # Group1: license (end)
            \n
            \#\ limitations\ under\ the\ License.
            \n
        )
        (.*?) # Group2 : all notes and comments (to be replaced)
        (\nimport[ ]) # Group3: import section (start)
        """,
        repl=rf"\1{warning_message}\3",
        string=code,
        count=1,
        flags=re.DOTALL | re.VERBOSE,
    )


def _add_imports(code: str) -> str:
    # global imports
    code = re.sub(
        r"(\nimport .*?\n)",
        repl=(
            r"\1" + "from .._common import _async_yield_from, _import_aiohttp\n" + "from typing import AsyncIterable\n"
        ),
        string=code,
        count=1,
        flags=re.DOTALL,
    )

    # type-checking imports
    code = re.sub(
        r"(\nif TYPE_CHECKING:\n)",
        repl=r"\1    from aiohttp import ClientResponse, ClientSession\n",
        string=code,
        count=1,
        flags=re.DOTALL,
    )

    return code


def _rename_to_AsyncInferenceClient(code: str) -> str:
    return code.replace("class InferenceClient:", "class AsyncInferenceClient:", 1)


ASYNC_POST_CODE = """
        aiohttp = _import_aiohttp()

        url = self._resolve_url(model, task)

        if data is not None and json is not None:
            warnings.warn("Ignoring `json` as `data` is passed as binary.")

        t0 = time.time()
        timeout = self.timeout
        while True:
            with _open_as_binary(data) as data_as_binary:
                # Do not use context manager as we don't want to close the connection immediately when returning
                # a stream
                client = aiohttp.ClientSession(
                    headers=self.headers, cookies=self.cookies, timeout=aiohttp.ClientTimeout(self.timeout)
                )

                try:
                    response = await client.post(url, headers=build_hf_headers(), json=json, data=data_as_binary)
                    response_error_payload = None
                    if response.status != 200:
                        try:
                            response_error_payload = await response.json()  # get payload before connection closed
                        except Exception:
                            pass
                    response.raise_for_status()
                    if stream:
                        return _async_yield_from(client, response)
                    else:
                        content = await response.read()
                        await client.close()
                        return content
                except TimeoutError as error:
                    await client.close()
                    # Convert any `TimeoutError` to a `InferenceTimeoutError`
                    raise InferenceTimeoutError(f"Inference call timed out: {url}") from error
                except aiohttp.ClientResponseError as error:
                    error.response_error_payload = response_error_payload
                    await client.close()
                    if response.status == 503:
                        # If Model is unavailable, either raise a TimeoutError...
                        if timeout is not None and time.time() - t0 > timeout:
                            raise InferenceTimeoutError(
                                f"Model not loaded on the server: {url}. Please retry with a higher timeout"
                                f" (current: {self.timeout})."
                            ) from error
                        # ...or wait 1s and retry
                        logger.info(f"Waiting for model to be loaded on the server: {error}")
                        time.sleep(1)
                        if timeout is not None:
                            timeout = max(self.timeout - (time.time() - t0), 1)  # type: ignore
                        continue
                    raise error"""


def _make_post_async(code: str) -> str:
    # Update AsyncInferenceClient.post() implementation (use aiohttp instead of requests)
    code = re.sub(
        r"""
        def[ ]post\( # definition
            (\n.*?\"\"\".*?\"\"\"\n) # Group1: docstring
            .*? # implementation (to be overwritten)
        (\n\W*def ) # Group2: next method
        """,
        repl=rf"async def post(\1{ASYNC_POST_CODE}\2",
        string=code,
        count=1,
        flags=re.DOTALL | re.VERBOSE,
    )
    # Update `post`'s type annotations
    return code.replace("Iterable[bytes]", "AsyncIterable[bytes]", 2)


def _rename_HTTPError_to_ClientResponseError_in_docstring(code: str) -> str:
    # Update `raises`-part in docstrings
    return code.replace("`HTTPError`:", "`aiohttp.ClientResponseError`:")


def _make_tasks_methods_async(code: str) -> str:
    # Add `async` keyword in front of public methods (of AsyncClientInference)
    return re.sub(
        r"""
        # Group 1: newline  + 4-spaces indent
        (\n\ {4})
        # Group 2: def + method name + parenthesis + optionally type: ignore + self
        (
            def[ ] # def
            [a-z]\w*? # method name (not starting by _)
            \( # parenthesis
            (\s*\#[ ]type:[ ]ignore)? # optionally type: ignore
            \s*self, # expect self, as first arg
        )""",
        repl=r"\1async \2",  # insert "async" keyword
        string=code,
        flags=re.DOTALL | re.VERBOSE,
    )


def _adapt_text_generation_to_async(code: str) -> str:
    # Text-generation task has to be handled specifically since it has a recursive call mechanism (to retry on non-tgi
    # servers)

    # Catch `aiohttp` error instead of `requests` error
    code = code.replace(
        """
        except HTTPError as e:
            if isinstance(e, BadRequestError) and "The following `model_kwargs` are not used by the model" in str(e):
    """,
        """
        except _import_aiohttp().ClientResponseError as e:
            error_message = getattr(e, "response_error_payload", {}).get("error", "")
            if e.code == 400 and "The following `model_kwargs` are not used by the model" in error_message:
    """,
    )

    # Await recursive call
    code = code.replace("return self.text_generation", "return await self.text_generation")

    # Update return types: Iterable -> AsyncIterable
    code = code.replace(") -> Iterable[str]:", ") -> AsyncIterable[str]:")
    code = code.replace(
        ") -> Iterable[TextGenerationStreamResponse]:", ") -> AsyncIterable[TextGenerationStreamResponse]:"
    )
    code = code.replace(
        ") -> Union[str, TextGenerationResponse, Iterable[str], Iterable[TextGenerationStreamResponse]]:",
        ") -> Union[str, TextGenerationResponse, AsyncIterable[str], AsyncIterable[TextGenerationStreamResponse]]:",
    )

    return code


def _await_post_method_call(code: str) -> str:
    return code.replace("self.post(", "await self.post(")


def _update_example_code_block(code_block: str) -> str:
    """Update an atomic code block example from a docstring."""
    code_block = "\n        # Must be run in an async context" + code_block
    code_block = code_block.replace("InferenceClient", "AsyncInferenceClient")
    code_block = code_block.replace("client.", "await client.")
    code_block = code_block.replace(" for ", " async for ")
    return code_block


def _update_examples_in_public_methods(code: str) -> str:
    for match in re.finditer(
        r"""
        \n\s*
        Example:\n\s* # example section
        ```py # start
        (.*?) # code block
        ``` # end
        \n
        """,
        string=code,
        flags=re.DOTALL | re.VERBOSE,
    ):
        # Example, including code block
        full_match = match.group()

        # Code block alone
        code_block = match.group(1)

        # Update code block in example
        updated_match = full_match.replace(code_block, _update_example_code_block(code_block))

        # Update example in full script
        code = code.replace(full_match, updated_match)

    return code


def _use_async_streaming_util(code: str) -> str:
    return code.replace(
        "_stream_text_generation_response",
        "_async_stream_text_generation_response",
    )


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--update",
        action="store_true",
        help=(
            "Whether to re-generate `./src/huggingface_hub/inference/_generated/_async_client.py` if a change is"
            " detected."
        ),
    )
    args = parser.parse_args()

    check_async_client(update=args.update)
