# tests/fixtures/tiny_dataset/test.py

import datasets


_CITATION = """\
"""

_DESCRIPTION = """\
This is a test dataset.
"""

_URLS = {"train": "https://pastebin.com/raw/HvpE1CnA", "dev": "some_text.txt"}


class Test(datasets.GeneratorBasedBuilder):
    """SQUAD: The Stanford Question Answering Dataset. Version 1.1."""

    def _info(self):
        return datasets.DatasetInfo(
            description=_DESCRIPTION,
            features=datasets.Features(
                {
                    "text": datasets.Value("string"),
                }
            ),
            supervised_keys=None,
            homepage="https://huggingface.co/datasets/lhoestq/test",
            citation=_CITATION,
        )

    def _split_generators(self, dl_manager):
        downloaded_files = dl_manager.download_and_extract(_URLS)

        return [
            datasets.SplitGenerator(
                name=datasets.Split.TRAIN,
                gen_kwargs={"filepath": downloaded_files["train"]},
            ),
            datasets.SplitGenerator(
                name=datasets.Split.VALIDATION,
                gen_kwargs={"filepath": downloaded_files["dev"]},
            ),
        ]

    def _generate_examples(self, filepath):
        """This function returns the examples in the raw (text) form."""
        for _id, line in enumerate(open(filepath, encoding="utf-8")):
            yield _id, {"text": line.rstrip()}


# tests/test_cache_layout.py

import os
import tempfile
import time
import unittest
import uuid
from io import BytesIO

from huggingface_hub import (
    HfApi,
    create_repo,
    delete_repo,
    hf_hub_download,
    snapshot_download,
    upload_file,
)
from huggingface_hub.utils import logging

from .testing_constants import ENDPOINT_STAGING, TOKEN, USER
from .testing_utils import with_production_testing


logger = logging.get_logger(__name__)
MODEL_IDENTIFIER = "hf-internal-testing/hfh-cache-layout"


def repo_name(id=uuid.uuid4().hex[:6]):
    return "repo-{0}-{1}".format(id, int(time.time() * 10e3))


def get_file_contents(path):
    with open(path) as f:
        content = f.read()

    return content


@with_production_testing
class CacheFileLayoutHfHubDownload(unittest.TestCase):
    def test_file_downloaded_in_cache(self):
        with tempfile.TemporaryDirectory() as cache:
            hf_hub_download(MODEL_IDENTIFIER, "file_0.txt", cache_dir=cache)

            expected_directory_name = f'models--{MODEL_IDENTIFIER.replace("/", "--")}'
            expected_path = os.path.join(cache, expected_directory_name)

            refs = os.listdir(os.path.join(expected_path, "refs"))
            snapshots = os.listdir(os.path.join(expected_path, "snapshots"))

            expected_reference = "main"

            # Only reference should be `main`.
            self.assertListEqual(refs, [expected_reference])

            with open(os.path.join(expected_path, "refs", expected_reference)) as f:
                snapshot_name = f.readline().strip()

            # The `main` reference should point to the only snapshot we have downloaded
            self.assertListEqual(snapshots, [snapshot_name])

            snapshot_path = os.path.join(expected_path, "snapshots", snapshot_name)
            snapshot_content = os.listdir(snapshot_path)

            # Only a single file in the snapshot
            self.assertEqual(len(snapshot_content), 1)

            snapshot_content_path = os.path.join(snapshot_path, snapshot_content[0])

            # The snapshot content should link to a blob
            self.assertTrue(os.path.islink(snapshot_content_path))

            resolved_blob_relative = os.readlink(snapshot_content_path)
            resolved_blob_absolute = os.path.normpath(
                os.path.join(snapshot_path, resolved_blob_relative)
            )

            with open(resolved_blob_absolute) as f:
                blob_contents = f.read().strip()

            # The contents of the file should be 'File 0'.
            self.assertEqual(blob_contents, "File 0")

    def test_file_downloaded_in_cache_with_revision(self):
        with tempfile.TemporaryDirectory() as cache:
            hf_hub_download(
                MODEL_IDENTIFIER, "file_0.txt", cache_dir=cache, revision="file-2"
            )

            expected_directory_name = f'models--{MODEL_IDENTIFIER.replace("/", "--")}'
            expected_path = os.path.join(cache, expected_directory_name)

            refs = os.listdir(os.path.join(expected_path, "refs"))
            snapshots = os.listdir(os.path.join(expected_path, "snapshots"))

            expected_reference = "file-2"

            # Only reference should be `file-2`.
            self.assertListEqual(refs, [expected_reference])

            with open(os.path.join(expected_path, "refs", expected_reference)) as f:
                snapshot_name = f.read().strip()

            # The `main` reference should point to the only snapshot we have downloaded
            self.assertListEqual(snapshots, [snapshot_name])

            snapshot_path = os.path.join(expected_path, "snapshots", snapshot_name)
            snapshot_content = os.listdir(snapshot_path)

            # Only a single file in the snapshot
            self.assertEqual(len(snapshot_content), 1)

            snapshot_content_path = os.path.join(snapshot_path, snapshot_content[0])

            # The snapshot content should link to a blob
            self.assertTrue(os.path.islink(snapshot_content_path))

            resolved_blob_relative = os.readlink(snapshot_content_path)
            resolved_blob_absolute = os.path.normpath(
                os.path.join(snapshot_path, resolved_blob_relative)
            )

            with open(resolved_blob_absolute) as f:
                blob_contents = f.readline().strip()

            # The contents of the file should be 'File 0'.
            self.assertEqual(blob_contents, "File 0")

    def test_file_download_happens_once(self):
        # Tests that a file is only downloaded once if it's not updated.

        with tempfile.TemporaryDirectory() as cache:
            path = hf_hub_download(MODEL_IDENTIFIER, "file_0.txt", cache_dir=cache)
            creation_time_0 = os.path.getmtime(path)

            time.sleep(2)

            path = hf_hub_download(MODEL_IDENTIFIER, "file_0.txt", cache_dir=cache)
            creation_time_1 = os.path.getmtime(path)

            self.assertEqual(creation_time_0, creation_time_1)

    def test_file_download_happens_once_intra_revision(self):
        # Tests that a file is only downloaded once if it's not updated, even across different revisions.

        with tempfile.TemporaryDirectory() as cache:
            path = hf_hub_download(MODEL_IDENTIFIER, "file_0.txt", cache_dir=cache)
            creation_time_0 = os.path.getmtime(path)

            time.sleep(2)

            path = hf_hub_download(
                MODEL_IDENTIFIER, "file_0.txt", cache_dir=cache, revision="file-2"
            )
            creation_time_1 = os.path.getmtime(path)

            self.assertEqual(creation_time_0, creation_time_1)

    def test_multiple_refs_for_same_file(self):
        with tempfile.TemporaryDirectory() as cache:
            hf_hub_download(MODEL_IDENTIFIER, "file_0.txt", cache_dir=cache)
            hf_hub_download(
                MODEL_IDENTIFIER, "file_0.txt", cache_dir=cache, revision="file-2"
            )

            expected_directory_name = f'models--{MODEL_IDENTIFIER.replace("/", "--")}'
            expected_path = os.path.join(cache, expected_directory_name)

            refs = os.listdir(os.path.join(expected_path, "refs"))
            refs.sort()

            snapshots = os.listdir(os.path.join(expected_path, "snapshots"))
            snapshots.sort()

            # Directory should contain two revisions
            self.assertListEqual(refs, ["file-2", "main"])

            refs_contents = [
                get_file_contents(os.path.join(expected_path, "refs", f)) for f in refs
            ]
            refs_contents.sort()

            # snapshots directory should contain two snapshots
            self.assertListEqual(refs_contents, snapshots)

            snapshot_links = [
                os.readlink(
                    os.path.join(expected_path, "snapshots", filename, "file_0.txt")
                )
                for filename in snapshots
            ]

            # All snapshot links should point to the same file.
            self.assertEqual(*snapshot_links)


@with_production_testing
class CacheFileLayoutSnapshotDownload(unittest.TestCase):
    def test_file_downloaded_in_cache(self):
        with tempfile.TemporaryDirectory() as cache:
            snapshot_download(MODEL_IDENTIFIER, cache_dir=cache)

            expected_directory_name = f'models--{MODEL_IDENTIFIER.replace("/", "--")}'
            expected_path = os.path.join(cache, expected_directory_name)

            refs = os.listdir(os.path.join(expected_path, "refs"))

            snapshots = os.listdir(os.path.join(expected_path, "snapshots"))
            snapshots.sort()

            # Directory should contain two revisions
            self.assertListEqual(refs, ["main"])

            ref_content = get_file_contents(
                os.path.join(expected_path, "refs", refs[0])
            )

            # snapshots directory should contain two snapshots
            self.assertListEqual([ref_content], snapshots)

            snapshot_path = os.path.join(expected_path, "snapshots", snapshots[0])

            files_in_snapshot = os.listdir(snapshot_path)

            snapshot_links = [
                os.readlink(os.path.join(snapshot_path, filename))
                for filename in files_in_snapshot
            ]

            resolved_snapshot_links = [
                os.path.normpath(os.path.join(snapshot_path, link))
                for link in snapshot_links
            ]

            self.assertTrue(all([os.path.isfile(l) for l in resolved_snapshot_links]))

    def test_file_downloaded_in_cache_several_revisions(self):
        with tempfile.TemporaryDirectory() as cache:
            snapshot_download(MODEL_IDENTIFIER, cache_dir=cache, revision="file-3")
            snapshot_download(MODEL_IDENTIFIER, cache_dir=cache, revision="file-2")

            expected_directory_name = f'models--{MODEL_IDENTIFIER.replace("/", "--")}'
            expected_path = os.path.join(cache, expected_directory_name)

            refs = os.listdir(os.path.join(expected_path, "refs"))
            refs.sort()

            snapshots = os.listdir(os.path.join(expected_path, "snapshots"))
            snapshots.sort()

            # Directory should contain two revisions
            self.assertListEqual(refs, ["file-2", "file-3"])

            refs_content = [
                get_file_contents(os.path.join(expected_path, "refs", ref))
                for ref in refs
            ]
            refs_content.sort()

            # snapshots directory should contain two snapshots
            self.assertListEqual(refs_content, snapshots)

            snapshots_paths = [
                os.path.join(expected_path, "snapshots", s) for s in snapshots
            ]

            files_in_snapshots = {s: os.listdir(s) for s in snapshots_paths}
            links_in_snapshots = {
                k: [os.readlink(os.path.join(k, _v)) for _v in v]
                for k, v in files_in_snapshots.items()
            }

            resolved_snapshots_links = {
                k: [os.path.normpath(os.path.join(k, link)) for link in v]
                for k, v in links_in_snapshots.items()
            }

            all_links = [b for a in resolved_snapshots_links.values() for b in a]
            all_unique_links = set(all_links)

            # [ 100]  .
            # ├── [ 140]  blobs
            # │   ├── [   7]  4475433e279a71203927cbe80125208a3b5db560
            # │   ├── [   7]  50fcd26d6ce3000f9d5f12904e80eccdc5685dd1
            # │   ├── [   7]  80146afc836c60e70ba67933fec439ab05b478f6
            # │   ├── [   7]  8cf9e18f080becb674b31c21642538269fe886a4
            # │   └── [1.1K]  ac481c8eb05e4d2496fbe076a38a7b4835dd733d
            # ├── [  80]  refs
            # │   ├── [  40]  file-2
            # │   └── [  40]  file-3
            # └── [  80]  snapshots
            #     ├── [ 120]  5e23cb3ae7f904919a442e1b27dcddae6c6bc292
            #     │   ├── [  52]  file_0.txt -> ../../blobs/80146afc836c60e70ba67933fec439ab05b478f6
            #     │   ├── [  52]  file_1.txt -> ../../blobs/50fcd26d6ce3000f9d5f12904e80eccdc5685dd1
            #     │   ├── [  52]  file_2.txt -> ../../blobs/4475433e279a71203927cbe80125208a3b5db560
            #     │   └── [  52]  .gitattributes -> ../../blobs/ac481c8eb05e4d2496fbe076a38a7b4835dd733d
            #     └── [ 120]  78aa2ebdb60bba086496a8792ba506e58e587b4c
            #         ├── [  52]  file_0.txt -> ../../blobs/80146afc836c60e70ba67933fec439ab05b478f6
            #         ├── [  52]  file_1.txt -> ../../blobs/50fcd26d6ce3000f9d5f12904e80eccdc5685dd1
            #         ├── [  52]  file_3.txt -> ../../blobs/8cf9e18f080becb674b31c21642538269fe886a4
            #         └── [  52]  .gitattributes -> ../../blobs/ac481c8eb05e4d2496fbe076a38a7b4835dd733d

            # Across the two revisions, there should be 8 total links
            self.assertEqual(len(all_links), 8)

            # Across the two revisions, there should only be 5 unique files.
            self.assertEqual(len(all_unique_links), 5)


class ReferenceUpdates(unittest.TestCase):
    _api = HfApi(endpoint=ENDPOINT_STAGING)

    @classmethod
    def setUpClass(cls):
        """
        Share this valid token in all tests below.
        """
        cls._token = TOKEN
        cls._api.set_access_token(TOKEN)

    def test_update_reference(self):
        repo_id = f"{USER}/{repo_name()}"
        create_repo(repo_id, token=self._token, exist_ok=True)

        try:
            upload_file(
                path_or_fileobj=BytesIO(b"Some string"),
                path_in_repo="file.txt",
                repo_id=repo_id,
                token=self._token,
            )

            with tempfile.TemporaryDirectory() as cache:
                hf_hub_download(repo_id, "file.txt", cache_dir=cache)

                expected_directory_name = f'models--{repo_id.replace("/", "--")}'
                expected_path = os.path.join(cache, expected_directory_name)

                refs = os.listdir(os.path.join(expected_path, "refs"))

                # Directory should contain two revisions
                self.assertListEqual(refs, ["main"])

                initial_ref_content = get_file_contents(
                    os.path.join(expected_path, "refs", refs[0])
                )

                # Upload a new file on the same branch
                upload_file(
                    path_or_fileobj=BytesIO(b"Some new string"),
                    path_in_repo="file.txt",
                    repo_id=repo_id,
                    token=self._token,
                )

                hf_hub_download(repo_id, "file.txt", cache_dir=cache)

                final_ref_content = get_file_contents(
                    os.path.join(expected_path, "refs", refs[0])
                )

                # The `main` reference should point to two different, but existing snapshots which contain
                # a 'file.txt'
                self.assertNotEqual(initial_ref_content, final_ref_content)
                self.assertTrue(
                    os.path.isdir(
                        os.path.join(expected_path, "snapshots", initial_ref_content)
                    )
                )
                self.assertTrue(
                    os.path.isfile(
                        os.path.join(
                            expected_path, "snapshots", initial_ref_content, "file.txt"
                        )
                    )
                )
                self.assertTrue(
                    os.path.isdir(
                        os.path.join(expected_path, "snapshots", final_ref_content)
                    )
                )
                self.assertTrue(
                    os.path.isfile(
                        os.path.join(
                            expected_path, "snapshots", final_ref_content, "file.txt"
                        )
                    )
                )
        except Exception:
            raise
        finally:
            delete_repo(repo_id, token=self._token)


# tests/test_endpoint_helpers.py

# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest

import requests
from huggingface_hub.hf_api import HfApi
from huggingface_hub.utils.endpoint_helpers import (
    AttributeDictionary,
    DatasetTags,
    GeneralTags,
    ModelTags,
)

from .testing_utils import with_production_testing


class AttributeDictionaryCommonTest(unittest.TestCase):
    _attrdict = AttributeDictionary()


class AttributeDictionaryTest(AttributeDictionaryCommonTest):
    def setUp(self):
        self._attrdict.clear()

    def test_adding_item(self):
        self._attrdict["itemA"] = 2
        self.assertEqual(self._attrdict.itemA, 2)
        self.assertEqual(self._attrdict["itemA"], 2)
        # We should be able to both set a property and a key
        self._attrdict.itemB = 3
        self.assertEqual(self._attrdict.itemB, 3)
        self.assertEqual(self._attrdict["itemB"], 3)

    def test_removing_item(self):
        self._attrdict["itemA"] = 2
        delattr(self._attrdict, "itemA")
        with self.assertRaises(KeyError):
            _ = self._attrdict["itemA"]

        self._attrdict.itemB = 3
        del self._attrdict["itemB"]
        with self.assertRaises(AttributeError):
            _ = self._attrdict.itemB

    def test_dir(self):
        # Since we subclass dict, dir should have everything
        # from dict and the atttributes
        _dict_keys = dir(dict) + [
            "__dict__",
            "__getattr__",
            "__module__",
            "__weakref__",
        ]
        self._attrdict["itemA"] = 2
        self._attrdict.itemB = 3
        _dict_keys += ["itemA", "itemB"]
        _dict_keys.sort()

        full_dir = dir(self._attrdict)
        full_dir.sort()
        self.assertEqual(full_dir, _dict_keys)

    def test_dir_with_numbers(self):
        self._attrdict["1a"] = 4
        self.assertFalse("1a" in dir(self._attrdict))
        self.assertTrue("1a" in list(self._attrdict.keys()))

    def test_dir_with_special_characters(self):
        self._attrdict["1<2"] = 3
        self.assertFalse("1<2" in dir(self._attrdict))
        self.assertTrue("1<2" in list(self._attrdict.keys()))

        self._attrdict["?abc"] = 4
        self.assertFalse("?abc" in dir(self._attrdict))
        self.assertTrue("?abc" in list(self._attrdict.keys()))

    def test_repr(self):
        self._attrdict["itemA"] = 2
        self._attrdict.itemB = 3
        self._attrdict["1a"] = 2
        self._attrdict["itemA?"] = 4
        repr_string = (
            "Available Attributes or Keys:\n * 1a (Key only)\n * itemA\n * itemA? (Key"
            " only)\n * itemB\n"
        )
        self.assertEqual(repr_string, repr(self._attrdict))


class GeneralTagsCommonTest(unittest.TestCase):
    # Similar to the output from /api/***-tags-by-type
    # id = how we can search hfapi, such as `'id': 'languages:en'`
    # label = A human readable version assigned to everything, such as `"label":"en"`
    _tag_dictionary = {
        "languages": [
            {"id": "itemA", "label": "Item A"},
            {"id": "itemB", "label": "1Item-B"},
        ],
        "license": [
            {"id": "itemC", "label": "Item C"},
            {"id": "itemD", "label": "Item.D"},
        ],
    }


class GeneralTagsTest(GeneralTagsCommonTest):
    def test_init(self):
        _tags = GeneralTags(self._tag_dictionary)
        self.assertTrue(all(hasattr(_tags, kind) for kind in ["languages", "license"]))
        languages = getattr(_tags, "languages")
        licenses = getattr(_tags, "license")
        # Ensure they have the right bits

        self.assertEqual(
            languages,
            AttributeDictionary({"ItemA": "itemA", "1Item_B": "itemB"}),
        )

        self.assertTrue("1Item_B" not in dir(languages))

        self.assertEqual(
            licenses, AttributeDictionary({"ItemC": "itemC", "Item_D": "itemD"})
        )

    def test_filter(self):
        _tags = GeneralTags(self._tag_dictionary, keys=["license"])
        self.assertTrue(hasattr(_tags, "license"))
        with self.assertRaises(AttributeError):
            _ = getattr(_tags, "languages")
        self.assertEqual(
            _tags.license, AttributeDictionary({"ItemC": "itemC", "Item_D": "itemD"})
        )


class ModelTagsTest(unittest.TestCase):
    @with_production_testing
    def test_tags(self):
        _api = HfApi()
        path = f"{_api.endpoint}/api/models-tags-by-type"
        r = requests.get(path)
        r.raise_for_status()
        d = r.json()
        o = ModelTags(d)
        for kind in ["library", "language", "license", "dataset", "pipeline_tag"]:
            self.assertTrue(len(getattr(o, kind).keys()) > 0)


class DatasetTagsTest(unittest.TestCase):
    @with_production_testing
    def test_tags(self):
        _api = HfApi()
        path = f"{_api.endpoint}/api/datasets-tags-by-type"
        r = requests.get(path)
        r.raise_for_status()
        d = r.json()
        o = DatasetTags(d)
        for kind in [
            "languages",
            "multilinguality",
            "language_creators",
            "task_categories",
            "size_categories",
            "benchmark",
            "task_ids",
            "licenses",
        ]:
            self.assertTrue(len(getattr(o, kind).keys()) > 0)


# tests/test_fastai_integration.py

import os
import shutil
import time
import uuid
from unittest import TestCase, skip

from huggingface_hub import HfApi
from huggingface_hub.fastai_utils import (
    _save_pretrained_fastai,
    from_pretrained_fastai,
    push_to_hub_fastai,
)
from huggingface_hub.file_download import (
    is_fastai_available,
    is_fastcore_available,
    is_torch_available,
)

from .testing_constants import ENDPOINT_STAGING, TOKEN, USER
from .testing_utils import set_write_permission_and_retry


def repo_name(id=uuid.uuid4().hex[:6]):
    return "fastai-repo-{0}-{1}".format(id, int(time.time() * 10e3))


WORKING_REPO_SUBDIR = f"fixtures/working_repo_{__name__.split('.')[-1]}"
WORKING_REPO_DIR = os.path.join(
    os.path.dirname(os.path.abspath(__file__)), WORKING_REPO_SUBDIR
)

if is_fastai_available():
    from fastai.data.block import DataBlock
    from fastai.test_utils import synth_learner

if is_torch_available():
    import torch


def require_fastai_fastcore(test_case):
    """
    Decorator marking a test that requires fastai and fastcore.
    These tests are skipped when fastai and fastcore are not installed.
    """
    if not is_fastai_available():
        return skip("Test requires fastai")(test_case)
    elif not is_fastcore_available():
        return skip("Test requires fastcore")(test_case)
    else:
        return test_case


def fake_dataloaders(a=2, b=3, bs=16, n=10):
    def get_data(n):
        x = torch.randn(bs * n, 1)
        return torch.cat((x, a * x + b + 0.1 * torch.randn(bs * n, 1)), 1)

    ds = get_data(n)
    dblock = DataBlock()
    return dblock.dataloaders(ds)


if is_fastai_available():
    dummy_model = synth_learner(data=fake_dataloaders())
    dummy_config = dict(test="test_0")
else:
    dummy_model = None
    dummy_config = None


@require_fastai_fastcore
class TestFastaiUtils(TestCase):
    @classmethod
    def setUpClass(cls):
        """
        Share this valid token in all tests below.
        """
        cls._api = HfApi(endpoint=ENDPOINT_STAGING)
        cls._token = TOKEN
        cls._api.set_access_token(TOKEN)

    def tearDown(self) -> None:
        try:
            shutil.rmtree(WORKING_REPO_DIR, onerror=set_write_permission_and_retry)
        except FileNotFoundError:
            pass

    def test_save_pretrained_without_config(self):
        REPO_NAME = repo_name("save")
        _save_pretrained_fastai(dummy_model, f"{WORKING_REPO_DIR}/{REPO_NAME}")
        files = os.listdir(f"{WORKING_REPO_DIR}/{REPO_NAME}")
        self.assertTrue("model.pkl" in files)
        self.assertTrue("pyproject.toml" in files)
        self.assertTrue("README.md" in files)
        self.assertEqual(len(files), 3)

    def test_save_pretrained_with_config(self):
        REPO_NAME = repo_name("save")
        _save_pretrained_fastai(
            dummy_model, f"{WORKING_REPO_DIR}/{REPO_NAME}", config=dummy_config
        )
        files = os.listdir(f"{WORKING_REPO_DIR}/{REPO_NAME}")
        self.assertTrue("config.json" in files)
        self.assertEqual(len(files), 4)

    def test_push_to_hub_and_from_pretrained_fastai(self):
        REPO_NAME = repo_name("push_to_hub")
        push_to_hub_fastai(
            learner=dummy_model,
            repo_id=f"{USER}/{REPO_NAME}",
            token=self._token,
            config=dummy_config,
        )
        model_info = self._api.model_info(
            f"{USER}/{REPO_NAME}",
        )
        self.assertEqual(model_info.modelId, f"{USER}/{REPO_NAME}")

        loaded_model = from_pretrained_fastai(f"{USER}/{REPO_NAME}")
        self.assertEqual(
            dummy_model.show_training_loop(), loaded_model.show_training_loop()
        )
        self._api.delete_repo(repo_id=f"{REPO_NAME}", token=self._token)


# tests/test_file_download.py

# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest

from huggingface_hub.constants import (
    CONFIG_NAME,
    PYTORCH_WEIGHTS_NAME,
    REPO_TYPE_DATASET,
)
from huggingface_hub.file_download import (
    cached_download,
    filename_to_url,
    hf_hub_download,
    hf_hub_url,
)
from huggingface_hub.utils import (
    EntryNotFoundError,
    RepositoryNotFoundError,
    RevisionNotFoundError,
)

from .testing_utils import (
    DUMMY_MODEL_ID,
    DUMMY_MODEL_ID_PINNED_SHA1,
    DUMMY_MODEL_ID_PINNED_SHA256,
    DUMMY_MODEL_ID_REVISION_INVALID,
    DUMMY_MODEL_ID_REVISION_ONE_SPECIFIC_COMMIT,
    SAMPLE_DATASET_IDENTIFIER,
    OfflineSimulationMode,
    offline,
    with_production_testing,
)


REVISION_ID_DEFAULT = "main"
# Default branch name

DATASET_ID = SAMPLE_DATASET_IDENTIFIER
# An actual dataset hosted on huggingface.co


DATASET_REVISION_ID_ONE_SPECIFIC_COMMIT = "e25d55a1c4933f987c46cc75d8ffadd67f257c61"
# One particular commit for DATASET_ID
DATASET_SAMPLE_PY_FILE = "custom_squad.py"


@with_production_testing
class CachedDownloadTests(unittest.TestCase):
    def test_bogus_url(self):
        url = "https://bogus"
        with self.assertRaisesRegex(ValueError, "Connection error"):
            _ = cached_download(url, legacy_cache_layout=True)

    def test_no_connection(self):
        invalid_url = hf_hub_url(
            DUMMY_MODEL_ID,
            filename=CONFIG_NAME,
            revision=DUMMY_MODEL_ID_REVISION_INVALID,
        )
        valid_url = hf_hub_url(
            DUMMY_MODEL_ID, filename=CONFIG_NAME, revision=REVISION_ID_DEFAULT
        )
        self.assertIsNotNone(
            cached_download(valid_url, force_download=True, legacy_cache_layout=True)
        )
        for offline_mode in OfflineSimulationMode:
            with offline(mode=offline_mode):
                with self.assertRaisesRegex(ValueError, "Connection error"):
                    _ = cached_download(invalid_url, legacy_cache_layout=True)
                with self.assertRaisesRegex(ValueError, "Connection error"):
                    _ = cached_download(
                        valid_url, force_download=True, legacy_cache_layout=True
                    )
                self.assertIsNotNone(
                    cached_download(valid_url, legacy_cache_layout=True)
                )

    def test_file_not_found(self):
        # Valid revision (None) but missing file.
        url = hf_hub_url(DUMMY_MODEL_ID, filename="missing.bin")
        with self.assertRaisesRegex(
            EntryNotFoundError, "404 Client Error: Entry Not Found"
        ):
            _ = cached_download(url, legacy_cache_layout=True)

    def test_revision_not_found(self):
        # Valid file but missing revision
        url = hf_hub_url(
            DUMMY_MODEL_ID,
            filename=CONFIG_NAME,
            revision=DUMMY_MODEL_ID_REVISION_INVALID,
        )
        with self.assertRaisesRegex(
            RevisionNotFoundError, "404 Client Error: Revision Not Found"
        ):
            _ = cached_download(url, legacy_cache_layout=True)

    def test_repo_not_found(self):
        # Invalid model file.
        url = hf_hub_url("bert-base", filename="pytorch_model.bin")
        with self.assertRaisesRegex(
            RepositoryNotFoundError, "401 Client Error: Repository Not Found"
        ):
            _ = cached_download(url, legacy_cache_layout=True)

    def test_standard_object(self):
        url = hf_hub_url(
            DUMMY_MODEL_ID, filename=CONFIG_NAME, revision=REVISION_ID_DEFAULT
        )
        filepath = cached_download(url, force_download=True, legacy_cache_layout=True)
        metadata = filename_to_url(filepath, legacy_cache_layout=True)
        self.assertEqual(metadata, (url, f'"{DUMMY_MODEL_ID_PINNED_SHA1}"'))

    def test_standard_object_rev(self):
        # Same object, but different revision
        url = hf_hub_url(
            DUMMY_MODEL_ID,
            filename=CONFIG_NAME,
            revision=DUMMY_MODEL_ID_REVISION_ONE_SPECIFIC_COMMIT,
        )
        filepath = cached_download(url, force_download=True, legacy_cache_layout=True)
        metadata = filename_to_url(filepath, legacy_cache_layout=True)
        self.assertNotEqual(metadata[1], f'"{DUMMY_MODEL_ID_PINNED_SHA1}"')
        # Caution: check that the etag is *not* equal to the one from `test_standard_object`

    def test_lfs_object(self):
        url = hf_hub_url(
            DUMMY_MODEL_ID, filename=PYTORCH_WEIGHTS_NAME, revision=REVISION_ID_DEFAULT
        )
        filepath = cached_download(url, force_download=True, legacy_cache_layout=True)
        metadata = filename_to_url(filepath, legacy_cache_layout=True)
        self.assertEqual(metadata, (url, f'"{DUMMY_MODEL_ID_PINNED_SHA256}"'))

    def test_dataset_standard_object_rev(self):
        url = hf_hub_url(
            DATASET_ID,
            filename=DATASET_SAMPLE_PY_FILE,
            repo_type=REPO_TYPE_DATASET,
            revision=DATASET_REVISION_ID_ONE_SPECIFIC_COMMIT,
        )
        # We can also just get the same url by prefixing "datasets" to repo_id:
        url2 = hf_hub_url(
            repo_id=f"datasets/{DATASET_ID}",
            filename=DATASET_SAMPLE_PY_FILE,
            revision=DATASET_REVISION_ID_ONE_SPECIFIC_COMMIT,
        )
        self.assertEqual(url, url2)
        # now let's download
        filepath = cached_download(url, force_download=True, legacy_cache_layout=True)
        metadata = filename_to_url(filepath, legacy_cache_layout=True)
        self.assertNotEqual(metadata[1], f'"{DUMMY_MODEL_ID_PINNED_SHA1}"')

    def test_dataset_lfs_object(self):
        url = hf_hub_url(
            DATASET_ID,
            filename="dev-v1.1.json",
            repo_type=REPO_TYPE_DATASET,
            revision=DATASET_REVISION_ID_ONE_SPECIFIC_COMMIT,
        )
        filepath = cached_download(url, force_download=True, legacy_cache_layout=True)
        metadata = filename_to_url(filepath, legacy_cache_layout=True)
        self.assertEqual(
            metadata,
            (url, '"95aa6a52d5d6a735563366753ca50492a658031da74f301ac5238b03966972c9"'),
        )

    def test_hf_hub_download_legacy(self):
        filepath = hf_hub_download(
            DUMMY_MODEL_ID,
            filename=CONFIG_NAME,
            revision=REVISION_ID_DEFAULT,
            force_download=True,
            legacy_cache_layout=True,
        )
        metadata = filename_to_url(filepath, legacy_cache_layout=True)
        self.assertEqual(metadata[1], f'"{DUMMY_MODEL_ID_PINNED_SHA1}"')


# tests/test_hf_api.py

# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


import os
import shutil
import subprocess
import tempfile
import time
import unittest
import uuid
import warnings
from io import BytesIO

import pytest

import requests
from huggingface_hub.commands.user import _login
from huggingface_hub.constants import (
    REPO_TYPE_DATASET,
    REPO_TYPE_MODEL,
    REPO_TYPE_SPACE,
    SPACES_SDK_TYPES,
)
from huggingface_hub.file_download import cached_download, hf_hub_download
from huggingface_hub.hf_api import (
    USERNAME_PLACEHOLDER,
    DatasetInfo,
    DatasetSearchArguments,
    HfApi,
    HfFolder,
    MetricInfo,
    ModelInfo,
    ModelSearchArguments,
    erase_from_credential_store,
    read_from_credential_store,
    repo_type_and_id_from_hf_id,
)
from huggingface_hub.utils import logging
from huggingface_hub.utils.endpoint_helpers import (
    DatasetFilter,
    ModelFilter,
    _filter_emissions,
)
from requests.exceptions import HTTPError

from .testing_constants import (
    ENDPOINT_STAGING,
    ENDPOINT_STAGING_BASIC_AUTH,
    FULL_NAME,
    TOKEN,
    USER,
)
from .testing_utils import (
    DUMMY_DATASET_ID,
    DUMMY_DATASET_ID_REVISION_ONE_SPECIFIC_COMMIT,
    DUMMY_MODEL_ID,
    DUMMY_MODEL_ID_REVISION_ONE_SPECIFIC_COMMIT,
    require_git_lfs,
    retry_endpoint,
    set_write_permission_and_retry,
    with_production_testing,
)


logger = logging.get_logger(__name__)


def repo_name(id=uuid.uuid4().hex[:6]):
    return "my-model-{0}-{1}".format(id, int(time.time() * 10e3))


def repo_name_large_file(id=uuid.uuid4().hex[:6]):
    return "my-model-largefiles-{0}-{1}".format(id, int(time.time() * 10e3))


def dataset_repo_name(id=uuid.uuid4().hex[:6]):
    return "my-dataset-{0}-{1}".format(id, int(time.time() * 10e3))


def space_repo_name(id=uuid.uuid4().hex[:6]):
    return "my-space-{0}-{1}".format(id, int(time.time() * 10e3))


WORKING_REPO_DIR = os.path.join(
    os.path.dirname(os.path.abspath(__file__)), "fixtures/working_repo"
)
LARGE_FILE_14MB = "https://cdn-media.huggingface.co/lfs-largefiles/progit.epub"
LARGE_FILE_18MB = "https://cdn-media.huggingface.co/lfs-largefiles/progit.pdf"


class HfApiCommonTest(unittest.TestCase):
    _api = HfApi(endpoint=ENDPOINT_STAGING)


class HfApiLoginTest(HfApiCommonTest):
    def setUp(self) -> None:
        erase_from_credential_store(USERNAME_PLACEHOLDER)

    @classmethod
    def tearDownClass(cls) -> None:
        cls._api.set_access_token(TOKEN)

    def test_login_git_credentials(self):
        self.assertTupleEqual(
            read_from_credential_store(USERNAME_PLACEHOLDER), (None, None)
        )
        self._api.set_access_token(TOKEN)
        self.assertTupleEqual(
            read_from_credential_store(USERNAME_PLACEHOLDER),
            (USERNAME_PLACEHOLDER, TOKEN),
        )
        erase_from_credential_store(username=USERNAME_PLACEHOLDER)
        self.assertTupleEqual(
            read_from_credential_store(USERNAME_PLACEHOLDER), (None, None)
        )

    def test_login_cli(self):
        self._api.set_access_token(TOKEN)
        self.assertTupleEqual(
            read_from_credential_store(USERNAME_PLACEHOLDER),
            (USERNAME_PLACEHOLDER, TOKEN),
        )
        erase_from_credential_store(username=USERNAME_PLACEHOLDER)
        self.assertTupleEqual(
            read_from_credential_store(USERNAME_PLACEHOLDER), (None, None)
        )

        _login(self._api, token=TOKEN)
        self.assertTupleEqual(
            read_from_credential_store(USERNAME_PLACEHOLDER),
            (USERNAME_PLACEHOLDER, TOKEN),
        )
        erase_from_credential_store(username=USERNAME_PLACEHOLDER)
        self.assertTupleEqual(
            read_from_credential_store(USERNAME_PLACEHOLDER), (None, None)
        )

    def test_login_cli_org_fail(self):
        with pytest.raises(
            ValueError, match="You must use your personal account token."
        ):
            _login(self._api, token="api_org_dummy_token")


class HfApiCommonTestWithLogin(HfApiCommonTest):
    @classmethod
    def setUpClass(cls):
        """
        Share this valid token in all tests below.
        """
        cls._token = TOKEN
        cls._api.set_access_token(TOKEN)


@retry_endpoint
def test_repo_id_no_warning():
    # tests that passing repo_id as positional arg doesn't raise any warnings
    # for {create, delete}_repo and update_repo_visibility
    api = HfApi(endpoint=ENDPOINT_STAGING)
    REPO_NAME = repo_name("crud")

    args = [
        ("create_repo", {}),
        ("update_repo_visibility", {"private": False}),
        ("delete_repo", {}),
    ]

    for method, kwargs in args:
        with warnings.catch_warnings(record=True) as record:
            getattr(api, method)(
                REPO_NAME, token=TOKEN, repo_type=REPO_TYPE_MODEL, **kwargs
            )
        assert not len(record)


class HfApiEndpointsTest(HfApiCommonTestWithLogin):
    def test_whoami(self):
        info = self._api.whoami(token=self._token)
        self.assertEqual(info["name"], USER)
        self.assertEqual(info["fullname"], FULL_NAME)
        self.assertIsInstance(info["orgs"], list)
        valid_org = [org for org in info["orgs"] if org["name"] == "valid_org"][0]
        self.assertIsInstance(valid_org["apiToken"], str)

    @retry_endpoint
    def test_delete_repo_error_message(self):
        # test for #751
        with self.assertRaisesRegex(
            requests.exceptions.HTTPError,
            r"404 Client Error: Repository Not Found (.+) \(Request ID: .+\)",
        ):
            self._api.delete_repo("repo-that-does-not-exist", token=self._token)

    @retry_endpoint
    def test_create_update_and_delete_repo(self):
        REPO_NAME = repo_name("crud")
        self._api.create_repo(repo_id=REPO_NAME, token=self._token)
        res = self._api.update_repo_visibility(
            repo_id=REPO_NAME, token=self._token, private=True
        )
        self.assertTrue(res["private"])
        res = self._api.update_repo_visibility(
            repo_id=REPO_NAME, token=self._token, private=False
        )
        self.assertFalse(res["private"])
        self._api.delete_repo(repo_id=REPO_NAME, token=self._token)

    @retry_endpoint
    def test_create_update_and_delete_model_repo(self):
        REPO_NAME = repo_name("crud")
        self._api.create_repo(
            repo_id=REPO_NAME, token=self._token, repo_type=REPO_TYPE_MODEL
        )
        res = self._api.update_repo_visibility(
            repo_id=REPO_NAME,
            token=self._token,
            private=True,
            repo_type=REPO_TYPE_MODEL,
        )
        self.assertTrue(res["private"])
        res = self._api.update_repo_visibility(
            repo_id=REPO_NAME,
            token=self._token,
            private=False,
            repo_type=REPO_TYPE_MODEL,
        )
        self.assertFalse(res["private"])
        self._api.delete_repo(
            repo_id=REPO_NAME, token=self._token, repo_type=REPO_TYPE_MODEL
        )

    @retry_endpoint
    def test_create_update_and_delete_dataset_repo(self):
        DATASET_REPO_NAME = dataset_repo_name("crud")
        self._api.create_repo(
            repo_id=DATASET_REPO_NAME, token=self._token, repo_type=REPO_TYPE_DATASET
        )
        res = self._api.update_repo_visibility(
            repo_id=DATASET_REPO_NAME,
            token=self._token,
            private=True,
            repo_type=REPO_TYPE_DATASET,
        )
        self.assertTrue(res["private"])
        res = self._api.update_repo_visibility(
            repo_id=DATASET_REPO_NAME,
            token=self._token,
            private=False,
            repo_type=REPO_TYPE_DATASET,
        )
        self.assertFalse(res["private"])
        self._api.delete_repo(
            repo_id=DATASET_REPO_NAME, token=self._token, repo_type=REPO_TYPE_DATASET
        )

    @retry_endpoint
    def test_create_update_and_delete_space_repo(self):
        SPACE_REPO_NAME = space_repo_name("failing")
        with pytest.raises(ValueError, match=r"No space_sdk provided.*"):
            self._api.create_repo(
                token=self._token,
                repo_id=SPACE_REPO_NAME,
                repo_type=REPO_TYPE_SPACE,
                space_sdk=None,
            )
        with pytest.raises(ValueError, match=r"Invalid space_sdk.*"):
            self._api.create_repo(
                token=self._token,
                repo_id=SPACE_REPO_NAME,
                repo_type=REPO_TYPE_SPACE,
                space_sdk="asdfasdf",
            )

        for sdk in SPACES_SDK_TYPES:
            SPACE_REPO_NAME = space_repo_name(sdk)
            self._api.create_repo(
                repo_id=SPACE_REPO_NAME,
                token=self._token,
                repo_type=REPO_TYPE_SPACE,
                space_sdk=sdk,
            )
            res = self._api.update_repo_visibility(
                repo_id=SPACE_REPO_NAME,
                token=self._token,
                private=True,
                repo_type=REPO_TYPE_SPACE,
            )
            self.assertTrue(res["private"])
            res = self._api.update_repo_visibility(
                repo_id=SPACE_REPO_NAME,
                token=self._token,
                private=False,
                repo_type=REPO_TYPE_SPACE,
            )
            self.assertFalse(res["private"])
            self._api.delete_repo(
                repo_id=SPACE_REPO_NAME, token=self._token, repo_type=REPO_TYPE_SPACE
            )

    @retry_endpoint
    def test_move_repo(self):
        REPO_NAME = repo_name("crud")
        repo_id = f"__DUMMY_TRANSFORMERS_USER__/{REPO_NAME}"
        NEW_REPO_NAME = repo_name("crud2")
        new_repo_id = f"__DUMMY_TRANSFORMERS_USER__/{NEW_REPO_NAME}"

        for repo_type in [REPO_TYPE_MODEL, REPO_TYPE_DATASET, REPO_TYPE_SPACE]:
            self._api.create_repo(
                repo_id=REPO_NAME,
                token=self._token,
                repo_type=repo_type,
                space_sdk="static",
            )

            with pytest.raises(ValueError, match=r"Invalid repo_id*"):
                self._api.move_repo(
                    from_id=repo_id,
                    to_id="invalid_repo_id",
                    token=self._token,
                    repo_type=repo_type,
                )

            # Should raise an error if it fails
            self._api.move_repo(
                from_id=repo_id,
                to_id=new_repo_id,
                token=self._token,
                repo_type=repo_type,
            )
            self._api.delete_repo(
                repo_id=NEW_REPO_NAME, token=self._token, repo_type=repo_type
            )


class HfApiUploadFileTest(HfApiCommonTestWithLogin):
    def setUp(self) -> None:
        super().setUp()
        self.tmp_dir = tempfile.mkdtemp()
        self.tmp_file = os.path.join(self.tmp_dir, "temp")
        self.tmp_file_content = "Content of the file"
        with open(self.tmp_file, "w+") as f:
            f.write(self.tmp_file_content)
        self.addCleanup(
            lambda: shutil.rmtree(self.tmp_dir, onerror=set_write_permission_and_retry)
        )

    @retry_endpoint
    def test_upload_file_validation(self):
        REPO_NAME = repo_name("upload")
        with self.assertRaises(ValueError, msg="Wrong repo type"):
            self._api.upload_file(
                path_or_fileobj=self.tmp_file,
                path_in_repo="README.md",
                repo_id=f"{USER}/{REPO_NAME}",
                repo_type="this type does not exist",
                token=self._token,
            )

        with self.assertRaises(ValueError, msg="File opened in text mode"):
            with open(self.tmp_file, "rt") as ftext:
                self._api.upload_file(
                    path_or_fileobj=ftext,
                    path_in_repo="README.md",
                    repo_id=f"{USER}/{REPO_NAME}",
                    token=self._token,
                )

        with self.assertRaises(
            ValueError, msg="path_or_fileobj is str but does not point to a file"
        ):
            self._api.upload_file(
                path_or_fileobj=os.path.join(self.tmp_dir, "nofile.pth"),
                path_in_repo="README.md",
                repo_id=f"{USER}/{REPO_NAME}",
                token=self._token,
            )

    @retry_endpoint
    def test_upload_file_path(self):
        REPO_NAME = repo_name("path")
        self._api.create_repo(token=self._token, repo_id=REPO_NAME)
        try:
            self._api.upload_file(
                path_or_fileobj=self.tmp_file,
                path_in_repo="temp/new_file.md",
                repo_id=f"{USER}/{REPO_NAME}",
                token=self._token,
            )
            url = "{}/{user}/{repo}/resolve/main/temp/new_file.md".format(
                ENDPOINT_STAGING,
                user=USER,
                repo=REPO_NAME,
            )
            filepath = cached_download(
                url, force_download=True, legacy_cache_layout=True
            )
            with open(filepath) as downloaded_file:
                content = downloaded_file.read()
            self.assertEqual(content, self.tmp_file_content)

        except Exception as err:
            self.fail(err)
        finally:
            self._api.delete_repo(repo_id=REPO_NAME, token=self._token)

    @retry_endpoint
    def test_upload_file_fileobj(self):
        REPO_NAME = repo_name("fileobj")
        self._api.create_repo(repo_id=REPO_NAME, token=self._token)
        try:
            with open(self.tmp_file, "rb") as filestream:
                self._api.upload_file(
                    path_or_fileobj=filestream,
                    path_in_repo="temp/new_file.md",
                    repo_id=f"{USER}/{REPO_NAME}",
                    token=self._token,
                )
            url = "{}/{user}/{repo}/resolve/main/temp/new_file.md".format(
                ENDPOINT_STAGING,
                user=USER,
                repo=REPO_NAME,
            )
            filepath = cached_download(
                url, force_download=True, legacy_cache_layout=True
            )
            with open(filepath) as downloaded_file:
                content = downloaded_file.read()
            self.assertEqual(content, self.tmp_file_content)

        except Exception as err:
            self.fail(err)
        finally:
            self._api.delete_repo(repo_id=REPO_NAME, token=self._token)

    @retry_endpoint
    def test_upload_file_bytesio(self):
        REPO_NAME = repo_name("bytesio")
        self._api.create_repo(repo_id=REPO_NAME, token=self._token)
        try:
            filecontent = BytesIO(b"File content, but in bytes IO")
            self._api.upload_file(
                path_or_fileobj=filecontent,
                path_in_repo="temp/new_file.md",
                repo_id=f"{USER}/{REPO_NAME}",
                token=self._token,
            )
            url = "{}/{user}/{repo}/resolve/main/temp/new_file.md".format(
                ENDPOINT_STAGING,
                user=USER,
                repo=REPO_NAME,
            )
            filepath = cached_download(
                url, force_download=True, legacy_cache_layout=True
            )
            with open(filepath) as downloaded_file:
                content = downloaded_file.read()
            self.assertEqual(content, filecontent.getvalue().decode())

        except Exception as err:
            self.fail(err)
        finally:
            self._api.delete_repo(repo_id=REPO_NAME, token=self._token)

    @retry_endpoint
    def test_create_repo_org_token_fail(self):
        REPO_NAME = repo_name("org")
        with pytest.raises(
            ValueError, match="You must use your personal account token."
        ):
            self._api.create_repo(repo_id=REPO_NAME, token="api_org_dummy_token")

    @retry_endpoint
    def test_create_repo_org_token_none_fail(self):
        REPO_NAME = repo_name("org")
        HfFolder.save_token("api_org_dummy_token")
        with pytest.raises(
            ValueError, match="You must use your personal account token."
        ):
            self._api.create_repo(repo_id=REPO_NAME)

    @retry_endpoint
    def test_upload_buffer(self):
        REPO_NAME = repo_name("buffer")
        self._api.create_repo(repo_id=REPO_NAME, token=self._token)
        try:
            buffer = BytesIO()
            buffer.write(self.tmp_file_content.encode())
            self._api.upload_file(
                path_or_fileobj=buffer.getvalue(),
                path_in_repo="temp/new_file.md",
                repo_id=f"{USER}/{REPO_NAME}",
                token=self._token,
            )
            url = "{}/{user}/{repo}/resolve/main/temp/new_file.md".format(
                ENDPOINT_STAGING,
                user=USER,
                repo=REPO_NAME,
            )
            filepath = cached_download(
                url, force_download=True, legacy_cache_layout=True
            )
            with open(filepath) as downloaded_file:
                content = downloaded_file.read()
            self.assertEqual(content, self.tmp_file_content)

        except Exception as err:
            self.fail(err)
        finally:
            self._api.delete_repo(repo_id=REPO_NAME, token=self._token)

    @retry_endpoint
    def test_delete_file(self):
        REPO_NAME = repo_name("delete")
        self._api.create_repo(token=self._token, repo_id=REPO_NAME)
        try:
            self._api.upload_file(
                path_or_fileobj=self.tmp_file,
                path_in_repo="temp/new_file.md",
                repo_id=f"{USER}/{REPO_NAME}",
                token=self._token,
            )
            self._api.delete_file(
                path_in_repo="temp/new_file.md",
                repo_id=f"{USER}/{REPO_NAME}",
                token=self._token,
            )

            with self.assertRaises(HTTPError):
                # Should raise a 404
                hf_hub_download(f"{USER}/{REPO_NAME}", "temp/new_file.md")

        except Exception as err:
            self.fail(err)
        finally:
            self._api.delete_repo(repo_id=REPO_NAME, token=self._token)

    def test_get_full_repo_name(self):
        repo_name_with_no_org = self._api.get_full_repo_name("model", token=self._token)
        self.assertEqual(repo_name_with_no_org, f"{USER}/model")

        repo_name_with_no_org = self._api.get_full_repo_name(
            "model", organization="org", token=self._token
        )
        self.assertEqual(repo_name_with_no_org, "org/model")


class HfApiPublicTest(unittest.TestCase):
    def test_staging_list_models(self):
        _api = HfApi(endpoint=ENDPOINT_STAGING)
        _ = _api.list_models()

    @with_production_testing
    def test_list_models(self):
        _api = HfApi()
        models = _api.list_models()
        self.assertGreater(len(models), 100)
        self.assertIsInstance(models[0], ModelInfo)

    @with_production_testing
    def test_list_models_author(self):
        _api = HfApi()
        models = _api.list_models(author="google")
        self.assertGreater(len(models), 10)
        self.assertIsInstance(models[0], ModelInfo)
        [self.assertTrue("google" in model.author for model in models)]

    @with_production_testing
    def test_list_models_search(self):
        _api = HfApi()
        models = _api.list_models(search="bert")
        self.assertGreater(len(models), 10)
        self.assertIsInstance(models[0], ModelInfo)
        [self.assertTrue("bert" in model.modelId.lower()) for model in models]

    @with_production_testing
    def test_list_models_complex_query(self):
        # Let's list the 10 most recent models
        # with tags "bert" and "jax",
        # ordered by last modified date.
        _api = HfApi()
        models = _api.list_models(
            filter=("bert", "jax"), sort="lastModified", direction=-1, limit=10
        )
        # we have at least 1 models
        self.assertGreater(len(models), 1)
        self.assertLessEqual(len(models), 10)
        model = models[0]
        self.assertIsInstance(model, ModelInfo)
        self.assertTrue(all(tag in model.tags for tag in ["bert", "jax"]))

    @with_production_testing
    def test_list_models_with_config(self):
        _api = HfApi()
        models = _api.list_models(
            filter="adapter-transformers", fetch_config=True, limit=20
        )
        found_configs = 0
        for model in models:
            if model.config:
                found_configs = found_configs + 1
        self.assertGreater(found_configs, 0)

    @with_production_testing
    def test_model_info(self):
        _api = HfApi()
        model = _api.model_info(repo_id=DUMMY_MODEL_ID)
        self.assertIsInstance(model, ModelInfo)
        self.assertNotEqual(model.sha, DUMMY_MODEL_ID_REVISION_ONE_SPECIFIC_COMMIT)
        # One particular commit (not the top of `main`)
        model = _api.model_info(
            repo_id=DUMMY_MODEL_ID, revision=DUMMY_MODEL_ID_REVISION_ONE_SPECIFIC_COMMIT
        )
        self.assertIsInstance(model, ModelInfo)
        self.assertEqual(model.sha, DUMMY_MODEL_ID_REVISION_ONE_SPECIFIC_COMMIT)

    @with_production_testing
    def test_model_info_with_security(self):
        _api = HfApi()
        model = _api.model_info(
            repo_id=DUMMY_MODEL_ID,
            revision=DUMMY_MODEL_ID_REVISION_ONE_SPECIFIC_COMMIT,
            securityStatus=True,
        )
        self.assertEqual(
            getattr(model, "securityStatus"),
            {"containsInfected": False},
        )

    @with_production_testing
    def test_list_repo_files(self):
        _api = HfApi()
        files = _api.list_repo_files(repo_id=DUMMY_MODEL_ID)
        expected_files = [
            ".gitattributes",
            "README.md",
            "config.json",
            "flax_model.msgpack",
            "merges.txt",
            "pytorch_model.bin",
            "tf_model.h5",
            "vocab.json",
        ]
        self.assertListEqual(files, expected_files)

    def test_staging_list_datasets(self):
        _api = HfApi(endpoint=ENDPOINT_STAGING)
        _ = _api.list_datasets()

    @with_production_testing
    def test_list_datasets(self):
        _api = HfApi()
        datasets = _api.list_datasets()
        self.assertGreater(len(datasets), 100)
        self.assertIsInstance(datasets[0], DatasetInfo)

    @with_production_testing
    def test_filter_datasets_by_author_and_name(self):
        _api = HfApi()
        f = DatasetFilter(author="huggingface", dataset_name="DataMeasurementsFiles")
        datasets = _api.list_datasets(filter=f)
        self.assertEqual(len(datasets), 1)
        self.assertTrue("huggingface" in datasets[0].author)
        self.assertTrue("DataMeasurementsFiles" in datasets[0].id)

    @with_production_testing
    def test_filter_datasets_by_benchmark(self):
        _api = HfApi()
        f = DatasetFilter(benchmark="raft")
        datasets = _api.list_datasets(filter=f)
        self.assertGreater(len(datasets), 0)
        self.assertTrue("benchmark:raft" in datasets[0].tags)

    @with_production_testing
    def test_filter_datasets_by_language_creator(self):
        _api = HfApi()
        f = DatasetFilter(language_creators="crowdsourced")
        datasets = _api.list_datasets(filter=f)
        self.assertGreater(len(datasets), 0)
        self.assertTrue("language_creators:crowdsourced" in datasets[0].tags)

    @with_production_testing
    def test_filter_datasets_by_language(self):
        _api = HfApi()
        f = DatasetFilter(languages="en")
        datasets = _api.list_datasets(filter=f)
        self.assertGreater(len(datasets), 0)
        self.assertTrue("languages:en" in datasets[0].tags)
        args = DatasetSearchArguments()
        f = DatasetFilter(languages=(args.languages.en, args.languages.fr))
        datasets = _api.list_datasets(filter=f)
        self.assertGreater(len(datasets), 0)
        self.assertTrue("languages:en" in datasets[0].tags)
        self.assertTrue("languages:fr" in datasets[0].tags)

    @with_production_testing
    def test_filter_datasets_by_multilinguality(self):
        _api = HfApi()
        f = DatasetFilter(multilinguality="yes")
        datasets = _api.list_datasets(filter=f)
        self.assertGreater(len(datasets), 0)
        self.assertTrue("multilinguality:yes" in datasets[0].tags)

    @with_production_testing
    def test_filter_datasets_by_size_categories(self):
        _api = HfApi()
        f = DatasetFilter(size_categories="100K<n<1M")
        datasets = _api.list_datasets(filter=f)
        self.assertGreater(len(datasets), 0)
        self.assertTrue("size_categories:100K<n<1M" in datasets[0].tags)

    @with_production_testing
    def test_filter_datasets_by_task_categories(self):
        _api = HfApi()
        f = DatasetFilter(task_categories="audio-classification")
        datasets = _api.list_datasets(filter=f)
        self.assertGreater(len(datasets), 0)
        self.assertTrue("task_categories:audio-classification" in datasets[0].tags)

    @with_production_testing
    def test_filter_datasets_by_task_ids(self):
        _api = HfApi()
        f = DatasetFilter(task_ids="automatic-speech-recognition")
        datasets = _api.list_datasets(filter=f)
        self.assertGreater(len(datasets), 0)
        self.assertTrue("task_ids:automatic-speech-recognition" in datasets[0].tags)

    @with_production_testing
    def test_list_datasets_full(self):
        _api = HfApi()
        datasets = _api.list_datasets(full=True)
        self.assertGreater(len(datasets), 100)
        dataset = datasets[0]
        self.assertIsInstance(dataset, DatasetInfo)
        self.assertTrue(any(dataset.cardData for dataset in datasets))

    @with_production_testing
    def test_list_datasets_author(self):
        _api = HfApi()
        datasets = _api.list_datasets(author="huggingface")
        self.assertGreater(len(datasets), 1)
        self.assertIsInstance(datasets[0], DatasetInfo)

    @with_production_testing
    def test_list_datasets_search(self):
        _api = HfApi()
        datasets = _api.list_datasets(search="wikipedia")
        self.assertGreater(len(datasets), 10)
        self.assertIsInstance(datasets[0], DatasetInfo)

    @with_production_testing
    def test_filter_datasets_with_cardData(self):
        _api = HfApi()
        datasets = _api.list_datasets(cardData=True)
        self.assertGreater(
            sum(
                [getattr(dataset, "cardData", None) is not None for dataset in datasets]
            ),
            0,
        )
        datasets = _api.list_datasets()
        self.assertTrue(
            all([getattr(dataset, "cardData", None) is None for dataset in datasets])
        )

    @with_production_testing
    def test_dataset_info(self):
        _api = HfApi()
        dataset = _api.dataset_info(repo_id=DUMMY_DATASET_ID)
        self.assertTrue(
            isinstance(dataset.cardData, dict) and len(dataset.cardData) > 0
        )
        self.assertTrue(
            isinstance(dataset.siblings, list) and len(dataset.siblings) > 0
        )
        self.assertIsInstance(dataset, DatasetInfo)
        self.assertNotEqual(dataset.sha, DUMMY_DATASET_ID_REVISION_ONE_SPECIFIC_COMMIT)
        dataset = _api.dataset_info(
            repo_id=DUMMY_DATASET_ID,
            revision=DUMMY_DATASET_ID_REVISION_ONE_SPECIFIC_COMMIT,
        )
        self.assertIsInstance(dataset, DatasetInfo)
        self.assertEqual(dataset.sha, DUMMY_DATASET_ID_REVISION_ONE_SPECIFIC_COMMIT)

    def test_staging_list_metrics(self):
        _api = HfApi(endpoint=ENDPOINT_STAGING)
        _ = _api.list_metrics()

    @with_production_testing
    def test_list_metrics(self):
        _api = HfApi()
        metrics = _api.list_metrics()
        self.assertGreater(len(metrics), 10)
        self.assertIsInstance(metrics[0], MetricInfo)
        self.assertTrue(any(metric.description for metric in metrics))

    @with_production_testing
    def test_filter_models_by_author(self):
        _api = HfApi()
        f = ModelFilter(author="muellerzr")
        models = _api.list_models(filter=f)
        self.assertGreater(len(models), 0)
        self.assertTrue("muellerzr" in models[0].modelId)

    @with_production_testing
    def test_filter_models_by_author_and_name(self):
        # Test we can search by an author and a name, but the model is not found
        _api = HfApi()
        f = ModelFilter("facebook", model_name="bart-base")
        models = _api.list_models(filter=f)
        self.assertTrue("facebook/bart-base" in models[0].modelId)

    @with_production_testing
    def test_failing_filter_models_by_author_and_model_name(self):
        # Test we can search by an author and a name, but the model is not found
        _api = HfApi()
        f = ModelFilter(author="muellerzr", model_name="testme")
        models = _api.list_models(filter=f)
        self.assertEqual(len(models), 0)

    @with_production_testing
    def test_filter_models_with_library(self):
        _api = HfApi()
        f = ModelFilter("microsoft", model_name="wavlm-base-sd", library="tensorflow")
        models = _api.list_models(filter=f)
        self.assertGreater(1, len(models))
        f = ModelFilter("microsoft", model_name="wavlm-base-sd", library="pytorch")
        models = _api.list_models(filter=f)
        self.assertGreater(len(models), 0)

    @with_production_testing
    def test_filter_models_with_task(self):
        _api = HfApi()
        f = ModelFilter(task="fill-mask", model_name="albert-base-v2")
        models = _api.list_models(filter=f)
        self.assertTrue("fill-mask" == models[0].pipeline_tag)
        self.assertTrue("albert-base-v2" in models[0].modelId)
        f = ModelFilter(task="dummytask")
        models = _api.list_models(filter=f)
        self.assertGreater(1, len(models))

    @with_production_testing
    def test_filter_models_by_language(self):
        _api = HfApi()
        f_fr = ModelFilter(language="fr")
        res_fr = _api.list_models(filter=f_fr)

        f_en = ModelFilter(language="en")
        res_en = _api.list_models(filter=f_en)

        assert len(res_fr) != len(res_en)

    @with_production_testing
    def test_filter_models_with_complex_query(self):
        _api = HfApi()
        args = ModelSearchArguments()
        f = ModelFilter(
            task=args.pipeline_tag.TextClassification,
            library=[args.library.PyTorch, args.library.TensorFlow],
        )
        models = _api.list_models(filter=f)
        self.assertGreater(len(models), 1)
        self.assertTrue(
            [
                "text-classification" in model.pipeline_tag
                or "text-classification" in model.tags
                for model in models
            ]
        )
        self.assertTrue(
            ["pytorch" in model.tags and "tf" in model.tags for model in models]
        )

    @with_production_testing
    def test_filter_models_with_cardData(self):
        _api = HfApi()
        models = _api.list_models(filter="co2_eq_emissions", cardData=True)
        self.assertTrue([hasattr(model, "cardData") for model in models])
        models = _api.list_models(filter="co2_eq_emissions")
        self.assertTrue(all([not hasattr(model, "cardData") for model in models]))

    def test_filter_emissions_dict(self):
        # tests that dictionary is handled correctly as "emissions" and that
        # 17g is accepted and parsed correctly as a value
        # regression test for #753
        model = ModelInfo(cardData={"co2_eq_emissions": {"emissions": "17g"}})
        res = _filter_emissions([model], -1, 100)
        assert len(res) == 1

    @with_production_testing
    def test_filter_emissions_with_max(self):
        _api = HfApi()
        models = _api.list_models(emissions_thresholds=(None, 100), cardData=True)
        self.assertTrue(
            all(
                [
                    model.cardData["co2_eq_emissions"] <= 100
                    for model in models
                    if isinstance(model.cardData["co2_eq_emissions"], (float, int))
                ]
            )
        )

    @with_production_testing
    def test_filter_emissions_with_min(self):
        _api = HfApi()
        models = _api.list_models(emissions_thresholds=(5, None), cardData=True)
        self.assertTrue(
            all(
                [
                    model.cardData["co2_eq_emissions"] >= 5
                    for model in models
                    if isinstance(model.cardData["co2_eq_emissions"], (float, int))
                ]
            )
        )

    @with_production_testing
    def test_filter_emissions_with_min_and_max(self):
        _api = HfApi()
        models = _api.list_models(emissions_thresholds=(5, 100), cardData=True)
        self.assertTrue(
            all(
                [
                    model.cardData["co2_eq_emissions"] >= 5
                    for model in models
                    if isinstance(model.cardData["co2_eq_emissions"], (float, int))
                ]
            )
        )
        self.assertTrue(
            all(
                [
                    model.cardData["co2_eq_emissions"] <= 100
                    for model in models
                    if isinstance(model.cardData["co2_eq_emissions"], (float, int))
                ]
            )
        )


class HfApiPrivateTest(HfApiCommonTestWithLogin):
    @retry_endpoint
    def setUp(self) -> None:
        super().setUp()
        self.REPO_NAME = repo_name("private")
        self._api.create_repo(repo_id=self.REPO_NAME, token=self._token, private=True)
        self._api.create_repo(
            repo_id=self.REPO_NAME, token=self._token, private=True, repo_type="dataset"
        )

    def tearDown(self) -> None:
        self._api.delete_repo(repo_id=self.REPO_NAME, token=self._token)
        self._api.delete_repo(
            repo_id=self.REPO_NAME, token=self._token, repo_type="dataset"
        )

    def test_model_info(self):
        shutil.rmtree(os.path.dirname(HfFolder.path_token))
        # Test we cannot access model info without a token
        with self.assertRaisesRegex(
            requests.exceptions.HTTPError,
            r"401 Client Error: Repository Not Found for url: (.+) \(Request ID: .+\)",
        ):
            _ = self._api.model_info(repo_id=f"{USER}/{self.REPO_NAME}")
        # Test we can access model info with a token
        model_info = self._api.model_info(
            repo_id=f"{USER}/{self.REPO_NAME}", token=self._token
        )
        self.assertIsInstance(model_info, ModelInfo)

    def test_list_private_datasets(self):
        orig = len(self._api.list_datasets())
        new = len(self._api.list_datasets(use_auth_token=self._token))
        self.assertGreater(new, orig)

    def test_list_private_models(self):
        orig = len(self._api.list_models())
        new = len(self._api.list_models(use_auth_token=self._token))
        self.assertGreater(new, orig)


class HfFolderTest(unittest.TestCase):
    def test_token_workflow(self):
        """
        Test the whole token save/get/delete workflow,
        with the desired behavior with respect to non-existent tokens.
        """
        token = "token-{}".format(int(time.time()))
        HfFolder.save_token(token)
        self.assertEqual(HfFolder.get_token(), token)
        HfFolder.delete_token()
        HfFolder.delete_token()
        # ^^ not an error, we test that the
        # second call does not fail.
        self.assertEqual(HfFolder.get_token(), None)
        # test TOKEN in env
        self.assertEqual(HfFolder.get_token(), None)
        with unittest.mock.patch.dict(os.environ, {"HUGGING_FACE_HUB_TOKEN": token}):
            self.assertEqual(HfFolder.get_token(), token)


@require_git_lfs
class HfLargefilesTest(HfApiCommonTest):
    @classmethod
    def setUpClass(cls):
        """
        Share this valid token in all tests below.
        """
        cls._token = TOKEN
        cls._api.set_access_token(TOKEN)

    def setUp(self):
        self.REPO_NAME_LARGE_FILE = repo_name_large_file()
        if os.path.exists(WORKING_REPO_DIR):
            shutil.rmtree(WORKING_REPO_DIR, onerror=set_write_permission_and_retry)
        logger.info(
            f"Does {WORKING_REPO_DIR} exist: {os.path.exists(WORKING_REPO_DIR)}"
        )

    def tearDown(self):
        self._api.delete_repo(repo_id=self.REPO_NAME_LARGE_FILE, token=self._token)

    def setup_local_clone(self, REMOTE_URL):
        REMOTE_URL_AUTH = REMOTE_URL.replace(
            ENDPOINT_STAGING, ENDPOINT_STAGING_BASIC_AUTH
        )
        subprocess.run(
            ["git", "clone", REMOTE_URL_AUTH, WORKING_REPO_DIR],
            check=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )
        subprocess.run(
            ["git", "lfs", "track", "*.pdf"], check=True, cwd=WORKING_REPO_DIR
        )
        subprocess.run(
            ["git", "lfs", "track", "*.epub"], check=True, cwd=WORKING_REPO_DIR
        )

    @retry_endpoint
    def test_end_to_end_thresh_6M(self):
        self._api._lfsmultipartthresh = 6 * 10**6
        REMOTE_URL = self._api.create_repo(
            repo_id=self.REPO_NAME_LARGE_FILE,
            token=self._token,
        )
        self._api._lfsmultipartthresh = None
        self.setup_local_clone(REMOTE_URL)

        subprocess.run(
            ["wget", LARGE_FILE_18MB],
            check=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=WORKING_REPO_DIR,
        )
        subprocess.run(["git", "add", "*"], check=True, cwd=WORKING_REPO_DIR)
        subprocess.run(
            ["git", "commit", "-m", "commit message"], check=True, cwd=WORKING_REPO_DIR
        )

        # This will fail as we haven't set up our custom transfer agent yet.
        failed_process = subprocess.run(
            ["git", "push"],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=WORKING_REPO_DIR,
        )
        self.assertEqual(failed_process.returncode, 1)
        self.assertIn("cli lfs-enable-largefiles", failed_process.stderr.decode())
        # ^ Instructions on how to fix this are included in the error message.

        subprocess.run(
            ["huggingface-cli", "lfs-enable-largefiles", WORKING_REPO_DIR], check=True
        )

        start_time = time.time()
        subprocess.run(["git", "push"], check=True, cwd=WORKING_REPO_DIR)
        print("took", time.time() - start_time)

        # To be 100% sure, let's download the resolved file
        pdf_url = f"{REMOTE_URL}/resolve/main/progit.pdf"
        DEST_FILENAME = "uploaded.pdf"
        subprocess.run(
            ["wget", pdf_url, "-O", DEST_FILENAME],
            check=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=WORKING_REPO_DIR,
        )
        dest_filesize = os.stat(os.path.join(WORKING_REPO_DIR, DEST_FILENAME)).st_size
        self.assertEqual(dest_filesize, 18685041)

    @retry_endpoint
    def test_end_to_end_thresh_16M(self):
        # Here we'll push one multipart and one non-multipart file in the same commit, and see what happens
        self._api._lfsmultipartthresh = 16 * 10**6
        REMOTE_URL = self._api.create_repo(
            repo_id=self.REPO_NAME_LARGE_FILE,
            token=self._token,
        )
        self._api._lfsmultipartthresh = None
        self.setup_local_clone(REMOTE_URL)

        subprocess.run(
            ["wget", LARGE_FILE_18MB],
            check=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=WORKING_REPO_DIR,
        )
        subprocess.run(
            ["wget", LARGE_FILE_14MB],
            check=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=WORKING_REPO_DIR,
        )
        subprocess.run(["git", "add", "*"], check=True, cwd=WORKING_REPO_DIR)
        subprocess.run(
            ["git", "commit", "-m", "both files in same commit"],
            check=True,
            cwd=WORKING_REPO_DIR,
        )

        subprocess.run(
            ["huggingface-cli", "lfs-enable-largefiles", WORKING_REPO_DIR], check=True
        )

        start_time = time.time()
        subprocess.run(["git", "push"], check=True, cwd=WORKING_REPO_DIR)
        print("took", time.time() - start_time)


class HfApiMiscTest(unittest.TestCase):
    def test_repo_type_and_id_from_hf_id(self):
        possible_values = {
            "https://huggingface.co/id": [None, None, "id"],
            "https://huggingface.co/user/id": [None, "user", "id"],
            "https://huggingface.co/datasets/user/id": ["dataset", "user", "id"],
            "https://huggingface.co/spaces/user/id": ["space", "user", "id"],
            "user/id": [None, "user", "id"],
            "dataset/user/id": ["dataset", "user", "id"],
            "space/user/id": ["space", "user", "id"],
            "id": [None, None, "id"],
        }

        for key, value in possible_values.items():
            self.assertEqual(
                repo_type_and_id_from_hf_id(key, hub_url="https://huggingface.co"),
                tuple(value),
            )


# tests/test_hubmixin.py

import os
import shutil
import time
import unittest
import uuid

from huggingface_hub import HfApi
from huggingface_hub.file_download import is_torch_available
from huggingface_hub.hub_mixin import PyTorchModelHubMixin
from huggingface_hub.utils import logging

from .testing_constants import ENDPOINT_STAGING, TOKEN, USER
from .testing_utils import set_write_permission_and_retry


def repo_name(id=uuid.uuid4().hex[:6]):
    return "mixin-repo-{0}-{1}".format(id, int(time.time() * 10e3))


logger = logging.get_logger(__name__)
WORKING_REPO_SUBDIR = "fixtures/working_repo_2"
WORKING_REPO_DIR = os.path.join(
    os.path.dirname(os.path.abspath(__file__)), WORKING_REPO_SUBDIR
)

if is_torch_available():
    import torch.nn as nn


def require_torch(test_case):
    """
    Decorator marking a test that requires PyTorch.

    These tests are skipped when PyTorch isn't installed.

    """
    if not is_torch_available():
        return unittest.skip("test requires PyTorch")(test_case)
    else:
        return test_case


if is_torch_available():

    class DummyModel(nn.Module, PyTorchModelHubMixin):
        def __init__(self, **kwargs):
            super().__init__()
            self.config = kwargs.pop("config", None)
            self.l1 = nn.Linear(2, 2)

        def forward(self, x):
            return self.l1(x)

else:
    DummyModel = None


@require_torch
class HubMixingCommonTest(unittest.TestCase):
    _api = HfApi(endpoint=ENDPOINT_STAGING)


@require_torch
class HubMixingTest(HubMixingCommonTest):
    def tearDown(self) -> None:
        if os.path.exists(WORKING_REPO_DIR):
            shutil.rmtree(WORKING_REPO_DIR, onerror=set_write_permission_and_retry)
        logger.info(
            f"Does {WORKING_REPO_DIR} exist: {os.path.exists(WORKING_REPO_DIR)}"
        )

    @classmethod
    def setUpClass(cls):
        """
        Share this valid token in all tests below.
        """
        cls._token = TOKEN
        cls._api.set_access_token(TOKEN)

    def test_save_pretrained(self):
        REPO_NAME = repo_name("save")
        model = DummyModel()

        model.save_pretrained(f"{WORKING_REPO_DIR}/{REPO_NAME}")
        files = os.listdir(f"{WORKING_REPO_DIR}/{REPO_NAME}")
        self.assertTrue("pytorch_model.bin" in files)
        self.assertEqual(len(files), 1)

        model.save_pretrained(
            f"{WORKING_REPO_DIR}/{REPO_NAME}", config={"num": 12, "act": "gelu"}
        )
        files = os.listdir(f"{WORKING_REPO_DIR}/{REPO_NAME}")
        self.assertTrue("config.json" in files)
        self.assertTrue("pytorch_model.bin" in files)
        self.assertEqual(len(files), 2)

    def test_rel_path_from_pretrained(self):
        model = DummyModel()
        model.save_pretrained(
            f"tests/{WORKING_REPO_SUBDIR}/FROM_PRETRAINED",
            config={"num": 10, "act": "gelu_fast"},
        )

        model = DummyModel.from_pretrained(
            f"tests/{WORKING_REPO_SUBDIR}/FROM_PRETRAINED"
        )
        self.assertTrue(model.config == {"num": 10, "act": "gelu_fast"})

    def test_abs_path_from_pretrained(self):
        REPO_NAME = repo_name("FROM_PRETRAINED")
        model = DummyModel()
        model.save_pretrained(
            f"{WORKING_REPO_DIR}/{REPO_NAME}",
            config={"num": 10, "act": "gelu_fast"},
        )

        model = DummyModel.from_pretrained(f"{WORKING_REPO_DIR}/{REPO_NAME}")
        self.assertDictEqual(model.config, {"num": 10, "act": "gelu_fast"})

    def test_push_to_hub(self):
        REPO_NAME = repo_name("PUSH_TO_HUB")
        model = DummyModel()
        model.push_to_hub(
            repo_path_or_name=f"{WORKING_REPO_DIR}/{REPO_NAME}",
            api_endpoint=ENDPOINT_STAGING,
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
            config={"num": 7, "act": "gelu_fast"},
        )

        model_info = self._api.model_info(f"{USER}/{REPO_NAME}", token=self._token)
        self.assertEqual(model_info.modelId, f"{USER}/{REPO_NAME}")

        self._api.delete_repo(repo_id=f"{REPO_NAME}", token=self._token)


# tests/test_inference_api.py

# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest

import datasets

from huggingface_hub.inference_api import InferenceApi

from .testing_utils import with_production_testing


class InferenceApiTest(unittest.TestCase):
    def read(self, filename: str) -> bytes:
        with open(filename, "rb") as f:
            bpayload = f.read()
        return bpayload

    @with_production_testing
    def test_simple_inference(self):
        api = InferenceApi("bert-base-uncased")
        inputs = "Hi, I think [MASK] is cool"
        results = api(inputs)
        self.assertIsInstance(results, list)

        result = results[0]
        self.assertIsInstance(result, dict)
        self.assertTrue("sequence" in result)
        self.assertTrue("score" in result)

    @with_production_testing
    def test_inference_with_params(self):
        api = InferenceApi("typeform/distilbert-base-uncased-mnli")
        inputs = (
            "I bought a device but it is not working and I would like to get"
            " reimbursed!"
        )
        params = {"candidate_labels": ["refund", "legal", "faq"]}
        result = api(inputs, params)
        self.assertIsInstance(result, dict)
        self.assertTrue("sequence" in result)
        self.assertTrue("scores" in result)

    @with_production_testing
    def test_inference_with_dict_inputs(self):
        api = InferenceApi("deepset/roberta-base-squad2")
        inputs = {
            "question": "What's my name?",
            "context": "My name is Clara and I live in Berkeley.",
        }
        result = api(inputs)
        self.assertIsInstance(result, dict)
        self.assertTrue("score" in result)
        self.assertTrue("answer" in result)

    @with_production_testing
    def test_inference_with_audio(self):
        api = InferenceApi("facebook/wav2vec2-base-960h")
        dataset = datasets.load_dataset(
            "patrickvonplaten/librispeech_asr_dummy", "clean", split="validation"
        )
        data = self.read(dataset["file"][0])
        result = api(data=data)
        self.assertIsInstance(result, dict)
        self.assertTrue("text" in result, f"We received {result} instead")

    @with_production_testing
    def test_inference_with_image(self):
        api = InferenceApi("google/vit-base-patch16-224")
        dataset = datasets.load_dataset("Narsil/image_dummy", "image", split="test")
        data = self.read(dataset["file"][0])
        result = api(data=data)
        self.assertIsInstance(result, list)
        for classification in result:
            self.assertIsInstance(classification, dict)
            self.assertTrue("score" in classification)
            self.assertTrue("label" in classification)

    @with_production_testing
    def test_inference_overriding_task(self):
        api = InferenceApi(
            "sentence-transformers/paraphrase-albert-small-v2",
            task="feature-extraction",
        )
        inputs = "This is an example again"
        result = api(inputs)
        self.assertIsInstance(result, list)

    @with_production_testing
    def test_inference_overriding_invalid_task(self):
        with self.assertRaises(
            ValueError, msg="Invalid task invalid-task. Make sure it's valid."
        ):
            InferenceApi("bert-base-uncased", task="invalid-task")

    @with_production_testing
    def test_inference_missing_input(self):
        api = InferenceApi("deepset/roberta-base-squad2")
        result = api({"question": "What's my name?"})
        self.assertIsInstance(result, dict)
        self.assertTrue("error" in result)


# tests/test_keras_integration.py

import json
import os
import shutil
import tempfile
import time
import unittest
import uuid

import pytest

from huggingface_hub import HfApi
from huggingface_hub.file_download import (
    is_graphviz_available,
    is_pydot_available,
    is_tf_available,
)
from huggingface_hub.keras_mixin import (
    KerasModelHubMixin,
    from_pretrained_keras,
    push_to_hub_keras,
    save_pretrained_keras,
)
from huggingface_hub.utils import logging

from .testing_constants import ENDPOINT_STAGING, TOKEN, USER
from .testing_utils import retry_endpoint, set_write_permission_and_retry


def repo_name(id=uuid.uuid4().hex[:6]):
    return "keras-repo-{0}-{1}".format(id, int(time.time() * 10e3))


logger = logging.get_logger(__name__)

WORKING_REPO_SUBDIR = f"fixtures/working_repo_{__name__.split('.')[-1]}"
WORKING_REPO_DIR = os.path.join(
    os.path.dirname(os.path.abspath(__file__)), WORKING_REPO_SUBDIR
)

if is_tf_available():
    import tensorflow as tf


def require_tf(test_case):
    """
    Decorator marking a test that requires TensorFlow, graphviz and pydot.

    These tests are skipped when TensorFlow, graphviz and pydot are installed.

    """
    if not is_tf_available() or not is_pydot_available() or not is_graphviz_available():
        return unittest.skip("test requires Tensorflow, graphviz and pydot.")(test_case)
    else:
        return test_case


if is_tf_available():

    # Define dummy mixin model...
    class DummyModel(tf.keras.Model, KerasModelHubMixin):
        def __init__(self, **kwargs):
            super().__init__()
            self.l1 = tf.keras.layers.Dense(2, activation="relu")
            dummy_batch_size = input_dim = 2
            self.dummy_inputs = tf.ones([dummy_batch_size, input_dim])

        def call(self, x):
            return self.l1(x)

else:
    DummyModel = None


@require_tf
class HubMixingTestKeras(unittest.TestCase):
    def tearDown(self) -> None:
        if os.path.exists(WORKING_REPO_DIR):
            shutil.rmtree(WORKING_REPO_DIR, onerror=set_write_permission_and_retry)
        logger.info(
            f"Does {WORKING_REPO_DIR} exist: {os.path.exists(WORKING_REPO_DIR)}"
        )

    @classmethod
    def setUpClass(cls):
        """
        Share this valid token in all tests below.
        """
        cls._api = HfApi(endpoint=ENDPOINT_STAGING)
        cls._token = TOKEN
        cls._api.set_access_token(TOKEN)

    def test_save_pretrained(self):
        REPO_NAME = repo_name("save")
        model = DummyModel()
        model(model.dummy_inputs)
        model.save_pretrained(f"{WORKING_REPO_DIR}/{REPO_NAME}")
        files = os.listdir(f"{WORKING_REPO_DIR}/{REPO_NAME}")
        self.assertTrue("saved_model.pb" in files)
        self.assertTrue("keras_metadata.pb" in files)
        self.assertTrue("README.md" in files)
        self.assertTrue("model.png" in files)
        self.assertEqual(len(files), 6)

        model.save_pretrained(
            f"{WORKING_REPO_DIR}/{REPO_NAME}", config={"num": 12, "act": "gelu"}
        )
        files = os.listdir(f"{WORKING_REPO_DIR}/{REPO_NAME}")
        self.assertTrue("config.json" in files)
        self.assertTrue("saved_model.pb" in files)
        self.assertEqual(len(files), 7)

    def test_keras_from_pretrained_weights(self):
        model = DummyModel()
        model(model.dummy_inputs)

        model.save_pretrained(f"{WORKING_REPO_DIR}/FROM_PRETRAINED")
        new_model = DummyModel.from_pretrained(f"{WORKING_REPO_DIR}/FROM_PRETRAINED")

        # Check the reloaded model's weights match the original model's weights
        self.assertTrue(tf.reduce_all(tf.equal(new_model.weights[0], model.weights[0])))

        # Check a new model's weights are not the same as the reloaded model's weights
        another_model = DummyModel()
        another_model(tf.ones([2, 2]))
        self.assertFalse(
            tf.reduce_all(tf.equal(new_model.weights[0], another_model.weights[0]))
            .numpy()
            .item()
        )

    def test_rel_path_from_pretrained(self):
        model = DummyModel()
        model(model.dummy_inputs)
        model.save_pretrained(
            f"tests/{WORKING_REPO_SUBDIR}/FROM_PRETRAINED",
            config={"num": 10, "act": "gelu_fast"},
        )

        model = DummyModel.from_pretrained(
            f"tests/{WORKING_REPO_SUBDIR}/FROM_PRETRAINED"
        )
        self.assertTrue(model.config == {"num": 10, "act": "gelu_fast"})

    def test_abs_path_from_pretrained(self):
        REPO_NAME = repo_name("FROM_PRETRAINED")
        model = DummyModel()
        model(model.dummy_inputs)
        model.save_pretrained(
            f"{WORKING_REPO_DIR}/{REPO_NAME}", config={"num": 10, "act": "gelu_fast"}
        )

        model = DummyModel.from_pretrained(f"{WORKING_REPO_DIR}/{REPO_NAME}")
        self.assertDictEqual(model.config, {"num": 10, "act": "gelu_fast"})

    @retry_endpoint
    def test_push_to_hub(self):
        REPO_NAME = repo_name("PUSH_TO_HUB")
        model = DummyModel()
        model(model.dummy_inputs)
        model.push_to_hub(
            repo_path_or_name=f"{WORKING_REPO_DIR}/{REPO_NAME}",
            api_endpoint=ENDPOINT_STAGING,
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
            config={"num": 7, "act": "gelu_fast"},
        )

        model_info = self._api.model_info(
            f"{USER}/{REPO_NAME}",
        )
        self.assertEqual(model_info.modelId, f"{USER}/{REPO_NAME}")

        self._api.delete_repo(repo_id=f"{REPO_NAME}", token=self._token)


@require_tf
class HubKerasSequentialTest(HubMixingTestKeras):
    def model_init(self):
        model = tf.keras.models.Sequential()
        model.add(tf.keras.layers.Dense(2, activation="relu"))
        model.compile(optimizer="adam", loss="mse")
        return model

    def model_fit(self, model):
        x = tf.constant([[0.44, 0.90], [0.65, 0.39]])
        y = tf.constant([[1, 1], [0, 0]])
        model.fit(x, y)
        return model

    def test_save_pretrained(self):
        REPO_NAME = repo_name("save")
        model = self.model_init()

        with pytest.raises(ValueError, match="Model should be built*"):
            save_pretrained_keras(model, f"{WORKING_REPO_DIR}/{REPO_NAME}")

        model.build((None, 2))

        save_pretrained_keras(
            model,
            f"{WORKING_REPO_DIR}/{REPO_NAME}",
        )
        files = os.listdir(f"{WORKING_REPO_DIR}/{REPO_NAME}")

        self.assertIn("saved_model.pb", files)
        self.assertIn("keras_metadata.pb", files)
        self.assertIn("model.png", files)
        self.assertIn("README.md", files)
        self.assertEqual(len(files), 6)
        loaded_model = from_pretrained_keras(f"{WORKING_REPO_DIR}/{REPO_NAME}")
        self.assertIsNone(loaded_model.optimizer)

    def test_save_pretrained_model_card_fit(self):
        REPO_NAME = repo_name("save")
        model = self.model_init()
        model = self.model_fit(model)

        save_pretrained_keras(
            model,
            f"{WORKING_REPO_DIR}/{REPO_NAME}",
        )
        files = os.listdir(f"{WORKING_REPO_DIR}/{REPO_NAME}")

        self.assertIn("saved_model.pb", files)
        self.assertIn("keras_metadata.pb", files)
        self.assertIn("model.png", files)
        self.assertIn("README.md", files)
        self.assertIn("history.json", files)
        with open(f"{WORKING_REPO_DIR}/{REPO_NAME}/history.json") as f:
            history = json.load(f)

        self.assertEqual(history, model.history.history)
        self.assertEqual(len(files), 7)

    def test_save_model_card_history_removal(self):
        REPO_NAME = repo_name("save")
        model = self.model_init()
        model = self.model_fit(model)
        with tempfile.TemporaryDirectory() as tmpdirname:
            os.makedirs(f"{tmpdirname}/{WORKING_REPO_DIR}/{REPO_NAME}")
            with open(
                f"{tmpdirname}/{WORKING_REPO_DIR}/{REPO_NAME}/history.json", "w+"
            ) as fp:
                fp.write("Keras FTW")

            with pytest.warns(
                UserWarning, match="`history.json` file already exists, *"
            ):
                save_pretrained_keras(
                    model,
                    f"{tmpdirname}/{WORKING_REPO_DIR}/{REPO_NAME}",
                )
                # assert that it's not the same as old history file and it's overridden
                with open(
                    f"{tmpdirname}/{WORKING_REPO_DIR}/{REPO_NAME}/history.json", "r"
                ) as f:
                    history_content = f.read()
                    self.assertNotEqual("Keras FTW", history_content)

            # Check the history is saved as a json in the repository.
            files = os.listdir(f"{tmpdirname}/{WORKING_REPO_DIR}/{REPO_NAME}")
            self.assertIn("history.json", files)

            # Check that there is no "Training Metrics" section in the model card.
            # This was done in an older version.
            with open(
                f"{tmpdirname}/{WORKING_REPO_DIR}/{REPO_NAME}/README.md", "r"
            ) as file:
                data = file.read()
            self.assertNotIn(data, "Training Metrics")

    def test_save_pretrained_optimizer_state(self):
        REPO_NAME = repo_name("save")
        model = self.model_init()

        model.build((None, 2))
        save_pretrained_keras(
            model, f"{WORKING_REPO_DIR}/{REPO_NAME}", include_optimizer=True
        )

        loaded_model = from_pretrained_keras(f"{WORKING_REPO_DIR}/{REPO_NAME}")
        self.assertIsNotNone(loaded_model.optimizer)

    def test_save_pretrained_kwargs_load_fails_without_traces(self):
        REPO_NAME = repo_name("save")
        model = self.model_init()

        model.build((None, 2))

        save_pretrained_keras(
            model,
            f"{WORKING_REPO_DIR}/{REPO_NAME}",
            include_optimizer=False,
            save_traces=False,
        )

        from_pretrained_keras(f"{WORKING_REPO_DIR}/{REPO_NAME}")
        self.assertRaises(ValueError, msg="Exception encountered when calling layer*")

    def test_from_pretrained_weights(self):
        REPO_NAME = repo_name("FROM_PRETRAINED")
        model = self.model_init()
        model.build((None, 2))

        save_pretrained_keras(model, f"{WORKING_REPO_DIR}/{REPO_NAME}")
        new_model = from_pretrained_keras(f"{WORKING_REPO_DIR}/{REPO_NAME}")

        # Check a new model's weights are not the same as the reloaded model's weights
        another_model = DummyModel()
        another_model(tf.ones([2, 2]))
        self.assertFalse(
            tf.reduce_all(tf.equal(new_model.weights[0], another_model.weights[0]))
            .numpy()
            .item()
        )

    def test_save_pretrained_task_name_deprecation(self):
        REPO_NAME = repo_name("save")
        model = self.model_init()
        model.build((None, 2))

        with pytest.warns(
            FutureWarning,
            match="`task_name` input argument is deprecated. Pass `tags` instead.",
        ):
            save_pretrained_keras(
                model,
                f"{WORKING_REPO_DIR}/{REPO_NAME}",
                tags=["test"],
                task_name="test",
                save_traces=True,
            )

    def test_rel_path_from_pretrained(self):
        model = self.model_init()
        model.build((None, 2))
        save_pretrained_keras(
            model,
            f"tests/{WORKING_REPO_SUBDIR}/FROM_PRETRAINED",
            config={"num": 10, "act": "gelu_fast"},
        )

        new_model = from_pretrained_keras(
            f"tests/{WORKING_REPO_SUBDIR}/FROM_PRETRAINED"
        )

        # Check the reloaded model's weights match the original model's weights
        self.assertTrue(tf.reduce_all(tf.equal(new_model.weights[0], model.weights[0])))

        # Check saved configuration is what we expect
        self.assertTrue(new_model.config == {"num": 10, "act": "gelu_fast"})

    def test_abs_path_from_pretrained(self):
        REPO_NAME = repo_name("FROM_PRETRAINED")
        model = self.model_init()
        model.build((None, 2))
        save_pretrained_keras(
            model,
            f"{WORKING_REPO_DIR}/{REPO_NAME}",
            config={"num": 10, "act": "gelu_fast"},
            plot_model=True,
            tags=None,
        )

        new_model = from_pretrained_keras(f"{WORKING_REPO_DIR}/{REPO_NAME}")
        self.assertTrue(tf.reduce_all(tf.equal(new_model.weights[0], model.weights[0])))
        self.assertTrue(new_model.config == {"num": 10, "act": "gelu_fast"})

    @retry_endpoint
    def test_push_to_hub(self):
        REPO_NAME = repo_name("PUSH_TO_HUB")
        model = self.model_init()
        model.build((None, 2))
        push_to_hub_keras(
            model,
            repo_path_or_name=f"{WORKING_REPO_DIR}/{REPO_NAME}",
            api_endpoint=ENDPOINT_STAGING,
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
            config={"num": 7, "act": "gelu_fast"},
            include_optimizer=False,
        )

        model_info = HfApi(endpoint=ENDPOINT_STAGING).model_info(
            f"{USER}/{REPO_NAME}",
        )
        self.assertEqual(model_info.modelId, f"{USER}/{REPO_NAME}")

        self._api.delete_repo(repo_id=f"{REPO_NAME}", token=self._token)

    @retry_endpoint
    def test_push_to_hub_model_card_build(self):
        REPO_NAME = repo_name("PUSH_TO_HUB")
        model = self.model_init()
        model.build((None, 2))
        push_to_hub_keras(
            model,
            repo_path_or_name=f"{WORKING_REPO_DIR}/{REPO_NAME}",
            api_endpoint=ENDPOINT_STAGING,
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )
        model_info = HfApi(endpoint=ENDPOINT_STAGING).model_info(
            f"{USER}/{REPO_NAME}",
        )
        self.assertTrue("README.md" in [f.rfilename for f in model_info.siblings])
        self.assertTrue("model.png" in [f.rfilename for f in model_info.siblings])
        self._api.delete_repo(repo_id=f"{REPO_NAME}", token=self._token)

    @retry_endpoint
    def test_push_to_hub_model_card_plot_false(self):
        REPO_NAME = repo_name("PUSH_TO_HUB")
        model = self.model_init()
        model = self.model_fit(model)
        push_to_hub_keras(
            model,
            repo_path_or_name=f"{WORKING_REPO_DIR}/{REPO_NAME}",
            api_endpoint=ENDPOINT_STAGING,
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
            plot_model=False,
        )
        model_info = HfApi(endpoint=ENDPOINT_STAGING).model_info(
            f"{USER}/{REPO_NAME}",
        )
        self.assertFalse("model.png" in [f.rfilename for f in model_info.siblings])
        self._api.delete_repo(repo_id=f"{REPO_NAME}", token=self._token)

    @retry_endpoint
    def test_override_tensorboard(self):
        REPO_NAME = repo_name("PUSH_TO_HUB")
        with tempfile.TemporaryDirectory() as tmpdirname:
            os.makedirs(f"{tmpdirname}/tb_log_dir")
            with open(f"{tmpdirname}/tb_log_dir/tensorboard.txt", "w") as fp:
                fp.write("Keras FTW")
            model = self.model_init()
            model.build((None, 2))
            push_to_hub_keras(
                model,
                repo_path_or_name=f"{WORKING_REPO_DIR}/{REPO_NAME}",
                log_dir=f"{tmpdirname}/tb_log_dir",
                api_endpoint=ENDPOINT_STAGING,
                use_auth_token=self._token,
                git_user="ci",
                git_email="ci@dummy.com",
            )
            os.makedirs(f"{tmpdirname}/tb_log_dir2")
            with open(f"{tmpdirname}/tb_log_dir2/override.txt", "w") as fp:
                fp.write("Keras FTW")
            push_to_hub_keras(
                model,
                repo_path_or_name=f"{WORKING_REPO_DIR}/{REPO_NAME}",
                log_dir=f"{tmpdirname}/tb_log_dir2",
                api_endpoint=ENDPOINT_STAGING,
                use_auth_token=self._token,
                git_user="ci",
                git_email="ci@dummy.com",
            )

            model_info = HfApi(endpoint=ENDPOINT_STAGING).model_info(
                f"{USER}/{REPO_NAME}",
            )
            self.assertTrue(
                "logs/override.txt" in [f.rfilename for f in model_info.siblings]
            )
            self.assertFalse(
                "logs/tensorboard.txt" in [f.rfilename for f in model_info.siblings]
            )

            self._api.delete_repo(repo_id=f"{REPO_NAME}", token=self._token)

    @retry_endpoint
    def test_push_to_hub_model_kwargs(self):
        REPO_NAME = repo_name("PUSH_TO_HUB")
        model = self.model_init()
        model = self.model_fit(model)
        push_to_hub_keras(
            model,
            repo_path_or_name=f"{WORKING_REPO_DIR}/{REPO_NAME}",
            api_endpoint=ENDPOINT_STAGING,
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
            config={"num": 7, "act": "gelu_fast"},
            include_optimizer=True,
            save_traces=False,
        )

        model_info = HfApi(endpoint=ENDPOINT_STAGING).model_info(
            f"{USER}/{REPO_NAME}",
        )
        self.assertEqual(model_info.modelId, f"{USER}/{REPO_NAME}")

        from_pretrained_keras(f"{WORKING_REPO_DIR}/{REPO_NAME}")
        self.assertRaises(ValueError, msg="Exception encountered when calling layer*")

        self._api.delete_repo(repo_id=f"{REPO_NAME}", token=self._token)


@require_tf
class HubKerasFunctionalTest(HubKerasSequentialTest):
    def model_init(self):
        inputs = tf.keras.layers.Input(shape=(2,))
        outputs = tf.keras.layers.Dense(2, activation="relu")(inputs)
        model = tf.keras.models.Model(inputs=inputs, outputs=outputs)
        model.compile(optimizer="adam", loss="mse")
        return model

    def test_save_pretrained(self):
        REPO_NAME = repo_name("functional")
        model = self.model_init()
        model.build((None, 2))
        self.assertTrue(model.built)

        save_pretrained_keras(model, f"{WORKING_REPO_DIR}/{REPO_NAME}")
        files = os.listdir(f"{WORKING_REPO_DIR}/{REPO_NAME}")

        self.assertIn("saved_model.pb", files)
        self.assertIn("keras_metadata.pb", files)
        self.assertEqual(len(files), 6)

    def test_save_pretrained_fit(self):
        REPO_NAME = repo_name("functional")
        model = self.model_init()
        model = self.model_fit(model)

        save_pretrained_keras(model, f"{WORKING_REPO_DIR}/{REPO_NAME}")
        files = os.listdir(f"{WORKING_REPO_DIR}/{REPO_NAME}")

        self.assertIn("saved_model.pb", files)
        self.assertIn("keras_metadata.pb", files)
        self.assertEqual(len(files), 7)


# tests/test_offline_utils.py

from io import BytesIO

import pytest

import requests
from huggingface_hub.file_download import http_get

from .testing_utils import (
    OfflineSimulationMode,
    RequestWouldHangIndefinitelyError,
    offline,
)


def test_offline_with_timeout():
    with offline(OfflineSimulationMode.CONNECTION_TIMES_OUT):
        with pytest.raises(RequestWouldHangIndefinitelyError):
            requests.request("GET", "https://huggingface.co")
        with pytest.raises(requests.exceptions.ConnectTimeout):
            requests.request("GET", "https://huggingface.co", timeout=1.0)
        with pytest.raises(requests.exceptions.ConnectTimeout):
            http_get("https://huggingface.co", BytesIO())


def test_offline_with_connection_error():
    with offline(OfflineSimulationMode.CONNECTION_FAILS):
        with pytest.raises(requests.exceptions.ConnectionError):
            requests.request("GET", "https://huggingface.co")
        with pytest.raises(requests.exceptions.ConnectionError):
            http_get("https://huggingface.co", BytesIO())


def test_offline_with_datasets_offline_mode_enabled():
    with offline(OfflineSimulationMode.HF_HUB_OFFLINE_SET_TO_1):
        with pytest.raises(ConnectionError):
            http_get("https://huggingface.co", BytesIO())


# tests/test_repocard.py

# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import os
import shutil
import tempfile
import time
import unittest
import uuid
from pathlib import Path

import pytest

import yaml
from huggingface_hub import (
    metadata_eval_result,
    metadata_load,
    metadata_save,
    metadata_update,
)
from huggingface_hub.constants import REPOCARD_NAME
from huggingface_hub.file_download import hf_hub_download
from huggingface_hub.hf_api import HfApi
from huggingface_hub.repository import Repository
from huggingface_hub.utils import logging

from .testing_constants import ENDPOINT_STAGING, TOKEN, USER
from .testing_utils import retry_endpoint, set_write_permission_and_retry


ROUND_TRIP_MODELCARD_CASE = """
---
language: no
datasets: CLUECorpusSmall
widget:
- text: 北京是[MASK]国的首都。
---

# Title
"""

DUMMY_MODELCARD = """

Hi

---
license: mit
datasets:
- foo
- bar
---

Hello
"""

DUMMY_MODELCARD_TARGET = """

Hi

---
meaning_of_life: 42
---

Hello
"""

DUMMY_MODELCARD_TARGET_NO_YAML = """---
meaning_of_life: 42
---
Hello
"""

DUMMY_NEW_MODELCARD_TARGET = """---
meaning_of_life: 42
---
"""

DUMMY_MODELCARD_TARGET_NO_TAGS = """
Hello
"""

DUMMY_MODELCARD_EVAL_RESULT = """---
model-index:
- name: RoBERTa fine-tuned on ReactionGIF
  results:
  - metrics:
    - type: accuracy
      value: 0.2662102282047272
      name: Accuracy
    task:
      type: text-classification
      name: Text Classification
    dataset:
      name: ReactionGIF
      type: julien-c/reactiongif
---
"""

logger = logging.get_logger(__name__)

REPOCARD_DIR = os.path.join(
    os.path.dirname(os.path.abspath(__file__)), "fixtures/repocard"
)


def repo_name(id=uuid.uuid4().hex[:6]):
    return "dummy-hf-hub-{0}-{1}".format(id, int(time.time() * 10e3))


class RepocardTest(unittest.TestCase):
    def setUp(self):
        os.makedirs(REPOCARD_DIR, exist_ok=True)

    def tearDown(self) -> None:
        if os.path.exists(REPOCARD_DIR):
            shutil.rmtree(REPOCARD_DIR, onerror=set_write_permission_and_retry)
        logger.info(f"Does {REPOCARD_DIR} exist: {os.path.exists(REPOCARD_DIR)}")

    def test_metadata_load(self):
        filepath = Path(REPOCARD_DIR) / REPOCARD_NAME
        filepath.write_text(DUMMY_MODELCARD)
        data = metadata_load(filepath)
        self.assertDictEqual(data, {"license": "mit", "datasets": ["foo", "bar"]})

    def test_metadata_save(self):
        filename = "dummy_target.md"
        filepath = Path(REPOCARD_DIR) / filename
        filepath.write_text(DUMMY_MODELCARD)
        metadata_save(filepath, {"meaning_of_life": 42})
        content = filepath.read_text()
        self.assertEqual(content, DUMMY_MODELCARD_TARGET)

    def test_metadata_save_from_file_no_yaml(self):
        filename = "dummy_target_2.md"
        filepath = Path(REPOCARD_DIR) / filename
        filepath.write_text("Hello\n")
        metadata_save(filepath, {"meaning_of_life": 42})
        content = filepath.read_text()
        self.assertEqual(content, DUMMY_MODELCARD_TARGET_NO_YAML)

    def test_metadata_save_new_file(self):
        filename = "new_dummy_target.md"
        filepath = Path(REPOCARD_DIR) / filename
        metadata_save(filepath, {"meaning_of_life": 42})
        content = filepath.read_text()
        self.assertEqual(content, DUMMY_NEW_MODELCARD_TARGET)

    def test_no_metadata_returns_none(self):
        filename = "dummy_target_3.md"
        filepath = Path(REPOCARD_DIR) / filename
        filepath.write_text(DUMMY_MODELCARD_TARGET_NO_TAGS)
        data = metadata_load(filepath)
        self.assertEqual(data, None)

    def test_metadata_eval_result(self):
        data = metadata_eval_result(
            model_pretty_name="RoBERTa fine-tuned on ReactionGIF",
            task_pretty_name="Text Classification",
            task_id="text-classification",
            metrics_pretty_name="Accuracy",
            metrics_id="accuracy",
            metrics_value=0.2662102282047272,
            dataset_pretty_name="ReactionGIF",
            dataset_id="julien-c/reactiongif",
        )
        filename = "eval_results.md"
        filepath = Path(REPOCARD_DIR) / filename
        metadata_save(filepath, data)
        content = filepath.read_text().splitlines()
        self.assertEqual(content, DUMMY_MODELCARD_EVAL_RESULT.splitlines())


class RepocardUpdateTest(unittest.TestCase):
    _api = HfApi(endpoint=ENDPOINT_STAGING)

    @classmethod
    def setUpClass(cls):
        """
        Share this valid token in all tests below.
        """
        cls._token = TOKEN
        cls._api.set_access_token(TOKEN)

    @retry_endpoint
    def setUp(self) -> None:
        self.repo_path = Path(tempfile.mkdtemp())
        self.REPO_NAME = repo_name()
        self.repo = Repository(
            self.repo_path / self.REPO_NAME,
            clone_from=f"{USER}/{self.REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        with self.repo.commit("Add README to main branch"):
            with open("README.md", "w+") as f:
                f.write(DUMMY_MODELCARD_EVAL_RESULT)

        self.existing_metadata = yaml.safe_load(
            DUMMY_MODELCARD_EVAL_RESULT.strip().strip("---")
        )

    def tearDown(self) -> None:
        self._api.delete_repo(repo_id=f"{self.REPO_NAME}", token=self._token)
        shutil.rmtree(self.repo_path)

    def test_update_dataset_name(self):
        new_datasets_data = {"datasets": "['test/test_dataset']"}
        metadata_update(
            f"{USER}/{self.REPO_NAME}", new_datasets_data, token=self._token
        )

        self.repo.git_pull()
        updated_metadata = metadata_load(self.repo_path / self.REPO_NAME / "README.md")
        expected_metadata = copy.deepcopy(self.existing_metadata)
        expected_metadata.update(new_datasets_data)
        self.assertDictEqual(updated_metadata, expected_metadata)

    def test_update_existing_result_with_overwrite(self):
        new_metadata = copy.deepcopy(self.existing_metadata)
        new_metadata["model-index"][0]["results"][0]["metrics"][0][
            "value"
        ] = 0.2862102282047272
        metadata_update(
            f"{USER}/{self.REPO_NAME}", new_metadata, token=self._token, overwrite=True
        )

        self.repo.git_pull()
        updated_metadata = metadata_load(self.repo_path / self.REPO_NAME / "README.md")
        self.assertDictEqual(updated_metadata, new_metadata)

    def test_metadata_update_upstream(self):
        new_metadata = copy.deepcopy(self.existing_metadata)
        new_metadata["model-index"][0]["results"][0]["metrics"][0]["value"] = 0.1

        path = hf_hub_download(
            f"{USER}/{self.REPO_NAME}",
            filename=REPOCARD_NAME,
            use_auth_token=self._token,
        )

        metadata_update(
            f"{USER}/{self.REPO_NAME}", new_metadata, token=self._token, overwrite=True
        )

        self.assertNotEqual(metadata_load(path), new_metadata)
        self.assertEqual(metadata_load(path), self.existing_metadata)

    def test_update_existing_result_without_overwrite(self):
        new_metadata = copy.deepcopy(self.existing_metadata)
        new_metadata["model-index"][0]["results"][0]["metrics"][0][
            "value"
        ] = 0.2862102282047272

        with pytest.raises(
            ValueError,
            match=(
                "You passed a new value for the existing metric 'name: Accuracy, type:"
                " accuracy'. Set `overwrite=True` to overwrite existing metrics."
            ),
        ):
            metadata_update(
                f"{USER}/{self.REPO_NAME}",
                new_metadata,
                token=self._token,
                overwrite=False,
            )

    def test_update_existing_field_without_overwrite(self):
        new_datasets_data = {"datasets": "['test/test_dataset']"}
        metadata_update(
            f"{USER}/{self.REPO_NAME}", new_datasets_data, token=self._token
        )

        with pytest.raises(
            ValueError,
            match=(
                "You passed a new value for the existing meta data field 'datasets'."
                " Set `overwrite=True` to overwrite existing metadata."
            ),
        ):
            new_datasets_data = {"datasets": "['test/test_dataset_2']"}
            metadata_update(
                f"{USER}/{self.REPO_NAME}",
                new_datasets_data,
                token=self._token,
                overwrite=False,
            )

    def test_update_new_result_existing_dataset(self):
        new_result = metadata_eval_result(
            model_pretty_name="RoBERTa fine-tuned on ReactionGIF",
            task_pretty_name="Text Classification",
            task_id="text-classification",
            metrics_pretty_name="Recall",
            metrics_id="recall",
            metrics_value=0.7762102282047272,
            dataset_pretty_name="ReactionGIF",
            dataset_id="julien-c/reactiongif",
        )

        metadata_update(
            f"{USER}/{self.REPO_NAME}", new_result, token=self._token, overwrite=False
        )

        expected_metadata = copy.deepcopy(self.existing_metadata)
        expected_metadata["model-index"][0]["results"][0]["metrics"].append(
            new_result["model-index"][0]["results"][0]["metrics"][0]
        )

        self.repo.git_pull()
        updated_metadata = metadata_load(self.repo_path / self.REPO_NAME / "README.md")
        self.assertDictEqual(updated_metadata, expected_metadata)

    def test_update_new_result_new_dataset(self):
        new_result = metadata_eval_result(
            model_pretty_name="RoBERTa fine-tuned on ReactionGIF",
            task_pretty_name="Text Classification",
            task_id="text-classification",
            metrics_pretty_name="Accuracy",
            metrics_id="accuracy",
            metrics_value=0.2662102282047272,
            dataset_pretty_name="ReactionJPEG",
            dataset_id="julien-c/reactionjpeg",
        )

        metadata_update(
            f"{USER}/{self.REPO_NAME}", new_result, token=self._token, overwrite=False
        )

        expected_metadata = copy.deepcopy(self.existing_metadata)
        expected_metadata["model-index"][0]["results"].append(
            new_result["model-index"][0]["results"][0]
        )
        self.repo.git_pull()
        updated_metadata = metadata_load(self.repo_path / self.REPO_NAME / "README.md")
        self.assertDictEqual(updated_metadata, expected_metadata)


# tests/test_repository.py

# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
import pathlib
import shutil
import subprocess
import tempfile
import time
import unittest
import uuid
from io import BytesIO

import pytest

import requests
from huggingface_hub.commands.user import currently_setup_credential_helpers
from huggingface_hub.hf_api import HfApi
from huggingface_hub.repository import (
    Repository,
    is_tracked_upstream,
    is_tracked_with_lfs,
)
from huggingface_hub.utils import logging

from .testing_constants import ENDPOINT_STAGING, TOKEN, USER
from .testing_utils import (
    retry_endpoint,
    set_write_permission_and_retry,
    with_production_testing,
)


logger = logging.get_logger(__name__)


def repo_name(id=uuid.uuid4().hex[:6]):
    return "repo-{0}-{1}".format(id, int(time.time() * 10e3))


WORKING_REPO_DIR = os.path.join(
    os.path.dirname(os.path.abspath(__file__)), "fixtures/working_repo_2"
)

DATASET_FIXTURE = os.path.join(
    os.path.dirname(os.path.abspath(__file__)), "fixtures/tiny_dataset"
)
WORKING_DATASET_DIR = os.path.join(
    os.path.dirname(os.path.abspath(__file__)), "fixtures/working_dataset"
)


class RepositoryCommonTest(unittest.TestCase):
    _api = HfApi(endpoint=ENDPOINT_STAGING)


class RepositoryTest(RepositoryCommonTest):
    @classmethod
    def setUpClass(cls):
        """
        Share this valid token in all tests below.
        """
        cls._api.set_access_token(TOKEN)
        cls._token = TOKEN

    @retry_endpoint
    def setUp(self):
        if os.path.exists(WORKING_REPO_DIR):
            shutil.rmtree(WORKING_REPO_DIR, onerror=set_write_permission_and_retry)
        logger.info(
            f"Does {WORKING_REPO_DIR} exist: {os.path.exists(WORKING_REPO_DIR)}"
        )
        self.REPO_NAME = repo_name()
        self._repo_url = self._api.create_repo(
            repo_id=self.REPO_NAME, token=self._token
        )
        self._api.upload_file(
            path_or_fileobj=BytesIO(b"some initial binary data: \x00\x01"),
            path_in_repo="random_file.txt",
            repo_id=f"{USER}/{self.REPO_NAME}",
            token=self._token,
        )

    def tearDown(self):
        try:
            self._api.delete_repo(repo_id=f"{USER}/{self.REPO_NAME}", token=self._token)
        except requests.exceptions.HTTPError:
            pass

        try:
            self._api.delete_repo(repo_id=self.REPO_NAME, token=self._token)
        except requests.exceptions.HTTPError:
            pass

        try:
            self._api.delete_repo(
                repo_id=f"valid_org/{self.REPO_NAME}", token=self._token
            )
        except requests.exceptions.HTTPError:
            pass

    def test_init_clone_from(self):
        temp_repo_url = self._api.create_repo(
            repo_id=f"{self.REPO_NAME}-temp",
            token=self._token,
            repo_type="space",
            space_sdk="static",
        )
        Repository(
            WORKING_REPO_DIR,
            clone_from=temp_repo_url,
            repo_type="space",
            use_auth_token=self._token,
        )
        self._api.delete_repo(
            repo_id=f"{USER}/{self.REPO_NAME}-temp",
            token=self._token,
            repo_type="space",
        )

    def test_clone_from_space(self):
        with pytest.raises(
            ValueError, match="Creating a Space through passing Space link*"
        ):
            Repository(
                WORKING_REPO_DIR,
                clone_from=f"{USER}/{uuid.uuid4()}",
                repo_type="space",
                use_auth_token=self._token,
            )

    def test_clone_from_model(self):
        temp_repo_url = self._api.create_repo(
            repo_id=f"{self.REPO_NAME}-temp", token=self._token, repo_type="model"
        )
        Repository(
            WORKING_REPO_DIR,
            clone_from=temp_repo_url,
            repo_type="model",
            use_auth_token=self._token,
        )
        self._api.delete_repo(
            repo_id=f"{USER}/{self.REPO_NAME}-temp", token=self._token
        )

    def test_init_from_existing_local_clone(self):
        subprocess.run(
            ["git", "clone", self._repo_url, WORKING_REPO_DIR],
            check=True,
        )

        repo = Repository(WORKING_REPO_DIR)
        repo.lfs_track(["*.pdf"])
        repo.lfs_enable_largefiles()
        repo.git_pull()

    def test_init_failure(self):
        with tempfile.TemporaryDirectory() as tmpdirname:
            with self.assertRaises(ValueError):
                _ = Repository(tmpdirname)

    @retry_endpoint
    def test_init_clone_in_empty_folder(self):
        repo = Repository(WORKING_REPO_DIR, clone_from=self._repo_url)
        repo.lfs_track(["*.pdf"])
        repo.lfs_enable_largefiles()
        repo.git_pull()

        self.assertIn("random_file.txt", os.listdir(WORKING_REPO_DIR))

    def test_git_lfs_filename(self):
        os.mkdir(WORKING_REPO_DIR)
        subprocess.run(
            ["git", "init"],
            stderr=subprocess.PIPE,
            stdout=subprocess.PIPE,
            check=True,
            cwd=WORKING_REPO_DIR,
        )

        repo = Repository(WORKING_REPO_DIR)

        large_file = [100] * int(4e6)
        with open(os.path.join(WORKING_REPO_DIR, "[].txt"), "w") as f:
            f.write(json.dumps(large_file))

        repo.git_add()

        repo.lfs_track(["[].txt"])
        self.assertFalse(is_tracked_with_lfs(f"{WORKING_REPO_DIR}/[].txt"))

        repo.lfs_track(["[].txt"], filename=True)
        self.assertTrue(is_tracked_with_lfs(f"{WORKING_REPO_DIR}/[].txt"))

    def test_init_clone_in_nonempty_folder(self):
        # Create dummy files
        # one is lfs-tracked, the other is not.
        os.makedirs(WORKING_REPO_DIR, exist_ok=True)
        with open(os.path.join(WORKING_REPO_DIR, "dummy.txt"), "w") as f:
            f.write("hello")
        with open(os.path.join(WORKING_REPO_DIR, "model.bin"), "w") as f:
            f.write("hello")
        self.assertRaises(
            OSError, Repository, WORKING_REPO_DIR, clone_from=self._repo_url
        )

    @retry_endpoint
    def test_init_clone_in_nonempty_non_linked_git_repo(self):
        # Create a new repository on the HF Hub
        temp_repo_url = self._api.create_repo(
            repo_id=f"{self.REPO_NAME}-temp", token=self._token
        )
        self._api.upload_file(
            path_or_fileobj=BytesIO(b"some initial binary data: \x00\x01"),
            path_in_repo="random_file_2.txt",
            repo_id=f"{USER}/{self.REPO_NAME}-temp",
            token=self._token,
        )

        # Clone the new repository
        os.makedirs(WORKING_REPO_DIR, exist_ok=True)
        Repository(WORKING_REPO_DIR, clone_from=self._repo_url)

        # Try and clone another repository within the same directory. Should error out due to mismatched remotes.
        self.assertRaises(
            EnvironmentError, Repository, WORKING_REPO_DIR, clone_from=temp_repo_url
        )

        self._api.delete_repo(repo_id=f"{self.REPO_NAME}-temp", token=self._token)

    @retry_endpoint
    def test_init_clone_in_nonempty_linked_git_repo_with_token(self):
        logger.info(
            f"Does {WORKING_REPO_DIR} exist: {os.path.exists(WORKING_REPO_DIR)}"
        )
        Repository(
            WORKING_REPO_DIR, clone_from=self._repo_url, use_auth_token=self._token
        )
        Repository(
            WORKING_REPO_DIR, clone_from=self._repo_url, use_auth_token=self._token
        )

    @retry_endpoint
    def test_init_clone_in_nonempty_linked_git_repo(self):
        # Clone the repository to disk
        Repository(WORKING_REPO_DIR, clone_from=self._repo_url)

        # Add to the remote repository without doing anything to the local repository.
        self._api.upload_file(
            path_or_fileobj=BytesIO(b"some initial binary data: \x00\x01"),
            path_in_repo="random_file_3.txt",
            repo_id=f"{USER}/{self.REPO_NAME}",
            token=self._token,
        )

        # Cloning the repository in the same directory should not result in a git pull.
        Repository(WORKING_REPO_DIR, clone_from=self._repo_url)
        self.assertNotIn("random_file_3.txt", os.listdir(WORKING_REPO_DIR))

    @retry_endpoint
    def test_init_clone_in_nonempty_linked_git_repo_unrelated_histories(self):
        # Clone the repository to disk
        repo = Repository(
            WORKING_REPO_DIR,
            clone_from=self._repo_url,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        with open(f"{WORKING_REPO_DIR}/random_file_3.txt", "w+") as f:
            f.write("New file.")

        repo.git_add()
        repo.git_commit("Unrelated commit")

        # Add to the remote repository without doing anything to the local repository.
        self._api.upload_file(
            path_or_fileobj=BytesIO(b"some initial binary data: \x00\x01"),
            path_in_repo="random_file_3.txt",
            repo_id=f"{USER}/{self.REPO_NAME}",
            token=self._token,
        )

        # The repo should initialize correctly as the remote is the same, even with unrelated historied
        Repository(WORKING_REPO_DIR, clone_from=self._repo_url)

    @retry_endpoint
    def test_add_commit_push(self):
        repo = Repository(
            WORKING_REPO_DIR,
            clone_from=self._repo_url,
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        # Create dummy files
        # one is lfs-tracked, the other is not.
        with open(os.path.join(WORKING_REPO_DIR, "dummy.txt"), "w") as f:
            f.write("hello")
        with open(os.path.join(WORKING_REPO_DIR, "model.bin"), "w") as f:
            f.write("hello")

        repo.git_add()
        repo.git_commit()
        url = repo.git_push()
        # Check that the returned commit url
        # actually exists.
        r = requests.head(url)
        r.raise_for_status()

    @retry_endpoint
    def test_add_commit_push_non_blocking(self):
        repo = Repository(
            WORKING_REPO_DIR,
            clone_from=self._repo_url,
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        # Create dummy files
        # one is lfs-tracked, the other is not.
        with open(os.path.join(WORKING_REPO_DIR, "dummy.txt"), "w") as f:
            f.write("hello")
        with open(os.path.join(WORKING_REPO_DIR, "model.bin"), "w") as f:
            f.write("hello")

        repo.git_add()
        repo.git_commit()
        url, result = repo.git_push(blocking=False)
        # Check that the returned commit url
        # actually exists.

        if result._process.poll() is None:
            self.assertEqual(result.status, -1)

        while not result.is_done:
            time.sleep(0.5)

        self.assertTrue(result.is_done)
        self.assertEqual(result.status, 0)

        r = requests.head(url)
        r.raise_for_status()

    @retry_endpoint
    def test_context_manager_non_blocking(self):
        repo = Repository(
            WORKING_REPO_DIR,
            clone_from=self._repo_url,
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        with repo.commit("New commit", blocking=False):
            with open(os.path.join(WORKING_REPO_DIR, "dummy.txt"), "w") as f:
                f.write("hello")

        while repo.commands_in_progress:
            time.sleep(1)

        self.assertEqual(len(repo.commands_in_progress), 0)
        self.assertEqual(len(repo.command_queue), 1)
        self.assertEqual(repo.command_queue[-1].status, 0)
        self.assertEqual(repo.command_queue[-1].is_done, True)
        self.assertEqual(repo.command_queue[-1].title, "push")

    @retry_endpoint
    def test_add_commit_push_non_blocking_process_killed(self):
        repo = Repository(
            WORKING_REPO_DIR,
            clone_from=self._repo_url,
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        # Create dummy files
        # one is lfs-tracked, the other is not.
        with open(os.path.join(WORKING_REPO_DIR, "dummy.txt"), "w") as f:
            f.write(str([[[1] * 10000] * 1000] * 10))

        repo.git_add(auto_lfs_track=True)
        repo.git_commit()
        url, result = repo.git_push(blocking=False)

        result._process.kill()

        while result._process.poll() is None:
            time.sleep(0.5)

        self.assertTrue(result.is_done)
        self.assertEqual(result.status, -9)

    @retry_endpoint
    def test_clone_with_endpoint(self):
        clone = Repository(
            f"{WORKING_REPO_DIR}/{self.REPO_NAME}",
            clone_from=f"{ENDPOINT_STAGING}/valid_org/{self.REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        with clone.commit("Commit"):
            with open("dummy.txt", "w") as f:
                f.write("hello")
            with open("model.bin", "w") as f:
                f.write("hello")

        shutil.rmtree(f"{WORKING_REPO_DIR}/{self.REPO_NAME}")

        Repository(
            f"{WORKING_REPO_DIR}/{self.REPO_NAME}",
            clone_from=f"{ENDPOINT_STAGING}/valid_org/{self.REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        files = os.listdir(f"{WORKING_REPO_DIR}/{self.REPO_NAME}")
        self.assertTrue("dummy.txt" in files)
        self.assertTrue("model.bin" in files)

    @retry_endpoint
    def test_clone_with_repo_name_and_org(self):
        clone = Repository(
            f"{WORKING_REPO_DIR}/{self.REPO_NAME}",
            clone_from=f"valid_org/{self.REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        with clone.commit("Commit"):
            with open("dummy.txt", "w") as f:
                f.write("hello")
            with open("model.bin", "w") as f:
                f.write("hello")

        shutil.rmtree(f"{WORKING_REPO_DIR}/{self.REPO_NAME}")

        Repository(
            f"{WORKING_REPO_DIR}/{self.REPO_NAME}",
            clone_from=f"valid_org/{self.REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        files = os.listdir(f"{WORKING_REPO_DIR}/{self.REPO_NAME}")
        self.assertTrue("dummy.txt" in files)
        self.assertTrue("model.bin" in files)

    @retry_endpoint
    def test_clone_with_repo_name_and_user_namespace(self):
        clone = Repository(
            f"{WORKING_REPO_DIR}/{self.REPO_NAME}",
            clone_from=f"{USER}/{self.REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        with clone.commit("Commit"):
            # Create dummy files
            # one is lfs-tracked, the other is not.
            with open("dummy.txt", "w") as f:
                f.write("hello")
            with open("model.bin", "w") as f:
                f.write("hello")

        shutil.rmtree(f"{WORKING_REPO_DIR}/{self.REPO_NAME}")

        Repository(
            f"{WORKING_REPO_DIR}/{self.REPO_NAME}",
            clone_from=f"{USER}/{self.REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        files = os.listdir(f"{WORKING_REPO_DIR}/{self.REPO_NAME}")
        self.assertTrue("dummy.txt" in files)
        self.assertTrue("model.bin" in files)

    @retry_endpoint
    def test_clone_with_repo_name_and_no_namespace(self):
        self.assertRaises(
            OSError,
            Repository,
            f"{WORKING_REPO_DIR}/{self.REPO_NAME}",
            clone_from=self.REPO_NAME,
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

    @retry_endpoint
    def test_clone_with_repo_name_user_and_no_auth_token(self):
        # Create repo
        Repository(
            f"{WORKING_REPO_DIR}/{self.REPO_NAME}",
            clone_from=f"{USER}/{self.REPO_NAME}",
            git_user="ci",
            git_email="ci@dummy.com",
        )

        # Instantiate it without token
        Repository(
            f"{WORKING_REPO_DIR}/{self.REPO_NAME}",
            clone_from=f"{USER}/{self.REPO_NAME}",
            git_user="ci",
            git_email="ci@dummy.com",
        )

    @retry_endpoint
    def test_clone_with_repo_name_org_and_no_auth_token(self):
        # Create repo
        Repository(
            f"{WORKING_REPO_DIR}/{self.REPO_NAME}",
            use_auth_token=self._token,
            clone_from=f"valid_org/{self.REPO_NAME}",
            git_user="ci",
            git_email="ci@dummy.com",
        )

        # Instantiate it without token
        Repository(
            f"{WORKING_REPO_DIR}/{self.REPO_NAME}",
            clone_from=f"valid_org/{self.REPO_NAME}",
            git_user="ci",
            git_email="ci@dummy.com",
        )

    @retry_endpoint
    def test_clone_not_hf_url(self):
        # Should not error out
        Repository(
            f"{WORKING_REPO_DIR}/{self.REPO_NAME}",
            clone_from=(
                "https://hf.co/hf-internal-testing/huggingface-hub-dummy-repository"
            ),
        )

    @with_production_testing
    @retry_endpoint
    def test_clone_repo_at_root(self):
        Repository(
            f"{WORKING_REPO_DIR}/{self.REPO_NAME}",
            clone_from="bert-base-cased",
            skip_lfs_files=True,
        )

        shutil.rmtree(f"{WORKING_REPO_DIR}/{self.REPO_NAME}")

        Repository(
            f"{WORKING_REPO_DIR}/{self.REPO_NAME}",
            clone_from="https://huggingface.co/bert-base-cased",
            skip_lfs_files=True,
        )

    @retry_endpoint
    def test_skip_lfs_files(self):
        repo = Repository(
            self.REPO_NAME,
            clone_from=f"{USER}/{self.REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        with repo.commit("Add LFS file"):
            with open("file.bin", "w+") as f:
                f.write("Bin file")

        shutil.rmtree(repo.local_dir)

        repo = Repository(
            self.REPO_NAME,
            clone_from=f"{USER}/{self.REPO_NAME}",
            use_auth_token=self._token,
            skip_lfs_files=True,
        )

        with open(pathlib.Path(repo.local_dir) / "file.bin", "r") as f:
            content = f.read()
            self.assertTrue(content.startswith("version"))

        repo.git_pull(lfs=True)

        with open(pathlib.Path(repo.local_dir) / "file.bin", "r") as f:
            content = f.read()
            self.assertEqual(content, "Bin file")

    @retry_endpoint
    def test_is_tracked_upstream(self):
        repo = Repository(
            self.REPO_NAME,
            clone_from=f"{USER}/{self.REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        self.assertTrue(is_tracked_upstream(repo.local_dir))

    @retry_endpoint
    def test_push_errors_on_wrong_checkout(self):
        repo = Repository(
            WORKING_REPO_DIR,
            clone_from=f"{USER}/{self.REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        head_commit_ref = (
            subprocess.run(
                "git show --oneline -s".split(),
                stderr=subprocess.PIPE,
                stdout=subprocess.PIPE,
                check=True,
                cwd=repo.local_dir,
            )
            .stdout.decode()
            .split()[0]
        )

        repo.git_checkout(head_commit_ref)

        with self.assertRaises(OSError):
            with repo.commit("New commit"):
                with open("new_file", "w+") as f:
                    f.write("Ok")

    @retry_endpoint
    def test_commits_on_correct_branch(self):
        repo = Repository(
            WORKING_REPO_DIR,
            clone_from=f"{USER}/{self.REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )
        branch = repo.current_branch
        repo.git_checkout("new-branch", create_branch_ok=True)
        repo.git_checkout(branch)

        with repo.commit("New commit"):
            with open("file.txt", "w+") as f:
                f.write("Ok")

        repo.git_checkout("new-branch")

        with repo.commit("New commit"):
            with open("new_file.txt", "w+") as f:
                f.write("Ok")

        with tempfile.TemporaryDirectory() as tmp:
            clone = Repository(
                tmp,
                clone_from=f"{USER}/{self.REPO_NAME}",
                use_auth_token=self._token,
                git_user="ci",
                git_email="ci@dummy.com",
            )
            files = os.listdir(clone.local_dir)
            self.assertTrue("file.txt" in files)
            self.assertFalse("new_file.txt" in files)

            clone.git_checkout("new-branch")
            files = os.listdir(clone.local_dir)
            self.assertFalse("file.txt" in files)
            self.assertTrue("new_file.txt" in files)

    @retry_endpoint
    def test_repo_checkout_push(self):
        repo = Repository(
            WORKING_REPO_DIR,
            clone_from=f"{USER}/{self.REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        repo.git_checkout("new-branch", create_branch_ok=True)
        repo.git_checkout("main")

        with open(os.path.join(repo.local_dir, "file.txt"), "w+") as f:
            f.write("Ok")

        repo.push_to_hub("Commit #1")
        repo.git_checkout("new-branch", create_branch_ok=True)

        with open(os.path.join(repo.local_dir, "new_file.txt"), "w+") as f:
            f.write("Ok")

        repo.push_to_hub("Commit #2")

        with tempfile.TemporaryDirectory() as tmp:
            clone = Repository(
                tmp,
                clone_from=f"{USER}/{self.REPO_NAME}",
                use_auth_token=self._token,
                git_user="ci",
                git_email="ci@dummy.com",
            )
            files = os.listdir(clone.local_dir)
            self.assertTrue("file.txt" in files)
            self.assertFalse("new_file.txt" in files)

            clone.git_checkout("new-branch")
            files = os.listdir(clone.local_dir)
            self.assertFalse("file.txt" in files)
            self.assertTrue("new_file.txt" in files)

    @retry_endpoint
    def test_repo_checkout_commit_context_manager(self):
        repo = Repository(
            WORKING_REPO_DIR,
            clone_from=f"{USER}/{self.REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
            revision="main",
        )

        with repo.commit("Commit #1", branch="new-branch"):
            with open(os.path.join(repo.local_dir, "file.txt"), "w+") as f:
                f.write("Ok")

        with repo.commit("Commit #2", branch="main"):
            with open(os.path.join(repo.local_dir, "new_file.txt"), "w+") as f:
                f.write("Ok")

        # Maintains lastly used branch
        with repo.commit("Commit #3"):
            with open(os.path.join(repo.local_dir, "new_file-2.txt"), "w+") as f:
                f.write("Ok")

        with tempfile.TemporaryDirectory() as tmp:
            clone = Repository(
                tmp,
                clone_from=f"{USER}/{self.REPO_NAME}",
                use_auth_token=self._token,
                git_user="ci",
                git_email="ci@dummy.com",
            )
            files = os.listdir(clone.local_dir)
            self.assertFalse("file.txt" in files)
            self.assertTrue("new_file-2.txt" in files)
            self.assertTrue("new_file.txt" in files)

            clone.git_checkout("new-branch")
            files = os.listdir(clone.local_dir)
            self.assertTrue("file.txt" in files)
            self.assertFalse("new_file.txt" in files)
            self.assertFalse("new_file-2.txt" in files)

    @retry_endpoint
    def test_add_tag(self):
        # Eventually see why this is needed
        if os.path.exists(WORKING_REPO_DIR):
            shutil.rmtree(WORKING_REPO_DIR, onerror=set_write_permission_and_retry)
        repo = Repository(
            WORKING_REPO_DIR,
            clone_from=f"{USER}/{self.REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
            revision="main",
        )

        repo.add_tag("v4.6.0", remote="origin")
        self.assertTrue(repo.tag_exists("v4.6.0", remote="origin"))

    @retry_endpoint
    def test_add_annotated_tag(self):
        repo = Repository(
            WORKING_REPO_DIR,
            clone_from=f"{USER}/{self.REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
            revision="main",
        )

        repo.add_tag("v4.5.0", message="This is an annotated tag", remote="origin")

        # Unfortunately git offers no built-in way to check the annotated
        # message of a remote tag.
        # In order to check that the remote tag was correctly annotated,
        # we delete the local tag before pulling the remote tag (which
        # should be the same). We then check that this tag is correctly
        # annotated.
        repo.delete_tag("v4.5.0")

        self.assertTrue(repo.tag_exists("v4.5.0", remote="origin"))
        self.assertFalse(repo.tag_exists("v4.5.0"))

        subprocess.run(
            ["git", "pull", "--tags"],
            stderr=subprocess.PIPE,
            stdout=subprocess.PIPE,
            check=True,
            encoding="utf-8",
            cwd=repo.local_dir,
        )

        self.assertTrue(repo.tag_exists("v4.5.0"))

        result = subprocess.run(
            ["git", "tag", "-n9"],
            stderr=subprocess.PIPE,
            stdout=subprocess.PIPE,
            check=True,
            encoding="utf-8",
            cwd=repo.local_dir,
        ).stdout.strip()

        self.assertIn("This is an annotated tag", result)

    @retry_endpoint
    def test_delete_tag(self):
        repo = Repository(
            WORKING_REPO_DIR,
            clone_from=f"{USER}/{self.REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
            revision="main",
        )

        repo.add_tag("v4.6.0", message="This is an annotated tag", remote="origin")
        self.assertTrue(repo.tag_exists("v4.6.0", remote="origin"))

        repo.delete_tag("v4.6.0")
        self.assertFalse(repo.tag_exists("v4.6.0"))
        self.assertTrue(repo.tag_exists("v4.6.0", remote="origin"))

        repo.delete_tag("v4.6.0", remote="origin")
        self.assertFalse(repo.tag_exists("v4.6.0", remote="origin"))

    @retry_endpoint
    def test_lfs_prune(self):
        repo = Repository(
            WORKING_REPO_DIR,
            clone_from=f"{USER}/{self.REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
            revision="main",
        )

        with repo.commit("Committing LFS file"):
            with open("file.bin", "w+") as f:
                f.write("Random string 1")

        with repo.commit("Committing LFS file"):
            with open("file.bin", "w+") as f:
                f.write("Random string 2")

        root_directory = pathlib.Path(repo.local_dir) / ".git" / "lfs"
        git_lfs_files_size = sum(
            f.stat().st_size for f in root_directory.glob("**/*") if f.is_file()
        )
        repo.lfs_prune()
        post_prune_git_lfs_files_size = sum(
            f.stat().st_size for f in root_directory.glob("**/*") if f.is_file()
        )

        # Size of the directory holding LFS files was reduced
        self.assertLess(post_prune_git_lfs_files_size, git_lfs_files_size)

    @retry_endpoint
    def test_lfs_prune_git_push(self):
        repo = Repository(
            WORKING_REPO_DIR,
            clone_from=f"{USER}/{self.REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
            revision="main",
        )

        with repo.commit("Committing LFS file"):
            with open("file.bin", "w+") as f:
                f.write("Random string 1")

        root_directory = pathlib.Path(repo.local_dir) / ".git" / "lfs"
        git_lfs_files_size = sum(
            f.stat().st_size for f in root_directory.glob("**/*") if f.is_file()
        )

        with open(os.path.join(repo.local_dir, "file.bin"), "w+") as f:
            f.write("Random string 2")

        repo.git_add()
        repo.git_commit("New commit")
        repo.git_push(auto_lfs_prune=True)

        post_prune_git_lfs_files_size = sum(
            f.stat().st_size for f in root_directory.glob("**/*") if f.is_file()
        )

        # Size of the directory holding LFS files is the exact same
        self.assertEqual(post_prune_git_lfs_files_size, git_lfs_files_size)

    @retry_endpoint
    def test_lfs_prune_git_push_non_blocking(self):
        repo = Repository(
            WORKING_REPO_DIR,
            clone_from=f"{USER}/{self.REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
            revision="main",
        )

        with repo.commit("Committing LFS file"):
            with open("file.bin", "w+") as f:
                f.write("Random string 1")

        root_directory = pathlib.Path(repo.local_dir) / ".git" / "lfs"
        git_lfs_files_size = sum(
            f.stat().st_size for f in root_directory.glob("**/*") if f.is_file()
        )

        with open(os.path.join(repo.local_dir, "file.bin"), "w+") as f:
            f.write("Random string 2")

        repo.git_add()
        repo.git_commit("New commit")
        repo.git_push(blocking=False, auto_lfs_prune=True)

        while len(repo.commands_in_progress):
            time.sleep(0.2)

        post_prune_git_lfs_files_size = sum(
            f.stat().st_size for f in root_directory.glob("**/*") if f.is_file()
        )

        # Size of the directory holding LFS files is the exact same
        self.assertEqual(post_prune_git_lfs_files_size, git_lfs_files_size)

    @retry_endpoint
    def test_lfs_prune_context_manager(self):
        repo = Repository(
            WORKING_REPO_DIR,
            clone_from=f"{USER}/{self.REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
            revision="main",
        )

        with repo.commit("Committing LFS file"):
            with open("file.bin", "w+") as f:
                f.write("Random string 1")

        root_directory = pathlib.Path(repo.local_dir) / ".git" / "lfs"
        git_lfs_files_size = sum(
            f.stat().st_size for f in root_directory.glob("**/*") if f.is_file()
        )

        with repo.commit("Committing LFS file", auto_lfs_prune=True):
            with open("file.bin", "w+") as f:
                f.write("Random string 2")

        post_prune_git_lfs_files_size = sum(
            f.stat().st_size for f in root_directory.glob("**/*") if f.is_file()
        )

        # Size of the directory holding LFS files is the exact same
        self.assertEqual(post_prune_git_lfs_files_size, git_lfs_files_size)

    @retry_endpoint
    def test_lfs_prune_context_manager_non_blocking(self):
        repo = Repository(
            WORKING_REPO_DIR,
            clone_from=f"{USER}/{self.REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
            revision="main",
        )

        with repo.commit("Committing LFS file"):
            with open("file.bin", "w+") as f:
                f.write("Random string 1")

        root_directory = pathlib.Path(repo.local_dir) / ".git" / "lfs"
        git_lfs_files_size = sum(
            f.stat().st_size for f in root_directory.glob("**/*") if f.is_file()
        )

        with repo.commit("Committing LFS file", auto_lfs_prune=True, blocking=False):
            with open("file.bin", "w+") as f:
                f.write("Random string 2")

        while len(repo.commands_in_progress):
            time.sleep(0.2)

        post_prune_git_lfs_files_size = sum(
            f.stat().st_size for f in root_directory.glob("**/*") if f.is_file()
        )

        # Size of the directory holding LFS files is the exact same
        self.assertEqual(post_prune_git_lfs_files_size, git_lfs_files_size)


class RepositoryOfflineTest(RepositoryCommonTest):
    @classmethod
    def setUpClass(cls) -> None:
        if os.path.exists(WORKING_REPO_DIR):
            shutil.rmtree(WORKING_REPO_DIR, onerror=set_write_permission_and_retry)
        logger.info(
            f"Does {WORKING_REPO_DIR} exist: {os.path.exists(WORKING_REPO_DIR)}"
        )
        os.makedirs(WORKING_REPO_DIR, exist_ok=True)
        subprocess.run(
            ["git", "init"],
            stderr=subprocess.PIPE,
            stdout=subprocess.PIPE,
            check=True,
            cwd=WORKING_REPO_DIR,
        )

        repo = Repository(WORKING_REPO_DIR, git_user="ci", git_email="ci@dummy.ci")

        with open(f"{WORKING_REPO_DIR}/.gitattributes", "w+") as f:
            f.write("*.pt filter=lfs diff=lfs merge=lfs -text")

        repo.git_add(".gitattributes")
        repo.git_commit("Add .gitattributes")

    def tearDown(self):
        subprocess.run(
            ["git", "reset", "--hard"],
            stderr=subprocess.PIPE,
            stdout=subprocess.PIPE,
            check=True,
            cwd=WORKING_REPO_DIR,
        )
        subprocess.run(
            ["git", "clean", "-fdx"],
            stderr=subprocess.PIPE,
            stdout=subprocess.PIPE,
            check=True,
            cwd=WORKING_REPO_DIR,
        )

        all_local_tags = subprocess.run(
            ["git", "tag", "-l"],
            stderr=subprocess.PIPE,
            stdout=subprocess.PIPE,
            check=True,
            cwd=WORKING_REPO_DIR,
        ).stdout.strip()

        if len(all_local_tags):
            subprocess.run(
                ["git", "tag", "-d", all_local_tags],
                stderr=subprocess.PIPE,
                stdout=subprocess.PIPE,
                check=True,
                cwd=WORKING_REPO_DIR,
            )

    @classmethod
    def tearDownClass(cls) -> None:
        if os.path.exists(WORKING_REPO_DIR):
            shutil.rmtree(WORKING_REPO_DIR, onerror=set_write_permission_and_retry)
        logger.info(
            f"Does {WORKING_REPO_DIR} exist: {os.path.exists(WORKING_REPO_DIR)}"
        )

    def test_is_tracked_with_lfs(self):
        repo = Repository(WORKING_REPO_DIR)

        # This content is under 10MB
        small_file = [100]

        with open(f"{WORKING_REPO_DIR}/small_file.txt", "w+") as f:
            f.write(json.dumps(small_file))

        with open(f"{WORKING_REPO_DIR}/small_file_2.txt", "w+") as f:
            f.write(json.dumps(small_file))

        with open(f"{WORKING_REPO_DIR}/model.pt", "w+") as f:
            f.write(json.dumps(small_file))

        repo.lfs_track("small_file.txt")

        self.assertTrue(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "small_file.txt"))
        )
        self.assertFalse(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "small_file_2.txt"))
        )
        self.assertTrue(is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "model.pt")))

    def test_is_tracked_with_lfs_with_pattern(self):
        repo = Repository(WORKING_REPO_DIR)

        # This content is 5MB (under 10MB)
        small_file = [100] * int(1e6)

        # This content is 20MB (over 10MB)
        large_file = [100] * int(4e6)

        with open(f"{WORKING_REPO_DIR}/large_file.txt", "w+") as f:
            f.write(json.dumps(large_file))

        with open(f"{WORKING_REPO_DIR}/small_file.txt", "w+") as f:
            f.write(json.dumps(small_file))

        os.makedirs(f"{WORKING_REPO_DIR}/dir", exist_ok=True)

        with open(f"{WORKING_REPO_DIR}/dir/large_file.txt", "w+") as f:
            f.write(json.dumps(large_file))

        with open(f"{WORKING_REPO_DIR}/dir/small_file.txt", "w+") as f:
            f.write(json.dumps(small_file))

        repo.auto_track_large_files("dir")

        self.assertFalse(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "large_file.txt"))
        )
        self.assertFalse(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "small_file.txt"))
        )
        self.assertTrue(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "dir/large_file.txt"))
        )
        self.assertFalse(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "dir/small_file.txt"))
        )

    def test_auto_track_large_files(self):
        repo = Repository(WORKING_REPO_DIR)

        # This content is 5MB (under 10MB)
        small_file = [100] * int(1e6)

        # This content is 20MB (over 10MB)
        large_file = [100] * int(4e6)

        with open(f"{WORKING_REPO_DIR}/large_file.txt", "w+") as f:
            f.write(json.dumps(large_file))

        with open(f"{WORKING_REPO_DIR}/small_file.txt", "w+") as f:
            f.write(json.dumps(small_file))

        repo.auto_track_large_files()

        self.assertTrue(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "large_file.txt"))
        )
        self.assertFalse(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "small_file.txt"))
        )

    def test_auto_track_binary_files(self):
        repo = Repository(WORKING_REPO_DIR)

        # This content is non-binary
        non_binary_file = [100] * int(1e6)

        # This content is binary (contains the null character)
        binary_file = "\x00\x00\x00\x00"

        with open(f"{WORKING_REPO_DIR}/non_binary_file.txt", "w+") as f:
            f.write(json.dumps(non_binary_file))

        with open(f"{WORKING_REPO_DIR}/binary_file.txt", "w+") as f:
            f.write(binary_file)

        repo.auto_track_binary_files()

        self.assertFalse(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "non_binary)file.txt"))
        )
        self.assertTrue(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "binary_file.txt"))
        )

    def test_auto_track_large_files_ignored_with_gitignore(self):
        repo = Repository(WORKING_REPO_DIR)

        # This content is 20MB (over 10MB)
        large_file = [100] * int(4e6)

        # Test nested gitignores
        os.makedirs(f"{WORKING_REPO_DIR}/directory")

        with open(f"{WORKING_REPO_DIR}/.gitignore", "w+") as f:
            f.write("large_file.txt")

        with open(f"{WORKING_REPO_DIR}/directory/.gitignore", "w+") as f:
            f.write("large_file_3.txt")

        with open(f"{WORKING_REPO_DIR}/large_file.txt", "w+") as f:
            f.write(json.dumps(large_file))

        with open(f"{WORKING_REPO_DIR}/large_file_2.txt", "w+") as f:
            f.write(json.dumps(large_file))

        with open(f"{WORKING_REPO_DIR}/directory/large_file_3.txt", "w+") as f:
            f.write(json.dumps(large_file))

        with open(f"{WORKING_REPO_DIR}/directory/large_file_4.txt", "w+") as f:
            f.write(json.dumps(large_file))

        repo.auto_track_large_files()

        # Large files
        self.assertFalse(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "large_file.txt"))
        )
        self.assertTrue(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "large_file_2.txt"))
        )

        self.assertFalse(
            is_tracked_with_lfs(
                os.path.join(WORKING_REPO_DIR, "directory/large_file_3.txt")
            )
        )
        self.assertTrue(
            is_tracked_with_lfs(
                os.path.join(WORKING_REPO_DIR, "directory/large_file_4.txt")
            )
        )

    def test_auto_track_binary_files_ignored_with_gitignore(self):
        repo = Repository(WORKING_REPO_DIR)

        # This content is binary (contains the null character)
        binary_file = "\x00\x00\x00\x00"

        # Test nested gitignores
        os.makedirs(f"{WORKING_REPO_DIR}/directory")

        with open(f"{WORKING_REPO_DIR}/.gitignore", "w+") as f:
            f.write("binary_file.txt")

        with open(f"{WORKING_REPO_DIR}/directory/.gitignore", "w+") as f:
            f.write("binary_file_3.txt")

        with open(f"{WORKING_REPO_DIR}/binary_file.txt", "w+") as f:
            f.write(binary_file)

        with open(f"{WORKING_REPO_DIR}/binary_file_2.txt", "w+") as f:
            f.write(binary_file)

        with open(f"{WORKING_REPO_DIR}/directory/binary_file_3.txt", "w+") as f:
            f.write(binary_file)

        with open(f"{WORKING_REPO_DIR}/directory/binary_file_4.txt", "w+") as f:
            f.write(binary_file)

        repo.auto_track_binary_files()

        # Binary files
        self.assertFalse(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "binary_file.txt"))
        )
        self.assertTrue(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "binary_file_2.txt"))
        )

        self.assertFalse(
            is_tracked_with_lfs(
                os.path.join(WORKING_REPO_DIR, "directory/binary_file_3.txt")
            )
        )
        self.assertTrue(
            is_tracked_with_lfs(
                os.path.join(WORKING_REPO_DIR, "directory/binary_file_4.txt")
            )
        )

    def test_auto_track_large_files_through_git_add(self):
        repo = Repository(WORKING_REPO_DIR)

        # This content is 5MB (under 10MB)
        small_file = [100] * int(1e6)

        # This content is 20MB (over 10MB)
        large_file = [100] * int(4e6)

        with open(f"{WORKING_REPO_DIR}/large_file.txt", "w+") as f:
            f.write(json.dumps(large_file))

        with open(f"{WORKING_REPO_DIR}/small_file.txt", "w+") as f:
            f.write(json.dumps(small_file))

        repo.git_add(auto_lfs_track=True)

        self.assertTrue(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "large_file.txt"))
        )
        self.assertFalse(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "small_file.txt"))
        )

    def test_auto_track_binary_files_through_git_add(self):
        repo = Repository(WORKING_REPO_DIR)

        # This content is non binary
        non_binary_file = [100] * int(1e6)

        # This content is binary (contains the null character)
        binary_file = "\x00\x00\x00\x00"

        with open(f"{WORKING_REPO_DIR}/small_file.txt", "w+") as f:
            f.write(json.dumps(non_binary_file))

        with open(f"{WORKING_REPO_DIR}/binary_file.txt", "w+") as f:
            f.write(binary_file)

        repo.git_add(auto_lfs_track=True)

        self.assertFalse(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "non_binary_file.txt"))
        )
        self.assertTrue(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "binary_file.txt"))
        )

    def test_auto_no_track_large_files_through_git_add(self):
        repo = Repository(WORKING_REPO_DIR)

        # This content is 5MB (under 10MB)
        small_file = [100] * int(1e6)

        # This content is 20MB (over 10MB)
        large_file = [100] * int(4e6)

        with open(f"{WORKING_REPO_DIR}/large_file.txt", "w+") as f:
            f.write(json.dumps(large_file))

        with open(f"{WORKING_REPO_DIR}/small_file.txt", "w+") as f:
            f.write(json.dumps(small_file))

        repo.git_add(auto_lfs_track=False)

        self.assertFalse(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "large_file.txt"))
        )
        self.assertFalse(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "small_file.txt"))
        )

    def test_auto_no_track_binary_files_through_git_add(self):
        repo = Repository(WORKING_REPO_DIR)

        # This content is non-binary
        non_binary_file = [100] * int(1e6)

        # This content is binary (contains the null character)
        binary_file = "\x00\x00\x00\x00"

        with open(f"{WORKING_REPO_DIR}/small_file.txt", "w+") as f:
            f.write(json.dumps(non_binary_file))

        with open(f"{WORKING_REPO_DIR}/binary_file.txt", "w+") as f:
            f.write(binary_file)

        repo.git_add(auto_lfs_track=False)

        self.assertFalse(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "non_binary_file.txt"))
        )
        self.assertFalse(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "binary_file.txt"))
        )

    def test_auto_track_updates_removed_gitattributes(self):
        repo = Repository(WORKING_REPO_DIR)

        # This content is 5MB (under 10MB)
        small_file = [100] * int(1e6)

        # This content is 20MB (over 10MB)
        large_file = [100] * int(4e6)

        with open(f"{WORKING_REPO_DIR}/large_file.txt", "w+") as f:
            f.write(json.dumps(large_file))

        with open(f"{WORKING_REPO_DIR}/small_file.txt", "w+") as f:
            f.write(json.dumps(small_file))

        repo.git_add(auto_lfs_track=True)

        self.assertTrue(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "large_file.txt"))
        )
        self.assertFalse(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "small_file.txt"))
        )

        # Remove large file
        os.remove(f"{WORKING_REPO_DIR}/large_file.txt")

        # Auto track should remove the entry from .gitattributes
        repo.auto_track_large_files()

        # Recreate the large file with smaller contents
        with open(f"{WORKING_REPO_DIR}/large_file.txt", "w+") as f:
            f.write(json.dumps(large_file))

        # Ensure the file is not LFS tracked anymore
        self.assertFalse(
            is_tracked_with_lfs(os.path.join(WORKING_REPO_DIR, "large_file.txt"))
        )

    def test_checkout_non_existant_branch(self):
        repo = Repository(WORKING_REPO_DIR)
        self.assertRaises(EnvironmentError, repo.git_checkout, "brand-new-branch")

    def test_checkout_new_branch(self):
        repo = Repository(WORKING_REPO_DIR)
        repo.git_checkout("new-branch", create_branch_ok=True)

        self.assertEqual(repo.current_branch, "new-branch")

    def test_is_not_tracked_upstream(self):
        repo = Repository(WORKING_REPO_DIR)
        repo.git_checkout("new-branch", create_branch_ok=True)
        self.assertFalse(is_tracked_upstream(repo.local_dir))

    def test_no_branch_checked_out_raises(self):
        repo = Repository(WORKING_REPO_DIR)

        head_commit_ref = (
            subprocess.run(
                "git show --oneline -s".split(),
                stderr=subprocess.PIPE,
                stdout=subprocess.PIPE,
                check=True,
                cwd=WORKING_REPO_DIR,
            )
            .stdout.decode()
            .split()[0]
        )

        repo.git_checkout(head_commit_ref)
        self.assertRaises(OSError, is_tracked_upstream, repo.local_dir)

    def test_repo_init_checkout_default_revision(self):
        # Instantiate repository on a given revision
        repo = Repository(WORKING_REPO_DIR, revision="new-branch")
        self.assertEqual(repo.current_branch, "new-branch")

        # The revision should be kept when re-initializing the repo
        repo_2 = Repository(WORKING_REPO_DIR)
        self.assertEqual(repo_2.current_branch, "new-branch")

    def test_repo_init_checkout_revision(self):
        # Instantiate repository on a given revision
        repo = Repository(WORKING_REPO_DIR)
        current_head_hash = repo.git_head_hash()

        with open(os.path.join(repo.local_dir, "file.txt"), "w+") as f:
            f.write("File")

        repo.git_add()
        repo.git_commit("Add file.txt")

        new_head_hash = repo.git_head_hash()

        self.assertNotEqual(current_head_hash, new_head_hash)

        previous_head_repo = Repository(WORKING_REPO_DIR, revision=current_head_hash)
        files = os.listdir(previous_head_repo.local_dir)
        self.assertNotIn("file.txt", files)

        current_head_repo = Repository(WORKING_REPO_DIR, revision=new_head_hash)
        files = os.listdir(current_head_repo.local_dir)
        self.assertIn("file.txt", files)

    def test_repo_user(self):
        api = HfApi(endpoint=ENDPOINT_STAGING)
        token = TOKEN
        api.set_access_token(TOKEN)

        repo = Repository(WORKING_REPO_DIR, use_auth_token=token)
        user = api.whoami(token)

        username = subprocess.run(
            ["git", "config", "user.name"],
            stderr=subprocess.PIPE,
            stdout=subprocess.PIPE,
            check=True,
            encoding="utf-8",
            cwd=repo.local_dir,
        ).stdout.strip()
        email = subprocess.run(
            ["git", "config", "user.email"],
            stderr=subprocess.PIPE,
            stdout=subprocess.PIPE,
            check=True,
            encoding="utf-8",
            cwd=repo.local_dir,
        ).stdout.strip()

        self.assertEqual(username, user["fullname"])
        self.assertEqual(email, user["email"])

    def test_repo_passed_user(self):
        api = HfApi(endpoint=ENDPOINT_STAGING)
        token = TOKEN
        api.set_access_token(TOKEN)
        repo = Repository(
            WORKING_REPO_DIR,
            git_user="RANDOM_USER",
            git_email="EMAIL@EMAIL.EMAIL",
            use_auth_token=token,
        )
        username = subprocess.run(
            ["git", "config", "user.name"],
            stderr=subprocess.PIPE,
            stdout=subprocess.PIPE,
            check=True,
            encoding="utf-8",
            cwd=repo.local_dir,
        ).stdout.strip()
        email = subprocess.run(
            ["git", "config", "user.email"],
            stderr=subprocess.PIPE,
            stdout=subprocess.PIPE,
            check=True,
            encoding="utf-8",
            cwd=repo.local_dir,
        ).stdout.strip()

        self.assertEqual(username, "RANDOM_USER")
        self.assertEqual(email, "EMAIL@EMAIL.EMAIL")

    def test_correct_helper(self):
        subprocess.run(
            ["git", "config", "--global", "credential.helper", "get"],
            stderr=subprocess.PIPE,
            stdout=subprocess.PIPE,
            check=True,
            encoding="utf-8",
        )
        repo = Repository(WORKING_REPO_DIR)
        self.assertListEqual(
            currently_setup_credential_helpers(repo.local_dir), ["get", "store"]
        )
        self.assertEqual(currently_setup_credential_helpers(), ["get"])

    def test_add_tag(self):
        repo = Repository(
            WORKING_REPO_DIR,
            git_user="RANDOM_USER",
            git_email="EMAIL@EMAIL.EMAIL",
        )

        repo.add_tag("v4.6.0")
        self.assertTrue(repo.tag_exists("v4.6.0"))

    def test_add_annotated_tag(self):
        repo = Repository(
            WORKING_REPO_DIR,
            git_user="RANDOM_USER",
            git_email="EMAIL@EMAIL.EMAIL",
        )

        repo.add_tag("v4.6.0", message="This is an annotated tag")
        self.assertTrue(repo.tag_exists("v4.6.0"))

        result = subprocess.run(
            ["git", "tag", "-n9"],
            stderr=subprocess.PIPE,
            stdout=subprocess.PIPE,
            check=True,
            encoding="utf-8",
            cwd=repo.local_dir,
        ).stdout.strip()

        self.assertIn("This is an annotated tag", result)

    def test_delete_tag(self):
        repo = Repository(
            WORKING_REPO_DIR,
            git_user="RANDOM_USER",
            git_email="EMAIL@EMAIL.EMAIL",
        )

        repo.add_tag("v4.6.0", message="This is an annotated tag")
        self.assertTrue(repo.tag_exists("v4.6.0"))

        repo.delete_tag("v4.6.0")
        self.assertFalse(repo.tag_exists("v4.6.0"))

    def test_repo_clean(self):
        repo = Repository(
            WORKING_REPO_DIR,
            git_user="RANDOM_USER",
            git_email="EMAIL@EMAIL.EMAIL",
        )

        self.assertTrue(repo.is_repo_clean())

        with open(os.path.join(repo.local_dir, "file.txt"), "w+") as f:
            f.write("Test")

        self.assertFalse(repo.is_repo_clean())


class RepositoryDatasetTest(RepositoryCommonTest):
    @classmethod
    def setUpClass(cls):
        """
        Share this valid token in all tests below.
        """
        cls._token = TOKEN
        cls._api.set_access_token(TOKEN)

    def setUp(self):
        self.REPO_NAME = repo_name()
        if os.path.exists(f"{WORKING_DATASET_DIR}/{self.REPO_NAME}"):
            shutil.rmtree(
                f"{WORKING_DATASET_DIR}/{self.REPO_NAME}",
                onerror=set_write_permission_and_retry,
            )
        logger.info(
            f"Does {WORKING_DATASET_DIR}/{self.REPO_NAME} exist:"
            f" {os.path.exists(f'{WORKING_DATASET_DIR}/{self.REPO_NAME}')}"
        )
        self._api.create_repo(
            token=self._token, repo_id=self.REPO_NAME, repo_type="dataset"
        )

    def tearDown(self):
        try:
            self._api.delete_repo(
                repo_id=self.REPO_NAME, token=self._token, repo_type="dataset"
            )
        except requests.exceptions.HTTPError:
            try:
                self._api.delete_repo(
                    repo_id=f"valid_org/{self.REPO_NAME}",
                    token=self._token,
                    repo_type="dataset",
                )
            except requests.exceptions.HTTPError:
                pass

    @retry_endpoint
    def test_clone_with_endpoint(self):
        clone = Repository(
            f"{WORKING_DATASET_DIR}/{self.REPO_NAME}",
            clone_from=f"{ENDPOINT_STAGING}/datasets/{USER}/{self.REPO_NAME}",
            repo_type="dataset",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        with clone.commit("Commit"):
            for file in os.listdir(DATASET_FIXTURE):
                shutil.copyfile(pathlib.Path(DATASET_FIXTURE) / file, file)

        shutil.rmtree(f"{WORKING_DATASET_DIR}/{self.REPO_NAME}")

        Repository(
            f"{WORKING_DATASET_DIR}/{self.REPO_NAME}",
            clone_from=f"{ENDPOINT_STAGING}/datasets/{USER}/{self.REPO_NAME}",
            use_auth_token=self._token,
            repo_type="dataset",
            git_user="ci",
            git_email="ci@dummy.com",
        )

        files = os.listdir(f"{WORKING_DATASET_DIR}/{self.REPO_NAME}")
        self.assertTrue("some_text.txt" in files)
        self.assertTrue("test.py" in files)

    @retry_endpoint
    def test_clone_with_repo_name_and_org(self):
        clone = Repository(
            f"{WORKING_DATASET_DIR}/{self.REPO_NAME}",
            clone_from=f"valid_org/{self.REPO_NAME}",
            repo_type="dataset",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        with clone.commit("Commit"):
            for file in os.listdir(DATASET_FIXTURE):
                shutil.copyfile(pathlib.Path(DATASET_FIXTURE) / file, file)

        shutil.rmtree(f"{WORKING_DATASET_DIR}/{self.REPO_NAME}")

        Repository(
            f"{WORKING_DATASET_DIR}/{self.REPO_NAME}",
            clone_from=f"valid_org/{self.REPO_NAME}",
            use_auth_token=self._token,
            repo_type="dataset",
            git_user="ci",
            git_email="ci@dummy.com",
        )

        files = os.listdir(f"{WORKING_DATASET_DIR}/{self.REPO_NAME}")
        self.assertTrue("some_text.txt" in files)
        self.assertTrue("test.py" in files)

    @retry_endpoint
    def test_clone_with_repo_name_and_user_namespace(self):
        clone = Repository(
            f"{WORKING_DATASET_DIR}/{self.REPO_NAME}",
            clone_from=f"{USER}/{self.REPO_NAME}",
            repo_type="dataset",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        with clone.commit("Commit"):
            for file in os.listdir(DATASET_FIXTURE):
                shutil.copyfile(pathlib.Path(DATASET_FIXTURE) / file, file)

        shutil.rmtree(f"{WORKING_DATASET_DIR}/{self.REPO_NAME}")

        Repository(
            f"{WORKING_DATASET_DIR}/{self.REPO_NAME}",
            clone_from=f"{USER}/{self.REPO_NAME}",
            use_auth_token=self._token,
            repo_type="dataset",
            git_user="ci",
            git_email="ci@dummy.com",
        )

        files = os.listdir(f"{WORKING_DATASET_DIR}/{self.REPO_NAME}")
        self.assertTrue("some_text.txt" in files)
        self.assertTrue("test.py" in files)

    @retry_endpoint
    def test_clone_with_repo_name_and_no_namespace(self):
        self.assertRaises(
            OSError,
            Repository,
            f"{WORKING_DATASET_DIR}/{self.REPO_NAME}",
            clone_from=self.REPO_NAME,
            repo_type="dataset",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

    @retry_endpoint
    def test_clone_with_repo_name_user_and_no_auth_token(self):
        # Create repo
        Repository(
            f"{WORKING_DATASET_DIR}/{self.REPO_NAME}",
            clone_from=f"{USER}/{self.REPO_NAME}",
            repo_type="dataset",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        # Instantiate it without token
        Repository(
            f"{WORKING_DATASET_DIR}/{self.REPO_NAME}",
            clone_from=f"{USER}/{self.REPO_NAME}",
            repo_type="dataset",
            git_user="ci",
            git_email="ci@dummy.com",
        )

    @retry_endpoint
    def test_clone_with_repo_name_org_and_no_auth_token(self):
        # Create repo
        Repository(
            f"{WORKING_DATASET_DIR}/{self.REPO_NAME}",
            clone_from=f"valid_org/{self.REPO_NAME}",
            repo_type="dataset",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        # Instantiate it without token
        Repository(
            f"{WORKING_DATASET_DIR}/{self.REPO_NAME}",
            clone_from=f"valid_org/{self.REPO_NAME}",
            repo_type="dataset",
            git_user="ci",
            git_email="ci@dummy.com",
        )


# tests/test_snapshot_download.py

import os
import shutil
import tempfile
import time
import unittest

import pytest

import requests
from huggingface_hub import HfApi, Repository, snapshot_download
from huggingface_hub.hf_api import HfFolder
from huggingface_hub.utils import logging

from .testing_constants import ENDPOINT_STAGING, TOKEN, USER
from .testing_utils import retry_endpoint, set_write_permission_and_retry


logger = logging.get_logger(__name__)

REPO_NAME = "dummy-hf-hub-{}".format(int(time.time() * 10e3))


class SnapshotDownloadTests(unittest.TestCase):
    _api = HfApi(endpoint=ENDPOINT_STAGING)

    @classmethod
    def setUpClass(cls):
        """
        Share this valid token in all tests below.
        """
        cls._token = TOKEN
        cls._api.set_access_token(TOKEN)

    @retry_endpoint
    def setUp(self) -> None:
        if os.path.exists(REPO_NAME):
            shutil.rmtree(REPO_NAME, onerror=set_write_permission_and_retry)
        logger.info(f"Does {REPO_NAME} exist: {os.path.exists(REPO_NAME)}")
        repo = Repository(
            REPO_NAME,
            clone_from=f"{USER}/{REPO_NAME}",
            use_auth_token=self._token,
            git_user="ci",
            git_email="ci@dummy.com",
        )

        with repo.commit("Add file to main branch"):
            with open("dummy_file.txt", "w+") as f:
                f.write("v1")

        self.first_commit_hash = repo.git_head_hash()

        with repo.commit("Add file to main branch"):
            with open("dummy_file.txt", "w+") as f:
                f.write("v2")
            with open("dummy_file_2.txt", "w+") as f:
                f.write("v3")

        self.second_commit_hash = repo.git_head_hash()

        with repo.commit("Add file to other branch", branch="other"):
            with open("dummy_file_2.txt", "w+") as f:
                f.write("v4")

        self.third_commit_hash = repo.git_head_hash()

    def tearDown(self) -> None:
        self._api.delete_repo(repo_id=REPO_NAME, token=self._token)
        shutil.rmtree(REPO_NAME)

    def test_download_model(self):
        # Test `main` branch
        with tempfile.TemporaryDirectory() as tmpdirname:
            storage_folder = snapshot_download(
                f"{USER}/{REPO_NAME}", revision="main", cache_dir=tmpdirname
            )

            # folder contains the two files contributed and the .gitattributes
            folder_contents = os.listdir(storage_folder)
            self.assertEqual(len(folder_contents), 3)
            self.assertTrue("dummy_file.txt" in folder_contents)
            self.assertTrue("dummy_file_2.txt" in folder_contents)
            self.assertTrue(".gitattributes" in folder_contents)

            with open(os.path.join(storage_folder, "dummy_file.txt"), "r") as f:
                contents = f.read()
                self.assertEqual(contents, "v2")

            # folder name contains the revision's commit sha.
            self.assertTrue(self.second_commit_hash in storage_folder)

        # Test with specific revision
        with tempfile.TemporaryDirectory() as tmpdirname:
            storage_folder = snapshot_download(
                f"{USER}/{REPO_NAME}",
                revision=self.first_commit_hash,
                cache_dir=tmpdirname,
            )

            # folder contains the two files contributed and the .gitattributes
            folder_contents = os.listdir(storage_folder)
            self.assertEqual(len(folder_contents), 2)
            self.assertTrue("dummy_file.txt" in folder_contents)
            self.assertTrue(".gitattributes" in folder_contents)

            with open(os.path.join(storage_folder, "dummy_file.txt"), "r") as f:
                contents = f.read()
                self.assertEqual(contents, "v1")

            # folder name contains the revision's commit sha.
            self.assertTrue(self.first_commit_hash in storage_folder)

    def test_download_private_model(self):
        self._api.update_repo_visibility(
            token=self._token, repo_id=REPO_NAME, private=True
        )

        # Test download fails without token
        with tempfile.TemporaryDirectory() as tmpdirname:
            with self.assertRaisesRegex(
                requests.exceptions.HTTPError, "401 Client Error"
            ):
                _ = snapshot_download(
                    f"{USER}/{REPO_NAME}", revision="main", cache_dir=tmpdirname
                )

        # Test we can download with token from cache
        with tempfile.TemporaryDirectory() as tmpdirname:
            HfFolder.save_token(self._token)
            storage_folder = snapshot_download(
                f"{USER}/{REPO_NAME}",
                revision="main",
                cache_dir=tmpdirname,
                use_auth_token=True,
            )

            # folder contains the two files contributed and the .gitattributes
            folder_contents = os.listdir(storage_folder)
            self.assertEqual(len(folder_contents), 3)
            self.assertTrue("dummy_file.txt" in folder_contents)
            self.assertTrue("dummy_file_2.txt" in folder_contents)
            self.assertTrue(".gitattributes" in folder_contents)

            with open(os.path.join(storage_folder, "dummy_file.txt"), "r") as f:
                contents = f.read()
                self.assertEqual(contents, "v2")

            # folder name contains the revision's commit sha.
            self.assertTrue(self.second_commit_hash in storage_folder)

        # Test we can download with explicit token
        with tempfile.TemporaryDirectory() as tmpdirname:
            storage_folder = snapshot_download(
                f"{USER}/{REPO_NAME}",
                revision="main",
                cache_dir=tmpdirname,
                use_auth_token=self._token,
            )

            # folder contains the two files contributed and the .gitattributes
            folder_contents = os.listdir(storage_folder)
            self.assertEqual(len(folder_contents), 3)
            self.assertTrue("dummy_file.txt" in folder_contents)
            self.assertTrue("dummy_file_2.txt" in folder_contents)
            self.assertTrue(".gitattributes" in folder_contents)

            with open(os.path.join(storage_folder, "dummy_file.txt"), "r") as f:
                contents = f.read()
                self.assertEqual(contents, "v2")

            # folder name contains the revision's commit sha.
            self.assertTrue(self.second_commit_hash in storage_folder)

        self._api.update_repo_visibility(
            token=self._token, repo_id=REPO_NAME, private=False
        )

    def test_download_model_local_only(self):
        # Test no branch specified
        with tempfile.TemporaryDirectory() as tmpdirname:
            # first download folder to cache it
            snapshot_download(f"{USER}/{REPO_NAME}", cache_dir=tmpdirname)

            # now load from cache
            storage_folder = snapshot_download(
                f"{USER}/{REPO_NAME}",
                cache_dir=tmpdirname,
                local_files_only=True,
            )

            # folder contains the two files contributed and the .gitattributes
            folder_contents = os.listdir(storage_folder)
            self.assertEqual(len(folder_contents), 3)
            self.assertTrue("dummy_file.txt" in folder_contents)
            self.assertTrue("dummy_file_2.txt" in folder_contents)
            self.assertTrue(".gitattributes" in folder_contents)

            with open(os.path.join(storage_folder, "dummy_file.txt"), "r") as f:
                contents = f.read()
                self.assertEqual(contents, "v2")

            # folder name contains the revision's commit sha.
            self.assertTrue(self.second_commit_hash in storage_folder)

        # Test with specific revision branch
        with tempfile.TemporaryDirectory() as tmpdirname:
            # first download folder to cache it
            snapshot_download(
                f"{USER}/{REPO_NAME}",
                revision="other",
                cache_dir=tmpdirname,
            )

            # now load from cache
            storage_folder = snapshot_download(
                f"{USER}/{REPO_NAME}",
                revision="other",
                cache_dir=tmpdirname,
                local_files_only=True,
            )

            # folder contains the two files contributed and the .gitattributes
            folder_contents = os.listdir(storage_folder)
            self.assertEqual(len(folder_contents), 3)
            self.assertTrue("dummy_file.txt" in folder_contents)
            self.assertTrue(".gitattributes" in folder_contents)

            with open(os.path.join(storage_folder, "dummy_file.txt"), "r") as f:
                contents = f.read()
                self.assertEqual(contents, "v2")

            # folder name contains the revision's commit sha.
            self.assertTrue(self.third_commit_hash in storage_folder)

        # Test with specific revision hash
        with tempfile.TemporaryDirectory() as tmpdirname:
            # first download folder to cache it
            snapshot_download(
                f"{USER}/{REPO_NAME}",
                revision=self.first_commit_hash,
                cache_dir=tmpdirname,
            )

            # now load from cache
            storage_folder = snapshot_download(
                f"{USER}/{REPO_NAME}",
                revision=self.first_commit_hash,
                cache_dir=tmpdirname,
                local_files_only=True,
            )

            # folder contains the two files contributed and the .gitattributes
            folder_contents = os.listdir(storage_folder)
            self.assertEqual(len(folder_contents), 2)
            self.assertTrue("dummy_file.txt" in folder_contents)
            self.assertTrue(".gitattributes" in folder_contents)

            with open(os.path.join(storage_folder, "dummy_file.txt"), "r") as f:
                contents = f.read()
                self.assertEqual(contents, "v1")

            # folder name contains the revision's commit sha.
            self.assertTrue(self.first_commit_hash in storage_folder)

    def test_download_model_local_only_multiple(self):
        # Test `main` branch
        with tempfile.TemporaryDirectory() as tmpdirname:
            # download both from branch and from commit
            snapshot_download(
                f"{USER}/{REPO_NAME}",
                cache_dir=tmpdirname,
            )

            snapshot_download(
                f"{USER}/{REPO_NAME}",
                revision=self.first_commit_hash,
                cache_dir=tmpdirname,
            )

        # cache multiple commits and make sure correct commit is taken
        with tempfile.TemporaryDirectory() as tmpdirname:
            # first download folder to cache it
            snapshot_download(
                f"{USER}/{REPO_NAME}",
                cache_dir=tmpdirname,
            )

            # now load folder from another branch
            snapshot_download(
                f"{USER}/{REPO_NAME}",
                revision="other",
                cache_dir=tmpdirname,
            )

            # now make sure that loading "main" branch gives correct branch
            storage_folder = snapshot_download(
                f"{USER}/{REPO_NAME}",
                cache_dir=tmpdirname,
                local_files_only=True,
            )

            # folder contains the two files contributed and the .gitattributes
            folder_contents = os.listdir(storage_folder)
            self.assertEqual(len(folder_contents), 3)
            self.assertTrue("dummy_file.txt" in folder_contents)
            self.assertTrue(".gitattributes" in folder_contents)

            with open(os.path.join(storage_folder, "dummy_file.txt"), "r") as f:
                contents = f.read()
                self.assertEqual(contents, "v2")

            # folder name contains the 2nd commit sha and not the 3rd
            self.assertTrue(self.second_commit_hash in storage_folder)

    def check_download_model_with_regex(self, regex, allow=True):
        # Test `main` branch
        allow_regex = regex if allow else None
        ignore_regex = regex if not allow else None

        with tempfile.TemporaryDirectory() as tmpdirname:
            storage_folder = snapshot_download(
                f"{USER}/{REPO_NAME}",
                revision="main",
                cache_dir=tmpdirname,
                allow_regex=allow_regex,
                ignore_regex=ignore_regex,
            )

            # folder contains the two files contributed and the .gitattributes
            folder_contents = os.listdir(storage_folder)
            self.assertEqual(len(folder_contents), 2)
            self.assertTrue("dummy_file.txt" in folder_contents)
            self.assertTrue("dummy_file_2.txt" in folder_contents)
            self.assertTrue(".gitattributes" not in folder_contents)

            with open(os.path.join(storage_folder, "dummy_file.txt"), "r") as f:
                contents = f.read()
                self.assertEqual(contents, "v2")

            # folder name contains the revision's commit sha.
            self.assertTrue(self.second_commit_hash in storage_folder)

    def test_download_model_with_allow_regex(self):
        self.check_download_model_with_regex("*.txt")

    def test_download_model_with_allow_regex_list(self):
        self.check_download_model_with_regex(["dummy_file.txt", "dummy_file_2.txt"])

    def test_download_model_with_ignore_regex(self):
        self.check_download_model_with_regex(".gitattributes", allow=False)

    def test_download_model_with_ignore_regex_list(self):
        self.check_download_model_with_regex(["*.git*", "*.pt"], allow=False)


def test_snapshot_download_import():
    with pytest.warns(FutureWarning, match="has been made private"):
        from huggingface_hub.snapshot_download import snapshot_download as x  # noqa

    assert x is snapshot_download


def test_snapshot_download_import_constant_not_raise():
    from huggingface_hub.snapshot_download import REPO_ID_SEPARATOR  # noqa


# tests/testing_constants.py

USER = "__DUMMY_TRANSFORMERS_USER__"
FULL_NAME = "Dummy User"
PASS = "__DUMMY_TRANSFORMERS_PASS__"

# Not critical, only usable on the sandboxed CI instance.
TOKEN = "hf_94wBhPGp6KrrTH3KDchhKpRxZwd6dmHWLL"

ENDPOINT_PRODUCTION = "https://huggingface.co"
ENDPOINT_STAGING = "https://hub-ci.huggingface.co"
ENDPOINT_STAGING_BASIC_AUTH = f"https://{USER}:{PASS}@hub-ci.huggingface.co"

ENDPOINT_PRODUCTION_URL_SCHEME = (
    ENDPOINT_PRODUCTION + "/{repo_id}/resolve/{revision}/{filename}"
)


# tests/testing_utils.py

import os
import stat
import time
import unittest
from contextlib import contextmanager
from distutils.util import strtobool
from enum import Enum
from unittest.mock import patch

from huggingface_hub.utils import logging
from requests.exceptions import HTTPError
from tests.testing_constants import ENDPOINT_PRODUCTION, ENDPOINT_PRODUCTION_URL_SCHEME


logger = logging.get_logger(__name__)

SMALL_MODEL_IDENTIFIER = "julien-c/bert-xsmall-dummy"
DUMMY_DIFF_TOKENIZER_IDENTIFIER = "julien-c/dummy-diff-tokenizer"
# Example model ids

# An actual model hosted on huggingface.co,
# w/ more details.
DUMMY_MODEL_ID = "julien-c/dummy-unknown"
DUMMY_MODEL_ID_REVISION_ONE_SPECIFIC_COMMIT = "f2c752cfc5c0ab6f4bdec59acea69eefbee381c2"
# One particular commit (not the top of `main`)
DUMMY_MODEL_ID_REVISION_INVALID = "aaaaaaa"
# This commit does not exist, so we should 404.
DUMMY_MODEL_ID_PINNED_SHA1 = "d9e9f15bc825e4b2c9249e9578f884bbcb5e3684"
# Sha-1 of config.json on the top of `main`, for checking purposes
DUMMY_MODEL_ID_PINNED_SHA256 = (
    "4b243c475af8d0a7754e87d7d096c92e5199ec2fe168a2ee7998e3b8e9bcb1d3"
)
# Sha-256 of pytorch_model.bin on the top of `main`, for checking purposes


SAMPLE_DATASET_IDENTIFIER = "lhoestq/custom_squad"
# Example dataset ids
DUMMY_DATASET_ID = "lhoestq/test"
DUMMY_DATASET_ID_REVISION_ONE_SPECIFIC_COMMIT = (  # on branch "test-branch"
    "81d06f998585f8ee10e6e3a2ea47203dc75f2a16"
)


def parse_flag_from_env(key, default=False):
    try:
        value = os.environ[key]
    except KeyError:
        # KEY isn't set, default to `default`.
        _value = default
    else:
        # KEY is set, convert it to True or False.
        try:
            _value = strtobool(value)
        except ValueError:
            # More values are supported, but let's keep the message simple.
            raise ValueError("If set, {} must be yes or no.".format(key))
    return _value


def parse_int_from_env(key, default=None):
    try:
        value = os.environ[key]
    except KeyError:
        _value = default
    else:
        try:
            _value = int(value)
        except ValueError:
            raise ValueError("If set, {} must be a int.".format(key))
    return _value


_run_git_lfs_tests = parse_flag_from_env("RUN_GIT_LFS_TESTS", default=False)


def require_git_lfs(test_case):
    """
    Decorator marking a test that requires git-lfs.

    git-lfs requires additional dependencies, and tests are skipped by default. Set the RUN_GIT_LFS_TESTS environment
    variable to a truthy value to run them.
    """
    if not _run_git_lfs_tests:
        return unittest.skip("test of git lfs workflow")(test_case)
    else:
        return test_case


class RequestWouldHangIndefinitelyError(Exception):
    pass


class OfflineSimulationMode(Enum):
    CONNECTION_FAILS = 0
    CONNECTION_TIMES_OUT = 1
    HF_HUB_OFFLINE_SET_TO_1 = 2


@contextmanager
def offline(mode=OfflineSimulationMode.CONNECTION_FAILS, timeout=1e-16):
    """
    Simulate offline mode.

    There are three offline simulatiom modes:

    CONNECTION_FAILS (default mode): a ConnectionError is raised for each network call.
        Connection errors are created by mocking socket.socket
    CONNECTION_TIMES_OUT: the connection hangs until it times out.
        The default timeout value is low (1e-16) to speed up the tests.
        Timeout errors are created by mocking requests.request
    HF_HUB_OFFLINE_SET_TO_1: the HF_HUB_OFFLINE_SET_TO_1 environment variable is set to 1.
        This makes the http/ftp calls of the library instantly fail and raise an OfflineModeEmabled error.
    """
    import socket

    from requests import request as online_request

    def timeout_request(method, url, **kwargs):
        # Change the url to an invalid url so that the connection hangs
        invalid_url = "https://10.255.255.1"
        if kwargs.get("timeout") is None:
            raise RequestWouldHangIndefinitelyError(
                f"Tried a call to {url} in offline mode with no timeout set. Please set"
                " a timeout."
            )
        kwargs["timeout"] = timeout
        try:
            return online_request(method, invalid_url, **kwargs)
        except Exception as e:
            # The following changes in the error are just here to make the offline timeout error prettier
            e.request.url = url
            max_retry_error = e.args[0]
            max_retry_error.args = (
                max_retry_error.args[0].replace("10.255.255.1", f"OfflineMock[{url}]"),
            )
            e.args = (max_retry_error,)
            raise

    def offline_socket(*args, **kwargs):
        raise socket.error("Offline mode is enabled.")

    if mode is OfflineSimulationMode.CONNECTION_FAILS:
        # inspired from https://stackoverflow.com/a/18601897
        with patch("socket.socket", offline_socket):
            yield
    elif mode is OfflineSimulationMode.CONNECTION_TIMES_OUT:
        # inspired from https://stackoverflow.com/a/904609
        with patch("requests.request", timeout_request):
            yield
    elif mode is OfflineSimulationMode.HF_HUB_OFFLINE_SET_TO_1:
        with patch("huggingface_hub.constants.HF_HUB_OFFLINE", True):
            yield
    else:
        raise ValueError("Please use a value from the OfflineSimulationMode enum.")


def set_write_permission_and_retry(func, path, excinfo):
    os.chmod(path, stat.S_IWRITE)
    func(path)


def with_production_testing(func):
    file_download = patch(
        "huggingface_hub.file_download.HUGGINGFACE_CO_URL_TEMPLATE",
        ENDPOINT_PRODUCTION_URL_SCHEME,
    )

    hf_api = patch(
        "huggingface_hub.hf_api.ENDPOINT",
        ENDPOINT_PRODUCTION,
    )

    return hf_api(file_download(func))


def retry_endpoint(function, number_of_tries: int = 3, wait_time: int = 5):
    """
    Retries test if failure, waiting `wait_time`.
    Should be added to any test hitting the `moon-staging` endpoint that is
    downloading Repositories or uploading data

    Args:
        number_of_tries: Number of tries to attempt a passing test
        wait_time: Time to wait in-between attempts in seconds
    """

    def decorator(*args, **kwargs):
        retry_count = 1
        while retry_count < number_of_tries:
            try:
                return function(*args, **kwargs)
            except HTTPError as e:
                if e.response.status_code == 504:
                    logger.info(
                        f"Attempt {retry_count} failed with a 504 error. Retrying new"
                        f" execution in {wait_time} second(s)..."
                    )
                    time.sleep(5)
                    retry_count += 1
            except OSError:
                logger.info(
                    "Race condition met where we tried to `clone` before fully"
                    " deleting a repository. Retrying new execution in"
                    f" {wait_time} second(s)..."
                )
                retry_count += 1
            # Preserve original traceback
            return function(*args, **kwargs)

    return decorator


# setup.py

from setuptools import find_packages, setup


def get_version() -> str:
    rel_path = "src/huggingface_hub/__init__.py"
    with open(rel_path, "r") as fp:
        for line in fp.read().splitlines():
            if line.startswith("__version__"):
                delim = '"' if '"' in line else "'"
                return line.split(delim)[1]
    raise RuntimeError("Unable to find version string.")


install_requires = [
    "filelock",
    "requests",
    "tqdm",
    "pyyaml>=5.1",
    "typing-extensions>=3.7.4.3",  # to be able to import TypeAlias
    "importlib_metadata;python_version<'3.8'",
    "packaging>=20.9",
]

extras = {}

extras["torch"] = [
    "torch",
]

extras["fastai"] = [
    "toml",
    "fastai>=2.4",
    "fastcore>=1.3.27",
]

extras["tensorflow"] = ["tensorflow", "pydot", "graphviz"]

extras["testing"] = [
    "pytest",
    "pytest-cov",
    "datasets",
    "soundfile",
]

extras["quality"] = [
    "black~=22.0",
    "isort>=5.5.4",
    "flake8>=3.8.3",
]

extras["all"] = extras["testing"] + extras["quality"]

extras["dev"] = extras["all"]


setup(
    name="huggingface_hub",
    version=get_version(),
    author="Hugging Face, Inc.",
    author_email="julien@huggingface.co",
    description=(
        "Client library to download and publish models, datasets and other repos on the"
        " huggingface.co hub"
    ),
    long_description=open("README.md", "r", encoding="utf-8").read(),
    long_description_content_type="text/markdown",
    keywords=(
        "model-hub machine-learning models natural-language-processing deep-learning"
        " pytorch pretrained-models"
    ),
    license="Apache",
    url="https://github.com/huggingface/huggingface_hub",
    package_dir={"": "src"},
    packages=find_packages("src"),
    extras_require=extras,
    entry_points={
        "console_scripts": [
            "huggingface-cli=huggingface_hub.commands.huggingface_cli:main"
        ]
    },
    python_requires=">=3.7.0",
    install_requires=install_requires,
    classifiers=[
        "Intended Audience :: Developers",
        "Intended Audience :: Education",
        "Intended Audience :: Science/Research",
        "License :: OSI Approved :: Apache Software License",
        "Operating System :: OS Independent",
        "Programming Language :: Python :: 3",
        "Topic :: Scientific/Engineering :: Artificial Intelligence",
    ],
)


# src/huggingface_hub/commands/__init__.py

# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from abc import ABC, abstractmethod
from argparse import ArgumentParser


class BaseHuggingfaceCLICommand(ABC):
    @staticmethod
    @abstractmethod
    def register_subcommand(parser: ArgumentParser):
        raise NotImplementedError()

    @abstractmethod
    def run(self):
        raise NotImplementedError()


# src/huggingface_hub/commands/huggingface_cli.py

#!/usr/bin/env python
# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from argparse import ArgumentParser

from huggingface_hub.commands.lfs import LfsCommands
from huggingface_hub.commands.user import UserCommands


def main():
    parser = ArgumentParser(
        "huggingface-cli", usage="huggingface-cli <command> [<args>]"
    )
    commands_parser = parser.add_subparsers(help="huggingface-cli command helpers")

    # Register commands
    UserCommands.register_subcommand(commands_parser)
    LfsCommands.register_subcommand(commands_parser)

    # Let's go
    args = parser.parse_args()

    if not hasattr(args, "func"):
        parser.print_help()
        exit(1)

    # Run
    service = args.func(args)
    service.run()


if __name__ == "__main__":
    main()


# src/huggingface_hub/commands/lfs.py

"""
Implementation of a custom transfer agent for the transfer type "multipart" for
git-lfs.

Inspired by:
github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py

Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md


To launch debugger while developing:

``` [lfs "customtransfer.multipart"]
path = /path/to/huggingface_hub/.env/bin/python args = -m debugpy --listen 5678
--wait-for-client
/path/to/huggingface_hub/src/huggingface_hub/commands/huggingface_cli.py
lfs-multipart-upload ```"""

import json
import os
import subprocess
import sys
from argparse import ArgumentParser
from contextlib import AbstractContextManager
from typing import Dict, List, Optional

import requests
from huggingface_hub.commands import BaseHuggingfaceCLICommand
from huggingface_hub.lfs import LFS_MULTIPART_UPLOAD_COMMAND

from ..utils import logging
from ..utils._errors import _raise_with_request_id


logger = logging.get_logger(__name__)


class LfsCommands(BaseHuggingfaceCLICommand):
    """
    Implementation of a custom transfer agent for the transfer type "multipart"
    for git-lfs. This lets users upload large files >5GB 🔥. Spec for LFS custom
    transfer agent is:
    https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md

    This introduces two commands to the CLI:

    1. $ huggingface-cli lfs-enable-largefiles

    This should be executed once for each model repo that contains a model file
    >5GB. It's documented in the error message you get if you just try to git
    push a 5GB file without having enabled it before.

    2. $ huggingface-cli lfs-multipart-upload

    This command is called by lfs directly and is not meant to be called by the
    user.
    """

    @staticmethod
    def register_subcommand(parser: ArgumentParser):
        enable_parser = parser.add_parser(
            "lfs-enable-largefiles",
            help="Configure your repository to enable upload of files > 5GB.",
        )
        enable_parser.add_argument(
            "path", type=str, help="Local path to repository you want to configure."
        )
        enable_parser.set_defaults(func=lambda args: LfsEnableCommand(args))

        upload_parser = parser.add_parser(
            LFS_MULTIPART_UPLOAD_COMMAND,
            help="Command will get called by git-lfs, do not call it directly.",
        )
        upload_parser.set_defaults(func=lambda args: LfsUploadCommand(args))


class LfsEnableCommand:
    def __init__(self, args):
        self.args = args

    def run(self):
        local_path = os.path.abspath(self.args.path)
        if not os.path.isdir(local_path):
            print("This does not look like a valid git repo.")
            exit(1)
        subprocess.run(
            "git config lfs.customtransfer.multipart.path huggingface-cli".split(),
            check=True,
            cwd=local_path,
        )
        subprocess.run(
            "git config lfs.customtransfer.multipart.args"
            f" {LFS_MULTIPART_UPLOAD_COMMAND}".split(),
            check=True,
            cwd=local_path,
        )
        print("Local repo set up for largefiles")


def write_msg(msg: Dict):
    """Write out the message in Line delimited JSON."""
    msg = json.dumps(msg) + "\n"
    sys.stdout.write(msg)
    sys.stdout.flush()


def read_msg() -> Optional[Dict]:
    """Read Line delimited JSON from stdin."""
    msg = json.loads(sys.stdin.readline().strip())

    if "terminate" in (msg.get("type"), msg.get("event")):
        # terminate message received
        return None

    if msg.get("event") not in ("download", "upload"):
        logger.critical("Received unexpected message")
        sys.exit(1)

    return msg


class FileSlice(AbstractContextManager):
    """
    File-like object that only reads a slice of a file

    Inspired by stackoverflow.com/a/29838711/593036
    """

    def __init__(self, filepath: str, seek_from: int, read_limit: int):
        self.filepath = filepath
        self.seek_from = seek_from
        self.read_limit = read_limit
        self.n_seen = 0

    def __enter__(self):
        self.f = open(self.filepath, "rb")
        self.f.seek(self.seek_from)
        return self

    def __len__(self):
        total_length = os.fstat(self.f.fileno()).st_size
        return min(self.read_limit, total_length - self.seek_from)

    def read(self, n=-1):
        if self.n_seen >= self.read_limit:
            return b""
        remaining_amount = self.read_limit - self.n_seen
        data = self.f.read(remaining_amount if n < 0 else min(n, remaining_amount))
        self.n_seen += len(data)
        return data

    def __iter__(self):
        yield self.read(n=4 * 1024 * 1024)

    def __exit__(self, *args):
        self.f.close()


class LfsUploadCommand:
    def __init__(self, args):
        self.args = args

    def run(self):
        # Immediately after invoking a custom transfer process, git-lfs
        # sends initiation data to the process over stdin.
        # This tells the process useful information about the configuration.
        init_msg = json.loads(sys.stdin.readline().strip())
        if not (
            init_msg.get("event") == "init" and init_msg.get("operation") == "upload"
        ):
            write_msg({"error": {"code": 32, "message": "Wrong lfs init operation"}})
            sys.exit(1)

        # The transfer process should use the information it needs from the
        # initiation structure, and also perform any one-off setup tasks it
        # needs to do. It should then respond on stdout with a simple empty
        # confirmation structure, as follows:
        write_msg({})

        # After the initiation exchange, git-lfs will send any number of
        # transfer requests to the stdin of the transfer process, in a serial sequence.
        while True:
            msg = read_msg()
            if msg is None:
                # When all transfers have been processed, git-lfs will send
                # a terminate event to the stdin of the transfer process.
                # On receiving this message the transfer process should
                # clean up and terminate. No response is expected.
                sys.exit(0)

            oid = msg["oid"]
            filepath = msg["path"]
            completion_url = msg["action"]["href"]
            header = msg["action"]["header"]
            chunk_size = int(header.pop("chunk_size"))
            presigned_urls: List[str] = list(header.values())

            # Send a "started" progress event to allow other workers to start.
            # Otherwise they're delayed until first "progress" event is reported,
            # i.e. after the first 5GB by default (!)
            write_msg(
                {
                    "event": "progress",
                    "oid": oid,
                    "bytesSoFar": 1,
                    "bytesSinceLast": 0,
                }
            )

            parts = []
            for i, presigned_url in enumerate(presigned_urls):
                with FileSlice(
                    filepath, seek_from=i * chunk_size, read_limit=chunk_size
                ) as data:
                    r = requests.put(presigned_url, data=data)
                    _raise_with_request_id(r)
                    parts.append(
                        {
                            "etag": r.headers.get("etag"),
                            "partNumber": i + 1,
                        }
                    )
                    # In order to support progress reporting while data is uploading / downloading,
                    # the transfer process should post messages to stdout
                    write_msg(
                        {
                            "event": "progress",
                            "oid": oid,
                            "bytesSoFar": (i + 1) * chunk_size,
                            "bytesSinceLast": chunk_size,
                        }
                    )
                    # Not precise but that's ok.

            r = requests.post(
                completion_url,
                json={
                    "oid": oid,
                    "parts": parts,
                },
            )
            _raise_with_request_id(r)

            write_msg({"event": "complete", "oid": oid})


# src/huggingface_hub/commands/user.py

# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import subprocess
from argparse import ArgumentParser
from getpass import getpass
from typing import List, Union

from huggingface_hub.commands import BaseHuggingfaceCLICommand
from huggingface_hub.constants import (
    REPO_TYPES,
    REPO_TYPES_URL_PREFIXES,
    SPACES_SDK_TYPES,
)
from huggingface_hub.hf_api import HfApi, HfFolder
from requests.exceptions import HTTPError

from ..utils import run_subprocess


class UserCommands(BaseHuggingfaceCLICommand):
    @staticmethod
    def register_subcommand(parser: ArgumentParser):
        login_parser = parser.add_parser(
            "login", help="Log in using the same credentials as on huggingface.co"
        )
        login_parser.set_defaults(func=lambda args: LoginCommand(args))
        whoami_parser = parser.add_parser(
            "whoami", help="Find out which huggingface.co account you are logged in as."
        )
        whoami_parser.set_defaults(func=lambda args: WhoamiCommand(args))
        logout_parser = parser.add_parser("logout", help="Log out")
        logout_parser.set_defaults(func=lambda args: LogoutCommand(args))

        # new system: git-based repo system
        repo_parser = parser.add_parser(
            "repo",
            help=(
                "{create, ls-files} Commands to interact with your huggingface.co"
                " repos."
            ),
        )
        repo_subparsers = repo_parser.add_subparsers(
            help="huggingface.co repos related commands"
        )
        repo_create_parser = repo_subparsers.add_parser(
            "create", help="Create a new repo on huggingface.co"
        )
        repo_create_parser.add_argument(
            "name",
            type=str,
            help=(
                "Name for your repo. Will be namespaced under your username to build"
                " the repo id."
            ),
        )
        repo_create_parser.add_argument(
            "--type",
            type=str,
            help=(
                'Optional: repo_type: set to "dataset" or "space" if creating a dataset'
                " or space, default is model."
            ),
        )
        repo_create_parser.add_argument(
            "--organization", type=str, help="Optional: organization namespace."
        )
        repo_create_parser.add_argument(
            "--space_sdk",
            type=str,
            help=(
                "Optional: Hugging Face Spaces SDK type. Required when --type is set to"
                ' "space".'
            ),
            choices=SPACES_SDK_TYPES,
        )
        repo_create_parser.add_argument(
            "-y",
            "--yes",
            action="store_true",
            help="Optional: answer Yes to the prompt",
        )
        repo_create_parser.set_defaults(func=lambda args: RepoCreateCommand(args))


class ANSI:
    """
    Helper for en.wikipedia.org/wiki/ANSI_escape_code
    """

    _bold = "\u001b[1m"
    _red = "\u001b[31m"
    _gray = "\u001b[90m"
    _reset = "\u001b[0m"

    @classmethod
    def bold(cls, s):
        return f"{cls._bold}{s}{cls._reset}"

    @classmethod
    def red(cls, s):
        return f"{cls._bold + cls._red}{s}{cls._reset}"

    @classmethod
    def gray(cls, s):
        return f"{cls._gray}{s}{cls._reset}"


def tabulate(rows: List[List[Union[str, int]]], headers: List[str]) -> str:
    """
    Inspired by:

    - stackoverflow.com/a/8356620/593036
    - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data
    """
    col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)]
    row_format = ("{{:{}}} " * len(headers)).format(*col_widths)
    lines = []
    lines.append(row_format.format(*headers))
    lines.append(row_format.format(*["-" * w for w in col_widths]))
    for row in rows:
        lines.append(row_format.format(*row))
    return "\n".join(lines)


def currently_setup_credential_helpers(directory=None) -> List[str]:
    try:
        output = run_subprocess(
            "git config --list".split(),
            directory,
        ).stdout.split("\n")

        current_credential_helpers = []
        for line in output:
            if "credential.helper" in line:
                current_credential_helpers.append(line.split("=")[-1])
    except subprocess.CalledProcessError as exc:
        raise EnvironmentError(exc.stderr)

    return current_credential_helpers


class BaseUserCommand:
    def __init__(self, args):
        self.args = args
        self._api = HfApi()


class LoginCommand(BaseUserCommand):
    def run(self):
        print(  # docstyle-ignore
            """
        _|    _|  _|    _|    _|_|_|    _|_|_|  _|_|_|  _|      _|    _|_|_|      _|_|_|_|    _|_|      _|_|_|  _|_|_|_|
        _|    _|  _|    _|  _|        _|          _|    _|_|    _|  _|            _|        _|    _|  _|        _|
        _|_|_|_|  _|    _|  _|  _|_|  _|  _|_|    _|    _|  _|  _|  _|  _|_|      _|_|_|    _|_|_|_|  _|        _|_|_|
        _|    _|  _|    _|  _|    _|  _|    _|    _|    _|    _|_|  _|    _|      _|        _|    _|  _|        _|
        _|    _|    _|_|      _|_|_|    _|_|_|  _|_|_|  _|      _|    _|_|_|      _|        _|    _|    _|_|_|  _|_|_|_|

        To login, `huggingface_hub` now requires a token generated from https://huggingface.co/settings/tokens .
        """
        )

        token = getpass("Token: ")
        _login(self._api, token=token)


class WhoamiCommand(BaseUserCommand):
    def run(self):
        token = HfFolder.get_token()
        if token is None:
            print("Not logged in")
            exit()
        try:
            info = self._api.whoami(token)
            print(info["name"])
            orgs = [org["name"] for org in info["orgs"]]
            if orgs:
                print(ANSI.bold("orgs: "), ",".join(orgs))
        except HTTPError as e:
            print(e)
            print(ANSI.red(e.response.text))
            exit(1)


class LogoutCommand(BaseUserCommand):
    def run(self):
        token = HfFolder.get_token()
        if token is None:
            print("Not logged in")
            exit()
        HfFolder.delete_token()
        HfApi.unset_access_token()
        print("Successfully logged out.")


class RepoCreateCommand(BaseUserCommand):
    def run(self):
        token = HfFolder.get_token()
        if token is None:
            print("Not logged in")
            exit(1)
        try:
            stdout = subprocess.check_output(["git", "--version"]).decode("utf-8")
            print(ANSI.gray(stdout.strip()))
        except FileNotFoundError:
            print("Looks like you do not have git installed, please install.")

        try:
            stdout = subprocess.check_output(["git-lfs", "--version"]).decode("utf-8")
            print(ANSI.gray(stdout.strip()))
        except FileNotFoundError:
            print(
                ANSI.red(
                    "Looks like you do not have git-lfs installed, please install."
                    " You can install from https://git-lfs.github.com/."
                    " Then run `git lfs install` (you only have to do this once)."
                )
            )
        print("")

        user = self._api.whoami(token)["name"]
        namespace = (
            self.args.organization if self.args.organization is not None else user
        )

        repo_id = f"{namespace}/{self.args.name}"

        if self.args.type not in REPO_TYPES:
            print("Invalid repo --type")
            exit(1)

        if self.args.type in REPO_TYPES_URL_PREFIXES:
            prefixed_repo_id = REPO_TYPES_URL_PREFIXES[self.args.type] + repo_id
        else:
            prefixed_repo_id = repo_id

        print(f"You are about to create {ANSI.bold(prefixed_repo_id)}")

        if not self.args.yes:
            choice = input("Proceed? [Y/n] ").lower()
            if not (choice == "" or choice == "y" or choice == "yes"):
                print("Abort")
                exit()
        try:

            url = self._api.create_repo(
                repo_id=repo_id,
                token=token,
                repo_type=self.args.type,
                space_sdk=self.args.space_sdk,
            )
        except HTTPError as e:
            print(e)
            print(ANSI.red(e.response.text))
            exit(1)
        print("\nYour repo now lives at:")
        print(f"  {ANSI.bold(url)}")
        print(
            "\nYou can clone it locally with the command below,"
            " and commit/push as usual."
        )
        print(f"\n  git clone {url}")
        print("")


NOTEBOOK_LOGIN_PASSWORD_HTML = """<center> <img
src=https://huggingface.co/front/assets/huggingface_logo-noborder.svg
alt='Hugging Face'> <br> Immediately click login after typing your password or
it might be stored in plain text in this notebook file. </center>"""


NOTEBOOK_LOGIN_TOKEN_HTML_START = """<center> <img
src=https://huggingface.co/front/assets/huggingface_logo-noborder.svg
alt='Hugging Face'> <br> Copy a token from <a
href="https://huggingface.co/settings/tokens" target="_blank">your Hugging Face
tokens page</a> and paste it below. <br> Immediately click login after copying
your token or it might be stored in plain text in this notebook file. </center>"""


NOTEBOOK_LOGIN_TOKEN_HTML_END = """
<b>Pro Tip:</b> If you don't already have one, you can create a dedicated
'notebooks' token with 'write' access, that you can then easily reuse for all
notebooks. </center>"""


def notebook_login():
    """
    Displays a widget to login to the HF website and store the token.
    """
    try:
        import ipywidgets.widgets as widgets
        from IPython.display import clear_output, display
    except ImportError:
        raise ImportError(
            "The `notebook_login` function can only be used in a notebook (Jupyter or"
            " Colab) and you need the `ipywidgets` module: `pip install ipywidgets`."
        )

    box_layout = widgets.Layout(
        display="flex", flex_flow="column", align_items="center", width="50%"
    )

    token_widget = widgets.Password(description="Token:")
    token_finish_button = widgets.Button(description="Login")

    login_token_widget = widgets.VBox(
        [
            widgets.HTML(NOTEBOOK_LOGIN_TOKEN_HTML_START),
            token_widget,
            token_finish_button,
            widgets.HTML(NOTEBOOK_LOGIN_TOKEN_HTML_END),
        ],
        layout=box_layout,
    )
    display(login_token_widget)

    # On click events
    def login_token_event(t):
        token = token_widget.value
        # Erase token and clear value to make sure it's not saved in the notebook.
        token_widget.value = ""
        clear_output()
        _login(HfApi(), token=token)

    token_finish_button.on_click(login_token_event)


def _login(hf_api, token=None):
    token, name = hf_api._validate_or_retrieve_token(token)
    hf_api.set_access_token(token)
    HfFolder.save_token(token)
    print("Login successful")
    print("Your token has been saved to", HfFolder.path_token)
    helpers = currently_setup_credential_helpers()

    if "store" not in helpers:
        print(
            ANSI.red(
                "Authenticated through git-credential store but this isn't the helper"
                " defined on your machine.\nYou might have to re-authenticate when"
                " pushing to the Hugging Face Hub. Run the following command in your"
                " terminal in case you want to set this credential helper as the"
                " default\n\ngit config --global credential.helper store"
            )
        )


# src/huggingface_hub/utils/__init__.py

# flake8: noqa
#!/usr/bin/env python
# coding=utf-8
# Copyright 2021 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License

from ._errors import EntryNotFoundError, RepositoryNotFoundError, RevisionNotFoundError
from ._subprocess import run_subprocess


# src/huggingface_hub/utils/_deprecation.py

import warnings
from functools import wraps
from inspect import Parameter, signature


def _deprecate_positional_args(func=None, *, version="0.8"):
    """Decorator for methods that issues warnings for positional arguments.
    Using the keyword-only argument syntax in pep 3102, arguments after the
    * will issue a warning when passed as a positional argument.

    Args:
        func (``Callable``, `optional`):
            Function to check arguments on.
        version (``Callable``, `optional`, defaults to ``"0.8"``):
            The version when positional arguments will result in error.
    """

    def _inner_deprecate_positional_args(f):
        sig = signature(f)
        kwonly_args = []
        all_args = []
        for name, param in sig.parameters.items():
            if param.kind == Parameter.POSITIONAL_OR_KEYWORD:
                all_args.append(name)
            elif param.kind == Parameter.KEYWORD_ONLY:
                kwonly_args.append(name)

        @wraps(f)
        def inner_f(*args, **kwargs):
            extra_args = len(args) - len(all_args)
            if extra_args <= 0:
                return f(*args, **kwargs)
            # extra_args > 0
            args_msg = [
                f"{name}='{arg}'" if isinstance(arg, str) else f"{name}={arg}"
                for name, arg in zip(kwonly_args[:extra_args], args[-extra_args:])
            ]
            args_msg = ", ".join(args_msg)
            warnings.warn(
                f"Pass {args_msg} as keyword args. From version "
                f"{version} passing these as positional arguments "
                "will result in an error",
                FutureWarning,
            )
            kwargs.update(zip(sig.parameters, args))
            return f(**kwargs)

        return inner_f

    if func is not None:
        return _inner_deprecate_positional_args(func)
    return _inner_deprecate_positional_args


# src/huggingface_hub/utils/_errors.py

from requests import HTTPError


class RepositoryNotFoundError(HTTPError):
    """
    Raised when trying to access a hf.co URL with an invalid repository name, or
    with a private repo name the user does not have access to.

    Example:

    ```py
    >>> from huggingface_hub import model_info
    >>> model_info("<non_existant_repository>")
    huggingface_hub.utils._errors.RepositoryNotFoundError: 404 Client Error: Repository Not Found for url: <url>
    ```
    """


class RevisionNotFoundError(HTTPError):
    """
    Raised when trying to access a hf.co URL with a valid repository but an invalid
    revision.

    Example:

    ```py
    >>> from huggingface_hub import hf_hub_download
    >>> hf_hub_download('bert-base-cased', 'config.json', revision='<non-existant-revision>')
    huggingface_hub.utils._errors.RevisionNotFoundError: 404 Client Error: Revision Not Found for url: <url>
    ```

    """


class EntryNotFoundError(HTTPError):
    """
    Raised when trying to access a hf.co URL with a valid repository and revision
    but an invalid filename.

    Example:

    ```py
    >>> from huggingface_hub import hf_hub_download
    >>> hf_hub_download('bert-base-cased', '<non-existant-file>')
    huggingface_hub.utils._errors.EntryNotFoundError: 404 Client Error: Entry Not Found for url: <url>
    ```
    """


def _raise_for_status(request):
    """
    Internal version of `request.raise_for_status()` that will refine a
    potential HTTPError.
    """
    request_id = request.headers.get("X-Request-Id")

    if "X-Error-Code" in request.headers:
        error_code = request.headers["X-Error-Code"]
        if error_code == "RepoNotFound":
            raise RepositoryNotFoundError(
                f"404 Client Error: Repository Not Found for url: {request.url}. If the"
                " repo is private, make sure you are authenticated. (Request ID:"
                f" {request_id})"
            )
        elif error_code == "RevisionNotFound":
            raise RevisionNotFoundError(
                f"404 Client Error: Revision Not Found for url: {request.url}. (Request"
                f" ID: {request_id})"
            )
        elif error_code == "EntryNotFound":
            raise EntryNotFoundError(
                f"404 Client Error: Entry Not Found for url: {request.url}. (Request"
                f" ID: {request_id})"
            )

    if request.status_code == 401:
        # The repo was not found and the user is not Authenticated
        raise RepositoryNotFoundError(
            f"401 Client Error: Repository Not Found for url: {request.url}. If the"
            " repo is private, make sure you are authenticated. (Request ID:"
            f" {request_id})"
        )

    _raise_with_request_id(request)


def _raise_with_request_id(request):
    request_id = request.headers.get("X-Request-Id")
    try:
        request.raise_for_status()
    except Exception as e:
        if request_id is not None and len(e.args) > 0 and isinstance(e.args[0], str):
            e.args = (e.args[0] + f" (Request ID: {request_id})",) + e.args[1:]

        raise e


# src/huggingface_hub/utils/_fixes.py

# JSONDecodeError was introduced in requests=2.27 released in 2022.
# This allows us to support older requests for users
# More information: https://github.com/psf/requests/pull/5856
try:
    from requests import JSONDecodeError  # noqa
except ImportError:
    try:
        from simplejson import JSONDecodeError  # noqa
    except ImportError:
        from json import JSONDecodeError  # noqa


# src/huggingface_hub/utils/_subprocess.py

#!/usr/bin/env python
# coding=utf-8
# Copyright 2021 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License

import subprocess
from typing import List

from .logging import get_logger


logger = get_logger(__name__)


def run_subprocess(
    command: List[str], folder: str, check=True, **kwargs
) -> subprocess.CompletedProcess:
    """
    Method to run subprocesses. Calling this will capture the `stderr` and `stdout`,
    please call `subprocess.run` manually in case you would like for them not to
    be captured.

    Args:
        command (`List[str]`):
            The command to execute as a list of strings.
        folder (`str`):
            The folder in which to run the command.
        check (`bool`, *optional*, defaults to `True`):
            Setting `check` to `True` will raise a `subprocess.CalledProcessError`
            when the subprocess has a non-zero exit code.
        kwargs (`Dict[str]`):
            Keyword arguments to be passed to the `subprocess.run` underlying command.

    Returns:
        `subprocess.CompletedProcess`: The completed process.
    """
    if isinstance(command, str):
        raise ValueError("`run_subprocess` should be called with a list of strings.")

    return subprocess.run(
        command,
        stderr=subprocess.PIPE,
        stdout=subprocess.PIPE,
        check=check,
        encoding="utf-8",
        cwd=folder,
        **kwargs,
    )


# src/huggingface_hub/utils/endpoint_helpers.py

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Helpful utility functions and classes in relation to exploring API endpoints
with the aim for a user-friendly interface
"""

import math
import re
from dataclasses import dataclass
from typing import List, Union


def _filter_emissions(
    models,
    minimum_threshold: float = None,
    maximum_threshold: float = None,
):
    """Filters a list of models for those that include an emission tag
    and limit them to between two thresholds

    Args:
        models (`ModelInfo` or `List`):
            A list of `ModelInfo`'s to filter by.
        minimum_threshold (`float`, *optional*):
            A minimum carbon threshold to filter by, such as 1.
        maximum_threshold (`float`, *optional*):
            A maximum carbon threshold to filter by, such as 10.
    """
    if minimum_threshold is None and maximum_threshold is None:
        raise ValueError(
            "Both `minimum_threshold` and `maximum_threshold` cannot both be `None`"
        )
    if minimum_threshold is None:
        minimum_threshold = -1
    if maximum_threshold is None:
        maximum_threshold = math.inf
    emissions = []
    for i, model in enumerate(models):
        if hasattr(model, "cardData"):
            if isinstance(model.cardData, dict):
                emission = model.cardData.get("co2_eq_emissions", None)
                if isinstance(emission, dict):
                    emission = emission["emissions"]
                if emission:
                    emission = str(emission)
                    if any(char.isdigit() for char in emission):
                        emission = re.search("\d+\.\d+|\d+", emission).group(0)
                        emissions.append((i, float(emission)))
    filtered_results = []
    for idx, emission in emissions:
        if emission >= minimum_threshold and emission <= maximum_threshold:
            filtered_results.append(models[idx])
    return filtered_results


@dataclass
class DatasetFilter:
    """
    A class that converts human-readable dataset search parameters into ones
    compatible with the REST API. For all parameters capitalization does not
    matter.

    Args:
        author (`str`, *optional*):
            A string or list of strings that can be used to identify datasets on
            the Hub by the original uploader (author or organization), such as
            `facebook` or `huggingface`.
        benchmark (`str` or `List`, *optional*):
            A string or list of strings that can be used to identify datasets on
            the Hub by their official benchmark.
        dataset_name (`str`, *optional*):
            A string or list of strings that can be used to identify datasets on
            the Hub by its name, such as `SQAC` or `wikineural`
        language_creators (`str` or `List`, *optional*):
            A string or list of strings that can be used to identify datasets on
            the Hub with how the data was curated, such as `crowdsourced` or
            `machine_generated`.
        languages (`str` or `List`, *optional*):
            A string or list of strings representing a two-character language to
            filter datasets by on the Hub.
        multilinguality (`str` or `List`, *optional*):
            A string or list of strings representing a filter for datasets that
            contain multiple languages.
        size_categories (`str` or `List`, *optional*):
            A string or list of strings that can be used to identify datasets on
            the Hub by the size of the dataset such as `100K<n<1M` or
            `1M<n<10M`.
        task_categories (`str` or `List`, *optional*):
            A string or list of strings that can be used to identify datasets on
            the Hub by the designed task, such as `audio_classification` or
            `named_entity_recognition`.
        task_ids (`str` or `List`, *optional*):
            A string or list of strings that can be used to identify datasets on
            the Hub by the specific task such as `speech_emotion_recognition` or
            `paraphrase`.

    Examples:

    ```py
    >>> from huggingface_hub import DatasetFilter

    >>> # Using author
    >>> new_filter = DatasetFilter(author="facebook")

    >>> # Using benchmark
    >>> new_filter = DatasetFilter(benchmark="raft")

    >>> # Using dataset_name
    >>> new_filter = DatasetFilter(dataset_name="wikineural")

    >>> # Using language_creator
    >>> new_filter = DatasetFilter(language_creator="crowdsourced")

    >>> # Using language
    >>> new_filter = DatasetFilter(language="en")

    >>> # Using multilinguality
    >>> new_filter = DatasetFilter(multilinguality="yes")

    >>> # Using size_categories
    >>> new_filter = DatasetFilter(size_categories="100K<n<1M")

    >>> # Using task_categories
    >>> new_filter = DatasetFilter(task_categories="audio_classification")

    >>> # Using task_ids
    >>> new_filter = DatasetFilter(task_ids="paraphrase")
    ```
    """

    author: str = None
    benchmark: Union[str, List[str]] = None
    dataset_name: str = None
    language_creators: Union[str, List[str]] = None
    languages: Union[str, List[str]] = None
    multilinguality: Union[str, List[str]] = None
    size_categories: Union[str, List[str]] = None
    task_categories: Union[str, List[str]] = None
    task_ids: Union[str, List[str]] = None


@dataclass
class ModelFilter:
    """
    A class that converts human-readable model search parameters into ones
    compatible with the REST API. For all parameters capitalization does not
    matter.

    Args:
        author (`str`, *optional*):
            A string that can be used to identify models on the Hub by the
            original uploader (author or organization), such as `facebook` or
            `huggingface`.
        library (`str` or `List`, *optional*):
            A string or list of strings of foundational libraries models were
            originally trained from, such as pytorch, tensorflow, or allennlp.
        language (`str` or `List`, *optional*):
            A string or list of strings of languages, both by name and country
            code, such as "en" or "English"
        model_name (`str`, *optional*):
            A string that contain complete or partial names for models on the
            Hub, such as "bert" or "bert-base-cased"
        task (`str` or `List`, *optional*):
            A string or list of strings of tasks models were designed for, such
            as: "fill-mask" or "automatic-speech-recognition"
        tags (`str` or `List`, *optional*):
            A string tag or a list of tags to filter models on the Hub by, such
            as `text-generation` or `spacy`.
        trained_dataset (`str` or `List`, *optional*):
            A string tag or a list of string tags of the trained dataset for a
            model on the Hub.


    ```python
    >>> from huggingface_hub import ModelFilter

    >>> # For the author_or_organization
    >>> new_filter = ModelFilter(author_or_organization="facebook")

    >>> # For the library
    >>> new_filter = ModelFilter(library="pytorch")

    >>> # For the language
    >>> new_filter = ModelFilter(language="french")

    >>> # For the model_name
    >>> new_filter = ModelFilter(model_name="bert")

    >>> # For the task
    >>> new_filter = ModelFilter(task="text-classification")

    >>> # Retrieving tags using the `HfApi.get_model_tags` method
    >>> from huggingface_hub import HfApi

    >>> api = HfApi()
    # To list model tags

    >>> api.get_model_tags()
    # To list dataset tags

    >>> api.get_dataset_tags()
    >>> new_filter = ModelFilter(tags="benchmark:raft")

    >>> # Related to the dataset
    >>> new_filter = ModelFilter(trained_dataset="common_voice")
    ```
    """

    author: str = None
    library: Union[str, List[str]] = None
    language: Union[str, List[str]] = None
    model_name: str = None
    task: Union[str, List[str]] = None
    trained_dataset: Union[str, List[str]] = None
    tags: Union[str, List[str]] = None


class AttributeDictionary(dict):
    """
    `dict` subclass that also provides access to keys as attributes

    If a key starts with a number, it will exist in the dictionary but not as an
    attribute

    Example usage:

    ```python
    >>> d = AttributeDictionary()
    >>> d["test"] = "a"
    >>> print(d.test)  # prints "a"
    ```

    """

    def __getattr__(self, k):
        if k in self:
            return self[k]
        else:
            raise AttributeError(k)

    def __setattr__(self, k, v):
        (self.__setitem__, super().__setattr__)[k[0] == "_"](k, v)

    def __delattr__(self, k):
        if k in self:
            del self[k]
        else:
            raise AttributeError(k)

    def __dir__(self):
        keys = sorted(self.keys())
        keys = [key for key in keys if key.replace("_", "").isalpha()]
        return super().__dir__() + keys

    def __repr__(self):
        repr_str = "Available Attributes or Keys:\n"
        for key in sorted(self.keys()):
            repr_str += f" * {key}"
            if not key.replace("_", "").isalpha():
                repr_str += " (Key only)"
            repr_str += "\n"
        return repr_str


class GeneralTags(AttributeDictionary):
    """
    A namespace object holding all tags, filtered by `keys` If a tag starts with
    a number, it will only exist in the dictionary

    Example usage:
    ```python
    >>> a.b["1a"]  # will work
    >>> a["b"]["1a"]  # will work
    >>> # a.b.1a # will not work
    ```

    Args:
        tag_dictionary (`dict`):
            A dictionary of tags returned from the /api/***-tags-by-type api
            endpoint
        keys (`list`):
            A list of keys to unpack the `tag_dictionary` with, such as
            `["library","language"]`
    """

    def __init__(self, tag_dictionary: dict, keys: list = None):
        self._tag_dictionary = tag_dictionary
        if keys is None:
            keys = list(self._tag_dictionary.keys())
        for key in keys:
            self._unpack_and_assign_dictionary(key)

    def _unpack_and_assign_dictionary(self, key: str):
        "Assignes nested attributes to `self.key` containing information as an `AttributeDictionary`"
        setattr(self, key, AttributeDictionary())
        for item in self._tag_dictionary[key]:
            ref = getattr(self, key)
            item["label"] = (
                item["label"].replace(" ", "").replace("-", "_").replace(".", "_")
            )
            setattr(ref, item["label"], item["id"])


class ModelTags(GeneralTags):
    """
    A namespace object holding all available model tags If a tag starts with a
    number, it will only exist in the dictionary

    Example usage:

    ```python
    >>> a.dataset["1_5BArabicCorpus"]  # will work
    >>> a["dataset"]["1_5BArabicCorpus"]  # will work
    >>> # o.dataset.1_5BArabicCorpus # will not work
    ```

    Args:
        model_tag_dictionary (`dict`):
            A dictionary of valid model tags, returned from the
            /api/models-tags-by-type api endpoint
    """

    def __init__(self, model_tag_dictionary: dict):
        keys = ["library", "language", "license", "dataset", "pipeline_tag"]
        super().__init__(model_tag_dictionary, keys)


class DatasetTags(GeneralTags):
    """
    A namespace object holding all available dataset tags If a tag starts with a
    number, it will only exist in the dictionary

    Example

    ```python
    >>> a.size_categories["100K<n<1M"]  # will work
    >>> a["size_categories"]["100K<n<1M"]  # will work
    >>> # o.size_categories.100K<n<1M # will not work
    ```

    Args:
        dataset_tag_dictionary (`dict`):
            A dictionary of valid dataset tags, returned from the
            /api/datasets-tags-by-type api endpoint
    """

    def __init__(self, dataset_tag_dictionary: dict):
        keys = [
            "languages",
            "multilinguality",
            "language_creators",
            "task_categories",
            "size_categories",
            "benchmark",
            "task_ids",
            "licenses",
        ]
        super().__init__(dataset_tag_dictionary, keys)


# src/huggingface_hub/utils/logging.py

# coding=utf-8
# Copyright 2020 Optuna, Hugging Face
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Logging utilities."""

import logging
import os
from logging import CRITICAL  # NOQA
from logging import DEBUG  # NOQA
from logging import ERROR  # NOQA
from logging import FATAL  # NOQA
from logging import INFO  # NOQA
from logging import NOTSET  # NOQA
from logging import WARN  # NOQA
from logging import WARNING  # NOQA
from typing import Optional


log_levels = {
    "debug": logging.DEBUG,
    "info": logging.INFO,
    "warning": logging.WARNING,
    "error": logging.ERROR,
    "critical": logging.CRITICAL,
}

_default_log_level = logging.WARNING


def _get_library_name() -> str:
    return __name__.split(".")[0]


def _get_library_root_logger() -> logging.Logger:

    return logging.getLogger(_get_library_name())


def _get_default_logging_level():
    """
    If HUGGINGFACE_HUB_VERBOSITY env var is set to one of the valid choices
    return that as the new default level. If it is not - fall back to
    `_default_log_level`
    """
    env_level_str = os.getenv("HUGGINGFACE_HUB_VERBOSITY", None)
    if env_level_str:
        if env_level_str in log_levels:
            return log_levels[env_level_str]
        else:
            logging.getLogger().warning(
                f"Unknown option HUGGINGFACE_HUB_VERBOSITY={env_level_str}, "
                f"has to be one of: { ', '.join(log_levels.keys()) }"
            )
    return _default_log_level


def _configure_library_root_logger() -> None:
    library_root_logger = _get_library_root_logger()
    library_root_logger.addHandler(logging.StreamHandler())
    library_root_logger.setLevel(_get_default_logging_level())


def _reset_library_root_logger() -> None:
    library_root_logger = _get_library_root_logger()
    library_root_logger.setLevel(logging.NOTSET)


def get_logger(name: Optional[str] = None) -> logging.Logger:
    """
        Returns a logger with the specified name. This function is not supposed
        to be directly accessed by library users.

        Args:
            name (`str`, *optional*):
                The name of the logger to get, usually the filename

        Example Usage:

    ```python
    >>> from huggingface_hub import get_logger

    >>> logger = get_logger(__file__)
    >>> logger.set_verbosity_info()
    ```
    """

    if name is None:
        name = _get_library_name()

    return logging.getLogger(name)


def get_verbosity() -> int:
    """Return the current level for the HuggingFace Hub's root logger.

    Returns:
        Logging level, e.g., `huggingface_hub.logging.DEBUG` and
        `huggingface_hub.logging.INFO`.

    <Tip>

    HuggingFace Hub has following logging levels:

    - `huggingface_hub.logging.CRITICAL`, `huggingface_hub.logging.FATAL`
    - `huggingface_hub.logging.ERROR`
    - `huggingface_hub.logging.WARNING`, `huggingface_hub.logging.WARN`
    - `huggingface_hub.logging.INFO`
    - `huggingface_hub.logging.DEBUG`

    </Tip>
    """
    return _get_library_root_logger().getEffectiveLevel()


def set_verbosity(verbosity: int) -> None:
    """
    Sets the level for the HuggingFace Hub's root logger.

    Args:
        verbosity (`int`):
            Logging level, e.g., `huggingface_hub.logging.DEBUG` and
            `huggingface_hub.logging.INFO`.
    """
    _get_library_root_logger().setLevel(verbosity)


def set_verbosity_info():
    """
    Sets the verbosity to `logging.INFO`.
    """
    return set_verbosity(INFO)


def set_verbosity_warning():
    """
    Sets the verbosity to `logging.WARNING`.
    """
    return set_verbosity(WARNING)


def set_verbosity_debug():
    """
    Sets the verbosity to `logging.DEBUG`.
    """
    return set_verbosity(DEBUG)


def set_verbosity_error():
    """
    Sets the verbosity to `logging.ERROR`.
    """
    return set_verbosity(ERROR)


def disable_propagation() -> None:
    """
    Disable propagation of the library log outputs. Note that log propagation is
    disabled by default.
    """
    _get_library_root_logger().propagate = False


def enable_propagation() -> None:
    """
    Enable propagation of the library log outputs. Please disable the
    HuggingFace Hub's default handler to prevent double logging if the root
    logger has been configured.
    """
    _get_library_root_logger().propagate = True


_configure_library_root_logger()


# src/huggingface_hub/__init__.py

# flake8: noqa
# There's no way to ignore "F401 '...' imported but unused" warnings in this
# module, but to preserve other warnings. So, don't check this module at all.

# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# ***********
# vendored from https://github.com/scientific-python/lazy_loader
import importlib
import importlib.util
import inspect
import os
import sys
import types
import warnings


class _LazyImportWarning(Warning):
    pass


def _attach(package_name, submodules=None, submod_attrs=None):
    """Attach lazily loaded submodules, functions, or other attributes.

    Typically, modules import submodules and attributes as follows:

    ```py
    import mysubmodule
    import anothersubmodule

    from .foo import someattr
    ```

    The idea is to replace a package's `__getattr__`, `__dir__`, and
    `__all__`, such that all imports work exactly the way they would
    with normal imports, except that the import occurs upon first use.

    The typical way to call this function, replacing the above imports, is:

    ```python
    __getattr__, __dir__, __all__ = lazy.attach(
        __name__,
        ['mysubmodule', 'anothersubmodule'],
        {'foo': ['someattr']}
    )
    ```
    This functionality requires Python 3.7 or higher.

    Args:
        package_name (`str`):
            Typically use `__name__`.
        submodules (`set`):
            List of submodules to attach.
        submod_attrs (`dict`):
            Dictionary of submodule -> list of attributes / functions.
            These attributes are imported as they are used.

    Returns:
        __getattr__, __dir__, __all__

    """
    if submod_attrs is None:
        submod_attrs = {}

    if submodules is None:
        submodules = set()
    else:
        submodules = set(submodules)

    attr_to_modules = {
        attr: mod for mod, attrs in submod_attrs.items() for attr in attrs
    }

    __all__ = list(submodules | attr_to_modules.keys())

    def __getattr__(name):
        if name in submodules:
            return importlib.import_module(f"{package_name}.{name}")
        elif name in attr_to_modules:
            submod_path = f"{package_name}.{attr_to_modules[name]}"
            submod = importlib.import_module(submod_path)
            attr = getattr(submod, name)

            # If the attribute lives in a file (module) with the same
            # name as the attribute, ensure that the attribute and *not*
            # the module is accessible on the package.
            if name == attr_to_modules[name]:
                pkg = sys.modules[package_name]
                pkg.__dict__[name] = attr

            return attr
        else:
            raise AttributeError(f"No {package_name} attribute {name}")

    def __dir__():
        return __all__

    if os.environ.get("EAGER_IMPORT", ""):
        for attr in set(attr_to_modules.keys()) | submodules:
            __getattr__(attr)

    return __getattr__, __dir__, list(__all__)


# ************

__version__ = "0.8.0.dev0"


__getattr__, __dir__, __all__ = _attach(
    __name__,
    submodules=[],
    submod_attrs={
        "commands.user": ["notebook_login"],
        "constants": [
            "CONFIG_NAME",
            "FLAX_WEIGHTS_NAME",
            "HUGGINGFACE_CO_URL_HOME",
            "HUGGINGFACE_CO_URL_TEMPLATE",
            "PYTORCH_WEIGHTS_NAME",
            "REPO_TYPE_DATASET",
            "REPO_TYPE_MODEL",
            "REPO_TYPE_SPACE",
            "TF2_WEIGHTS_NAME",
            "TF_WEIGHTS_NAME",
        ],
        "fastai_utils": [
            "_save_pretrained_fastai",
            "from_pretrained_fastai",
            "push_to_hub_fastai",
        ],
        "file_download": ["cached_download", "hf_hub_download", "hf_hub_url"],
        "hf_api": [
            "DatasetSearchArguments",
            "HfApi",
            "HfFolder",
            "ModelSearchArguments",
            "create_repo",
            "dataset_info",
            "delete_file",
            "delete_repo",
            "get_dataset_tags",
            "get_full_repo_name",
            "get_model_tags",
            "list_datasets",
            "list_metrics",
            "list_models",
            "list_repo_files",
            "login",
            "logout",
            "model_info",
            "move_repo",
            "repo_type_and_id_from_hf_id",
            "set_access_token",
            "space_info",
            "unset_access_token",
            "update_repo_visibility",
            "upload_file",
            "whoami",
        ],
        "hub_mixin": ["ModelHubMixin", "PyTorchModelHubMixin"],
        "inference_api": ["InferenceApi"],
        "keras_mixin": [
            "KerasModelHubMixin",
            "from_pretrained_keras",
            "push_to_hub_keras",
            "save_pretrained_keras",
        ],
        "repository": ["Repository"],
        "_snapshot_download": ["snapshot_download"],
        "utils": ["logging"],
        "utils.endpoint_helpers": ["DatasetFilter", "ModelFilter"],
        "repocard": [
            "metadata_eval_result",
            "metadata_load",
            "metadata_save",
            "metadata_update",
        ],
    },
)


# src/huggingface_hub/_snapshot_download.py

import os
from fnmatch import fnmatch
from pathlib import Path
from typing import Dict, List, Optional, Union

from .constants import DEFAULT_REVISION, HUGGINGFACE_HUB_CACHE, REPO_TYPES
from .file_download import REGEX_COMMIT_HASH, hf_hub_download, repo_folder_name
from .hf_api import HfApi, HfFolder
from .utils import logging


logger = logging.get_logger(__name__)


def _filter_repo_files(
    *,
    repo_files: List[str],
    allow_regex: Optional[Union[List[str], str]] = None,
    ignore_regex: Optional[Union[List[str], str]] = None,
) -> List[str]:
    allow_regex = [allow_regex] if isinstance(allow_regex, str) else allow_regex
    ignore_regex = [ignore_regex] if isinstance(ignore_regex, str) else ignore_regex
    filtered_files = []
    for repo_file in repo_files:
        # if there's an allowlist, skip download if file does not match any regex
        if allow_regex is not None and not any(
            fnmatch(repo_file, r) for r in allow_regex
        ):
            continue

        # if there's a denylist, skip download if file does matches any regex
        if ignore_regex is not None and any(
            fnmatch(repo_file, r) for r in ignore_regex
        ):
            continue

        filtered_files.append(repo_file)
    return filtered_files


def snapshot_download(
    repo_id: str,
    *,
    revision: Optional[str] = None,
    repo_type: Optional[str] = None,
    cache_dir: Union[str, Path, None] = None,
    library_name: Optional[str] = None,
    library_version: Optional[str] = None,
    user_agent: Optional[Union[Dict, str]] = None,
    proxies: Optional[Dict] = None,
    etag_timeout: Optional[float] = 10,
    resume_download: Optional[bool] = False,
    use_auth_token: Optional[Union[bool, str]] = None,
    local_files_only: Optional[bool] = False,
    allow_regex: Optional[Union[List[str], str]] = None,
    ignore_regex: Optional[Union[List[str], str]] = None,
) -> str:
    """Download all files of a repo.

    Downloads a whole snapshot of a repo's files at the specified revision. This
    is useful when you want all files from a repo, because you don't know which
    ones you will need a priori. All files are nested inside a folder in order
    to keep their actual filename relative to that folder.

    An alternative would be to just clone a repo but this would require that the
    user always has git and git-lfs installed, and properly configured.

    Args:
        repo_id (`str`):
            A user or an organization name and a repo name separated by a `/`.
        revision (`str`, *optional*):
            An optional Git revision id which can be a branch name, a tag, or a
            commit hash.
        repo_type (`str`, *optional*):
            Set to `"dataset"` or `"space"` if uploading to a dataset or space,
            `None` or `"model"` if uploading to a model. Default is `None`.
        cache_dir (`str`, `Path`, *optional*):
            Path to the folder where cached files are stored.
        library_name (`str`, *optional*):
            The name of the library to which the object corresponds.
        library_version (`str`, *optional*):
            The version of the library.
        user_agent (`str`, `dict`, *optional*):
            The user-agent info in the form of a dictionary or a string.
        proxies (`dict`, *optional*):
            Dictionary mapping protocol to the URL of the proxy passed to
            `requests.request`.
        etag_timeout (`float`, *optional*, defaults to `10`):
            When fetching ETag, how many seconds to wait for the server to send
            data before giving up which is passed to `requests.request`.
        resume_download (`bool`, *optional*, defaults to `False):
            If `True`, resume a previously interrupted download.
        use_auth_token (`str`, `bool`, *optional*):
            A token to be used for the download.
                - If `True`, the token is read from the HuggingFace config
                  folder.
                - If a string, it's used as the authentication token.
        local_files_only (`bool`, *optional*, defaults to `False`):
            If `True`, avoid downloading the file and return the path to the
            local cached file if it exists.
        allow_regex (`list of str`, `str`, *optional*):
            If provided, only files matching this regex are downloaded.
        ignore_regex (`list of str`, `str`, *optional*):
            If provided, files matching this regex are not downloaded.

    Returns:
        Local folder path (string) of repo snapshot

    <Tip>

    Raises the following errors:

    - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
      if `use_auth_token=True` and the token cannot be found.
    - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) if
      ETag cannot be determined.
    - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
      if some parameter value is invalid

    </Tip>
    """

    if cache_dir is None:
        cache_dir = HUGGINGFACE_HUB_CACHE
    if revision is None:
        revision = DEFAULT_REVISION
    if isinstance(cache_dir, Path):
        cache_dir = str(cache_dir)

    if isinstance(use_auth_token, str):
        token = use_auth_token
    elif use_auth_token:
        token = HfFolder.get_token()
        if token is None:
            raise EnvironmentError(
                "You specified use_auth_token=True, but a Hugging Face token was not"
                " found."
            )
    else:
        token = None

    if repo_type is None:
        repo_type = "model"
    if repo_type not in REPO_TYPES:
        raise ValueError(
            f"Invalid repo type: {repo_type}. Accepted repo types are:"
            f" {str(REPO_TYPES)}"
        )

    storage_folder = os.path.join(
        cache_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type)
    )

    # if we have no internet connection we will look for an
    # appropriate folder in the cache
    # If the specified revision is a commit hash, look inside "snapshots".
    # If the specified revision is a branch or tag, look inside "refs".
    if local_files_only:

        if REGEX_COMMIT_HASH.match(revision):
            commit_hash = revision
        else:
            # retrieve commit_hash from file
            ref_path = os.path.join(storage_folder, "refs", revision)
            with open(ref_path) as f:
                commit_hash = f.read()

        snapshot_folder = os.path.join(storage_folder, "snapshots", commit_hash)

        if os.path.exists(snapshot_folder):
            return snapshot_folder

        raise ValueError(
            "Cannot find an appropriate cached snapshot folder for the specified"
            " revision on the local disk and outgoing traffic has been disabled. To"
            " enable repo look-ups and downloads online, set 'local_files_only' to"
            " False."
        )

    # if we have internet connection we retrieve the correct folder name from the huggingface api
    _api = HfApi()
    repo_info = _api.repo_info(
        repo_id=repo_id, repo_type=repo_type, revision=revision, token=token
    )
    filtered_repo_files = _filter_repo_files(
        repo_files=[f.rfilename for f in repo_info.siblings],
        allow_regex=allow_regex,
        ignore_regex=ignore_regex,
    )
    commit_hash = repo_info.sha
    snapshot_folder = os.path.join(storage_folder, "snapshots", commit_hash)
    # if passed revision is not identical to commit_hash
    # then revision has to be a branch name or tag name.
    # In that case store a ref.
    if revision != commit_hash:
        ref_path = os.path.join(storage_folder, "refs", revision)
        os.makedirs(os.path.dirname(ref_path), exist_ok=True)
        with open(ref_path, "w") as f:
            f.write(commit_hash)

    # we pass the commit_hash to hf_hub_download
    # so no network call happens if we already
    # have the file locally.

    for repo_file in filtered_repo_files:
        _ = hf_hub_download(
            repo_id,
            filename=repo_file,
            repo_type=repo_type,
            revision=commit_hash,
            cache_dir=cache_dir,
            library_name=library_name,
            library_version=library_version,
            user_agent=user_agent,
            proxies=proxies,
            etag_timeout=etag_timeout,
            resume_download=resume_download,
            use_auth_token=use_auth_token,
        )

    return snapshot_folder


# src/huggingface_hub/constants.py

import os


# Possible values for env variables

ENV_VARS_TRUE_VALUES = {"1", "ON", "YES", "TRUE"}
ENV_VARS_TRUE_AND_AUTO_VALUES = ENV_VARS_TRUE_VALUES.union({"AUTO"})

# Constants for file downloads

PYTORCH_WEIGHTS_NAME = "pytorch_model.bin"
TF2_WEIGHTS_NAME = "tf_model.h5"
TF_WEIGHTS_NAME = "model.ckpt"
FLAX_WEIGHTS_NAME = "flax_model.msgpack"
CONFIG_NAME = "config.json"
REPOCARD_NAME = "README.md"

DEFAULT_REVISION = "main"

HUGGINGFACE_CO_URL_HOME = "https://huggingface.co/"

ENV_VARS_TRUE_VALUES = {"1", "ON", "YES", "TRUE"}
_staging_mode = (
    os.environ.get("HUGGINGFACE_CO_STAGING", "NO").upper() in ENV_VARS_TRUE_VALUES
)

ENDPOINT = os.getenv("HF_ENDPOINT") or (
    "https://hub-ci.huggingface.co" if _staging_mode else "https://huggingface.co"
)

HUGGINGFACE_CO_URL_TEMPLATE = ENDPOINT + "/{repo_id}/resolve/{revision}/{filename}"
HUGGINGFACE_HEADER_X_REPO_COMMIT = "X-Repo-Commit"
HUGGINGFACE_HEADER_X_LINKED_ETAG = "X-Linked-Etag"

REPO_ID_SEPARATOR = "--"
# ^ this substring is not allowed in repo_ids on hf.co
# and is the canonical one we use for serialization of repo ids elsewhere.


REPO_TYPE_DATASET = "dataset"
REPO_TYPE_SPACE = "space"
REPO_TYPE_MODEL = "model"
REPO_TYPES = [None, REPO_TYPE_MODEL, REPO_TYPE_DATASET, REPO_TYPE_SPACE]
SPACES_SDK_TYPES = ["gradio", "streamlit", "static"]

REPO_TYPES_URL_PREFIXES = {
    REPO_TYPE_DATASET: "datasets/",
    REPO_TYPE_SPACE: "spaces/",
}
REPO_TYPES_MAPPING = {
    "datasets": REPO_TYPE_DATASET,
    "spaces": REPO_TYPE_SPACE,
    "models": REPO_TYPE_MODEL,
}


# default cache
hf_cache_home = os.path.expanduser(
    os.getenv(
        "HF_HOME", os.path.join(os.getenv("XDG_CACHE_HOME", "~/.cache"), "huggingface")
    )
)
default_cache_path = os.path.join(hf_cache_home, "hub")

HUGGINGFACE_HUB_CACHE = os.getenv("HUGGINGFACE_HUB_CACHE", default_cache_path)

HF_HUB_OFFLINE = os.environ.get("HF_HUB_OFFLINE", "AUTO").upper()
if HF_HUB_OFFLINE in ENV_VARS_TRUE_VALUES:
    HF_HUB_OFFLINE = True
else:
    HF_HUB_OFFLINE = False


# src/huggingface_hub/fastai_utils.py

import json
import os
from pathlib import Path
from pickle import DEFAULT_PROTOCOL, PicklingError
from typing import Any, Dict, Optional

from packaging import version

from huggingface_hub import snapshot_download
from huggingface_hub.constants import CONFIG_NAME
from huggingface_hub.file_download import (
    _PY_VERSION,
    get_fastai_version,
    get_fastcore_version,
)
from huggingface_hub.hf_api import HfApi, HfFolder
from huggingface_hub.repository import Repository
from huggingface_hub.utils import logging


logger = logging.get_logger(__name__)


def _check_fastai_fastcore_versions(
    fastai_min_version: Optional[str] = "2.4",
    fastcore_min_version: Optional[str] = "1.3.27",
):
    """
    Checks that the installed fastai and fastcore versions are compatible for pickle serialization.

    Args:
        fastai_min_version (`str`, *optional*):
            The minimum fastai version supported.
        fastcore_min_version (`str`, *optional*):
            The minimum fastcore version supported.

    <Tip>
    Raises the following error:

        - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
          if the fastai or fastcore libraries are not available or are of an invalid version.

    </Tip>
    """

    if (get_fastcore_version() or get_fastai_version()) == "N/A":
        raise ImportError(
            f"fastai>={fastai_min_version} and fastcore>={fastcore_min_version} are"
            f" required. Currently using fastai=={get_fastai_version()} and"
            f" fastcore=={get_fastcore_version()}."
        )

    current_fastai_version = version.Version(get_fastai_version())
    current_fastcore_version = version.Version(get_fastcore_version())

    if current_fastai_version < version.Version(fastai_min_version):
        raise ImportError(
            "`push_to_hub_fastai` and `from_pretrained_fastai` require a"
            f" fastai>={fastai_min_version} version, but you are using fastai version"
            f" {get_fastai_version()} which is incompatible. Upgrade with `pip install"
            " fastai==2.5.6`."
        )

    if current_fastcore_version < version.Version(fastcore_min_version):
        raise ImportError(
            "`push_to_hub_fastai` and `from_pretrained_fastai` require a"
            f" fastcore>={fastcore_min_version} version, but you are using fastcore"
            f" version {get_fastcore_version()} which is incompatible. Upgrade with"
            " `pip install fastcore==1.3.27`."
        )


def _check_fastai_fastcore_pyproject_versions(
    storage_folder: str,
    fastai_min_version: Optional[str] = "2.4",
    fastcore_min_version: Optional[str] = "1.3.27",
):
    """
    Checks that the `pyproject.toml` file in the directory `storage_folder` has fastai and fastcore versions
    that are compatible with `from_pretrained_fastai` and `push_to_hub_fastai`. If `pyproject.toml` does not exist
    or does not contain versions for fastai and fastcore, then it logs a warning.

    Args:
        storage_folder (`str`):
            Folder to look for the `pyproject.toml` file.
        fastai_min_version (`str`, *optional*):
            The minimum fastai version supported.
        fastcore_min_version (`str`, *optional*):
            The minimum fastcore version supported.

    <Tip>
    Raises the following errors:

        - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
          if the `toml` module is not installed.
        - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
          if the `pyproject.toml` indicates a lower than minimum supported version of fastai or fastcore.

    </Tip>
    """

    try:
        import toml
    except ModuleNotFoundError:
        raise ImportError(
            "`push_to_hub_fastai` and `from_pretrained_fastai` require the toml module."
            " Install it with `pip install toml`."
        )

    # Checks that a `pyproject.toml`, with `build-system` and `requires` sections, exists in the repository. If so, get a list of required packages.
    if not os.path.isfile(f"{storage_folder}/pyproject.toml"):
        logger.warning(
            "There is no `pyproject.toml` in the repository that contains the fastai"
            " `Learner`. The `pyproject.toml` would allow us to verify that your fastai"
            " and fastcore versions are compatible with those of the model you want to"
            " load."
        )
        return
    pyproject_toml = toml.load(f"{storage_folder}/pyproject.toml")

    if "build-system" not in pyproject_toml.keys():
        logger.warning(
            "There is no `build-system` section in the pyproject.toml of the repository"
            " that contains the fastai `Learner`. The `build-system` would allow us to"
            " verify that your fastai and fastcore versions are compatible with those"
            " of the model you want to load."
        )
        return
    build_system_toml = pyproject_toml["build-system"]

    if "requires" not in build_system_toml.keys():
        logger.warning(
            "There is no `requires` section in the pyproject.toml of the repository"
            " that contains the fastai `Learner`. The `requires` would allow us to"
            " verify that your fastai and fastcore versions are compatible with those"
            " of the model you want to load."
        )
        return
    package_versions = build_system_toml["requires"]

    # Extracts contains fastai and fastcore versions from `pyproject.toml` if available.
    # If the package is specified but not the version (e.g. "fastai" instead of "fastai=2.4"), the default versions are the highest.
    fastai_packages = [pck for pck in package_versions if pck.startswith("fastai")]
    if len(fastai_packages) == 0:
        logger.warning(
            "The repository does not have a fastai version specified in the"
            " `pyproject.toml`."
        )
    # fastai_version is an empty string if not specified
    else:
        fastai_version = str(fastai_packages[0]).partition("=")[2]
        if fastai_version != "" and version.Version(fastai_version) < version.Version(
            fastai_min_version
        ):
            raise ImportError(
                "`from_pretrained_fastai` requires"
                f" fastai>={fastai_min_version} version but the model to load uses"
                f" {fastai_version} which is incompatible."
            )

    fastcore_packages = [pck for pck in package_versions if pck.startswith("fastcore")]
    if len(fastcore_packages) == 0:
        logger.warning(
            "The repository does not have a fastcore version specified in the"
            " `pyproject.toml`."
        )
    # fastcore_version is an empty string if not specified
    else:
        fastcore_version = str(fastcore_packages[0]).partition("=")[2]
        if fastcore_version != "" and version.Version(
            fastcore_version
        ) < version.Version(fastcore_min_version):
            raise ImportError(
                "`from_pretrained_fastai` requires"
                f" fastcore>={fastcore_min_version} version, but you are using fastcore"
                f" version {fastcore_version} which is incompatible."
            )


README_TEMPLATE = """---
tags:
- fastai
---

# Amazing!

🥳 Congratulations on hosting your fastai model on the Hugging Face Hub!

# Some next steps
1. Fill out this model card with more information (see the template below and the [documentation here](https://huggingface.co/docs/hub/model-repos))!

2. Create a demo in Gradio or Streamlit using 🤗 Spaces ([documentation here](https://huggingface.co/docs/hub/spaces)).

3. Join the fastai community on the [Fastai Discord](https://discord.com/invite/YKrxeNn)!

Greetings fellow fastlearner 🤝! Don't forget to delete this content from your model card.


---


# Model card

## Model description
More information needed

## Intended uses & limitations
More information needed

## Training and evaluation data
More information needed
"""

PYPROJECT_TEMPLATE = f"""[build-system]
requires = ["setuptools>=40.8.0", "wheel", "python={_PY_VERSION}", "fastai={get_fastai_version()}", "fastcore={get_fastcore_version()}"]
build-backend = "setuptools.build_meta:__legacy__"
"""


def _create_model_card(repo_dir: Path):
    """
    Creates a model card for the repository.

    Args:
        repo_dir (`Path`):
            Directory where model card is created.
    """
    readme_path = repo_dir / "README.md"

    if not readme_path.exists():
        with readme_path.open("w", encoding="utf-8") as f:
            f.write(README_TEMPLATE)


def _create_model_pyproject(repo_dir: Path):
    """
    Creates a `pyproject.toml` for the repository.

    Args:
        repo_dir (`Path`):
            Directory where `pyproject.toml` is created.
    """
    pyproject_path = repo_dir / "pyproject.toml"

    if not pyproject_path.exists():
        with pyproject_path.open("w", encoding="utf-8") as f:
            f.write(PYPROJECT_TEMPLATE)


def _save_pretrained_fastai(
    learner,
    save_directory: str,
    config: Optional[Dict[str, Any]] = None,
):
    """
    Saves a fastai learner to `save_directory` in pickle format using the default pickle protocol for the version of python used.

    Args:
        learner (`Learner`):
            The `fastai.Learner` you'd like to save.
        save_directory (`str`):
            Specific directory in which you want to save the fastai learner.
        config (`dict`, *optional*):
            Configuration object. Will be uploaded as a .json file. Example: 'https://huggingface.co/espejelomar/fastai-pet-breeds-classification/blob/main/config.json'.

    <Tip>

    Raises the following error:

        - [`RuntimeError`](https://docs.python.org/3/library/exceptions.html#RuntimeError)
          if the config file provided is not a dictionary.

    </Tip>
    """
    _check_fastai_fastcore_versions()

    os.makedirs(save_directory, exist_ok=True)

    # if the user provides config then we update it with the fastai and fastcore versions in CONFIG_TEMPLATE.
    if config is not None:
        if not isinstance(config, dict):
            raise RuntimeError(
                f"Provided config should be a dict. Got: '{type(config)}'"
            )
        path = os.path.join(save_directory, CONFIG_NAME)
        with open(path, "w") as f:
            json.dump(config, f)

    _create_model_card(Path(save_directory))
    _create_model_pyproject(Path(save_directory))

    # learner.export saves the model in `self.path`.
    learner.path = Path(save_directory)
    os.makedirs(save_directory, exist_ok=True)
    try:
        learner.export(
            fname="model.pkl",
            pickle_protocol=DEFAULT_PROTOCOL,
        )
    except PicklingError:
        raise PicklingError(
            "You are using a lambda function, i.e., an anonymous function. `pickle`"
            " cannot pickle function objects and requires that all functions have"
            " names. One possible solution is to name the function."
        )


def from_pretrained_fastai(
    repo_id: str,
    revision: Optional[str] = None,
):
    """
    Load pretrained fastai model from the Hub or from a local directory.

    Args:
        repo_id (`str`):
            The location where the pickled fastai.Learner is. It can be either of the two:
                - Hosted on the Hugging Face Hub. E.g.: 'espejelomar/fatai-pet-breeds-classification' or 'distilgpt2'.
                  You can add a `revision` by appending `@` at the end of `repo_id`. E.g.: `dbmdz/bert-base-german-cased@main`.
                  Revision is the specific model version to use. Since we use a git-based system for storing models and other
                  artifacts on the Hugging Face Hub, it can be a branch name, a tag name, or a commit id.
                - Hosted locally. `repo_id` would be a directory containing the pickle and a pyproject.toml
                  indicating the fastai and fastcore versions used to build the `fastai.Learner`. E.g.: `./my_model_directory/`.
        revision (`str`, *optional*):
            Revision at which the repo's files are downloaded. See documentation of `snapshot_download`.

    Returns:
        The `fastai.Learner` model in the `repo_id` repo.
    """
    _check_fastai_fastcore_versions()

    # Load the `repo_id` repo.
    # `snapshot_download` returns the folder where the model was stored.
    # `cache_dir` will be the default '/root/.cache/huggingface/hub'
    if not os.path.isdir(repo_id):
        storage_folder = snapshot_download(
            repo_id=repo_id,
            revision=revision,
            library_name="fastai",
            library_version=get_fastai_version(),
        )
    else:
        storage_folder = repo_id

    _check_fastai_fastcore_pyproject_versions(storage_folder)

    from fastai.learner import load_learner

    return load_learner(os.path.join(storage_folder, "model.pkl"))


def push_to_hub_fastai(
    learner,
    repo_id: str,
    commit_message: Optional[str] = "Add model",
    private: Optional[bool] = None,
    token: Optional[str] = None,
    config: Optional[dict] = None,
    **kwargs,
):
    """
    Upload learner checkpoint files to the Hub while synchronizing a local clone of the repo in
    :obj:`repo_id`.

    Args:
        learner (`Learner`):
            The `fastai.Learner' you'd like to push to the Hub.
        repo_id (`str`):
            The repository id for your model in Hub in the format of "namespace/repo_name". The namespace can be your individual account or an organization to which you have write access (for example, 'stanfordnlp/stanza-de').
        commit_message (`str`, *optional*):
            Message to commit while pushing. Will default to :obj:`"add model"`.
        private (`bool`, *optional*):
            Whether or not the repository created should be private.
        token (`str`, *optional*):
            The Hugging Face account token to use as HTTP bearer authorization for remote files. If :obj:`None`, the token will be asked by a prompt.
        config (`dict`, *optional*):
            Configuration object to be saved alongside the model weights.

    Keyword Args:
        api_endpoint (`str`, *optional*):
            The API endpoint to use when pushing the model to the hub.
        git_user (`str`, *optional*):
            Will override the ``git config user.name`` for committing and pushing files to the hub.
        git_email (`str`, *optional*):
            Will override the ``git config user.email`` for committing and pushing files to the hub.

    Returns:
        The url of the commit of your model in the given repository.

    <Tip>

    Raises the following error:

        - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
          if the user is not log on to the Hugging Face Hub.

    </Tip>
    """

    _check_fastai_fastcore_versions()

    api_endpoint: str = kwargs.get("api_endpoint", None)
    git_user: str = kwargs.get("git_user", None)
    git_email: str = kwargs.get("git_email", None)

    if token is None:
        token = HfFolder.get_token()

    if token is None:
        raise ValueError(
            "You must login to the Hugging Face Hub. There are two options: "
            "(1) Type `huggingface-cli login` in your terminal and enter your token. "
            "(2) Enter your token in the `token` argument. "
            "Your token is available in the Settings of your Hugging Face account. "
        )

    # Create repo using `HfApi()`.
    repo_url = HfApi(endpoint=api_endpoint).create_repo(
        repo_id,
        token=token,
        private=private,
        repo_type=None,
        exist_ok=True,
    )

    # If repository exists in the Hugging Face Hub then clone it locally in `repo_id`.
    repo = Repository(
        repo_id,
        clone_from=repo_url,
        use_auth_token=token,
        git_user=git_user,
        git_email=git_email,
    )
    repo.git_pull(rebase=True)

    _save_pretrained_fastai(learner, repo_id, config=config)

    return repo.push_to_hub(commit_message=commit_message)


# src/huggingface_hub/file_download.py

import copy
import fnmatch
import io
import json
import os
import re
import sys
import tempfile
import time
import warnings
from contextlib import contextmanager
from functools import partial
from hashlib import sha256
from pathlib import Path
from typing import BinaryIO, Dict, Optional, Tuple, Union

import packaging.version
from tqdm.auto import tqdm

import requests
from filelock import FileLock
from huggingface_hub import constants

from . import __version__
from .constants import (
    DEFAULT_REVISION,
    HUGGINGFACE_CO_URL_TEMPLATE,
    HUGGINGFACE_HEADER_X_LINKED_ETAG,
    HUGGINGFACE_HEADER_X_REPO_COMMIT,
    HUGGINGFACE_HUB_CACHE,
    REPO_ID_SEPARATOR,
    REPO_TYPES,
    REPO_TYPES_URL_PREFIXES,
)
from .hf_api import HfFolder
from .utils import logging
from .utils._errors import _raise_for_status


logger = logging.get_logger(__name__)

_PY_VERSION: str = sys.version.split()[0].rstrip("+")

if packaging.version.Version(_PY_VERSION) < packaging.version.Version("3.8.0"):
    import importlib_metadata
else:
    import importlib.metadata as importlib_metadata

_torch_version = "N/A"
_torch_available = False
try:
    _torch_version = importlib_metadata.version("torch")
    _torch_available = True
except importlib_metadata.PackageNotFoundError:
    pass

_pydot_available = False

try:
    _pydot_version = importlib_metadata.version("pydot")
    _pydot_available = True
except importlib_metadata.PackageNotFoundError:
    pass


def is_pydot_available():
    return _pydot_available


_graphviz_available = False

try:
    _graphviz_version = importlib_metadata.version("graphviz")
    _graphviz_available = True
except importlib_metadata.PackageNotFoundError:
    pass


def is_graphviz_available():
    return _graphviz_available


_tf_version = "N/A"
_tf_available = False
_tf_candidates = (
    "tensorflow",
    "tensorflow-cpu",
    "tensorflow-gpu",
    "tf-nightly",
    "tf-nightly-cpu",
    "tf-nightly-gpu",
    "intel-tensorflow",
    "intel-tensorflow-avx512",
    "tensorflow-rocm",
    "tensorflow-macos",
)
for package_name in _tf_candidates:
    try:
        _tf_version = importlib_metadata.version(package_name)
        _tf_available = True
        break
    except importlib_metadata.PackageNotFoundError:
        pass

_fastai_version = "N/A"
_fastai_available = False
try:
    _fastai_version: str = importlib_metadata.version("fastai")
    _fastai_available = True
except importlib_metadata.PackageNotFoundError:
    pass

_fastcore_version = "N/A"
_fastcore_available = False
try:
    _fastcore_version: str = importlib_metadata.version("fastcore")
    _fastcore_available = True
except importlib_metadata.PackageNotFoundError:
    pass


def is_torch_available():
    return _torch_available


def is_tf_available():
    return _tf_available


def get_tf_version():
    return _tf_version


def is_fastai_available():
    return _fastai_available


def get_fastai_version():
    return _fastai_version


def is_fastcore_available():
    return _fastcore_available


def get_fastcore_version():
    return _fastcore_version


REGEX_COMMIT_HASH = re.compile(r"^[0-9a-f]{40}$")


def hf_hub_url(
    repo_id: str,
    filename: str,
    *,
    subfolder: Optional[str] = None,
    repo_type: Optional[str] = None,
    revision: Optional[str] = None,
) -> str:
    """Construct the URL of a file from the given information.

    The resolved address can either be a huggingface.co-hosted url, or a link to
    Cloudfront (a Content Delivery Network, or CDN) for large files which are
    more than a few MBs.

    Args:
        repo_id (`str`):
            A namespace (user or an organization) name and a repo name separated
            by a `/`.
        filename (`str`):
            The name of the file in the repo.
        subfolder (`str`, *optional*):
            An optional value corresponding to a folder inside the repo.
        repo_type (`str`, *optional*):
            Set to `"dataset"` or `"space"` if uploading to a dataset or space,
            `None` or `"model"` if uploading to a model. Default is `None`.
        revision (`str`, *optional*):
            An optional Git revision id which can be a branch name, a tag, or a
            commit hash.

    Example:

    ```python
    >>> from huggingface_hub import hf_hub_url

    >>> hf_hub_url(
    ...     repo_id="julien-c/EsperBERTo-small", filename="pytorch_model.bin"
    ... )
    'https://huggingface.co/julien-c/EsperBERTo-small/resolve/main/pytorch_model.bin'
    ```

    <Tip>

    Notes:

        Cloudfront is replicated over the globe so downloads are way faster for
        the end user (and it also lowers our bandwidth costs).

        Cloudfront aggressively caches files by default (default TTL is 24
        hours), however this is not an issue here because we implement a
        git-based versioning system on huggingface.co, which means that we store
        the files on S3/Cloudfront in a content-addressable way (i.e., the file
        name is its hash). Using content-addressable filenames means cache can't
        ever be stale.

        In terms of client-side caching from this library, we base our caching
        on the objects' entity tag (`ETag`), which is an identifier of a
        specific version of a resource [1]_. An object's ETag is: its git-sha1
        if stored in git, or its sha256 if stored in git-lfs.

    </Tip>

    References:

    -  [1] https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
    """
    if subfolder is not None:
        filename = f"{subfolder}/{filename}"

    if repo_type not in REPO_TYPES:
        raise ValueError("Invalid repo type")

    if repo_type in REPO_TYPES_URL_PREFIXES:
        repo_id = REPO_TYPES_URL_PREFIXES[repo_type] + repo_id

    if revision is None:
        revision = DEFAULT_REVISION
    return HUGGINGFACE_CO_URL_TEMPLATE.format(
        repo_id=repo_id, revision=revision, filename=filename
    )


def url_to_filename(url: str, etag: Optional[str] = None) -> str:
    """Generate a local filename from a url.

    Convert `url` into a hashed filename in a reproducible way. If `etag` is
    specified, append its hash to the url's, delimited by a period. If the url
    ends with .h5 (Keras HDF5 weights) adds '.h5' to the name so that TF 2.0 can
    identify it as a HDF5 file (see
    https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1380)

    Args:
        url (`str`):
            The address to the file.
        etag (`str`, *optional*):
            The ETag of the file.

    Returns:
        The generated filename.
    """
    url_bytes = url.encode("utf-8")
    filename = sha256(url_bytes).hexdigest()

    if etag:
        etag_bytes = etag.encode("utf-8")
        filename += "." + sha256(etag_bytes).hexdigest()

    if url.endswith(".h5"):
        filename += ".h5"

    return filename


def filename_to_url(
    filename,
    cache_dir: Optional[str] = None,
    legacy_cache_layout: Optional[bool] = False,
) -> Tuple[str, str]:
    """
    Return the url and etag (which may be `None`) stored for `filename`. Raise
    `EnvironmentError` if `filename` or its stored metadata do not exist.

    Args:
        filename (`str`):
            The name of the file
        cache_dir (`str`, *optional*):
            The cache directory to use instead of the default one.
        legacy_cache_layout (`bool`, *optional*, defaults to `False`):
            If `True`, uses the legacy file cache layout i.e. just call `hf_hub_url`
            then `cached_download`. This is deprecated as the new cache layout is
            more powerful.
    """
    if not legacy_cache_layout:
        warnings.warn(
            "`filename_to_url` uses the legacy way cache file layout",
            FutureWarning,
        )

    if cache_dir is None:
        cache_dir = HUGGINGFACE_HUB_CACHE
    if isinstance(cache_dir, Path):
        cache_dir = str(cache_dir)

    cache_path = os.path.join(cache_dir, filename)
    if not os.path.exists(cache_path):
        raise EnvironmentError(f"file {cache_path} not found")

    meta_path = cache_path + ".json"
    if not os.path.exists(meta_path):
        raise EnvironmentError(f"file {meta_path} not found")

    with open(meta_path, encoding="utf-8") as meta_file:
        metadata = json.load(meta_file)
    url = metadata["url"]
    etag = metadata["etag"]

    return url, etag


def http_user_agent(
    *,
    library_name: Optional[str] = None,
    library_version: Optional[str] = None,
    user_agent: Union[Dict, str, None] = None,
) -> str:
    """Formats a user-agent string with basic info about a request.

    Args:
        library_name (`str`, *optional*):
            The name of the library to which the object corresponds.
        library_version (`str`, *optional*):
            The version of the library.
        user_agent (`str`, `dict`, *optional*):
            The user agent info in the form of a dictionary or a single string.

    Returns:
        The formatted user-agent string.
    """
    if library_name is not None:
        ua = f"{library_name}/{library_version}"
    else:
        ua = "unknown/None"
    ua += f"; hf_hub/{__version__}"
    ua += f"; python/{_PY_VERSION}"
    if is_torch_available():
        ua += f"; torch/{_torch_version}"
    if is_tf_available():
        ua += f"; tensorflow/{_tf_version}"
    if is_fastai_available():
        ua += f"; fastai/{_fastai_version}"
    if is_fastcore_available():
        ua += f"; fastcore/{_fastcore_version}"
    if isinstance(user_agent, dict):
        ua += "; " + "; ".join(f"{k}/{v}" for k, v in user_agent.items())
    elif isinstance(user_agent, str):
        ua += "; " + user_agent
    return ua


class OfflineModeIsEnabled(ConnectionError):
    pass


def _raise_if_offline_mode_is_enabled(msg: Optional[str] = None):
    """Raise a OfflineModeIsEnabled error (subclass of ConnectionError) if
    HF_HUB_OFFLINE is True."""
    if constants.HF_HUB_OFFLINE:
        raise OfflineModeIsEnabled(
            "Offline mode is enabled."
            if msg is None
            else "Offline mode is enabled. " + str(msg)
        )


def _request_with_retry(
    method: str,
    url: str,
    *,
    max_retries: int = 0,
    base_wait_time: float = 0.5,
    max_wait_time: float = 2,
    timeout: float = 10.0,
    **params,
) -> requests.Response:
    """Wrapper around requests to retry in case it fails with a `ConnectTimeout`, with
    exponential backoff.

        Note that if the environment variable HF_HUB_OFFLINE is set to 1, then a
        `OfflineModeIsEnabled` error is raised.

        Args:
            method (`str`):
                HTTP method, such as 'GET' or 'HEAD'.
            url (`str`):
                The URL of the resource to fetch.
            max_retries (`int`, *optional*, defaults to `0`):
                Maximum number of retries, defaults to 0 (no retries).
            base_wait_time (`float`, *optional*, defaults to `0.5`):
                Duration (in seconds) to wait before retrying the first time.
                Wait time between retries then grows exponentially, capped by
                `max_wait_time`.
            max_wait_time (`float`, *optional*, defaults to `2`):
                Maximum amount of time between two retries, in seconds.
            timeout (`float`, *optional*, defaults to `10`):
                How many seconds to wait for the server to send data before
                giving up which is passed to `requests.request`.
            **params (`dict`, *optional*):
                Params to pass to `requests.request`.
    """
    _raise_if_offline_mode_is_enabled(f"Tried to reach {url}")
    tries, success = 0, False
    while not success:
        tries += 1
        try:
            response = requests.request(
                method=method.upper(), url=url, timeout=timeout, **params
            )
            success = True
        except requests.exceptions.ConnectTimeout as err:
            if tries > max_retries:
                raise err
            else:
                logger.info(
                    f"{method} request to {url} timed out, retrying..."
                    f" [{tries/max_retries}]"
                )
                sleep_time = min(
                    max_wait_time, base_wait_time * 2 ** (tries - 1)
                )  # Exponential backoff
                time.sleep(sleep_time)
    return response


def http_get(
    url: str,
    temp_file: BinaryIO,
    *,
    proxies=None,
    resume_size=0,
    headers: Optional[Dict[str, str]] = None,
    timeout=10.0,
    max_retries=0,
):
    """
    Donwload a remote file. Do not gobble up errors, and will return errors tailored to the Hugging Face Hub.
    """
    headers = copy.deepcopy(headers)
    if resume_size > 0:
        headers["Range"] = "bytes=%d-" % (resume_size,)
    r = _request_with_retry(
        method="GET",
        url=url,
        stream=True,
        proxies=proxies,
        headers=headers,
        timeout=timeout,
        max_retries=max_retries,
    )
    _raise_for_status(r)
    content_length = r.headers.get("Content-Length")
    total = resume_size + int(content_length) if content_length is not None else None
    progress = tqdm(
        unit="B",
        unit_scale=True,
        total=total,
        initial=resume_size,
        desc="Downloading",
        disable=bool(logger.getEffectiveLevel() == logging.NOTSET),
    )
    for chunk in r.iter_content(chunk_size=1024):
        if chunk:  # filter out keep-alive new chunks
            progress.update(len(chunk))
            temp_file.write(chunk)
    progress.close()


def cached_download(
    url: str,
    *,
    library_name: Optional[str] = None,
    library_version: Optional[str] = None,
    cache_dir: Union[str, Path, None] = None,
    user_agent: Union[Dict, str, None] = None,
    force_download: Optional[bool] = False,
    force_filename: Optional[str] = None,
    proxies: Optional[Dict] = None,
    etag_timeout: Optional[float] = 10,
    resume_download: Optional[bool] = False,
    use_auth_token: Union[bool, str, None] = None,
    local_files_only: Optional[bool] = False,
    legacy_cache_layout: Optional[bool] = False,
) -> Optional[str]:  # pragma: no cover
    """
    Download from a given URL and cache it if it's not already present in the
    local cache.

    Given a URL, this function looks for the corresponding file in the local
    cache. If it's not there, download it. Then return the path to the cached
    file.

    Will raise errors tailored to the Hugging Face Hub.

    Args:
        url (`str`):
            The path to the file to be downloaded.
        library_name (`str`, *optional*):
            The name of the library to which the object corresponds.
        library_version (`str`, *optional*):
            The version of the library.
        cache_dir (`str`, `Path`, *optional*):
            Path to the folder where cached files are stored.
        user_agent (`dict`, `str`, *optional*):
            The user-agent info in the form of a dictionary or a string.
        force_download (`bool`, *optional*, defaults to `False`):
            Whether the file should be downloaded even if it already exists in
            the local cache.
        force_filename (`str`, *optional*):
            Use this name instead of a generated file name.
        proxies (`dict`, *optional*):
            Dictionary mapping protocol to the URL of the proxy passed to
            `requests.request`.
        etag_timeout (`float`, *optional* defaults to `10`):
            When fetching ETag, how many seconds to wait for the server to send
            data before giving up which is passed to `requests.request`.
        resume_download (`bool`, *optional*, defaults to `False`):
            If `True`, resume a previously interrupted download.
        use_auth_token (`bool`, `str`, *optional*):
            A token to be used for the download.
                - If `True`, the token is read from the HuggingFace config
                  folder.
                - If a string, it's used as the authentication token.
        local_files_only (`bool`, *optional*, defaults to `False`):
            If `True`, avoid downloading the file and return the path to the
            local cached file if it exists.
        legacy_cache_layout (`bool`, *optional*, defaults to `False`):
            Set this parameter to `True` to mention that you'd like to continue
            the old cache layout. Putting this to `True` manually will not raise
            any warning when using `cached_download`. We recommend using
            `hf_hub_download` to take advantage of the new cache.

    Returns:
        Local path (string) of file or if networking is off, last version of
        file cached on disk.

    <Tip>

    Raises the following errors:

        - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
          if `use_auth_token=True` and the token cannot be found.
        - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError)
          if ETag cannot be determined.
        - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
          if some parameter value is invalid
        - [`~huggingface_hub.utils.RepositoryNotFoundError`]
          If the repository to download from cannot be found. This may be because it doesn't exist,
          or because it is set to `private` and you do not have access.
        - [`~huggingface_hub.utils.RevisionNotFoundError`]
          If the revision to download from cannot be found.
        - [`~huggingface_hub.utils.EntryNotFoundError`]
          If the file to download cannot be found.

    </Tip>
    """
    if not legacy_cache_layout:
        warnings.warn(
            "`cached_download` is the legacy way to download files from the HF hub,"
            " please consider upgrading to `hf_hub_download`",
            FutureWarning,
        )

    if cache_dir is None:
        cache_dir = HUGGINGFACE_HUB_CACHE
    if isinstance(cache_dir, Path):
        cache_dir = str(cache_dir)

    os.makedirs(cache_dir, exist_ok=True)

    headers = {
        "user-agent": http_user_agent(
            library_name=library_name,
            library_version=library_version,
            user_agent=user_agent,
        )
    }
    if isinstance(use_auth_token, str):
        headers["authorization"] = f"Bearer {use_auth_token}"
    elif use_auth_token:
        token = HfFolder.get_token()
        if token is None:
            raise EnvironmentError(
                "You specified use_auth_token=True, but a huggingface token was not"
                " found."
            )
        headers["authorization"] = f"Bearer {token}"

    url_to_download = url
    etag = None
    if not local_files_only:
        try:
            r = _request_with_retry(
                method="HEAD",
                url=url,
                headers=headers,
                allow_redirects=False,
                proxies=proxies,
                timeout=etag_timeout,
            )
            _raise_for_status(r)
            etag = r.headers.get("X-Linked-Etag") or r.headers.get("ETag")
            # We favor a custom header indicating the etag of the linked resource, and
            # we fallback to the regular etag header.
            # If we don't have any of those, raise an error.
            if etag is None:
                raise OSError(
                    "Distant resource does not have an ETag, we won't be able to"
                    " reliably ensure reproducibility."
                )
            # In case of a redirect,
            # save an extra redirect on the request.get call,
            # and ensure we download the exact atomic version even if it changed
            # between the HEAD and the GET (unlikely, but hey).
            if 300 <= r.status_code <= 399:
                url_to_download = r.headers["Location"]
        except (requests.exceptions.SSLError, requests.exceptions.ProxyError):
            # Actually raise for those subclasses of ConnectionError
            raise
        except (
            requests.exceptions.ConnectionError,
            requests.exceptions.Timeout,
            OfflineModeIsEnabled,
        ):
            # Otherwise, our Internet connection is down.
            # etag is None
            pass

    filename = (
        force_filename if force_filename is not None else url_to_filename(url, etag)
    )

    # get cache path to put the file
    cache_path = os.path.join(cache_dir, filename)

    # etag is None == we don't have a connection or we passed local_files_only.
    # try to get the last downloaded one
    if etag is None:
        if os.path.exists(cache_path) and not force_download:
            return cache_path
        else:
            matching_files = [
                file
                for file in fnmatch.filter(
                    os.listdir(cache_dir), filename.split(".")[0] + ".*"
                )
                if not file.endswith(".json") and not file.endswith(".lock")
            ]
            if (
                len(matching_files) > 0
                and not force_download
                and force_filename is None
            ):
                return os.path.join(cache_dir, matching_files[-1])
            else:
                # If files cannot be found and local_files_only=True,
                # the models might've been found if local_files_only=False
                # Notify the user about that
                if local_files_only:
                    raise ValueError(
                        "Cannot find the requested files in the cached path and"
                        " outgoing traffic has been disabled. To enable model look-ups"
                        " and downloads online, set 'local_files_only' to False."
                    )
                else:
                    raise ValueError(
                        "Connection error, and we cannot find the requested files in"
                        " the cached path. Please try again or make sure your Internet"
                        " connection is on."
                    )

    # From now on, etag is not None.
    if os.path.exists(cache_path) and not force_download:
        return cache_path

    # Prevent parallel downloads of the same file with a lock.
    lock_path = cache_path + ".lock"

    # Some Windows versions do not allow for paths longer than 255 characters.
    # In this case, we must specify it is an extended path by using the "\\?\" prefix.
    if os.name == "nt" and len(os.path.abspath(lock_path)) > 255:
        lock_path = "\\\\?\\" + os.path.abspath(lock_path)

    if os.name == "nt" and len(os.path.abspath(cache_path)) > 255:
        cache_path = "\\\\?\\" + os.path.abspath(cache_path)

    with FileLock(lock_path):

        # If the download just completed while the lock was activated.
        if os.path.exists(cache_path) and not force_download:
            # Even if returning early like here, the lock will be released.
            return cache_path

        if resume_download:
            incomplete_path = cache_path + ".incomplete"

            @contextmanager
            def _resumable_file_manager() -> "io.BufferedWriter":
                with open(incomplete_path, "ab") as f:
                    yield f

            temp_file_manager = _resumable_file_manager
            if os.path.exists(incomplete_path):
                resume_size = os.stat(incomplete_path).st_size
            else:
                resume_size = 0
        else:
            temp_file_manager = partial(
                tempfile.NamedTemporaryFile, mode="wb", dir=cache_dir, delete=False
            )
            resume_size = 0

        # Download to temporary file, then copy to cache dir once finished.
        # Otherwise you get corrupt cache entries if the download gets interrupted.
        with temp_file_manager() as temp_file:
            logger.info("downloading %s to %s", url, temp_file.name)

            http_get(
                url_to_download,
                temp_file,
                proxies=proxies,
                resume_size=resume_size,
                headers=headers,
            )

        logger.info("storing %s in cache at %s", url, cache_path)
        os.replace(temp_file.name, cache_path)

        if force_filename is None:
            logger.info("creating metadata file for %s", cache_path)
            meta = {"url": url, "etag": etag}
            meta_path = cache_path + ".json"
            with open(meta_path, "w") as meta_file:
                json.dump(meta, meta_file)

    return cache_path


def _normalize_etag(etag: str) -> str:
    """Normalize ETag HTTP header, so it can be used to create nice filepaths.

    The HTTP spec allows two forms of ETag:
      ETag: W/"<etag_value>"
      ETag: "<etag_value>"

    The hf.co hub guarantees to only send the second form.

    Args:
        etag (`str`): HTTP header

    Returns:
        `str`: string that can be used as a nice directory name.
    """
    return etag.strip('"')


def _create_relative_symlink(src: str, dst: str) -> None:
    """Create a symbolic link named dst pointing to src as a relative path to dst.

    The relative part is mostly because it seems more elegant to the author.

    The result layout looks something like
        └── [ 128]  snapshots
            ├── [ 128]  2439f60ef33a0d46d85da5001d52aeda5b00ce9f
            │   ├── [  52]  README.md -> ../../blobs/d7edf6bd2a681fb0175f7735299831ee1b22b812
            │   └── [  76]  pytorch_model.bin -> ../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd
    """
    relative_src = os.path.relpath(src, start=os.path.dirname(dst))
    try:
        os.remove(dst)
    except OSError:
        pass
    try:
        os.symlink(relative_src, dst)
    except OSError:
        # Likely running on Windows
        if os.name == "nt":
            raise OSError(
                "Windows requires Developer Mode to be activated, or to run Python as "
                "an administrator, in order to create symlinks.\nIn order to "
                "activate Developer Mode, see this article: "
                "https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development"
            )
        else:
            raise


def repo_folder_name(*, repo_id: str, repo_type: str) -> str:
    """Return a serialized version of a hf.co repo name and type, safe for disk storage
    as a single non-nested folder.

    Example: models--julien-c--EsperBERTo-small
    """
    # remove all `/` occurrences to correctly convert repo to directory name
    parts = [f"{repo_type}s", *repo_id.split("/")]
    return REPO_ID_SEPARATOR.join(parts)


def hf_hub_download(
    repo_id: str,
    filename: str,
    *,
    subfolder: Optional[str] = None,
    repo_type: Optional[str] = None,
    revision: Optional[str] = None,
    library_name: Optional[str] = None,
    library_version: Optional[str] = None,
    cache_dir: Union[str, Path, None] = None,
    user_agent: Union[Dict, str, None] = None,
    force_download: Optional[bool] = False,
    force_filename: Optional[str] = None,
    proxies: Optional[Dict] = None,
    etag_timeout: Optional[float] = 10,
    resume_download: Optional[bool] = False,
    use_auth_token: Union[bool, str, None] = None,
    local_files_only: Optional[bool] = False,
    legacy_cache_layout: Optional[bool] = False,
):
    """Download a given file if it's not already present in the local cache.

    The new cache file layout looks like this:
    - The cache directory contains one subfolder per repo_id (namespaced by repo type)
    - inside each repo folder:
        - refs is a list of the latest known revision => commit_hash pairs
        - blobs contains the actual file blobs (identified by their git-sha or sha256, depending on
          whether they're LFS files or not)
        - snapshots contains one subfolder per commit, each "commit" contains the subset of the files
          that have been resolved at that particular commit. Each filename is a symlink to the blob
          at that particular commit.

    [  96]  .
    └── [ 160]  models--julien-c--EsperBERTo-small
        ├── [ 160]  blobs
        │   ├── [321M]  403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd
        │   ├── [ 398]  7cb18dc9bafbfcf74629a4b760af1b160957a83e
        │   └── [1.4K]  d7edf6bd2a681fb0175f7735299831ee1b22b812
        ├── [  96]  refs
        │   └── [  40]  main
        └── [ 128]  snapshots
            ├── [ 128]  2439f60ef33a0d46d85da5001d52aeda5b00ce9f
            │   ├── [  52]  README.md -> ../../blobs/d7edf6bd2a681fb0175f7735299831ee1b22b812
            │   └── [  76]  pytorch_model.bin -> ../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd
            └── [ 128]  bbc77c8132af1cc5cf678da3f1ddf2de43606d48
                ├── [  52]  README.md -> ../../blobs/7cb18dc9bafbfcf74629a4b760af1b160957a83e
                └── [  76]  pytorch_model.bin -> ../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd

    Args:
        repo_id (`str`):
            A user or an organization name and a repo name separated by a `/`.
        filename (`str`):
            The name of the file in the repo.
        subfolder (`str`, *optional*):
            An optional value corresponding to a folder inside the model repo.
        repo_type (`str`, *optional*):
            Set to `"dataset"` or `"space"` if uploading to a dataset or space,
            `None` or `"model"` if uploading to a model. Default is `None`.
        revision (`str`, *optional*):
            An optional Git revision id which can be a branch name, a tag, or a
            commit hash.
        library_name (`str`, *optional*):
            The name of the library to which the object corresponds.
        library_version (`str`, *optional*):
            The version of the library.
        cache_dir (`str`, `Path`, *optional*):
            Path to the folder where cached files are stored.
        user_agent (`dict`, `str`, *optional*):
            The user-agent info in the form of a dictionary or a string.
        force_download (`bool`, *optional*, defaults to `False`):
            Whether the file should be downloaded even if it already exists in
            the local cache.
        proxies (`dict`, *optional*):
            Dictionary mapping protocol to the URL of the proxy passed to
            `requests.request`.
        etag_timeout (`float`, *optional*, defaults to `10`):
            When fetching ETag, how many seconds to wait for the server to send
            data before giving up which is passed to `requests.request`.
        resume_download (`bool`, *optional*, defaults to `False`):
            If `True`, resume a previously interrupted download.
        use_auth_token (`str`, `bool`, *optional*):
            A token to be used for the download.
                - If `True`, the token is read from the HuggingFace config
                  folder.
                - If a string, it's used as the authentication token.
        local_files_only (`bool`, *optional*, defaults to `False`):
            If `True`, avoid downloading the file and return the path to the
            local cached file if it exists.
        legacy_cache_layout (`bool`, *optional*, defaults to `False`):
            If `True`, uses the legacy file cache layout i.e. just call [`hf_hub_url`]
            then `cached_download`. This is deprecated as the new cache layout is
            more powerful.

    Returns:
        Local path (string) of file or if networking is off, last version of
        file cached on disk.

    <Tip>

    Raises the following errors:

        - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
          if `use_auth_token=True` and the token cannot be found.
        - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError)
          if ETag cannot be determined.
        - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
          if some parameter value is invalid
        - [`~huggingface_hub.utils.RepositoryNotFoundError`]
          If the repository to download from cannot be found. This may be because it doesn't exist,
          or because it is set to `private` and you do not have access.
        - [`~huggingface_hub.utils.RevisionNotFoundError`]
          If the revision to download from cannot be found.
        - [`~huggingface_hub.utils.EntryNotFoundError`]
          If the file to download cannot be found.

    </Tip>
    """
    if force_filename is not None:
        warnings.warn(
            "The `force_filename` parameter is deprecated as a new caching system, "
            "which keeps the filenames as they are on the Hub, is now in place.",
            FutureWarning,
        )
        legacy_cache_layout = True

    if legacy_cache_layout:
        url = hf_hub_url(
            repo_id,
            filename,
            subfolder=subfolder,
            repo_type=repo_type,
            revision=revision,
        )

        return cached_download(
            url,
            library_name=library_name,
            library_version=library_version,
            cache_dir=cache_dir,
            user_agent=user_agent,
            force_download=force_download,
            force_filename=force_filename,
            proxies=proxies,
            etag_timeout=etag_timeout,
            resume_download=resume_download,
            use_auth_token=use_auth_token,
            local_files_only=local_files_only,
            legacy_cache_layout=legacy_cache_layout,
        )

    if cache_dir is None:
        cache_dir = HUGGINGFACE_HUB_CACHE
    if revision is None:
        revision = DEFAULT_REVISION
    if isinstance(cache_dir, Path):
        cache_dir = str(cache_dir)

    if subfolder is not None:
        # This is used to create a URL, and not a local path, hence the forward slash.
        filename = f"{subfolder}/{filename}"

    if repo_type is None:
        repo_type = "model"
    if repo_type not in REPO_TYPES:
        raise ValueError(
            f"Invalid repo type: {repo_type}. Accepted repo types are:"
            f" {str(REPO_TYPES)}"
        )

    storage_folder = os.path.join(
        cache_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type)
    )
    os.makedirs(storage_folder, exist_ok=True)

    # cross platform transcription of filename, to be used as a local file path.
    relative_filename = os.path.join(*filename.split("/"))

    # if user provides a commit_hash and they already have the file on disk,
    # shortcut everything.
    if REGEX_COMMIT_HASH.match(revision):
        pointer_path = os.path.join(
            storage_folder, "snapshots", revision, relative_filename
        )
        if os.path.exists(pointer_path):
            return pointer_path

    url = hf_hub_url(repo_id, filename, repo_type=repo_type, revision=revision)

    headers = {
        "user-agent": http_user_agent(
            library_name=library_name,
            library_version=library_version,
            user_agent=user_agent,
        )
    }
    if isinstance(use_auth_token, str):
        headers["authorization"] = f"Bearer {use_auth_token}"
    elif use_auth_token:
        token = HfFolder.get_token()
        if token is None:
            raise EnvironmentError(
                "You specified use_auth_token=True, but a huggingface token was not"
                " found."
            )
        headers["authorization"] = f"Bearer {token}"

    url_to_download = url
    etag = None
    commit_hash = None
    if not local_files_only:
        try:
            r = _request_with_retry(
                method="HEAD",
                url=url,
                headers=headers,
                allow_redirects=False,
                proxies=proxies,
                timeout=etag_timeout,
            )
            _raise_for_status(r)
            commit_hash = r.headers[HUGGINGFACE_HEADER_X_REPO_COMMIT]
            if commit_hash is None:
                raise OSError(
                    "Distant resource does not seem to be on huggingface.co (missing"
                    " commit header)."
                )
            etag = r.headers.get(HUGGINGFACE_HEADER_X_LINKED_ETAG) or r.headers.get(
                "ETag"
            )
            # We favor a custom header indicating the etag of the linked resource, and
            # we fallback to the regular etag header.
            # If we don't have any of those, raise an error.
            if etag is None:
                raise OSError(
                    "Distant resource does not have an ETag, we won't be able to"
                    " reliably ensure reproducibility."
                )
            etag = _normalize_etag(etag)
            # In case of a redirect,
            # save an extra redirect on the request.get call,
            # and ensure we download the exact atomic version even if it changed
            # between the HEAD and the GET (unlikely, but hey).
            if 300 <= r.status_code <= 399:
                url_to_download = r.headers["Location"]
        except (requests.exceptions.SSLError, requests.exceptions.ProxyError):
            # Actually raise for those subclasses of ConnectionError
            raise
        except (
            requests.exceptions.ConnectionError,
            requests.exceptions.Timeout,
            OfflineModeIsEnabled,
        ):
            # Otherwise, our Internet connection is down.
            # etag is None
            pass

    # etag is None == we don't have a connection or we passed local_files_only.
    # try to get the last downloaded one from the specified revision.
    # If the specified revision is a commit hash, look inside "snapshots".
    # If the specified revision is a branch or tag, look inside "refs".
    if etag is None:
        # In those cases, we cannot force download.
        if force_download:
            raise ValueError(
                "We have no connection or you passed local_files_only, so"
                " force_download is not an accepted option."
            )
        if REGEX_COMMIT_HASH.match(revision):
            commit_hash = revision
        else:
            ref_path = os.path.join(storage_folder, "refs", revision)
            with open(ref_path) as f:
                commit_hash = f.read()

        pointer_path = os.path.join(
            storage_folder, "snapshots", commit_hash, relative_filename
        )
        if os.path.exists(pointer_path):
            return pointer_path

        # If we couldn't find an appropriate file on disk,
        # raise an error.
        # If files cannot be found and local_files_only=True,
        # the models might've been found if local_files_only=False
        # Notify the user about that
        if local_files_only:
            raise ValueError(
                "Cannot find the requested files in the disk cache and"
                " outgoing traffic has been disabled. To enable hf.co look-ups"
                " and downloads online, set 'local_files_only' to False."
            )
        else:
            raise ValueError(
                "Connection error, and we cannot find the requested files in"
                " the disk cache. Please try again or make sure your Internet"
                " connection is on."
            )

    # From now on, etag and commit_hash are not None.
    blob_path = os.path.join(storage_folder, "blobs", etag)
    pointer_path = os.path.join(
        storage_folder, "snapshots", commit_hash, relative_filename
    )

    os.makedirs(os.path.dirname(blob_path), exist_ok=True)
    os.makedirs(os.path.dirname(pointer_path), exist_ok=True)
    # if passed revision is not identical to commit_hash
    # then revision has to be a branch name or tag name.
    # In that case store a ref.
    if revision != commit_hash:
        ref_path = os.path.join(storage_folder, "refs", revision)
        os.makedirs(os.path.dirname(ref_path), exist_ok=True)
        with open(ref_path, "w") as f:
            f.write(commit_hash)

    if os.path.exists(pointer_path) and not force_download:
        return pointer_path

    if os.path.exists(blob_path) and not force_download:
        # we have the blob already, but not the pointer
        logger.info("creating pointer to %s from %s", blob_path, pointer_path)
        _create_relative_symlink(blob_path, pointer_path)
        return pointer_path

    # Prevent parallel downloads of the same file with a lock.
    lock_path = blob_path + ".lock"

    # Some Windows versions do not allow for paths longer than 255 characters.
    # In this case, we must specify it is an extended path by using the "\\?\" prefix.
    if os.name == "nt" and len(os.path.abspath(lock_path)) > 255:
        lock_path = "\\\\?\\" + os.path.abspath(lock_path)

    if os.name == "nt" and len(os.path.abspath(blob_path)) > 255:
        blob_path = "\\\\?\\" + os.path.abspath(blob_path)

    with FileLock(lock_path):

        # If the download just completed while the lock was activated.
        if os.path.exists(pointer_path) and not force_download:
            # Even if returning early like here, the lock will be released.
            return pointer_path

        if resume_download:
            incomplete_path = blob_path + ".incomplete"

            @contextmanager
            def _resumable_file_manager() -> "io.BufferedWriter":
                with open(incomplete_path, "ab") as f:
                    yield f

            temp_file_manager = _resumable_file_manager
            if os.path.exists(incomplete_path):
                resume_size = os.stat(incomplete_path).st_size
            else:
                resume_size = 0
        else:
            temp_file_manager = partial(
                tempfile.NamedTemporaryFile, mode="wb", dir=cache_dir, delete=False
            )
            resume_size = 0

        # Download to temporary file, then copy to cache dir once finished.
        # Otherwise you get corrupt cache entries if the download gets interrupted.
        with temp_file_manager() as temp_file:
            logger.info("downloading %s to %s", url, temp_file.name)

            http_get(
                url_to_download,
                temp_file,
                proxies=proxies,
                resume_size=resume_size,
                headers=headers,
            )

        logger.info("storing %s in cache at %s", url, blob_path)
        os.replace(temp_file.name, blob_path)

        logger.info("creating pointer to %s from %s", blob_path, pointer_path)
        _create_relative_symlink(blob_path, pointer_path)

    try:
        os.remove(lock_path)
    except OSError:
        pass

    return pointer_path


# src/huggingface_hub/hf_api.py

# coding=utf-8
# Copyright 2019-present, the HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import re
import subprocess
import sys
import warnings
from io import BufferedIOBase, RawIOBase
from os.path import expanduser
from typing import IO, Dict, Iterable, List, Optional, Tuple, Union

import requests
from requests.exceptions import HTTPError

from .constants import (
    ENDPOINT,
    REPO_TYPES,
    REPO_TYPES_MAPPING,
    REPO_TYPES_URL_PREFIXES,
    SPACES_SDK_TYPES,
)
from .utils import logging
from .utils._deprecation import _deprecate_positional_args
from .utils._errors import _raise_for_status, _raise_with_request_id
from .utils._fixes import JSONDecodeError
from .utils.endpoint_helpers import (
    AttributeDictionary,
    DatasetFilter,
    DatasetTags,
    ModelFilter,
    ModelTags,
    _filter_emissions,
)


if sys.version_info >= (3, 8):
    from typing import Literal
else:
    from typing_extensions import Literal


USERNAME_PLACEHOLDER = "hf_user"

logger = logging.get_logger(__name__)


# TODO: remove after deprecation period is over (v0.10)
def _validate_repo_id_deprecation(repo_id, name, organization):
    """Returns (name, organization) from the input."""
    if repo_id and not name and organization:
        # this means the user had passed name as positional, now mapped to
        # repo_id and is passing organization as well. This wouldn't be an
        # issue if they pass everything as kwarg. So we switch the parameters
        # here:
        repo_id, name = name, repo_id

    if not (repo_id or name):
        raise ValueError(
            "No name provided. Please pass `repo_id` with a valid repository name."
        )

    if repo_id and (name or organization):
        raise ValueError(
            "Only pass `repo_id` and leave deprecated `name` and "
            "`organization` to be None."
        )
    elif name or organization:
        warnings.warn(
            "`name` and `organization` input arguments are deprecated and "
            "will be removed in v0.10. Pass `repo_id` instead.",
            FutureWarning,
        )
    else:
        if "/" in repo_id:
            organization, name = repo_id.split("/")
        else:
            organization, name = None, repo_id
    return name, organization


def repo_type_and_id_from_hf_id(
    hf_id: str, hub_url: Optional[str] = None
) -> Tuple[Optional[str], Optional[str], str]:
    """
    Returns the repo type and ID from a huggingface.co URL linking to a
    repository

    Args:
        hf_id (`str`):
            An URL or ID of a repository on the HF hub. Accepted values are:

            - https://huggingface.co/<repo_type>/<namespace>/<repo_id>
            - https://huggingface.co/<namespace>/<repo_id>
            - <repo_type>/<namespace>/<repo_id>
            - <namespace>/<repo_id>
            - <repo_id>
        hub_url (`str`, *optional*):
            The URL of the HuggingFace Hub, defaults to https://huggingface.co
    """
    hub_url = re.sub(r"https?://", "", hub_url if hub_url is not None else ENDPOINT)
    is_hf_url = hub_url in hf_id and "@" not in hf_id
    url_segments = hf_id.split("/")
    is_hf_id = len(url_segments) <= 3

    if is_hf_url:
        namespace, repo_id = url_segments[-2:]
        if namespace == hub_url:
            namespace = None
        if len(url_segments) > 2 and hub_url not in url_segments[-3]:
            repo_type = url_segments[-3]
        else:
            repo_type = None
    elif is_hf_id:
        if len(url_segments) == 3:
            # Passed <repo_type>/<user>/<model_id> or <repo_type>/<org>/<model_id>
            repo_type, namespace, repo_id = url_segments[-3:]
        elif len(url_segments) == 2:
            # Passed <user>/<model_id> or <org>/<model_id>
            namespace, repo_id = hf_id.split("/")[-2:]
            repo_type = None
        else:
            # Passed <model_id>
            repo_id = url_segments[0]
            namespace, repo_type = None, None
    else:
        raise ValueError(
            f"Unable to retrieve user and repo ID from the passed HF ID: {hf_id}"
        )

    repo_type = (
        repo_type if repo_type in REPO_TYPES else REPO_TYPES_MAPPING.get(repo_type)
    )

    return repo_type, namespace, repo_id


class RepoFile:
    """
    Data structure that represents a public file inside a repo, accessible from
    huggingface.co

    Args:
        rfilename (str):
            file name, relative to the repo root. This is the only attribute
            that's guaranteed to be here, but under certain conditions there can
            certain other stuff.
    """

    def __init__(self, rfilename: str, **kwargs):
        self.rfilename = rfilename  # filename relative to the repo root
        for k, v in kwargs.items():
            setattr(self, k, v)

    def __repr__(self):
        items = (f"{k}='{v}'" for k, v in self.__dict__.items())
        return f"{self.__class__.__name__}({', '.join(items)})"


class ModelInfo:
    """
    Info about a model accessible from huggingface.co

    Args:
        modelId (`str`, *optional*):
            ID of model repository.
        sha (`str`, *optional*):
            repo sha at this particular revision
        lastModified (`str`, *optional*):
            date of last commit to repo
        tags (`Listr[str]`, *optional*):
            List of tags.
        pipeline_tag (`str`, *optional*):
            Pipeline tag to identify the correct widget.
        siblings (`List[Dict]`, *optional*):
            list of files that constitute the Space
        private (`bool`, *optional*):
            is the repo private
        author (`str`, *optional*):
            repo author
        config (`Dict`, *optional*):
            Model configuration information
        kwargs (`Dict`, *optional*):
            Kwargs that will be become attributes of the class.
    """

    def __init__(
        self,
        *,
        modelId: Optional[str] = None,
        sha: Optional[str] = None,
        lastModified: Optional[str] = None,
        tags: List[str] = [],
        pipeline_tag: Optional[str] = None,
        siblings: Optional[List[Dict]] = None,
        private: Optional[bool] = None,
        author: Optional[str] = None,
        config: Optional[Dict] = None,
        **kwargs,
    ):
        self.modelId = modelId
        self.sha = sha
        self.lastModified = lastModified
        self.tags = tags
        self.pipeline_tag = pipeline_tag
        self.siblings = (
            [RepoFile(**x) for x in siblings] if siblings is not None else None
        )
        self.private = private
        self.author = author
        self.config = config
        for k, v in kwargs.items():
            setattr(self, k, v)

    def __repr__(self):
        s = f"{self.__class__.__name__}:" + " {"
        for key, val in self.__dict__.items():
            s += f"\n\t{key}: {val}"
        return s + "\n}"

    def __str__(self):
        r = f"Model Name: {self.modelId}, Tags: {self.tags}"
        if self.pipeline_tag:
            r += f", Task: {self.pipeline_tag}"
        return r


class DatasetInfo:
    """
    Info about a dataset accessible from huggingface.co

    Args:
        id (`str`, *optional*):
            ID of dataset repository.
        sha (`str`, *optional*):
            repo sha at this particular revision
        lastModified (`str`, *optional*):
            date of last commit to repo
        tags (`Listr[str]`, *optional*):
            List of tags.
        siblings (`List[Dict]`, *optional*):
            list of files that constitute the Space
        private (`bool`, *optional*):
            is the repo private
        author (`str`, *optional*):
            repo author
        description (`str`, *optional*):
            Description of the dataset
        citation (`str`, *optional*):
            Dataset citation
        cardData (`Dict`, *optional*):
            Metadata of the model card as a dictionary.
        kwargs (`Dict`, *optional*):
            Kwargs that will be become attributes of the class.
    """

    def __init__(
        self,
        *,
        id: Optional[str] = None,
        sha: Optional[str] = None,
        lastModified: Optional[str] = None,
        tags: List[str] = [],
        siblings: Optional[List[Dict]] = None,
        private: Optional[bool] = None,
        author: Optional[str] = None,
        description: Optional[str] = None,
        citation: Optional[str] = None,
        cardData: Optional[dict] = None,
        **kwargs,
    ):
        self.id = id
        self.sha = sha
        self.lastModified = lastModified
        self.tags = tags
        self.private = private
        self.author = author
        self.description = description
        self.citation = citation
        self.cardData = cardData
        self.siblings = (
            [RepoFile(**x) for x in siblings] if siblings is not None else None
        )
        # Legacy stuff, "key" is always returned with an empty string
        # because of old versions of the datasets lib that need this field
        kwargs.pop("key", None)
        # Store all the other fields returned by the API
        for k, v in kwargs.items():
            setattr(self, k, v)

    def __repr__(self):
        s = f"{self.__class__.__name__}:" + " {"
        for key, val in self.__dict__.items():
            s += f"\n\t{key}: {val}"
        return s + "\n}"

    def __str__(self):
        r = f"Dataset Name: {self.id}, Tags: {self.tags}"
        return r


class SpaceInfo:
    """
    Info about a Space accessible from huggingface.co

    This is a "dataclass" like container that just sets on itself any attribute
    passed by the server.

    Args:
        id (`str`, *optional*):
            id of space
        sha (`str`, *optional*):
            repo sha at this particular revision
        lastModified (`str`, *optional*):
            date of last commit to repo
        siblings (`List[Dict]`, *optional*):
            list of files that constitute the Space
        private (`bool`, *optional*):
            is the repo private
        author (`str`, *optional*):
            repo author
        kwargs (`Dict`, *optional*):
            Kwargs that will be become attributes of the class.
    """

    def __init__(
        self,
        *,
        id: Optional[str] = None,
        sha: Optional[str] = None,
        lastModified: Optional[str] = None,
        siblings: Optional[List[Dict]] = None,
        private: Optional[bool] = None,
        author: Optional[str] = None,
        **kwargs,
    ):
        self.id = id
        self.sha = sha
        self.lastModified = lastModified
        self.siblings = (
            [RepoFile(**x) for x in siblings] if siblings is not None else None
        )
        self.private = private
        self.author = author
        for k, v in kwargs.items():
            setattr(self, k, v)

    def __repr__(self):
        s = f"{self.__class__.__name__}:" + " {"
        for key, val in self.__dict__.items():
            s += f"\n\t{key}: {val}"
        return s + "\n}"


class MetricInfo:
    """
    Info about a public metric accessible from huggingface.co
    """

    def __init__(
        self,
        *,
        id: Optional[str] = None,  # id of metric
        description: Optional[str] = None,
        citation: Optional[str] = None,
        **kwargs,
    ):
        self.id = id
        self.description = description
        self.citation = citation
        # Legacy stuff, "key" is always returned with an empty string
        # because of old versions of the datasets lib that need this field
        kwargs.pop("key", None)
        # Store all the other fields returned by the API
        for k, v in kwargs.items():
            setattr(self, k, v)

    def __repr__(self):
        s = f"{self.__class__.__name__}:" + " {"
        for key, val in self.__dict__.items():
            s += f"\n\t{key}: {val}"
        return s + "\n}"

    def __str__(self):
        r = f"Metric Name: {self.id}"
        return r


class ModelSearchArguments(AttributeDictionary):
    """
    A nested namespace object holding all possible values for properties of
    models currently hosted in the Hub with tab-completion. If a value starts
    with a number, it will only exist in the dictionary

    Example:

    ```python
    >>> args = ModelSearchArguments()
    >>> args.author_or_organization.huggingface
    >>> args.language.en
    ```
    """

    def __init__(self):
        self._api = HfApi()
        tags = self._api.get_model_tags()
        super().__init__(tags)
        self._process_models()

    def _process_models(self):
        def clean(s: str):
            return s.replace(" ", "").replace("-", "_").replace(".", "_")

        models = self._api.list_models()
        author_dict, model_name_dict = AttributeDictionary(), AttributeDictionary()
        for model in models:
            if "/" in model.modelId:
                author, name = model.modelId.split("/")
                author_dict[author] = clean(author)
            else:
                name = model.modelId
            model_name_dict[name] = clean(name)
        self["model_name"] = model_name_dict
        self["author"] = author_dict


class DatasetSearchArguments(AttributeDictionary):
    """
    A nested namespace object holding all possible values for properties of
    datasets currently hosted in the Hub with tab-completion. If a value starts
    with a number, it will only exist in the dictionary

    Example:

    ```python
    >>> args = DatasetSearchArguments()
    >>> args.author_or_organization.huggingface
    >>> args.language.en
    ```
    """

    def __init__(self):
        self._api = HfApi()
        tags = self._api.get_dataset_tags()
        super().__init__(tags)
        self._process_models()

    def _process_models(self):
        def clean(s: str):
            return s.replace(" ", "").replace("-", "_").replace(".", "_")

        datasets = self._api.list_datasets()
        author_dict, dataset_name_dict = AttributeDictionary(), AttributeDictionary()
        for dataset in datasets:
            if "/" in dataset.id:
                author, name = dataset.id.split("/")
                author_dict[author] = clean(author)
            else:
                name = dataset.id
            dataset_name_dict[name] = clean(name)
        self["dataset_name"] = dataset_name_dict
        self["author"] = author_dict


def write_to_credential_store(username: str, password: str):
    with subprocess.Popen(
        "git credential-store store".split(),
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
    ) as process:
        input_username = f"username={username.lower()}"
        input_password = f"password={password}"

        process.stdin.write(
            f"url={ENDPOINT}\n{input_username}\n{input_password}\n\n".encode("utf-8")
        )
        process.stdin.flush()


def read_from_credential_store(
    username=None,
) -> Tuple[Union[str, None], Union[str, None]]:
    """
    Reads the credential store relative to huggingface.co. If no `username` is
    specified, will read the first entry for huggingface.co, otherwise will read
    the entry corresponding to the username specified.

    The username returned will be all lowercase.
    """
    with subprocess.Popen(
        "git credential-store get".split(),
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
    ) as process:
        standard_input = f"url={ENDPOINT}\n"

        if username is not None:
            standard_input += f"username={username.lower()}\n"

        standard_input += "\n"

        process.stdin.write(standard_input.encode("utf-8"))
        process.stdin.flush()
        output = process.stdout.read()
        output = output.decode("utf-8")

    if len(output) == 0:
        return None, None

    username, password = [line for line in output.split("\n") if len(line) != 0]
    return username.split("=")[1], password.split("=")[1]


def erase_from_credential_store(username=None):
    """
    Erases the credential store relative to huggingface.co. If no `username` is
    specified, will erase the first entry for huggingface.co, otherwise will
    erase the entry corresponding to the username specified.
    """
    with subprocess.Popen(
        "git credential-store erase".split(),
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
    ) as process:
        standard_input = f"url={ENDPOINT}\n"

        if username is not None:
            standard_input += f"username={username.lower()}\n"

        standard_input += "\n"

        process.stdin.write(standard_input.encode("utf-8"))
        process.stdin.flush()


class HfApi:
    def __init__(self, endpoint=None):
        self.endpoint = endpoint if endpoint is not None else ENDPOINT

    def whoami(self, token: Optional[str] = None) -> Dict:
        """
        Call HF API to know "whoami".

        Args:
            token (`str`, *optional*):
                Hugging Face token. Will default to the locally saved token if
                not provided.
        """
        if token is None:
            token = HfFolder.get_token()
        if token is None:
            raise ValueError(
                "You need to pass a valid `token` or login by using `huggingface-cli "
                "login`"
            )
        path = f"{self.endpoint}/api/whoami-v2"
        r = requests.get(path, headers={"authorization": f"Bearer {token}"})
        try:
            _raise_with_request_id(r)
        except HTTPError as e:
            raise HTTPError(
                "Invalid user token. If you didn't pass a user token, make sure you "
                "are properly logged in by executing `huggingface-cli login`, and "
                "if you did pass a user token, double-check it's correct."
            ) from e
        return r.json()

    def _is_valid_token(self, token: str):
        """
        Determines whether `token` is a valid token or not.

        Args:
            token (`str`):
                The token to check for validity.

        Returns:
            `bool`: `True` if valid, `False` otherwise.
        """
        try:
            self.whoami(token=token)
            return True
        except HTTPError:
            return False

    def _validate_or_retrieve_token(
        self,
        token: Optional[str] = None,
        name: Optional[str] = None,
        function_name: Optional[str] = None,
    ):
        """
        Retrieves and validates stored token or validates passed token.
        Args:
            token (``str``, `optional`):
                Hugging Face token. Will default to the locally saved token if not provided.
            name (``str``, `optional`):
                Name of the repository. This is deprecated in favor of repo_id and will be removed in v0.8.
            function_name (``str``, `optional`):
                If _validate_or_retrieve_token is called from a function, name of that function to be passed inside deprecation warning.
        Returns:
            Validated token and the name of the repository.
        Raises:
            :class:`EnvironmentError`: If the token is not passed and there's no token saved locally.
            :class:`ValueError`: If organization token or invalid token is passed.
        """
        if token is None or token is True:
            token = HfFolder.get_token()
            if token is None:
                raise EnvironmentError(
                    "You need to provide a `token` or be logged in to Hugging "
                    "Face with `huggingface-cli login`."
                )
        if name is not None:
            if self._is_valid_token(name):
                # TODO(0.6) REMOVE
                warnings.warn(
                    f"`{function_name}` now takes `token` as an optional positional"
                    " argument. Be sure to adapt your code!",
                    FutureWarning,
                )
                token, name = name, token
        if isinstance(token, str):
            if token.startswith("api_org"):
                raise ValueError("You must use your personal account token.")
            if not self._is_valid_token(token):
                raise ValueError("Invalid token passed!")

        return token, name

    @staticmethod
    def set_access_token(access_token: str):
        """
        Saves the passed access token so git can correctly authenticate the
        user.

        Args:
            access_token (`str`):
                The access token to save.
        """
        write_to_credential_store(USERNAME_PLACEHOLDER, access_token)

    @staticmethod
    def unset_access_token():
        """
        Resets the user's access token.
        """
        erase_from_credential_store(USERNAME_PLACEHOLDER)

    def get_model_tags(self) -> ModelTags:
        "Gets all valid model tags as a nested namespace object"
        path = f"{self.endpoint}/api/models-tags-by-type"
        r = requests.get(path)
        _raise_with_request_id(r)
        d = r.json()
        return ModelTags(d)

    def get_dataset_tags(self) -> DatasetTags:
        """
        Gets all valid dataset tags as a nested namespace object.
        """
        path = f"{self.endpoint}/api/datasets-tags-by-type"
        r = requests.get(path)
        _raise_with_request_id(r)
        d = r.json()
        return DatasetTags(d)

    def list_models(
        self,
        *,
        filter: Union[ModelFilter, str, Iterable[str], None] = None,
        author: Optional[str] = None,
        search: Optional[str] = None,
        emissions_thresholds: Optional[Tuple[float, float]] = None,
        sort: Union[Literal["lastModified"], str, None] = None,
        direction: Optional[Literal[-1]] = None,
        limit: Optional[int] = None,
        full: Optional[bool] = None,
        cardData: Optional[bool] = None,
        fetch_config: Optional[bool] = None,
        use_auth_token: Optional[Union[bool, str]] = None,
    ) -> List[ModelInfo]:
        """
        Get the public list of all the models on huggingface.co

        Args:
            filter ([`ModelFilter`] or `str` or `Iterable`, *optional*):
                A string or [`ModelFilter`] which can be used to identify models
                on the Hub.
            author (`str`, *optional*):
                A string which identify the author (user or organization) of the
                returned models
            search (`str`, *optional*):
                A string that will be contained in the returned models Example
                usage:
            emissions_thresholds (`Tuple`, *optional*):
                A tuple of two ints or floats representing a minimum and maximum
                carbon footprint to filter the resulting models with in grams.
            sort (`Literal["lastModified"]` or `str`, *optional*):
                The key with which to sort the resulting models. Possible values
                are the properties of the `ModelInfo` class.
            direction (`Literal[-1]` or `int`, *optional*):
                Direction in which to sort. The value `-1` sorts by descending
                order while all other values sort by ascending order.
            limit (`int`, *optional*):
                The limit on the number of models fetched. Leaving this option
                to `None` fetches all models.
            full (`bool`, *optional*):
                Whether to fetch all model data, including the `lastModified`,
                the `sha`, the files and the `tags`. This is set to `True` by
                default when using a filter.
            cardData (`bool`, *optional*):
                Whether to grab the metadata for the model as well. Can contain
                useful information such as carbon emissions, metrics, and
                datasets trained on.
            fetch_config (`bool`, *optional*):
                Whether to fetch the model configs as well. This is not included
                in `full` due to its size.
            use_auth_token (`bool` or `str`, *optional*):
                Whether to use the `auth_token` provided from the
                `huggingface_hub` cli. If not logged in, a valid `auth_token`
                can be passed in as a string.

        Example usage with the `filter` argument:

        ```python
        >>> from huggingface_hub import HfApi

        >>> api = HfApi()

        >>> # List all models
        >>> api.list_models()

        >>> # Get all valid search arguments
        >>> args = ModelSearchArguments()

        >>> # List only the text classification models
        >>> api.list_models(filter="text-classification")
        >>> # Using the `ModelFilter`
        >>> filt = ModelFilter(task="text-classification")
        >>> # With `ModelSearchArguments`
        >>> filt = ModelFilter(task=args.pipeline_tags.TextClassification)
        >>> api.list_models(filter=filt)

        >>> # Using `ModelFilter` and `ModelSearchArguments` to find text classification in both PyTorch and TensorFlow
        >>> filt = ModelFilter(
        ...     task=args.pipeline_tags.TextClassification,
        ...     library=[args.library.PyTorch, args.library.TensorFlow],
        ... )
        >>> api.list_models(filter=filt)

        >>> # List only models from the AllenNLP library
        >>> api.list_models(filter="allennlp")
        >>> # Using `ModelFilter` and `ModelSearchArguments`
        >>> filt = ModelFilter(library=args.library.allennlp)
        ```

        Example usage with the `search` argument:

        ```python
        >>> from huggingface_hub import HfApi

        >>> api = HfApi()

        >>> # List all models with "bert" in their name
        >>> api.list_models(search="bert")

        >>> # List all models with "bert" in their name made by google
        >>> api.list_models(search="bert", author="google")
        ```
        """
        path = f"{self.endpoint}/api/models"
        if use_auth_token:
            token, name = self._validate_or_retrieve_token(use_auth_token)
        headers = {"authorization": f"Bearer {token}"} if use_auth_token else None
        params = {}
        if filter is not None:
            if isinstance(filter, ModelFilter):
                params = self._unpack_model_filter(filter)
            else:
                params.update({"filter": filter})
            params.update({"full": True})
        if author is not None:
            params.update({"author": author})
        if search is not None:
            params.update({"search": search})
        if sort is not None:
            params.update({"sort": sort})
        if direction is not None:
            params.update({"direction": direction})
        if limit is not None:
            params.update({"limit": limit})
        if full is not None:
            if full:
                params.update({"full": True})
            elif "full" in params:
                del params["full"]
        if fetch_config is not None:
            params.update({"config": fetch_config})
        if cardData is not None:
            params.update({"cardData": cardData})
        r = requests.get(path, params=params, headers=headers)
        _raise_with_request_id(r)
        d = r.json()
        res = [ModelInfo(**x) for x in d]
        if emissions_thresholds is not None:
            if cardData is None:
                raise ValueError(
                    "`emissions_thresholds` were passed without setting"
                    " `cardData=True`."
                )
            else:
                return _filter_emissions(res, *emissions_thresholds)
        return res

    def _unpack_model_filter(self, model_filter: ModelFilter):
        """
        Unpacks a [`ModelFilter`] into something readable for `list_models`
        """
        model_str = ""
        tags = []

        # Handling author
        if model_filter.author is not None:
            model_str = f"{model_filter.author}/"

        # Handling model_name
        if model_filter.model_name is not None:
            model_str += model_filter.model_name

        filter_tuple = []

        # Handling tasks
        if model_filter.task is not None:
            filter_tuple.extend(
                [model_filter.task]
                if isinstance(model_filter.task, str)
                else model_filter.task
            )

        # Handling dataset
        if model_filter.trained_dataset is not None:
            if not isinstance(model_filter.trained_dataset, (list, tuple)):
                model_filter.trained_dataset = [model_filter.trained_dataset]
            for dataset in model_filter.trained_dataset:
                if "dataset:" not in dataset:
                    dataset = f"dataset:{dataset}"
                filter_tuple.append(dataset)

        # Handling library
        if model_filter.library:
            filter_tuple.extend(
                [model_filter.library]
                if isinstance(model_filter.library, str)
                else model_filter.library
            )

        # Handling tags
        if model_filter.tags:
            tags.extend(
                [model_filter.tags]
                if isinstance(model_filter.tags, str)
                else model_filter.tags
            )

        query_dict = {}
        if model_str is not None:
            query_dict["search"] = model_str
        if len(tags) > 0:
            query_dict["tags"] = tags
        if model_filter.language is not None:
            filter_tuple.append(model_filter.language)
        query_dict["filter"] = tuple(filter_tuple)
        return query_dict

    def list_datasets(
        self,
        *,
        filter: Union[DatasetFilter, str, Iterable[str], None] = None,
        author: Optional[str] = None,
        search: Optional[str] = None,
        sort: Union[Literal["lastModified"], str, None] = None,
        direction: Optional[Literal[-1]] = None,
        limit: Optional[int] = None,
        cardData: Optional[bool] = None,
        full: Optional[bool] = None,
        use_auth_token: Optional[str] = None,
    ) -> List[DatasetInfo]:
        """
        Get the public list of all the datasets on huggingface.co

        Args:
            filter ([`DatasetFilter`] or `str` or `Iterable`, *optional*):
                A string or [`DatasetFilter`] which can be used to identify
                datasets on the hub.
            author (`str`, *optional*):
                A string which identify the author of the returned models
            search (`str`, *optional*):
                A string that will be contained in the returned models.
            sort (`Literal["lastModified"]` or `str`, *optional*):
                The key with which to sort the resulting datasets. Possible
                values are the properties of the `DatasetInfo` class.
            direction (`Literal[-1]` or `int`, *optional*):
                Direction in which to sort. The value `-1` sorts by descending
                order while all other values sort by ascending order.
            limit (`int`, *optional*):
                The limit on the number of datasets fetched. Leaving this option
                to `None` fetches all datasets.
            cardData (`bool`, *optional*):
                Whether to grab the metadata for the dataset as well. Can
                contain useful information such as the PapersWithCode ID.
            full (`bool`, *optional*):
                Whether to fetch all dataset data, including the `lastModified`
                and the `cardData`.
            use_auth_token (`bool` or `str`, *optional*):
                Whether to use the `auth_token` provided from the
                `huggingface_hub` cli. If not logged in, a valid `auth_token`
                can be passed in as a string.

        Example usage with the `filter` argument:

        ```python
        >>> from huggingface_hub import HfApi

        >>> api = HfApi()

        >>> # List all datasets
        >>> api.list_datasets()

        >>> # Get all valid search arguments
        >>> args = DatasetSearchArguments()

        >>> # List only the text classification datasets
        >>> api.list_datasets(filter="task_categories:text-classification")
        >>> # Using the `DatasetFilter`
        >>> filt = DatasetFilter(task_categories="text-classification")
        >>> # With `DatasetSearchArguments`
        >>> filt = DatasetFilter(task=args.task_categories.text_classification)
        >>> api.list_models(filter=filt)

        >>> # List only the datasets in russian for language modeling
        >>> api.list_datasets(
        ...     filter=("languages:ru", "task_ids:language-modeling")
        ... )
        >>> # Using the `DatasetFilter`
        >>> filt = DatasetFilter(languages="ru", task_ids="language-modeling")
        >>> # With `DatasetSearchArguments`
        >>> filt = DatasetFilter(
        ...     languages=args.languages.ru,
        ...     task_ids=args.task_ids.language_modeling,
        ... )
        >>> api.list_datasets(filter=filt)
        ```

        Example usage with the `search` argument:

        ```python
        >>> from huggingface_hub import HfApi

        >>> api = HfApi()

        >>> # List all datasets with "text" in their name
        >>> api.list_datasets(search="text")

        >>> # List all datasets with "text" in their name made by google
        >>> api.list_datasets(search="text", author="google")
        ```
        """
        path = f"{self.endpoint}/api/datasets"
        if use_auth_token:
            token, name = self._validate_or_retrieve_token(use_auth_token)
        headers = {"authorization": f"Bearer {token}"} if use_auth_token else None
        params = {}
        if filter is not None:
            if isinstance(filter, DatasetFilter):
                params = self._unpack_dataset_filter(filter)
            else:
                params.update({"filter": filter})
        if author is not None:
            params.update({"author": author})
        if search is not None:
            params.update({"search": search})
        if sort is not None:
            params.update({"sort": sort})
        if direction is not None:
            params.update({"direction": direction})
        if limit is not None:
            params.update({"limit": limit})
        if full is not None:
            if full:
                params.update({"full": True})
        if cardData is not None:
            if cardData:
                params.update({"full": True})
        r = requests.get(path, params=params, headers=headers)
        _raise_with_request_id(r)
        d = r.json()
        return [DatasetInfo(**x) for x in d]

    def _unpack_dataset_filter(self, dataset_filter: DatasetFilter):
        """
        Unpacks a [`DatasetFilter`] into something readable for `list_datasets`
        """
        dataset_str = ""

        # Handling author
        if dataset_filter.author is not None:
            dataset_str = f"{dataset_filter.author}/"

        # Handling dataset_name
        if dataset_filter.dataset_name is not None:
            dataset_str += dataset_filter.dataset_name

        filter_tuple = []
        data_attributes = [
            "benchmark",
            "language_creators",
            "languages",
            "multilinguality",
            "size_categories",
            "task_categories",
            "task_ids",
        ]

        for attr in data_attributes:
            curr_attr = getattr(dataset_filter, attr)
            if curr_attr is not None:
                if not isinstance(curr_attr, (list, tuple)):
                    curr_attr = [curr_attr]
                for data in curr_attr:
                    if f"{attr}:" not in data:
                        data = f"{attr}:{data}"
                    filter_tuple.append(data)

        query_dict = {}
        if dataset_str is not None:
            query_dict["search"] = dataset_str
        query_dict["filter"] = tuple(filter_tuple)
        return query_dict

    def list_metrics(self) -> List[MetricInfo]:
        """
        Get the public list of all the metrics on huggingface.co

        Returns:
            `List[MetricInfo]`: a list of [`MetricInfo`] objects which.
        """
        path = f"{self.endpoint}/api/metrics"
        params = {}
        r = requests.get(path, params=params)
        _raise_with_request_id(r)
        d = r.json()
        return [MetricInfo(**x) for x in d]

    def model_info(
        self,
        repo_id: str,
        *,
        revision: Optional[str] = None,
        token: Optional[str] = None,
        timeout: Optional[float] = None,
        securityStatus: Optional[bool] = None,
    ) -> ModelInfo:
        """
        Get info on one specific model on huggingface.co

        Model can be private if you pass an acceptable token or are logged in.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            revision (`str`, *optional*):
                The revision of the model repository from which to get the
                information.
            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)
            timeout (`float`, *optional*):
                Whether to set a timeout for the request to the Hub.
            securityStatus (`bool`, *optional*):
                Whether to retrieve the security status from the model
                repository as well.

        Returns:
            [`huggingface_hub.hf_api.ModelInfo`]: The model repository information.

        <Tip>

        Raises the following errors:

            - [`~huggingface_hub.utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.
            - [`~huggingface_hub.utils.RevisionNotFoundError`]
              If the revision to download from cannot be found.

        </Tip>
        """
        if token is None:
            token = HfFolder.get_token()

        path = (
            f"{self.endpoint}/api/models/{repo_id}"
            if revision is None
            else f"{self.endpoint}/api/models/{repo_id}/revision/{revision}"
        )
        headers = {"authorization": f"Bearer {token}"} if token is not None else None
        status_query_param = {"securityStatus": True} if securityStatus else None
        r = requests.get(
            path, headers=headers, timeout=timeout, params=status_query_param
        )
        _raise_for_status(r)
        d = r.json()
        return ModelInfo(**d)

    def dataset_info(
        self,
        repo_id: str,
        *,
        revision: Optional[str] = None,
        token: Optional[str] = None,
        timeout: Optional[float] = None,
    ) -> DatasetInfo:
        """
        Get info on one specific dataset on huggingface.co.

        Dataset can be private if you pass an acceptable token.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            revision (`str`, *optional*):
                The revision of the dataset repository from which to get the
                information.
            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)
            timeout (`float`, *optional*):
                Whether to set a timeout for the request to the Hub.

        Returns:
            [`DatasetInfo`]: The dataset repository information.

        <Tip>

        Raises the following errors:

            - [`~huggingface_hub.utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.
            - [`~huggingface_hub.utils.RevisionNotFoundError`]
              If the revision to download from cannot be found.

        </Tip>
        """
        if token is None:
            token = HfFolder.get_token()

        path = (
            f"{self.endpoint}/api/datasets/{repo_id}"
            if revision is None
            else f"{self.endpoint}/api/datasets/{repo_id}/revision/{revision}"
        )
        headers = {"authorization": f"Bearer {token}"} if token is not None else None
        r = requests.get(path, headers=headers, timeout=timeout)
        _raise_for_status(r)
        d = r.json()
        return DatasetInfo(**d)

    def space_info(
        self,
        repo_id: str,
        *,
        revision: Optional[str] = None,
        token: Optional[str] = None,
        timeout: Optional[float] = None,
    ) -> SpaceInfo:
        """
        Get info on one specific Space on huggingface.co.

        Space can be private if you pass an acceptable token.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            revision (`str`, *optional*):
                The revision of the space repository from which to get the
                information.
            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)
            timeout (`float`, *optional*):
                Whether to set a timeout for the request to the Hub.

        Returns:
            [`SpaceInfo`]: The space repository information.

        <Tip>

        Raises the following errors:

            - [`~huggingface_hub.utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.
            - [`~huggingface_hub.utils.RevisionNotFoundError`]
              If the revision to download from cannot be found.

        </Tip>
        """
        if token is None:
            token = HfFolder.get_token()

        path = (
            f"{self.endpoint}/api/spaces/{repo_id}"
            if revision is None
            else f"{self.endpoint}/api/spaces/{repo_id}/revision/{revision}"
        )
        headers = {"authorization": f"Bearer {token}"} if token is not None else None
        r = requests.get(path, headers=headers, timeout=timeout)
        _raise_for_status(r)
        d = r.json()
        return SpaceInfo(**d)

    def repo_info(
        self,
        repo_id: str,
        *,
        revision: Optional[str] = None,
        repo_type: Optional[str] = None,
        token: Optional[str] = None,
        timeout: Optional[float] = None,
    ) -> Union[ModelInfo, DatasetInfo, SpaceInfo]:
        """
        Get the info object for a given repo of a given type.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            revision (`str`, *optional*):
                The revision of the repository from which to get the
                information.
            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)
            timeout (`float`, *optional*):
                Whether to set a timeout for the request to the Hub.

        Returns:
            `Union[SpaceInfo, DatasetInfo, ModelInfo]`: The repository
            information.

        <Tip>

        Raises the following errors:

            - [`~huggingface_hub.utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.
            - [`~huggingface_hub.utils.RevisionNotFoundError`]
              If the revision to download from cannot be found.

        </Tip>
        """
        if repo_type is None or repo_type == "model":
            return self.model_info(
                repo_id, revision=revision, token=token, timeout=timeout
            )
        elif repo_type == "dataset":
            return self.dataset_info(
                repo_id, revision=revision, token=token, timeout=timeout
            )
        elif repo_type == "space":
            return self.space_info(
                repo_id, revision=revision, token=token, timeout=timeout
            )
        else:
            raise ValueError("Unsupported repo type.")

    def list_repo_files(
        self,
        repo_id: str,
        *,
        revision: Optional[str] = None,
        repo_type: Optional[str] = None,
        token: Optional[str] = None,
        timeout: Optional[float] = None,
    ) -> List[str]:
        """
        Get the list of files in a given repo.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            revision (`str`, *optional*):
                The revision of the model repository from which to get the
                information.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model. Default is
                `None`.
            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)
            timeout (`float`, *optional*):
                Whether to set a timeout for the request to the Hub.

        Returns:
            `List[str]`: the list of files in a given repository.
        """
        repo_info = self.repo_info(
            repo_id,
            revision=revision,
            repo_type=repo_type,
            token=token,
            timeout=timeout,
        )
        return [f.rfilename for f in repo_info.siblings]

    @_deprecate_positional_args
    def create_repo(
        self,
        repo_id: str = None,
        *,
        token: Optional[str] = None,
        organization: Optional[str] = None,
        private: Optional[bool] = None,
        repo_type: Optional[str] = None,
        exist_ok: Optional[bool] = False,
        space_sdk: Optional[str] = None,
        name: Optional[str] = None,
    ) -> str:
        """Create an empty repo on the HuggingFace Hub.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.

                <Tip>

                Version added: 0.5

                </Tip>

            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)
            private (`bool`, *optional*):
                Whether the model repo should be private.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model. Default is
                `None`.
            exist_ok (`bool`, *optional*, defaults to `False`):
                If `True`, do not raise an error if repo already exists.
            space_sdk (`str`, *optional*):
                Choice of SDK to use if repo_type is "space". Can be
                "streamlit", "gradio", or "static".

        Returns:
            `str`: URL to the newly created repo.
        """
        name, organization = _validate_repo_id_deprecation(repo_id, name, organization)

        path = f"{self.endpoint}/api/repos/create"

        token, name = self._validate_or_retrieve_token(
            token, name, function_name="create_repo"
        )

        checked_name = repo_type_and_id_from_hf_id(name)

        if (
            repo_type is not None
            and checked_name[0] is not None
            and repo_type != checked_name[0]
        ):
            raise ValueError(
                f"""Passed `repo_type` and found `repo_type` are not the same ({repo_type},
{checked_name[0]}).
            Please make sure you are expecting the right type of repository to
            exist."""
            )

        if (
            organization is not None
            and checked_name[1] is not None
            and organization != checked_name[1]
        ):
            raise ValueError(
                f"""Passed `organization` and `name` organization are not the same ({organization},
{checked_name[1]}).
            Please either include the organization in only `name` or the
            `organization` parameter, such as
            `api.create_repo({checked_name[0]}, organization={organization})` or
            `api.create_repo({checked_name[1]}/{checked_name[2]})`"""
            )

        repo_type = repo_type or checked_name[0]
        organization = organization or checked_name[1]
        name = checked_name[2]

        if repo_type not in REPO_TYPES:
            raise ValueError("Invalid repo type")

        json = {"name": name, "organization": organization, "private": private}
        if repo_type is not None:
            json["type"] = repo_type
        if repo_type == "space":
            if space_sdk is None:
                raise ValueError(
                    "No space_sdk provided. `create_repo` expects space_sdk to be one"
                    f" of {SPACES_SDK_TYPES} when repo_type is 'space'`"
                )
            if space_sdk not in SPACES_SDK_TYPES:
                raise ValueError(
                    f"Invalid space_sdk. Please choose one of {SPACES_SDK_TYPES}."
                )
            json["sdk"] = space_sdk
        if space_sdk is not None and repo_type != "space":
            warnings.warn(
                "Ignoring provided space_sdk because repo_type is not 'space'."
            )

        if getattr(self, "_lfsmultipartthresh", None):
            json["lfsmultipartthresh"] = self._lfsmultipartthresh
        r = requests.post(
            path,
            headers={"authorization": f"Bearer {token}"},
            json=json,
        )

        try:
            _raise_with_request_id(r)
        except HTTPError as err:
            if not (exist_ok and err.response.status_code == 409):
                try:
                    additional_info = r.json().get("error", None)
                    if additional_info:
                        new_err = f"{err.args[0]} - {additional_info}"
                        err.args = (new_err,) + err.args[1:]
                except ValueError:
                    pass

                raise err

        d = r.json()
        return d["url"]

    def delete_repo(
        self,
        repo_id: str = None,
        *,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
    ):
        """
        Delete a repo from the HuggingFace Hub. CAUTION: this is irreversible.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.

                <Tip>

                Version added: 0.5

                </Tip>

            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model.

        <Tip>

        Raises the following errors:

            - [`~huggingface_hub.utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.

        </Tip>
        """
        organization, name = repo_id.split("/") if "/" in repo_id else (None, repo_id)

        path = f"{self.endpoint}/api/repos/delete"

        token, name = self._validate_or_retrieve_token(
            token, name, function_name="delete_repo"
        )

        checked_name = repo_type_and_id_from_hf_id(name)

        if (
            repo_type is not None
            and checked_name[0] is not None
            and repo_type != checked_name[0]
        ):
            raise ValueError(
                f"""Passed `repo_type` and found `repo_type` are not the same ({repo_type},
{checked_name[0]}).
            Please make sure you are expecting the right type of repository to
            exist."""
            )

        if (
            organization is not None
            and checked_name[1] is not None
            and organization != checked_name[1]
        ):
            raise ValueError(
                "Passed `organization` and `name` organization are not the same"
                f" ({organization}, {checked_name[1]})."
                "\nPlease either include the organization in only `name` or the"
                " `organization` parameter, such as "
                f"`api.create_repo({checked_name[0]}, organization={organization})` "
                f"or `api.create_repo({checked_name[1]}/{checked_name[2]})`"
            )

        repo_type = repo_type or checked_name[0]
        organization = organization or checked_name[1]
        name = checked_name[2]

        if repo_type not in REPO_TYPES:
            raise ValueError("Invalid repo type")

        json = {"name": name, "organization": organization}
        if repo_type is not None:
            json["type"] = repo_type

        r = requests.delete(
            path,
            headers={"authorization": f"Bearer {token}"},
            json=json,
        )
        try:
            _raise_for_status(r)
        except requests.exceptions.RequestException as e:
            try:
                message = e.response.json()["error"]
            except JSONDecodeError:
                message = e.response.text
            except AttributeError:
                message = e.args[0]
            raise type(e)(message) from e

    def update_repo_visibility(
        self,
        repo_id: str = None,
        private: bool = False,
        *,
        token: Optional[str] = None,
        organization: Optional[str] = None,
        repo_type: Optional[str] = None,
        name: str = None,
    ) -> Dict[str, bool]:
        """Update the visibility setting of a repository.

        Args:
            repo_id (`str`, *optional*):
                A namespace (user or an organization) and a repo name separated
                by a `/`.

                <Tip>

                Version added: 0.5

                </Tip>

            private (`bool`, *optional*, defaults to `False`):
                Whether the model repo should be private.
            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model. Default is
                `None`.

        Returns:
            The HTTP response in json.

        <Tip>

        Raises the following errors:

            - [`~huggingface_hub.utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.

        </Tip>
        """
        if repo_type not in REPO_TYPES:
            raise ValueError("Invalid repo type")

        organization, name = repo_id.split("/") if "/" in repo_id else (None, repo_id)

        token, name = self._validate_or_retrieve_token(
            token, name, function_name="update_repo_visibility"
        )

        if organization is None:
            namespace = self.whoami(token)["name"]
        else:
            namespace = organization

        path_prefix = f"{self.endpoint}/api/"
        if repo_type in REPO_TYPES_URL_PREFIXES:
            path_prefix += REPO_TYPES_URL_PREFIXES[repo_type]

        path = f"{path_prefix}{namespace}/{name}/settings"

        json = {"private": private}

        r = requests.put(
            path,
            headers={"authorization": f"Bearer {token}"},
            json=json,
        )
        _raise_for_status(r)
        return r.json()

    def move_repo(
        self,
        from_id: str,
        to_id: str,
        *,
        repo_type: Optional[str] = None,
        token: Optional[str] = None,
    ):
        """
        Moving a repository from namespace1/repo_name1 to namespace2/repo_name2

        Note there are certain limitations. For more information about moving
        repositories, please see
        https://hf.co/docs/hub/main#how-can-i-rename-or-transfer-a-repo.

        Args:
            from_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`. Original repository identifier.
            to_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`. Final repository identifier.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model. Default is
                `None`.
            token (`str`, *optional*):
                An authentication token (See https://huggingface.co/settings/token)

        <Tip>

        Raises the following errors:

            - [`~huggingface_hub.utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.

        </Tip>
        """

        token, name = self._validate_or_retrieve_token(token)

        if len(from_id.split("/")) != 2:
            raise ValueError(
                f"Invalid repo_id: {from_id}. It should have a namespace"
                " (:namespace:/:repo_name:)"
            )

        if len(to_id.split("/")) != 2:
            raise ValueError(
                f"Invalid repo_id: {to_id}. It should have a namespace"
                " (:namespace:/:repo_name:)"
            )

        json = {"fromRepo": from_id, "toRepo": to_id, "type": repo_type}

        path = f"{self.endpoint}/api/repos/move"
        r = requests.post(
            path,
            headers={"authorization": f"Bearer {token}"},
            json=json,
        )
        try:
            _raise_for_status(r)
        except HTTPError as e:
            if r.text:
                raise HTTPError(
                    f"{r.status_code} Error Message: {r.text}. For additional"
                    " documentation please see"
                    " https://hf.co/docs/hub/main#how-can-i-rename-or-transfer-a-repo."
                ) from e
            else:
                raise e
        logger.info(
            "Accepted transfer request. You will get an email once this is successfully"
            " completed."
        )

    def upload_file(
        self,
        *,
        path_or_fileobj: Union[str, bytes, IO],
        path_in_repo: str,
        repo_id: str,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
        revision: Optional[str] = None,
        identical_ok: Optional[bool] = None,
    ) -> str:
        """
        Upload a local file (up to 5GB) to the given repo. The upload is done
        through a HTTP post request, and doesn't require git or git-lfs to be
        installed.

        Args:
            path_or_fileobj (`str`, `bytes`, or `IO`):
                Path to a file on the local machine or binary data stream /
                fileobj / buffer.
            path_in_repo (`str`):
                Relative filepath in the repo, for example:
                `"checkpoints/1fec34a/weights.bin"`
            repo_id (`str`):
                The repository to which the file will be uploaded, for example:
                `"username/custom_transformers"`
            token (`str`, *optional*):
                Authentication token, obtained with `HfApi.login` method. Will
                default to the stored token.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if uploading to a dataset or
                space, `None` or `"model"` if uploading to a model. Default is
                `None`.
            revision (`str`, *optional*):
                The git revision to commit from. Defaults to the head of the
                `"main"` branch.
            identical_ok (`bool`, *optional*, defaults to `True`):
                Deprecated: will be removed in 0.11.0.
                Changing this value has no effect.

        Returns:
            `str`: The URL to visualize the uploaded file on the hub

        <Tip>

        Raises the following errors:

            - [`HTTPError`](https://2.python-requests.org/en/master/api/#requests.HTTPError)
              if the HuggingFace API returned an error
            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
              if some parameter value is invalid
            - [`~huggingface_hub.utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.
            - [`~huggingface_hub.utils.RevisionNotFoundError`]
              If the revision to download from cannot be found.

        </Tip>

        Example usage:

        ```python
        >>> with open("./local/filepath", "rb") as fobj:
        ...     upload_file(
        ...         path_or_fileobj=fileobj,
        ...         path_in_repo="remote/file/path.h5",
        ...         repo_id="username/my-dataset",
        ...         repo_type="datasets",
        ...         token="my_token",
        ...     )
        "https://huggingface.co/datasets/username/my-dataset/blob/main/remote/file/path.h5"

        >>> upload_file(
        ...     path_or_fileobj=".\\\\local\\\\file\\\\path",
        ...     path_in_repo="remote/file/path.h5",
        ...     repo_id="username/my-model",
        ...     token="my_token",
        ... )
        "https://huggingface.co/username/my-model/blob/main/remote/file/path.h5"
        ```
        """
        if identical_ok is not None:
            warnings.warn(
                "`identical_ok` has no effect and is deprecated. It will be removed in"
                " 0.11.0.",
                FutureWarning,
            )
        if repo_type not in REPO_TYPES:
            raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")

        try:
            token, name = self._validate_or_retrieve_token(
                token, function_name="upload_file"
            )
        except ValueError:  # if token is invalid or organization token
            if self._is_valid_token(path_or_fileobj):
                warnings.warn(
                    "`upload_file` now takes `token` as an optional positional"
                    " argument. Be sure to adapt your code!",
                    FutureWarning,
                )
                token, path_or_fileobj, path_in_repo, repo_id = (
                    path_or_fileobj,
                    path_in_repo,
                    repo_id,
                    token,
                )
            else:
                raise ValueError("Invalid token passed!")

        # Validate path_or_fileobj
        if isinstance(path_or_fileobj, str):
            path_or_fileobj = os.path.normpath(os.path.expanduser(path_or_fileobj))
            if not os.path.isfile(path_or_fileobj):
                raise ValueError(f"Provided path: '{path_or_fileobj}' is not a file")
        elif not isinstance(path_or_fileobj, (RawIOBase, BufferedIOBase, bytes)):
            # ^^ Test from: https://stackoverflow.com/questions/44584829/how-to-determine-if-file-is-opened-in-binary-or-text-mode
            raise ValueError(
                "path_or_fileobj must be either an instance of str or BinaryIO. If you"
                " passed a fileobj, make sure you've opened the file in binary mode."
            )

        if repo_type in REPO_TYPES_URL_PREFIXES:
            repo_id = REPO_TYPES_URL_PREFIXES[repo_type] + repo_id

        revision = revision if revision is not None else "main"

        path = f"{self.endpoint}/api/{repo_id}/upload/{revision}/{path_in_repo}"

        headers = {"authorization": f"Bearer {token}"} if token is not None else None

        if isinstance(path_or_fileobj, str):
            with open(path_or_fileobj, "rb") as bytestream:
                r = requests.post(path, headers=headers, data=bytestream)
        else:
            r = requests.post(path, headers=headers, data=path_or_fileobj)

        _raise_for_status(r)
        d = r.json()
        return d["url"]

    def delete_file(
        self,
        path_in_repo: str,
        repo_id: str,
        *,
        token: Optional[str] = None,
        repo_type: Optional[str] = None,
        revision: Optional[str] = None,
    ):
        """
        Deletes a file in the given repo.

        Args:
            path_in_repo (`str`):
                Relative filepath in the repo, for example:
                `"checkpoints/1fec34a/weights.bin"`
            repo_id (`str`):
                The repository from which the file will be deleted, for example:
                `"username/custom_transformers"`
            token (`str`, *optional*):
                Authentication token, obtained with `HfApi.login` method. Will
                default to the stored token.
            repo_type (`str`, *optional*):
                Set to `"dataset"` or `"space"` if the file is in a dataset or
                space, `None` or `"model"` if in a model. Default is `None`.
            revision (`str`, *optional*):
                The git revision to commit from. Defaults to the head of the
                `"main"` branch.

        <Tip>

        Raises the following errors:

            - [`HTTPError`](https://2.python-requests.org/en/master/api/#requests.HTTPError)
              if the HuggingFace API returned an error
            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
              if some parameter value is invalid
            - [`~huggingface_hub.utils.RepositoryNotFoundError`]
              If the repository to download from cannot be found. This may be because it doesn't exist,
              or because it is set to `private` and you do not have access.
            - [`~huggingface_hub.utils.RevisionNotFoundError`]
              If the revision to download from cannot be found.
            - [`~huggingface_hub.utils.EntryNotFoundError`]
              If the file to download cannot be found.

        </Tip>

        """
        if repo_type not in REPO_TYPES:
            raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")

        token, name = self._validate_or_retrieve_token(token)

        if repo_type in REPO_TYPES_URL_PREFIXES:
            repo_id = REPO_TYPES_URL_PREFIXES[repo_type] + repo_id

        revision = revision if revision is not None else "main"

        path = f"{self.endpoint}/api/{repo_id}/delete/{revision}/{path_in_repo}"

        headers = {"authorization": f"Bearer {token}"}
        r = requests.delete(path, headers=headers)

        _raise_for_status(r)

    def get_full_repo_name(
        self,
        model_id: str,
        *,
        organization: Optional[str] = None,
        token: Optional[str] = None,
    ):
        """
        Returns the repository name for a given model ID and optional
        organization.

        Args:
            model_id (`str`):
                The name of the model.
            organization (`str`, *optional*):
                If passed, the repository name will be in the organization
                namespace instead of the user namespace.
            token (`str`, *optional*):
                The Hugging Face authentication token

        Returns:
            `str`: The repository name in the user's namespace
            ({username}/{model_id}) if no organization is passed, and under the
            organization namespace ({organization}/{model_id}) otherwise.
        """
        if organization is None:
            if "/" in model_id:
                username = model_id.split("/")[0]
            else:
                username = self.whoami(token=token)["name"]
            return f"{username}/{model_id}"
        else:
            return f"{organization}/{model_id}"


class HfFolder:
    path_token = expanduser("~/.huggingface/token")

    @classmethod
    def save_token(cls, token):
        """
        Save token, creating folder as needed.

        Args:
            token (`str`):
                The token to save to the [`HfFolder`]
        """
        os.makedirs(os.path.dirname(cls.path_token), exist_ok=True)
        with open(cls.path_token, "w+") as f:
            f.write(token)

    @classmethod
    def get_token(cls) -> Optional[str]:
        """
        Get token or None if not existent.

        Note that a token can be also provided using the `HUGGING_FACE_HUB_TOKEN`
        environment variable.

        Returns:
            `str` or `None`: The token, `None` if it doesn't exist.

        """
        token: Optional[str] = os.environ.get("HUGGING_FACE_HUB_TOKEN")
        if token is None:
            try:
                with open(cls.path_token, "r") as f:
                    token = f.read()
            except FileNotFoundError:
                pass
        return token

    @classmethod
    def delete_token(cls):
        """
        Deletes the token from storage. Does not fail if token does not exist.
        """
        try:
            os.remove(cls.path_token)
        except FileNotFoundError:
            pass


api = HfApi()

set_access_token = api.set_access_token
unset_access_token = api.unset_access_token

whoami = api.whoami

list_models = api.list_models
model_info = api.model_info

list_datasets = api.list_datasets
dataset_info = api.dataset_info

space_info = api.space_info

repo_info = api.repo_info
list_repo_files = api.list_repo_files

list_metrics = api.list_metrics

get_model_tags = api.get_model_tags
get_dataset_tags = api.get_dataset_tags

create_repo = api.create_repo
delete_repo = api.delete_repo
update_repo_visibility = api.update_repo_visibility
move_repo = api.move_repo
upload_file = api.upload_file
delete_file = api.delete_file
get_full_repo_name = api.get_full_repo_name


# src/huggingface_hub/hub_mixin.py

import json
import os
from pathlib import Path
from typing import Dict, Optional, Union

import requests

from .constants import CONFIG_NAME, PYTORCH_WEIGHTS_NAME
from .file_download import hf_hub_download, is_torch_available
from .hf_api import HfApi, HfFolder
from .repository import Repository
from .utils import logging


if is_torch_available():
    import torch


logger = logging.get_logger(__name__)


class ModelHubMixin:
    """
    A Generic Base Model Hub Mixin. Define your own mixin for anything by
    inheriting from this class and overwriting `_from_pretrained` and
    `_save_pretrained` to define custom logic for saving/loading your classes.
    See `huggingface_hub.PyTorchModelHubMixin` for an example.
    """

    def save_pretrained(
        self,
        save_directory: str,
        config: Optional[dict] = None,
        push_to_hub: bool = False,
        **kwargs,
    ):
        """
        Save weights in local directory.

                Parameters:
                    save_directory (`str`):
                        Specify directory in which you want to save weights.
                    config (`dict`, *optional*):
                        specify config (must be dict) in case you want to save
                        it.
                    push_to_hub (`bool`, *optional*, defaults to `False`):
                        Set it to `True` in case you want to push your weights
                        to huggingface_hub
                    kwargs (`Dict`, *optional*):
                        kwargs will be passed to `push_to_hub`
        """

        os.makedirs(save_directory, exist_ok=True)

        # saving config
        if isinstance(config, dict):
            path = os.path.join(save_directory, CONFIG_NAME)
            with open(path, "w") as f:
                json.dump(config, f)

        # saving model weights/files
        self._save_pretrained(save_directory)

        if push_to_hub:
            return self.push_to_hub(save_directory, **kwargs)

    def _save_pretrained(self, save_directory):
        """
        Overwrite this method in subclass to define how to save your model.
        """
        raise NotImplementedError

    @classmethod
    def from_pretrained(
        cls,
        pretrained_model_name_or_path: str,
        force_download: bool = False,
        resume_download: bool = False,
        proxies: Dict = None,
        use_auth_token: Optional[str] = None,
        cache_dir: Optional[str] = None,
        local_files_only: bool = False,
        **model_kwargs,
    ):
        r"""
        Instantiate a pretrained PyTorch model from a pre-trained model
                configuration from huggingface-hub. The model is set in
                evaluation mode by default using `model.eval()` (Dropout modules
                are deactivated). To train the model, you should first set it
                back in training mode with `model.train()`.

                Parameters:
                    pretrained_model_name_or_path (`str` or `os.PathLike`):
                        Can be either:
                            - A string, the `model id` of a pretrained model
                              hosted inside a model repo on huggingface.co.
                              Valid model ids can be located at the root-level,
                              like `bert-base-uncased`, or namespaced under a
                              user or organization name, like
                              `dbmdz/bert-base-german-cased`.
                            - You can add `revision` by appending `@` at the end
                              of model_id simply like this:
                              `dbmdz/bert-base-german-cased@main` Revision is
                              the specific model version to use. It can be a
                              branch name, a tag name, or a commit id, since we
                              use a git-based system for storing models and
                              other artifacts on huggingface.co, so `revision`
                              can be any identifier allowed by git.
                            - A path to a `directory` containing model weights
                              saved using
                              [`~transformers.PreTrainedModel.save_pretrained`],
                              e.g., `./my_model_directory/`.
                            - `None` if you are both providing the configuration
                              and state dictionary (resp. with keyword arguments
                              `config` and `state_dict`).
                    force_download (`bool`, *optional*, defaults to `False`):
                        Whether to force the (re-)download of the model weights
                        and configuration files, overriding the cached versions
                        if they exist.
                    resume_download (`bool`, *optional*, defaults to `False`):
                        Whether to delete incompletely received files. Will
                        attempt to resume the download if such a file exists.
                    proxies (`Dict[str, str]`, *optional*):
                        A dictionary of proxy servers to use by protocol or
                        endpoint, e.g., `{'http': 'foo.bar:3128',
                        'http://hostname': 'foo.bar:4012'}`. The proxies are
                        used on each request.
                    use_auth_token (`str` or `bool`, *optional*):
                        The token to use as HTTP bearer authorization for remote
                        files. If `True`, will use the token generated when
                        running `transformers-cli login` (stored in
                        `~/.huggingface`).
                    cache_dir (`Union[str, os.PathLike]`, *optional*):
                        Path to a directory in which a downloaded pretrained
                        model configuration should be cached if the standard
                        cache should not be used.
                    local_files_only(`bool`, *optional*, defaults to `False`):
                        Whether to only look at local files (i.e., do not try to
                        download the model).
                    model_kwargs (`Dict`, *optional*):
                        model_kwargs will be passed to the model during
                        initialization

                <Tip>

                Passing `use_auth_token=True` is required when you want to use a
                private model.

                </Tip>
        """

        model_id = pretrained_model_name_or_path

        revision = None
        if len(model_id.split("@")) == 2:
            model_id, revision = model_id.split("@")

        if os.path.isdir(model_id) and CONFIG_NAME in os.listdir(model_id):
            config_file = os.path.join(model_id, CONFIG_NAME)
        else:
            try:
                config_file = hf_hub_download(
                    repo_id=model_id,
                    filename=CONFIG_NAME,
                    revision=revision,
                    cache_dir=cache_dir,
                    force_download=force_download,
                    proxies=proxies,
                    resume_download=resume_download,
                    use_auth_token=use_auth_token,
                    local_files_only=local_files_only,
                )
            except requests.exceptions.RequestException:
                logger.warning(f"{CONFIG_NAME} not found in HuggingFace Hub")
                config_file = None

        if config_file is not None:
            with open(config_file, "r", encoding="utf-8") as f:
                config = json.load(f)
            model_kwargs.update({"config": config})

        return cls._from_pretrained(
            model_id,
            revision,
            cache_dir,
            force_download,
            proxies,
            resume_download,
            local_files_only,
            use_auth_token,
            **model_kwargs,
        )

    @classmethod
    def _from_pretrained(
        cls,
        model_id,
        revision,
        cache_dir,
        force_download,
        proxies,
        resume_download,
        local_files_only,
        use_auth_token,
        **model_kwargs,
    ):
        """Overwrite this method in subclass to define how to load your model from
        pretrained"""
        raise NotImplementedError

    def push_to_hub(
        self,
        repo_path_or_name: Optional[str] = None,
        repo_url: Optional[str] = None,
        commit_message: Optional[str] = "Add model",
        organization: Optional[str] = None,
        private: Optional[bool] = None,
        api_endpoint: Optional[str] = None,
        use_auth_token: Optional[Union[bool, str]] = None,
        git_user: Optional[str] = None,
        git_email: Optional[str] = None,
        config: Optional[dict] = None,
        skip_lfs_files: bool = False,
    ) -> str:
        """
        Upload model checkpoint or tokenizer files to the Hub while
        synchronizing a local clone of the repo in `repo_path_or_name`.

        Parameters:
            repo_path_or_name (`str`, *optional*):
                Can either be a repository name for your model or tokenizer in
                the Hub or a path to a local folder (in which case the
                repository will have the name of that local folder). If not
                specified, will default to the name given by `repo_url` and a
                local directory with that name will be created.
            repo_url (`str`, *optional*):
                Specify this in case you want to push to an existing repository
                in the hub. If unspecified, a new repository will be created in
                your namespace (unless you specify an `organization`) with
                `repo_name`.
            commit_message (`str`, *optional*):
                Message to commit while pushing. Will default to `"add config"`,
                `"add tokenizer"` or `"add model"` depending on the type of the
                class.
            organization (`str`, *optional*):
                Organization in which you want to push your model or tokenizer
                (you must be a member of this organization).
            private (`bool`, *optional*):
                Whether the repository created should be private.
            api_endpoint (`str`, *optional*):
                The API endpoint to use when pushing the model to the hub.
            use_auth_token (`bool` or `str`, *optional*):
                The token to use as HTTP bearer authorization for remote files.
                If `True`, will use the token generated when running
                `transformers-cli login` (stored in `~/.huggingface`). Will
                default to `True` if `repo_url` is not specified.
            git_user (`str`, *optional*):
                will override the `git config user.name` for committing and
                pushing files to the hub.
            git_email (`str`, *optional*):
                will override the `git config user.email` for committing and
                pushing files to the hub.
            config (`dict`, *optional*):
                Configuration object to be saved alongside the model weights.
            skip_lfs_files (`bool`, *optional*, defaults to `False`):
                Whether to skip git-LFS files or not.


        Returns:
            The url of the commit of your model in the given repository.
        """

        if repo_path_or_name is None and repo_url is None:
            raise ValueError(
                "You need to specify a `repo_path_or_name` or a `repo_url`."
            )

        if use_auth_token is None and repo_url is None:
            token = HfFolder.get_token()
            if token is None:
                raise ValueError(
                    "You must login to the Hugging Face hub on this computer by typing"
                    " `huggingface-cli login` and entering your credentials to use"
                    " `use_auth_token=True`. Alternatively, you can pass your own token"
                    " as the `use_auth_token` argument."
                )
        elif isinstance(use_auth_token, str):
            token = use_auth_token
        else:
            token = None

        if repo_path_or_name is None:
            repo_path_or_name = repo_url.split("/")[-1]

        # If no URL is passed and there's no path to a directory containing files, create a repo
        if repo_url is None and not os.path.exists(repo_path_or_name):
            repo_id = Path(repo_path_or_name).name
            if organization:
                repo_id = f"{organization}/{repo_id}"
            repo_url = HfApi(endpoint=api_endpoint).create_repo(
                repo_id=repo_id,
                token=token,
                private=private,
                repo_type=None,
                exist_ok=True,
            )

        repo = Repository(
            repo_path_or_name,
            clone_from=repo_url,
            use_auth_token=use_auth_token,
            git_user=git_user,
            git_email=git_email,
            skip_lfs_files=skip_lfs_files,
        )
        repo.git_pull(rebase=True)

        # Save the files in the cloned repo
        self.save_pretrained(repo_path_or_name, config=config)

        # Commit and push!
        repo.git_add()
        repo.git_commit(commit_message)
        return repo.git_push()


class PyTorchModelHubMixin(ModelHubMixin):
    def __init__(self, *args, **kwargs):
        """
        Mix this class with your torch-model class for ease process of saving &
        loading from huggingface-hub.

        Example usage:

        ```python
        >>> from huggingface_hub import PyTorchModelHubMixin


        >>> class MyModel(nn.Module, PyTorchModelHubMixin):
        ...     def __init__(self, **kwargs):
        ...         super().__init__()
        ...         self.config = kwargs.pop("config", None)
        ...         self.layer = ...

        ...     def forward(self, *args):
        ...         return ...


        >>> model = MyModel()
        >>> model.save_pretrained(
        ...     "mymodel", push_to_hub=False
        >>> )  # Saving model weights in the directory
        >>> model.push_to_hub(
        ...     "mymodel", "model-1"
        >>> )  # Pushing model-weights to hf-hub

        >>> # Downloading weights from hf-hub & model will be initialized from those weights
        >>> model = MyModel.from_pretrained("username/mymodel@main")
        ```
        """

    def _save_pretrained(self, save_directory):
        """
        Overwrite this method in case you don't want to save complete model,
        rather some specific layers
        """
        path = os.path.join(save_directory, PYTORCH_WEIGHTS_NAME)
        model_to_save = self.module if hasattr(self, "module") else self
        torch.save(model_to_save.state_dict(), path)

    @classmethod
    def _from_pretrained(
        cls,
        model_id,
        revision,
        cache_dir,
        force_download,
        proxies,
        resume_download,
        local_files_only,
        use_auth_token,
        map_location="cpu",
        strict=False,
        **model_kwargs,
    ):
        """
        Overwrite this method in case you wish to initialize your model in a
        different way.
        """
        map_location = torch.device(map_location)

        if os.path.isdir(model_id):
            print("Loading weights from local directory")
            model_file = os.path.join(model_id, PYTORCH_WEIGHTS_NAME)
        else:
            model_file = hf_hub_download(
                repo_id=model_id,
                filename=PYTORCH_WEIGHTS_NAME,
                revision=revision,
                cache_dir=cache_dir,
                force_download=force_download,
                proxies=proxies,
                resume_download=resume_download,
                use_auth_token=use_auth_token,
                local_files_only=local_files_only,
            )
        model = cls(**model_kwargs)

        state_dict = torch.load(model_file, map_location=map_location)
        model.load_state_dict(state_dict, strict=strict)
        model.eval()

        return model


# src/huggingface_hub/inference_api.py

from typing import Dict, List, Optional, Union

import requests

from .hf_api import HfApi
from .utils import logging


logger = logging.get_logger(__name__)


ENDPOINT = "https://api-inference.huggingface.co"

ALL_TASKS = [
    # NLP
    "text-classification",
    "token-classification",
    "table-question-answering",
    "question-answering",
    "zero-shot-classification",
    "translation",
    "summarization",
    "conversational",
    "feature-extraction",
    "text-generation",
    "text2text-generation",
    "fill-mask",
    "sentence-similarity",
    # Audio
    "text-to-speech",
    "automatic-speech-recognition",
    "audio-to-audio",
    "audio-classification",
    "voice-activity-detection",
    # Computer vision
    "image-classification",
    "object-detection",
    "image-segmentation",
    "text-to-image",
    # Others
    "structured-data-classification",
]


class InferenceApi:
    """Client to configure requests and make calls to the HuggingFace Inference API.

    Example:

    ```python
    >>> from huggingface_hub.inference_api import InferenceApi

    >>> # Mask-fill example
    >>> inference = InferenceApi("bert-base-uncased")
    >>> inference(inputs="The goal of life is [MASK].")
    [{'sequence': 'the goal of life is life.', 'score': 0.10933292657136917, 'token': 2166, 'token_str': 'life'}]

    >>> # Question Answering example
    >>> inference = InferenceApi("deepset/roberta-base-squad2")
    >>> inputs = {
    ...     "question": "What's my name?",
    ...     "context": "My name is Clara and I live in Berkeley.",
    ... }
    >>> inference(inputs)
    {'score': 0.9326569437980652, 'start': 11, 'end': 16, 'answer': 'Clara'}

    >>> # Zero-shot example
    >>> inference = InferenceApi("typeform/distilbert-base-uncased-mnli")
    >>> inputs = "Hi, I recently bought a device from your company but it is not working as advertised and I would like to get reimbursed!"
    >>> params = {"candidate_labels": ["refund", "legal", "faq"]}
    >>> inference(inputs, params)
    {'sequence': 'Hi, I recently bought a device from your company but it is not working as advertised and I would like to get reimbursed!', 'labels': ['refund', 'faq', 'legal'], 'scores': [0.9378499388694763, 0.04914155602455139, 0.013008488342165947]}

    >>> # Overriding configured task
    >>> inference = InferenceApi("bert-base-uncased", task="feature-extraction")
    ```
    """

    def __init__(
        self,
        repo_id: str,
        task: Optional[str] = None,
        token: Optional[str] = None,
        gpu: Optional[bool] = False,
    ):
        """Inits headers and API call information.

        Args:
            repo_id (``str``):
                Id of repository (e.g. `user/bert-base-uncased`).
            task (``str``, `optional`, defaults ``None``):
                Whether to force a task instead of using task specified in the
                repository.
            token (`str`, `optional`):
                The API token to use as HTTP bearer authorization. This is not
                the authentication token. You can find the token in
                https://huggingface.co/settings/token. Alternatively, you can
                find both your organizations and personal API tokens using
                `HfApi().whoami(token)`.
            gpu (``bool``, `optional`, defaults ``False``):
                Whether to use GPU instead of CPU for inference(requires Startup
                plan at least).
        .. note::
            Setting `token` is required when you want to use a private model.
        """
        self.options = {"wait_for_model": True, "use_gpu": gpu}

        self.headers = {}
        if isinstance(token, str):
            self.headers["Authorization"] = f"Bearer {token}"

        # Configure task
        model_info = HfApi().model_info(repo_id=repo_id, token=token)
        if not model_info.pipeline_tag and not task:
            raise ValueError(
                "Task not specified in the repository. Please add it to the model card"
                " using pipeline_tag"
                " (https://huggingface.co/docs#how-is-a-models-type-of-inference-api-and-widget-determined)"
            )

        if task and task != model_info.pipeline_tag:
            if task not in ALL_TASKS:
                raise ValueError(f"Invalid task {task}. Make sure it's valid.")

            logger.warning(
                "You're using a different task than the one specified in the"
                " repository. Be sure to know what you're doing :)"
            )
            self.task = task
        else:
            self.task = model_info.pipeline_tag

        self.api_url = f"{ENDPOINT}/pipeline/{self.task}/{repo_id}"

    def __repr__(self):
        items = (f"{k}='{v}'" for k, v in self.__dict__.items())
        return f"{self.__class__.__name__}({', '.join(items)})"

    def __call__(
        self,
        inputs: Optional[Union[str, Dict, List[str], List[List[str]]]] = None,
        params: Optional[Dict] = None,
        data: Optional[bytes] = None,
    ):
        payload = {
            "options": self.options,
        }

        if inputs:
            payload["inputs"] = inputs

        if params:
            payload["parameters"] = params

        # TODO: Decide if we should raise an error instead of
        # returning the json.
        response = requests.post(
            self.api_url, headers=self.headers, json=payload, data=data
        ).json()
        return response


# src/huggingface_hub/keras_mixin.py

import json
import os
import warnings
from pathlib import Path
from shutil import copytree, rmtree
from typing import Any, Dict, Optional, Union

import yaml
from huggingface_hub import ModelHubMixin, snapshot_download
from huggingface_hub.file_download import (
    get_tf_version,
    is_graphviz_available,
    is_pydot_available,
    is_tf_available,
)

from .constants import CONFIG_NAME
from .hf_api import HfApi, HfFolder
from .repository import Repository
from .utils import logging


logger = logging.get_logger(__name__)

if is_tf_available():
    import tensorflow as tf


def _create_hyperparameter_table(model):
    """Parse hyperparameter dictionary into a markdown table."""
    if model.optimizer is not None:
        optimizer_params = model.optimizer.get_config()
        optimizer_params[
            "training_precision"
        ] = tf.keras.mixed_precision.global_policy().name
        table = "|"
        for key in optimizer_params.keys():
            table += f" {key} |"

        table += "\n|"
        for key in optimizer_params.keys():
            table += "-" * len(key)
            table += "|"

        table += "\n|"
        for key in optimizer_params.keys():
            table += f"{optimizer_params[key]}|"
    else:
        table = None
    return table


def _plot_network(model, save_directory):
    tf.keras.utils.plot_model(
        model,
        to_file=f"{save_directory}/model.png",
        show_shapes=False,
        show_dtype=False,
        show_layer_names=True,
        rankdir="TB",
        expand_nested=False,
        dpi=96,
        layer_range=None,
    )


def _create_model_card(
    model,
    repo_dir: Path,
    plot_model: Optional[bool] = True,
    metadata: Optional[dict] = None,
):
    """
    Creates a model card for the repository.
    """
    hyperparameters = _create_hyperparameter_table(model)
    if plot_model and is_graphviz_available() and is_pydot_available():
        _plot_network(model, repo_dir)
    readme_path = f"{repo_dir}/README.md"
    metadata["library_name"] = "keras"
    model_card = "---\n"
    model_card += yaml.dump(metadata, default_flow_style=False)
    model_card += "---\n"
    model_card += "\n## Model description\n\nMore information needed\n"
    model_card += "\n## Intended uses & limitations\n\nMore information needed\n"
    model_card += "\n## Training and evaluation data\n\nMore information needed\n"
    if hyperparameters is not None:
        model_card += "\n## Training procedure\n"
        model_card += "\n### Training hyperparameters\n"
        model_card += "\nThe following hyperparameters were used during training:\n\n"
        model_card += hyperparameters
        model_card += "\n"
    if plot_model and os.path.exists(f"{repo_dir}/model.png"):
        model_card += "\n ## Model Plot\n"
        model_card += "\n<details>"
        model_card += "\n<summary>View Model Plot</summary>\n"
        path_to_plot = "./model.png"
        model_card += f"\n![Model Image]({path_to_plot})\n"
        model_card += "\n</details>"

    if os.path.exists(readme_path):
        with open(readme_path, "r", encoding="utf8") as f:
            readme = f.read()
    else:
        readme = model_card
    with open(readme_path, "w", encoding="utf-8") as f:
        f.write(readme)


def save_pretrained_keras(
    model,
    save_directory: str,
    config: Optional[Dict[str, Any]] = None,
    include_optimizer: Optional[bool] = False,
    plot_model: Optional[bool] = True,
    tags: Optional[Union[list, str]] = None,
    **model_save_kwargs,
):
    """
    Saves a Keras model to save_directory in SavedModel format. Use this if
    you're using the Functional or Sequential APIs.

    Args:
        model (`Keras.Model`):
            The [Keras
            model](https://www.tensorflow.org/api_docs/python/tf/keras/Model)
            you'd like to save. The model must be compiled and built.
        save_directory (`str`):
            Specify directory in which you want to save the Keras model.
        config (`dict`, *optional*):
            Configuration object to be saved alongside the model weights.
        include_optimizer(`bool`, *optional*, defaults to `False`):
            Whether or not to include optimizer in serialization.
        plot_model (`bool`, *optional*, defaults to `True`):
            Setting this to `True` will plot the model and put it in the model
            card. Requires graphviz and pydot to be installed.
        tags (Union[`str`,`list`], *optional*):
            List of tags that are related to model or string of a single tag. See example tags
            [here](https://github.com/huggingface/hub-docs/blame/main/modelcard.md).
        model_save_kwargs(`dict`, *optional*):
            model_save_kwargs will be passed to
            [`tf.keras.models.save_model()`](https://www.tensorflow.org/api_docs/python/tf/keras/models/save_model).
    """
    if is_tf_available():
        import tensorflow as tf
    else:
        raise ImportError(
            "Called a Tensorflow-specific function but could not import it."
        )

    if not model.built:
        raise ValueError("Model should be built before trying to save")

    os.makedirs(save_directory, exist_ok=True)

    # saving config
    if config:
        if not isinstance(config, dict):
            raise RuntimeError(
                "Provided config to save_pretrained_keras should be a dict. Got:"
                f" '{type(config)}'"
            )
        path = os.path.join(save_directory, CONFIG_NAME)
        with open(path, "w") as f:
            json.dump(config, f)

    metadata = {}
    if isinstance(tags, list):
        metadata["tags"] = tags
    elif isinstance(tags, str):
        metadata["tags"] = [tags]

    task_name = model_save_kwargs.pop("task_name", None)
    if task_name is not None:
        warnings.warn(
            "`task_name` input argument is deprecated. Pass `tags` instead.",
            FutureWarning,
        )
        if "tags" in metadata:
            metadata["tags"].append(task_name)
        else:
            metadata["tags"] = [task_name]

    if model.history is not None:
        if model.history.history != {}:
            path = os.path.join(save_directory, "history.json")
            if os.path.exists(path):
                warnings.warn(
                    "`history.json` file already exists, it will be overwritten by the"
                    " history of this version.",
                    UserWarning,
                )
            with open(path, "w", encoding="utf-8") as f:
                json.dump(model.history.history, f, indent=2, sort_keys=True)

    _create_model_card(model, save_directory, plot_model, metadata)
    tf.keras.models.save_model(
        model, save_directory, include_optimizer=include_optimizer, **model_save_kwargs
    )


def from_pretrained_keras(*args, **kwargs):
    r"""
    Instantiate a pretrained Keras model from a pre-trained model from the Hub.
    The model is expected to be in SavedModel format.```

    Parameters:
        pretrained_model_name_or_path (`str` or `os.PathLike`):
            Can be either:
                - A string, the `model id` of a pretrained model hosted inside a
                  model repo on huggingface.co. Valid model ids can be located
                  at the root-level, like `bert-base-uncased`, or namespaced
                  under a user or organization name, like
                  `dbmdz/bert-base-german-cased`.
                - You can add `revision` by appending `@` at the end of model_id
                  simply like this: `dbmdz/bert-base-german-cased@main` Revision
                  is the specific model version to use. It can be a branch name,
                  a tag name, or a commit id, since we use a git-based system
                  for storing models and other artifacts on huggingface.co, so
                  `revision` can be any identifier allowed by git.
                - A path to a `directory` containing model weights saved using
                  [`~transformers.PreTrainedModel.save_pretrained`], e.g.,
                  `./my_model_directory/`.
                - `None` if you are both providing the configuration and state
                  dictionary (resp. with keyword arguments `config` and
                  `state_dict`).
        force_download (`bool`, *optional*, defaults to `False`):
            Whether to force the (re-)download of the model weights and
            configuration files, overriding the cached versions if they exist.
        resume_download (`bool`, *optional*, defaults to `False`):
            Whether to delete incompletely received files. Will attempt to
            resume the download if such a file exists.
        proxies (`Dict[str, str]`, *optional*):
            A dictionary of proxy servers to use by protocol or endpoint, e.g.,
            `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The
            proxies are used on each request.
        use_auth_token (`str` or `bool`, *optional*):
            The token to use as HTTP bearer authorization for remote files. If
            `True`, will use the token generated when running `transformers-cli
            login` (stored in `~/.huggingface`).
        cache_dir (`Union[str, os.PathLike]`, *optional*):
            Path to a directory in which a downloaded pretrained model
            configuration should be cached if the standard cache should not be
            used.
        local_files_only(`bool`, *optional*, defaults to `False`):
            Whether to only look at local files (i.e., do not try to download
            the model).
        model_kwargs (`Dict`, *optional*):
            model_kwargs will be passed to the model during initialization

    <Tip>

    Passing `use_auth_token=True` is required when you want to use a private
    model.

    </Tip>
    """
    return KerasModelHubMixin.from_pretrained(*args, **kwargs)


def push_to_hub_keras(
    model,
    repo_path_or_name: Optional[str] = None,
    repo_url: Optional[str] = None,
    log_dir: Optional[str] = None,
    commit_message: Optional[str] = "Add model",
    organization: Optional[str] = None,
    private: Optional[bool] = None,
    api_endpoint: Optional[str] = None,
    use_auth_token: Optional[Union[bool, str]] = True,
    git_user: Optional[str] = None,
    git_email: Optional[str] = None,
    config: Optional[dict] = None,
    include_optimizer: Optional[bool] = False,
    tags: Optional[Union[list, str]] = None,
    plot_model: Optional[bool] = True,
    **model_save_kwargs,
):
    """
    Upload model checkpoint or tokenizer files to the Hub while synchronizing a
    local clone of the repo in `repo_path_or_name`.

    Parameters:
        model (`Keras.Model`):
            The [Keras
            model](`https://www.tensorflow.org/api_docs/python/tf/keras/Model`)
            you'd like to push to the Hub. The model must be compiled and built.
        repo_path_or_name (`str`, *optional*):
            Can either be a repository name for your model or tokenizer in the
            Hub or a path to a local folder (in which case the repository will
            have the name of that local folder). If not specified, will default
            to the name given by `repo_url` and a local directory with that name
            will be created.
        repo_url (`str`, *optional*):
            Specify this in case you want to push to an existing repository in
            the Hub. If unspecified, a new repository will be created in your
            namespace (unless you specify an `organization`) with `repo_name`.
        log_dir (`str`, *optional*):
            TensorBoard logging directory to be pushed. The Hub automatically
            hosts and displays a TensorBoard instance if log files are included
            in the repository.
        commit_message (`str`, *optional*, defaults to "Add message"):
            Message to commit while pushing.
        organization (`str`, *optional*):
            Organization in which you want to push your model or tokenizer (you
            must be a member of this organization).
        private (`bool`, *optional*):
            Whether the repository created should be private.
        api_endpoint (`str`, *optional*):
            The API endpoint to use when pushing the model to the hub.
        use_auth_token (`bool` or `str`, *optional*, defaults to `True`):
            The token to use as HTTP bearer authorization for remote files. If
            `True`, will use the token generated when running `transformers-cli
            login` (stored in `~/.huggingface`). Will default to `True`.
        git_user (`str`, *optional*):
            will override the `git config user.name` for committing and pushing
            files to the Hub.
        git_email (`str`, *optional*):
            will override the `git config user.email` for committing and pushing
            files to the Hub.
        config (`dict`, *optional*):
            Configuration object to be saved alongside the model weights.
        include_optimizer (`bool`, *optional*, defaults to `False`):
            Whether or not to include optimizer during serialization.
        tags (Union[`list`, `str`], *optional*):
            List of tags that are related to model or string of a single tag. See example tags
            [here](https://github.com/huggingface/hub-docs/blame/main/modelcard.md).
        plot_model (`bool`, *optional*, defaults to `True`):
            Setting this to `True` will plot the model and put it in the model
            card. Requires graphviz and pydot to be installed.
        model_save_kwargs(`dict`, *optional*):
            model_save_kwargs will be passed to
            [`tf.keras.models.save_model()`](https://www.tensorflow.org/api_docs/python/tf/keras/models/save_model).

    Returns:
        The url of the commit of your model in the given repository.
    """

    if repo_path_or_name is None and repo_url is None:
        raise ValueError("You need to specify a `repo_path_or_name` or a `repo_url`.")

    if isinstance(use_auth_token, bool) and use_auth_token:
        token = HfFolder.get_token()
    elif isinstance(use_auth_token, str):
        token = use_auth_token
    else:
        token = None

    if token is None:
        raise ValueError(
            "You must login to the Hugging Face hub on this computer by typing"
            " `huggingface-cli login` and entering your credentials to use"
            " `use_auth_token=True`. Alternatively, you can pass your own token as the"
            " `use_auth_token` argument."
        )

    if repo_path_or_name is None:
        repo_path_or_name = repo_url.split("/")[-1]

    # If no URL is passed and there's no path to a directory containing files, create a repo
    if repo_url is None and not os.path.exists(repo_path_or_name):
        repo_id = Path(repo_path_or_name).name
        if organization:
            repo_id = f"{organization}/{repo_id}"
        repo_url = HfApi(endpoint=api_endpoint).create_repo(
            repo_id=repo_id,
            token=token,
            private=private,
            repo_type=None,
            exist_ok=True,
        )

    repo = Repository(
        repo_path_or_name,
        clone_from=repo_url,
        use_auth_token=use_auth_token,
        git_user=git_user,
        git_email=git_email,
    )
    repo.git_pull(rebase=True)

    save_pretrained_keras(
        model,
        repo_path_or_name,
        config=config,
        include_optimizer=include_optimizer,
        tags=tags,
        plot_model=plot_model,
        **model_save_kwargs,
    )

    if log_dir is not None:
        if os.path.exists(f"{repo_path_or_name}/logs"):
            rmtree(f"{repo_path_or_name}/logs")
        copytree(log_dir, f"{repo_path_or_name}/logs")

    # Commit and push!
    repo.git_add(auto_lfs_track=True)
    repo.git_commit(commit_message)
    return repo.git_push()


class KerasModelHubMixin(ModelHubMixin):
    """
    Mixin to provide model Hub upload/download capabilities to Keras models.
    Override this class to obtain the following internal methods:
    - `_from_pretrained`, to load a model from the Hub or from local files.
    - `_save_pretrained`, to save a model in the `SavedModel` format.
    """

    def __init__(self, *args, **kwargs):
        """
        Mix this class with your keras-model class for ease process of saving &
        loading from huggingface-hub.


        ```python
        >>> from huggingface_hub import KerasModelHubMixin


        >>> class MyModel(tf.keras.Model, KerasModelHubMixin):
        ...     def __init__(self, **kwargs):
        ...         super().__init__()
        ...         self.config = kwargs.pop("config", None)
        ...         self.dummy_inputs = ...
        ...         self.layer = ...

        ...     def call(self, *args):
        ...         return ...


        >>> # Init and compile the model as you normally would
        >>> model = MyModel()
        >>> model.compile(...)
        >>> # Build the graph by training it or passing dummy inputs
        >>> _ = model(model.dummy_inputs)
        >>> # You can save your model like this
        >>> model.save_pretrained("local_model_dir/", push_to_hub=False)
        >>> # Or, you can push to a new public model repo like this
        >>> model.push_to_hub(
        ...     "super-cool-model",
        ...     git_user="your-hf-username",
        ...     git_email="you@somesite.com",
        ... )

        >>> # Downloading weights from hf-hub & model will be initialized from those weights
        >>> model = MyModel.from_pretrained("username/mymodel@main")
        ```
        """

    def _save_pretrained(self, save_directory):
        save_pretrained_keras(self, save_directory)

    @classmethod
    def _from_pretrained(
        cls,
        model_id,
        revision,
        cache_dir,
        force_download,
        proxies,
        resume_download,
        local_files_only,
        use_auth_token,
        **model_kwargs,
    ):
        """Here we just call from_pretrained_keras function so both the mixin and
        functional APIs stay in sync.

                TODO - Some args above aren't used since we are calling
                snapshot_download instead of hf_hub_download.
        """
        if is_tf_available():
            import tensorflow as tf
        else:
            raise ImportError(
                "Called a Tensorflow-specific function but could not import it."
            )

        # TODO - Figure out what to do about these config values. Config is not going to be needed to load model
        cfg = model_kwargs.pop("config", None)

        # Root is either a local filepath matching model_id or a cached snapshot
        if not os.path.isdir(model_id):
            storage_folder = snapshot_download(
                repo_id=model_id,
                revision=revision,
                cache_dir=cache_dir,
                library_name="keras",
                library_version=get_tf_version(),
            )
        else:
            storage_folder = model_id

        model = tf.keras.models.load_model(storage_folder, **model_kwargs)

        # For now, we add a new attribute, config, to store the config loaded from the hub/a local dir.
        model.config = cfg

        return model


# src/huggingface_hub/lfs.py

# coding=utf-8
# Copyright 2019-present, the HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import subprocess
import sys


LFS_MULTIPART_UPLOAD_COMMAND = "lfs-multipart-upload"


def install_lfs_in_userspace():
    """
    If in Linux, installs git-lfs in userspace (sometimes useful if you can't
    `sudo apt install` or equivalent).
    """
    if sys.platform != "linux":
        raise ValueError("Only implemented for Linux right now")
    GIT_LFS_TARBALL = "https://github.com/git-lfs/git-lfs/releases/download/v2.13.1/git-lfs-linux-amd64-v2.13.1.tar.gz"
    CWD = os.path.join(os.getcwd(), "install_lfs")
    os.makedirs(CWD, exist_ok=True)
    subprocess.run(
        ["wget", "-O", "tarball.tar.gz", GIT_LFS_TARBALL], check=True, cwd=CWD
    )
    subprocess.run(["tar", "-xvzf", "tarball.tar.gz"], check=True, cwd=CWD)
    subprocess.run(["bash", "install.sh"], check=True, cwd=CWD)


# src/huggingface_hub/repocard.py

import dataclasses
import os
import re
import shutil
import tempfile
from pathlib import Path
from typing import Any, Dict, Optional, Union

import yaml
from huggingface_hub.file_download import hf_hub_download
from huggingface_hub.hf_api import HfApi
from huggingface_hub.repocard_types import (
    ModelIndex,
    SingleMetric,
    SingleResult,
    SingleResultDataset,
    SingleResultTask,
)

from .constants import REPOCARD_NAME


# exact same regex as in the Hub server. Please keep in sync.
REGEX_YAML_BLOCK = re.compile(r"---[\n\r]+([\S\s]*?)[\n\r]+---[\n\r]")

UNIQUE_RESULT_FEATURES = ["dataset", "task"]
UNIQUE_METRIC_FEATURES = ["name", "type"]


def metadata_load(local_path: Union[str, Path]) -> Optional[Dict]:
    content = Path(local_path).read_text()
    match = REGEX_YAML_BLOCK.search(content)
    if match:
        yaml_block = match.group(1)
        data = yaml.safe_load(yaml_block)
        if isinstance(data, dict):
            return data
        else:
            raise ValueError("repo card metadata block should be a dict")
    else:
        return None


def metadata_save(local_path: Union[str, Path], data: Dict) -> None:
    """
    Save the metadata dict in the upper YAML part Trying to preserve newlines as
    in the existing file. Docs about open() with newline="" parameter:
    https://docs.python.org/3/library/functions.html?highlight=open#open Does
    not work with "^M" linebreaks, which are replaced by \n
    """
    line_break = "\n"
    content = ""
    # try to detect existing newline character
    if os.path.exists(local_path):
        with open(local_path, "r", newline="") as readme:
            if type(readme.newlines) is tuple:
                line_break = readme.newlines[0]
            if type(readme.newlines) is str:
                line_break = readme.newlines
            content = readme.read()

    # creates a new file if it not
    with open(local_path, "w", newline="") as readme:
        data_yaml = yaml.dump(data, sort_keys=False, line_break=line_break)
        # sort_keys: keep dict order
        match = REGEX_YAML_BLOCK.search(content)
        if match:
            output = (
                content[: match.start()]
                + f"---{line_break}{data_yaml}---{line_break}"
                + content[match.end() :]
            )
        else:
            output = f"---{line_break}{data_yaml}---{line_break}{content}"

        readme.write(output)
        readme.close()


def metadata_eval_result(
    model_pretty_name: str,
    task_pretty_name: str,
    task_id: str,
    metrics_pretty_name: str,
    metrics_id: str,
    metrics_value: Any,
    dataset_pretty_name: str,
    dataset_id: str,
) -> Dict:
    model_index = ModelIndex(
        name=model_pretty_name,
        results=[
            SingleResult(
                metrics=[
                    SingleMetric(
                        type=metrics_id,
                        name=metrics_pretty_name,
                        value=metrics_value,
                    ),
                ],
                task=SingleResultTask(type=task_id, name=task_pretty_name),
                dataset=SingleResultDataset(name=dataset_pretty_name, type=dataset_id),
            )
        ],
    )
    # use `dict_factory` to recursively ignore None values
    data = dataclasses.asdict(
        model_index, dict_factory=lambda x: {k: v for (k, v) in x if v is not None}
    )
    return {"model-index": [data]}


def metadata_update(
    repo_id: str,
    metadata: Dict,
    *,
    repo_type: str = None,
    overwrite: bool = False,
    token: str = None,
) -> str:
    """
    Updates the metadata in the README.md of a repository on the Hugging Face Hub.

    Example:
    >>> from huggingface_hub import metadata_update
    >>> metadata = {'model-index': [{'name': 'RoBERTa fine-tuned on ReactionGIF',
    ...             'results': [{'dataset': {'name': 'ReactionGIF',
    ...                                      'type': 'julien-c/reactiongif'},
    ...                           'metrics': [{'name': 'Recall',
    ...                                        'type': 'recall',
    ...                                        'value': 0.7762102282047272}],
    ...                          'task': {'name': 'Text Classification',
    ...                                   'type': 'text-classification'}}]}]}
    >>> update_metdata("julien-c/reactiongif-roberta", metadata)

    Args:
        repo_id (`str`):
            The name of the repository.
        metadata (`dict`):
            A dictionary containing the metadata to be updated.
        repo_type (`str`, *optional*):
            Set to `"dataset"` or `"space"` if updating to a dataset or space,
            `None` or `"model"` if updating to a model. Default is `None`.
        overwrite (`bool`, *optional*, defaults to `False`):
            If set to `True` an existing field can be overwritten, otherwise
            attempting to overwrite an existing field will cause an error.
        token (`str`, *optional*):
            The Hugging Face authentication token.

    Returns:
        `str`: URL of the commit which updated the card metadata.
    """

    upstream_filepath = hf_hub_download(
        repo_id,
        filename=REPOCARD_NAME,
        repo_type=repo_type,
        use_auth_token=token,
    )
    # work on a copy of the upstream file, to not mess up the cache
    with tempfile.TemporaryDirectory() as tmpdirname:
        filepath = shutil.copy(upstream_filepath, tmpdirname)

        existing_metadata = metadata_load(filepath)

        for key in metadata:
            # update model index containing the evaluation results
            if key == "model-index":
                if "model-index" not in existing_metadata:
                    existing_metadata["model-index"] = metadata["model-index"]
                else:
                    # the model-index contains a list of results as used by PwC but only has one element thus we take the first one
                    existing_metadata["model-index"][0][
                        "results"
                    ] = _update_metadata_model_index(
                        existing_metadata["model-index"][0]["results"],
                        metadata["model-index"][0]["results"],
                        overwrite=overwrite,
                    )
            # update all fields except model index
            else:
                if key in existing_metadata and not overwrite:
                    if existing_metadata[key] != metadata[key]:
                        raise ValueError(
                            f"""You passed a new value for the existing meta data field '{key}'. Set `overwrite=True` to overwrite existing metadata."""
                        )
                else:
                    existing_metadata[key] = metadata[key]

        # save and push to hub
        metadata_save(filepath, existing_metadata)

        return HfApi().upload_file(
            path_or_fileobj=filepath,
            path_in_repo=REPOCARD_NAME,
            repo_id=repo_id,
            repo_type=repo_type,
            token=token,
        )


def _update_metadata_model_index(existing_results, new_results, overwrite=False):
    """
    Updates the model-index fields in the metadata. If results with same unique
    features exist they are updated, else a new result is appended. Updating existing
    values is only possible if `overwrite=True`.

    Args:
        new_metrics (`List[dict]`):
            List of new metadata results.
        existing_metrics (`List[dict]`):
            List of existing metadata results.
        overwrite (`bool`, *optional*, defaults to `False`):
            If set to `True`, an existing metric values can be overwritten, otherwise
            attempting to overwrite an existing field will cause an error.

    Returns:
        `list`: List of updated metadata results
    """
    for new_result in new_results:
        result_found = False
        for existing_result_index, existing_result in enumerate(existing_results):
            if all(
                new_result[feat] == existing_result[feat]
                for feat in UNIQUE_RESULT_FEATURES
            ):
                result_found = True
                existing_results[existing_result_index][
                    "metrics"
                ] = _update_metadata_results_metric(
                    new_result["metrics"],
                    existing_result["metrics"],
                    overwrite=overwrite,
                )
        if not result_found:
            existing_results.append(new_result)
    return existing_results


def _update_metadata_results_metric(new_metrics, existing_metrics, overwrite=False):
    """
    Updates the metrics list of a result in the metadata. If metrics with same unique
    features exist their values are updated, else a new metric is appended. Updating
    existing values is only possible if `overwrite=True`.

    Args:
        new_metrics (`list`):
            List of new metrics.
        existing_metrics (`list`):
            List of existing metrics.
        overwrite (`bool`, *optional*, defaults to `False`):
            If set to `True`, an existing metric values can be overwritten, otherwise
            attempting to overwrite an existing field will cause an error.

    Returns:
        `list`: List of updated metrics
    """
    for new_metric in new_metrics:
        metric_exists = False
        for existing_metric_index, existing_metric in enumerate(existing_metrics):
            if all(
                new_metric[feat] == existing_metric[feat]
                for feat in UNIQUE_METRIC_FEATURES
            ):
                if overwrite:
                    existing_metrics[existing_metric_index]["value"] = new_metric[
                        "value"
                    ]
                else:
                    # if metric exists and value is not the same throw an error without overwrite flag
                    if (
                        existing_metrics[existing_metric_index]["value"]
                        != new_metric["value"]
                    ):
                        existing_str = ", ".join(
                            f"{feat}: {new_metric[feat]}"
                            for feat in UNIQUE_METRIC_FEATURES
                        )
                        raise ValueError(
                            "You passed a new value for the existing metric"
                            f" '{existing_str}'. Set `overwrite=True` to overwrite"
                            " existing metrics."
                        )
                metric_exists = True
        if not metric_exists:
            existing_metrics.append(new_metric)
    return existing_metrics


# src/huggingface_hub/repocard_types.py

# Code generated by jtd-codegen for Python v0.3.1

from dataclasses import dataclass, field
from typing import Any, List, Optional

from typing_extensions import TypeAlias


ModelIndexSet: TypeAlias = "List[ModelIndex]"


@dataclass
class ModelIndex:
    name: str
    results: "List[SingleResult]"


@dataclass
class SingleMetric:
    type: str
    """
    Example: wer
    """

    value: Any
    """
    Example: 20.0 or "20.0 ± 1.2"
    """

    args: Any = field(default=None)
    name: Optional[str] = field(default=None)


@dataclass
class SingleResultTask:
    type: str
    """
    Example: automatic-speech-recognition Use task id from
    https://github.com/huggingface/huggingface_hub/blob/main/js/src/lib/int
    erfaces/Types.ts
    """

    name: Optional[str] = None
    """
    Example: Speech Recognition
    """


@dataclass
class SingleResultDataset:
    """
    This will switch to required at some point. in any case, we need them to
    link to PWC
    """

    name: str
    """
    Example: Common Voice zh-CN Also encode config params into the name if
    relevant.
    """

    type: str
    """
    Example: common_voice. Use dataset id from https://hf.co/datasets
    """

    args: Any = None
    """
    Example: zh-CN
    """


@dataclass
class SingleResult:
    metrics: "List[SingleMetric]"
    task: "SingleResultTask"
    dataset: "Optional[SingleResultDataset]"
    """
    This will switch to required at some point. in any case, we need them to
    link to PWC
    """


# src/huggingface_hub/repository.py

import atexit
import os
import re
import subprocess
import tempfile
import threading
import time
from contextlib import contextmanager
from pathlib import Path
from typing import Callable, Dict, Iterator, List, Optional, Tuple, Union
from urllib.parse import urlparse

from tqdm.auto import tqdm

from huggingface_hub.constants import REPO_TYPES_URL_PREFIXES, REPOCARD_NAME
from huggingface_hub.repocard import metadata_load, metadata_save
from requests.exceptions import HTTPError

from .hf_api import HfApi, HfFolder, repo_type_and_id_from_hf_id
from .lfs import LFS_MULTIPART_UPLOAD_COMMAND
from .utils import logging, run_subprocess


logger = logging.get_logger(__name__)


class CommandInProgress:
    """
    Utility to follow commands launched asynchronously.
    """

    def __init__(
        self,
        title: str,
        is_done_method: Callable,
        status_method: Callable,
        process: subprocess.Popen,
        post_method: Optional[Callable] = None,
    ):
        self.title = title
        self._is_done = is_done_method
        self._status = status_method
        self._process = process
        self._stderr = ""
        self._stdout = ""
        self._post_method = post_method

    @property
    def is_done(self) -> bool:
        """
        Whether the process is done.
        """
        result = self._is_done()

        if result and self._post_method is not None:
            self._post_method()
            self._post_method = None

        return result

    @property
    def status(self) -> int:
        """
        The exit code/status of the current action. Will return `0` if the
        command has completed successfully, and a number between 1 and 255 if
        the process errored-out.

        Will return -1 if the command is still ongoing.
        """
        return self._status()

    @property
    def failed(self) -> bool:
        """
        Whether the process errored-out.
        """
        return self.status > 0

    @property
    def stderr(self) -> str:
        """
        The current output message on the standard error.
        """
        self._stderr += self._process.stderr.read()
        return self._stderr

    @property
    def stdout(self) -> str:
        """
        The current output message on the standard output.
        """
        self._stdout += self._process.stdout.read()
        return self._stdout

    def __repr__(self):
        status = self.status

        if status == -1:
            status = "running"

        return (
            f"[{self.title} command, status code: {status},"
            f" {'in progress.' if not self.is_done else 'finished.'} PID:"
            f" {self._process.pid}]"
        )


def is_git_repo(folder: Union[str, Path]) -> bool:
    """
    Check if the folder is the root or part of a git repository

    Args:
        folder (`str`):
            The folder in which to run the command.

    Returns:
        `bool`: `True` if the repository is part of a repository, `False`
        otherwise.
    """
    folder_exists = os.path.exists(os.path.join(folder, ".git"))
    git_branch = subprocess.run(
        "git branch".split(), cwd=folder, stdout=subprocess.PIPE, stderr=subprocess.PIPE
    )
    return folder_exists and git_branch.returncode == 0


def is_local_clone(folder: Union[str, Path], remote_url: str) -> bool:
    """
    Check if the folder is a local clone of the remote_url

    Args:
        folder (`str` or `Path`):
            The folder in which to run the command.
        remote_url (`str`):
            The url of a git repository.

    Returns:
        `bool`: `True` if the repository is a local clone of the remote
        repository specified, `False` otherwise.
    """
    if not is_git_repo(folder):
        return False

    remotes = run_subprocess("git remote -v".split(), folder).stdout

    # Remove token for the test with remotes.
    remote_url = re.sub(r"https://.*@", "https://", remote_url)
    remotes = [re.sub(r"https://.*@", "https://", remote) for remote in remotes.split()]
    return remote_url in remotes


def is_tracked_with_lfs(filename: Union[str, Path]) -> bool:
    """
    Check if the file passed is tracked with git-lfs.

    Args:
        filename (`str` or `Path`):
            The filename to check.

    Returns:
        `bool`: `True` if the file passed is tracked with git-lfs, `False`
        otherwise.
    """
    folder = Path(filename).parent
    filename = Path(filename).name

    try:
        p = run_subprocess("git check-attr -a".split() + [filename], folder)
        attributes = p.stdout.strip()
    except subprocess.CalledProcessError as exc:
        if not is_git_repo(folder):
            return False
        else:
            raise OSError(exc.stderr)

    if len(attributes) == 0:
        return False

    found_lfs_tag = {"diff": False, "merge": False, "filter": False}

    for attribute in attributes.split("\n"):
        for tag in found_lfs_tag.keys():
            if tag in attribute and "lfs" in attribute:
                found_lfs_tag[tag] = True

    return all(found_lfs_tag.values())


def is_git_ignored(filename: Union[str, Path]) -> bool:
    """
    Check if file is git-ignored. Supports nested .gitignore files.

    Args:
        filename (`str` or `Path`):
            The filename to check.

    Returns:
        `bool`: `True` if the file passed is ignored by `git`, `False`
        otherwise.
    """
    folder = Path(filename).parent
    filename = Path(filename).name

    try:
        p = run_subprocess("git check-ignore".split() + [filename], folder, check=False)
        # Will return exit code 1 if not gitignored
        is_ignored = not bool(p.returncode)
    except subprocess.CalledProcessError as exc:
        raise OSError(exc.stderr)

    return is_ignored


def is_binary_file(filename: Union[str, Path]) -> bool:
    """
    Check if file is a binary file.

    Args:
        filename (`str` or `Path`):
            The filename to check.

    Returns:
        `bool`: `True` if the file passed is a binary file, `False` otherwise.
    """
    try:
        with open(filename, "rb") as f:
            content = f.read(10 * (1024**2))  # Read a maximum of 10MB

        # Code sample taken from the following stack overflow thread
        # https://stackoverflow.com/questions/898669/how-can-i-detect-if-a-file-is-binary-non-text-in-python/7392391#7392391
        text_chars = bytearray(
            {7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7F}
        )
        return bool(content.translate(None, text_chars))
    except UnicodeDecodeError:
        return True


def files_to_be_staged(pattern: str, folder: Union[str, Path]) -> List[str]:
    """
    Returns a list of filenames that are to be staged.

    Args:
        pattern (`str` or `Path`):
            The pattern of filenames to check. Put `.` to get all files.
        folder (`str` or `Path`):
            The folder in which to run the command.

    Returns:
        `List[str]`: List of files that are to be staged.
    """
    try:
        p = run_subprocess("git ls-files -mo".split() + [pattern], folder)
        if len(p.stdout.strip()):
            files = p.stdout.strip().split("\n")
        else:
            files = []
    except subprocess.CalledProcessError as exc:
        raise EnvironmentError(exc.stderr)

    return files


def is_tracked_upstream(folder: Union[str, Path]) -> bool:
    """
    Check if the current checked-out branch is tracked upstream.

    Args:
        folder (`str` or `Path`):
            The folder in which to run the command.

    Returns:
        `bool`: `True` if the current checked-out branch is tracked upstream,
        `False` otherwise.
    """
    try:
        run_subprocess(
            "git rev-parse --symbolic-full-name --abbrev-ref @{u}".split(), folder
        )
        return True
    except subprocess.CalledProcessError as exc:
        if "HEAD" in exc.stderr:
            raise OSError("No branch checked out")

        return False


def commits_to_push(folder: Union[str, Path], upstream: Optional[str] = None) -> int:
    """
        Check the number of commits that would be pushed upstream

        Args:
            folder (`str` or `Path`):
                The folder in which to run the command.
            upstream (`str`, *optional*):
    The name of the upstream repository with which the comparison should be
    made.

        Returns:
            `int`: Number of commits that would be pushed upstream were a `git
            push` to proceed.
    """
    try:
        result = run_subprocess(f"git cherry -v {upstream or ''}".split(), folder)
        return len(result.stdout.split("\n")) - 1
    except subprocess.CalledProcessError as exc:
        raise EnvironmentError(exc.stderr)


@contextmanager
def _lfs_log_progress():
    """
    This is a context manager that will log the Git LFS progress of cleaning,
    smudging, pulling and pushing.
    """

    if logger.getEffectiveLevel() >= logging.ERROR:
        try:
            yield
        finally:
            return

    def output_progress(stopping_event: threading.Event):
        """
        To be launched as a separate thread with an event meaning it should stop
        the tail.
        """
        pbars = {}

        def close_pbars():
            for pbar in pbars.values():
                pbar["bar"].update(pbar["bar"].total - pbar["past_bytes"])
                pbar["bar"].refresh()
                pbar["bar"].close()

        def tail_file(filename) -> Iterator[str]:
            """
            Creates a generator to be iterated through, which will return each
            line one by one. Will stop tailing the file if the stopping_event is
            set.
            """
            with open(filename, "r") as file:
                current_line = ""
                while True:
                    if stopping_event.is_set():
                        close_pbars()
                        break

                    line_bit = file.readline()
                    if line_bit is not None and not len(line_bit.strip()) == 0:
                        current_line += line_bit
                        if current_line.endswith("\n"):
                            yield current_line
                            current_line = ""
                    else:
                        time.sleep(1)

        # If the file isn't created yet, wait for a few seconds before trying again.
        # Can be interrupted with the stopping_event.
        while not os.path.exists(os.environ["GIT_LFS_PROGRESS"]):
            if stopping_event.is_set():
                close_pbars()
                return

            time.sleep(2)

        for line in tail_file(os.environ["GIT_LFS_PROGRESS"]):
            state, file_progress, byte_progress, filename = line.split()
            description = f"{state.capitalize()} file {filename}"

            current_bytes, total_bytes = byte_progress.split("/")

            current_bytes = int(current_bytes)
            total_bytes = int(total_bytes)

            if pbars.get((state, filename)) is None:
                pbars[(state, filename)] = {
                    "bar": tqdm(
                        desc=description,
                        initial=current_bytes,
                        total=total_bytes,
                        unit="B",
                        unit_scale=True,
                        unit_divisor=1024,
                    ),
                    "past_bytes": current_bytes,
                }
            else:
                past_bytes = pbars[(state, filename)]["past_bytes"]
                pbars[(state, filename)]["bar"].update(current_bytes - past_bytes)
                pbars[(state, filename)]["past_bytes"] = current_bytes

    current_lfs_progress_value = os.environ.get("GIT_LFS_PROGRESS", "")

    with tempfile.TemporaryDirectory() as tmpdir:
        os.environ["GIT_LFS_PROGRESS"] = os.path.join(tmpdir, "lfs_progress")
        logger.debug(f"Following progress in {os.environ['GIT_LFS_PROGRESS']}")

        exit_event = threading.Event()
        x = threading.Thread(target=output_progress, args=(exit_event,), daemon=True)
        x.start()

        try:
            yield
        finally:
            exit_event.set()
            x.join()

            os.environ["GIT_LFS_PROGRESS"] = current_lfs_progress_value


class Repository:
    """
    Helper class to wrap the git and git-lfs commands.

    The aim is to facilitate interacting with huggingface.co hosted model or
    dataset repos, though not a lot here (if any) is actually specific to
    huggingface.co.
    """

    command_queue: List[CommandInProgress]

    def __init__(
        self,
        local_dir: str,
        clone_from: Optional[str] = None,
        repo_type: Optional[str] = None,
        use_auth_token: Union[bool, str] = True,
        git_user: Optional[str] = None,
        git_email: Optional[str] = None,
        revision: Optional[str] = None,
        private: bool = False,
        skip_lfs_files: bool = False,
        client: Optional[HfApi] = None,
    ):
        """
        Instantiate a local clone of a git repo.


        If specifying a `clone_from`, it will clone an existing remote
        repository, for instance one that was previously created using
        `HfApi().create_repo(repo_id=repo_name)`.

        `Repository` uses the local git credentials by default, but if required,
        the `huggingface_token` as well as the git `user` and the `email` can be
        explicitly specified.

        Args:
            local_dir (`str`):
                path (e.g. `'my_trained_model/'`) to the local directory, where
                the `Repository` will be initalized.
            clone_from (`str`, *optional*):
                repository url (e.g.
                `'https://huggingface.co/philschmid/playground-tests'`).
            repo_type (`str`, *optional*):
                To set when creating a repo: et to "dataset" or "space" if
                creating a dataset or space, default is model.
            use_auth_token (`str` or `bool`, *optional*, defaults to `True`):
                huggingface_token can be extract from `HfApi().login(username,
                password)` and is used to authenticate against the hub (useful
                from Google Colab for instance).
            git_user (`str`, *optional*):
                will override the `git config user.name` for committing and
                pushing files to the hub.
            git_email (`str`, *optional*):
                will override the `git config user.email` for committing and
                pushing files to the hub.
            revision (`str`, *optional*):
                Revision to checkout after initializing the repository. If the
                revision doesn't exist, a branch will be created with that
                revision name from the default branch's current HEAD.
            private (`bool`, *optional*, defaults to `False`):
                whether the repository is private or not.
            skip_lfs_files (`bool`, *optional*, defaults to `False`):
                whether to skip git-LFS files or not.
            client (`HfApi`, *optional*):
                Instance of HfApi to use when calling the HF Hub API. A new
                instance will be created if this is left to `None`.
        """

        os.makedirs(local_dir, exist_ok=True)
        self.local_dir = os.path.join(os.getcwd(), local_dir)
        self.repo_type = repo_type
        self.command_queue = []
        self.private = private
        self.skip_lfs_files = skip_lfs_files
        self.client = client if client is not None else HfApi()

        self.check_git_versions()

        if isinstance(use_auth_token, str):
            self.huggingface_token = use_auth_token
        elif use_auth_token:
            self.huggingface_token = HfFolder.get_token()
        else:
            self.huggingface_token = None

        if clone_from is not None:
            self.clone_from(repo_url=clone_from)
        else:
            if is_git_repo(self.local_dir):
                logger.debug("[Repository] is a valid git repo")
            else:
                raise ValueError(
                    "If not specifying `clone_from`, you need to pass Repository a"
                    " valid git clone."
                )

        if self.huggingface_token is not None and (
            git_email is None or git_user is None
        ):
            user = self.client.whoami(self.huggingface_token)

            if git_email is None:
                git_email = user["email"]

            if git_user is None:
                git_user = user["fullname"]

        if git_user is not None or git_email is not None:
            self.git_config_username_and_email(git_user, git_email)

        self.lfs_enable_largefiles()
        self.git_credential_helper_store()

        if revision is not None:
            self.git_checkout(revision, create_branch_ok=True)

        # This ensures that all commands exit before exiting the Python runtime.
        # This will ensure all pushes register on the hub, even if other errors happen in subsequent operations.
        atexit.register(self.wait_for_commands)

    @property
    def current_branch(self) -> str:
        """
        Returns the current checked out branch.

        Returns:
            `str`: Current checked out branch.
        """
        try:
            result = run_subprocess(
                "git rev-parse --abbrev-ref HEAD".split(), self.local_dir
            ).stdout.strip()
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

        return result

    def check_git_versions(self):
        """
        Checks that `git` and `git-lfs` can be run.

        Raises:
            `EnvironmentError`: if `git` or `git-lfs` are not installed.
        """
        try:
            git_version = run_subprocess(
                "git --version".split(), self.local_dir
            ).stdout.strip()
        except FileNotFoundError:
            raise EnvironmentError(
                "Looks like you do not have git installed, please install."
            )

        try:
            lfs_version = run_subprocess(
                "git-lfs --version".split(), self.local_dir
            ).stdout.strip()
        except FileNotFoundError:
            raise EnvironmentError(
                "Looks like you do not have git-lfs installed, please install."
                " You can install from https://git-lfs.github.com/."
                " Then run `git lfs install` (you only have to do this once)."
            )
        logger.info(git_version + "\n" + lfs_version)

    def clone_from(self, repo_url: str, use_auth_token: Union[bool, str, None] = None):
        """
        Clone from a remote. If the folder already exists, will try to clone the
        repository within it.

        If this folder is a git repository with linked history, will try to
        update the repository.

        Args:
            repo_url (`str`):
                The URL from which to clone the repository
            use_auth_token (`Union[str, bool]`, *optional*):
                Whether to use the authentication token. It can be:
                 - a string which is the token itself
                 - `False`, which would not use the authentication token
                 - `True`, which would fetch the authentication token from the
                   local folder and use it (you should be logged in for this to
                   work).
                - `None`, which would retrieve the value of
                  `self.huggingface_token`.

        Raises:
            `ValueError`: if the `token` cannot be identified and the `private`
            keyword is set to `True`. The `token`
                must be passed in order to handle private repositories.

        Raises:
            `EnvironmentError`: if you are trying to clone the repository in a
            non-empty folder, or if the `git`
                operations raise errors.

        """
        token = use_auth_token if use_auth_token is not None else self.huggingface_token
        if token is None and self.private:
            raise ValueError(
                "Couldn't load Hugging Face Authorization Token. Credentials are"
                " required to work with private repositories. Please login in using"
                " `huggingface-cli login` or provide your token manually with the"
                " `use_auth_token` key."
            )
        hub_url = self.client.endpoint
        if hub_url in repo_url or (
            "http" not in repo_url and len(repo_url.split("/")) <= 2
        ):
            repo_type, namespace, repo_id = repo_type_and_id_from_hf_id(
                repo_url, hub_url=hub_url
            )

            if repo_type is not None:
                self.repo_type = repo_type

            repo_url = hub_url + "/"

            if self.repo_type in REPO_TYPES_URL_PREFIXES:
                repo_url += REPO_TYPES_URL_PREFIXES[self.repo_type]

            if token is not None:
                whoami_info = self.client.whoami(token)
                user = whoami_info["name"]
                valid_organisations = [org["name"] for org in whoami_info["orgs"]]

                if namespace is not None:
                    repo_id = f"{namespace}/{repo_id}"
                repo_url += repo_id

                scheme = urlparse(repo_url).scheme
                repo_url = repo_url.replace(f"{scheme}://", f"{scheme}://user:{token}@")
                if namespace == user or namespace in valid_organisations:
                    try:
                        _ = HfApi().repo_info(
                            f"{repo_id}", repo_type=self.repo_type, token=token
                        )
                    except HTTPError:
                        if self.repo_type == "space":
                            raise ValueError(
                                "Creating a Space through passing Space link to"
                                " clone_from is not allowed. Make sure the Space exists"
                                " on Hugging Face Hub."
                            )
                        else:
                            self.client.create_repo(
                                repo_id=repo_id,
                                token=token,
                                repo_type=self.repo_type,
                                exist_ok=True,
                                private=self.private,
                            )

            else:
                if namespace is not None:
                    repo_url += f"{namespace}/"
                repo_url += repo_id

        # For error messages, it's cleaner to show the repo url without the token.
        clean_repo_url = re.sub(r"(https?)://.*@", r"\1://", repo_url)
        try:
            run_subprocess("git lfs install".split(), self.local_dir)

            # checks if repository is initialized in a empty repository or in one with files
            if len(os.listdir(self.local_dir)) == 0:
                logger.warning(f"Cloning {clean_repo_url} into local empty directory.")

                with _lfs_log_progress():
                    env = os.environ.copy()

                    if self.skip_lfs_files:
                        env.update({"GIT_LFS_SKIP_SMUDGE": "1"})

                    run_subprocess(
                        f"{'git clone' if self.skip_lfs_files else 'git lfs clone'} {repo_url} ."
                        .split(),
                        self.local_dir,
                        env=env,
                    )
            else:
                # Check if the folder is the root of a git repository
                in_repository = is_git_repo(self.local_dir)

                if in_repository:
                    if is_local_clone(self.local_dir, repo_url):
                        logger.warning(
                            f"{self.local_dir} is already a clone of {clean_repo_url}."
                            " Make sure you pull the latest changes with"
                            " `repo.git_pull()`."
                        )
                    else:
                        output = run_subprocess(
                            "git remote get-url origin".split(),
                            self.local_dir,
                            check=False,
                        )

                        error_msg = (
                            f"Tried to clone {clean_repo_url} in an unrelated git"
                            " repository.\nIf you believe this is an error, please add"
                            f" a remote with the following URL: {clean_repo_url}."
                        )
                        if output.returncode == 0:
                            clean_local_remote_url = re.sub(
                                r"https://.*@", "https://", output.stdout
                            )
                            error_msg += (
                                "\nLocal path has its origin defined as:"
                                f" {clean_local_remote_url}"
                            )

                        raise EnvironmentError(error_msg)

                if not in_repository:
                    raise EnvironmentError(
                        "Tried to clone a repository in a non-empty folder that isn't a"
                        " git repository. If you really want to do this, do it"
                        " manually:\ngit init && git remote add origin && git pull"
                        " origin main\n or clone repo to a new folder and move your"
                        " existing files there afterwards."
                    )

        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def git_config_username_and_email(
        self, git_user: Optional[str] = None, git_email: Optional[str] = None
    ):
        """
        Sets git username and email (only in the current repo).

        Args:
            git_user (`str`, *optional*):
                The username to register through `git`.
            git_email (`str`, *optional*):
                The email to register through `git`.
        """
        try:
            if git_user is not None:
                run_subprocess(
                    "git config user.name".split() + [git_user], self.local_dir
                )

            if git_email is not None:
                run_subprocess(
                    f"git config user.email {git_email}".split(), self.local_dir
                )
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def git_credential_helper_store(self):
        """
        Sets the git credential helper to `store`
        """
        try:
            run_subprocess("git config credential.helper store".split(), self.local_dir)
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def git_head_hash(self) -> str:
        """
        Get commit sha on top of HEAD.

        Returns:
            `str`: The current checked out commit SHA.
        """
        try:
            p = run_subprocess("git rev-parse HEAD".split(), self.local_dir)
            return p.stdout.strip()
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def git_remote_url(self) -> str:
        """
        Get URL to origin remote.

        Returns:
            `str`: The URL of the `origin` remote.
        """
        try:
            p = run_subprocess(
                "git config --get remote.origin.url".split(), self.local_dir
            )
            url = p.stdout.strip()
            # Strip basic auth info.
            return re.sub(r"https://.*@", "https://", url)
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def git_head_commit_url(self) -> str:
        """
        Get URL to last commit on HEAD. We assume it's been pushed, and the url
        scheme is the same one as for GitHub or HuggingFace.

        Returns:
            `str`: The URL to the current checked-out commit.
        """
        sha = self.git_head_hash()
        url = self.git_remote_url()
        if url.endswith("/"):
            url = url[:-1]
        return f"{url}/commit/{sha}"

    def list_deleted_files(self) -> List[str]:
        """
        Returns a list of the files that are deleted in the working directory or
        index.

        Returns:
            `List[str]`: A list of files that have been deleted in the working
            directory or index.
        """
        try:
            git_status = run_subprocess(
                "git status -s".split(), self.local_dir
            ).stdout.strip()
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

        if len(git_status) == 0:
            return []

        # Receives a status like the following
        #  D .gitignore
        #  D new_file.json
        # AD new_file1.json
        # ?? new_file2.json
        # ?? new_file4.json

        # Strip each line of whitespaces
        modified_files_statuses = [status.strip() for status in git_status.split("\n")]

        # Only keep files that are deleted using the D prefix
        deleted_files_statuses = [
            status for status in modified_files_statuses if "D" in status.split()[0]
        ]

        # Remove the D prefix and strip to keep only the relevant filename
        deleted_files = [
            status.split()[-1].strip() for status in deleted_files_statuses
        ]

        return deleted_files

    def lfs_track(
        self, patterns: Union[str, List[str]], filename: Optional[bool] = False
    ):
        """
        Tell git-lfs to track files according to a pattern.

        Setting the `filename` argument to `True` will treat the arguments as
        literal filenames, not as patterns. Any special glob characters in the
        filename will be escaped when writing to the `.gitattributes` file.

        Args:
            patterns (`Union[str, List[str]]`):
                The pattern, or list of patterns, to track with git-lfs.
            filename (`bool`, *optional*, defaults to `False`):
                Whether to use the patterns as literal filenames.
        """
        if isinstance(patterns, str):
            patterns = [patterns]
        try:
            for pattern in patterns:
                cmd = f"git lfs track {'--filename' if filename else ''}".split()
                cmd.append(pattern)
                run_subprocess(cmd, self.local_dir)
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def lfs_untrack(self, patterns: Union[str, List[str]]):
        """
        Tell git-lfs to untrack those files.

        Args:
            patterns (`Union[str, List[str]]`):
                The pattern, or list of patterns, to untrack with git-lfs.
        """
        if isinstance(patterns, str):
            patterns = [patterns]
        try:
            for pattern in patterns:
                run_subprocess("git lfs untrack".split() + [pattern], self.local_dir)
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def lfs_enable_largefiles(self):
        """
        HF-specific. This enables upload support of files >5GB.
        """
        try:
            lfs_config = "git config lfs.customtransfer.multipart"
            run_subprocess(f"{lfs_config}.path huggingface-cli".split(), self.local_dir)
            run_subprocess(
                f"{lfs_config}.args {LFS_MULTIPART_UPLOAD_COMMAND}".split(),
                self.local_dir,
            )
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def auto_track_binary_files(self, pattern: Optional[str] = ".") -> List[str]:
        """
        Automatically track binary files with git-lfs.

        Args:
            pattern (`str`, *optional*, defaults to "."):
                The pattern with which to track files that are binary.

        Returns:
            `List[str]`: List of filenames that are now tracked due to being
            binary files
        """
        files_to_be_tracked_with_lfs = []

        deleted_files = self.list_deleted_files()

        for filename in files_to_be_staged(pattern, folder=self.local_dir):
            if filename in deleted_files:
                continue

            path_to_file = os.path.join(os.getcwd(), self.local_dir, filename)

            if not (is_tracked_with_lfs(path_to_file) or is_git_ignored(path_to_file)):
                size_in_mb = os.path.getsize(path_to_file) / (1024 * 1024)

                if size_in_mb >= 10:
                    logger.warning(
                        "Parsing a large file to check if binary or not. Tracking large"
                        " files using `repository.auto_track_large_files` is"
                        " recommended so as to not load the full file in memory."
                    )

                is_binary = is_binary_file(path_to_file)

                if is_binary:
                    self.lfs_track(filename)
                    files_to_be_tracked_with_lfs.append(filename)

        # Cleanup the .gitattributes if files were deleted
        self.lfs_untrack(deleted_files)

        return files_to_be_tracked_with_lfs

    def auto_track_large_files(self, pattern: Optional[str] = ".") -> List[str]:
        """
        Automatically track large files (files that weigh more than 10MBs) with
        git-lfs.

        Args:
            pattern (`str`, *optional*, defaults to "."):
                The pattern with which to track files that are above 10MBs.

        Returns:
            `List[str]`: List of filenames that are now tracked due to their
            size.
        """
        files_to_be_tracked_with_lfs = []

        deleted_files = self.list_deleted_files()

        for filename in files_to_be_staged(pattern, folder=self.local_dir):
            if filename in deleted_files:
                continue

            path_to_file = os.path.join(os.getcwd(), self.local_dir, filename)
            size_in_mb = os.path.getsize(path_to_file) / (1024 * 1024)

            if (
                size_in_mb >= 10
                and not is_tracked_with_lfs(path_to_file)
                and not is_git_ignored(path_to_file)
            ):
                self.lfs_track(filename)
                files_to_be_tracked_with_lfs.append(filename)

        # Cleanup the .gitattributes if files were deleted
        self.lfs_untrack(deleted_files)

        return files_to_be_tracked_with_lfs

    def lfs_prune(self, recent=False):
        """
        git lfs prune

        Args:
            recent (`bool`, *optional*, defaults to `False`):
                Whether to prune files even if they were referenced by recent
                commits. See the following
                [link](https://github.com/git-lfs/git-lfs/blob/f3d43f0428a84fc4f1e5405b76b5a73ec2437e65/docs/man/git-lfs-prune.1.ronn#recent-files)
                for more information.
        """
        args = "git lfs prune".split()
        if recent:
            args.append("--recent")
        try:
            with _lfs_log_progress():
                result = run_subprocess(
                    args,
                    self.local_dir,
                )
                logger.info(result.stdout)
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def git_pull(self, rebase: Optional[bool] = False, lfs: Optional[bool] = False):
        """
        git pull

        Args:
            rebase (`bool`, *optional*, defaults to `False`):
                Whether to rebase the current branch on top of the upstream
                branch after fetching.
            lfs (`bool`, *optional*, defaults to `False`):
                Whether to fetch the LFS files too. This option only changes the
                behavior when a repository was cloned without fetching the LFS
                files; calling `repo.git_pull(lfs=True)` will then fetch the LFS
                file from the remote repository.
        """
        command = ("git pull" if not lfs else "git lfs pull").split()
        if rebase:
            command.append("--rebase")
        try:
            with _lfs_log_progress():
                result = run_subprocess(
                    command,
                    self.local_dir,
                )
                logger.info(result.stdout)
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def git_add(
        self, pattern: Optional[str] = ".", auto_lfs_track: Optional[bool] = False
    ):
        """
        git add

        Setting the `auto_lfs_track` parameter to `True` will automatically
        track files that are larger than 10MB with `git-lfs`.

        Args:
            pattern (`str`, *optional*, defaults to "."):
                The pattern with which to add files to staging.
            auto_lfs_track (`bool`, *optional*, defaults to `False`):
                Whether to automatically track large and binary files with
                git-lfs. Any file over 10MB in size, or in binary format, will
                be automatically tracked.
        """
        if auto_lfs_track:
            # Track files according to their size (>=10MB)
            tracked_files = self.auto_track_large_files(pattern)

            # Read the remaining files and track them if they're binary
            tracked_files.extend(self.auto_track_binary_files(pattern))

            if tracked_files:
                logger.warning(
                    f"Adding files tracked by Git LFS: {tracked_files}. This may take a"
                    " bit of time if the files are large."
                )

        try:
            result = run_subprocess("git add -v".split() + [pattern], self.local_dir)
            logger.info(f"Adding to index:\n{result.stdout}\n")
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

    def git_commit(self, commit_message: str = "commit files to HF hub"):
        """
        git commit

        Args:
            commit_message (`str`, *optional*, defaults to "commit files to HF hub"):
                The message attributed to the commit.
        """
        try:
            result = run_subprocess(
                "git commit -v -m".split() + [commit_message], self.local_dir
            )
            logger.info(f"Committed:\n{result.stdout}\n")
        except subprocess.CalledProcessError as exc:
            if len(exc.stderr) > 0:
                raise EnvironmentError(exc.stderr)
            else:
                raise EnvironmentError(exc.stdout)

    def git_push(
        self,
        upstream: Optional[str] = None,
        blocking: Optional[bool] = True,
        auto_lfs_prune: Optional[bool] = False,
    ) -> Union[str, Tuple[str, CommandInProgress]]:
        """
        git push

        If used without setting `blocking`, will return url to commit on remote
        repo. If used with `blocking=True`, will return a tuple containing the
        url to commit and the command object to follow for information about the
        process.

        Args:
            upstream (`str`, *optional*):
                Upstream to which this should push. If not specified, will push
                to the lastly defined upstream or to the default one (`origin
                main`).
            blocking (`bool`, *optional*, defaults to `True`):
                Whether the function should return only when the push has
                finished. Setting this to `False` will return an
                `CommandInProgress` object which has an `is_done` property. This
                property will be set to `True` when the push is finished.
            auto_lfs_prune (`bool`, *optional*, defaults to `False`):
                Whether to automatically prune files once they have been pushed
                to the remote.
        """
        command = "git push"

        if upstream:
            command += f" --set-upstream {upstream}"

        number_of_commits = commits_to_push(self.local_dir, upstream)

        if number_of_commits > 1:
            logger.warning(
                f"Several commits ({number_of_commits}) will be pushed upstream."
            )
            if blocking:
                logger.warning("The progress bars may be unreliable.")

        try:
            with _lfs_log_progress():
                process = subprocess.Popen(
                    command.split(),
                    stderr=subprocess.PIPE,
                    stdout=subprocess.PIPE,
                    encoding="utf-8",
                    cwd=self.local_dir,
                )

                if blocking:
                    stdout, stderr = process.communicate()
                    return_code = process.poll()
                    process.kill()

                    if len(stderr):
                        logger.warning(stderr)

                    if return_code:
                        raise subprocess.CalledProcessError(
                            return_code, process.args, output=stdout, stderr=stderr
                        )

        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

        if not blocking:

            def status_method():
                status = process.poll()
                if status is None:
                    return -1
                else:
                    return status

            command = CommandInProgress(
                "push",
                is_done_method=lambda: process.poll() is not None,
                status_method=status_method,
                process=process,
                post_method=self.lfs_prune if auto_lfs_prune else None,
            )

            self.command_queue.append(command)

            return self.git_head_commit_url(), command

        if auto_lfs_prune:
            self.lfs_prune()

        return self.git_head_commit_url()

    def git_checkout(self, revision: str, create_branch_ok: Optional[bool] = False):
        """
        git checkout a given revision

        Specifying `create_branch_ok` to `True` will create the branch to the
        given revision if that revision doesn't exist.

        Args:
            revision (`str`):
                The revision to checkout.
            create_branch_ok (`str`, *optional*, defaults to `False`):
                Whether creating a branch named with the `revision` passed at
                the current checked-out reference if `revision` isn't an
                existing revision is allowed.
        """
        command = f"git checkout {revision}"
        try:
            result = run_subprocess(command.split(), self.local_dir)
            logger.warning(f"Checked out {revision} from {self.current_branch}.")
            logger.warning(result.stdout)
        except subprocess.CalledProcessError as exc:
            if not create_branch_ok:
                raise EnvironmentError(exc.stderr)
            else:
                command = f"git checkout -b {revision}"
                try:
                    result = run_subprocess(command.split(), self.local_dir)
                    logger.warning(
                        f"Revision `{revision}` does not exist. Created and checked out"
                        f" branch `{revision}`."
                    )
                    logger.warning(result.stdout)
                except subprocess.CalledProcessError as exc:
                    raise EnvironmentError(exc.stderr)

    def tag_exists(self, tag_name: str, remote: Optional[str] = None) -> bool:
        """
        Check if a tag exists or not.

        Args:
            tag_name (`str`):
                The name of the tag to check.
            remote (`str`, *optional*):
                Whether to check if the tag exists on a remote. This parameter
                should be the identifier of the remote.

        Returns:
            `bool`: Whether the tag exists.
        """
        if remote:
            try:
                result = run_subprocess(
                    f"git ls-remote origin refs/tags/{tag_name}".split(), self.local_dir
                ).stdout.strip()
            except subprocess.CalledProcessError as exc:
                raise EnvironmentError(exc.stderr)

            return len(result) != 0
        else:
            try:
                git_tags = run_subprocess(
                    "git tag".split(), self.local_dir
                ).stdout.strip()
            except subprocess.CalledProcessError as exc:
                raise EnvironmentError(exc.stderr)

            git_tags = git_tags.split("\n")
            return tag_name in git_tags

    def delete_tag(self, tag_name: str, remote: Optional[str] = None) -> bool:
        """
        Delete a tag, both local and remote, if it exists

        Args:
            tag_name (`str`):
                The tag name to delete.
            remote (`str`, *optional*):
                The remote on which to delete the tag.

        Returns:
             `bool`: `True` if deleted, `False` if the tag didn't exist.
                If remote is not passed, will just be updated locally
        """
        delete_locally = True
        delete_remotely = True

        if not self.tag_exists(tag_name):
            delete_locally = False

        if not self.tag_exists(tag_name, remote=remote):
            delete_remotely = False

        if delete_locally:
            try:
                run_subprocess(
                    ["git", "tag", "-d", tag_name], self.local_dir
                ).stdout.strip()
            except subprocess.CalledProcessError as exc:
                raise EnvironmentError(exc.stderr)

        if remote and delete_remotely:
            try:
                run_subprocess(
                    f"git push {remote} --delete {tag_name}".split(), self.local_dir
                ).stdout.strip()
            except subprocess.CalledProcessError as exc:
                raise EnvironmentError(exc.stderr)

        return True

    def add_tag(self, tag_name: str, message: str = None, remote: Optional[str] = None):
        """
        Add a tag at the current head and push it

        If remote is None, will just be updated locally

        If no message is provided, the tag will be lightweight. if a message is
        provided, the tag will be annotated.

        Args:
            tag_name (`str`):
                The name of the tag to be added.
            message (`str`, *optional*):
                The message that accompanies the tag. The tag will turn into an
                annotated tag if a message is passed.
            remote (`str`, *optional*):
                The remote on which to add the tag.
        """
        if message:
            tag_args = ["git", "tag", "-a", tag_name, "-m", message]
        else:
            tag_args = ["git", "tag", tag_name]

        try:
            run_subprocess(tag_args, self.local_dir).stdout.strip()
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

        if remote:
            try:
                run_subprocess(
                    f"git push {remote} {tag_name}".split(), self.local_dir
                ).stdout.strip()
            except subprocess.CalledProcessError as exc:
                raise EnvironmentError(exc.stderr)

    def is_repo_clean(self) -> bool:
        """
        Return whether or not the git status is clean or not

        Returns:
            `bool`: `True` if the git status is clean, `False` otherwise.
        """
        try:
            git_status = run_subprocess(
                "git status --porcelain".split(), self.local_dir
            ).stdout.strip()
        except subprocess.CalledProcessError as exc:
            raise EnvironmentError(exc.stderr)

        return len(git_status) == 0

    def push_to_hub(
        self,
        commit_message: Optional[str] = "commit files to HF hub",
        blocking: Optional[bool] = True,
        clean_ok: Optional[bool] = True,
        auto_lfs_prune: Optional[bool] = False,
    ) -> Optional[str]:
        """
        Helper to add, commit, and push files to remote repository on the
        HuggingFace Hub. Will automatically track large files (>10MB).

        Args:
            commit_message (`str`):
                Message to use for the commit.
            blocking (`bool`, *optional*, defaults to `True`):
                Whether the function should return only when the `git push` has
                finished.
            clean_ok (`bool`, *optional*, defaults to `True`):
                If True, this function will return None if the repo is
                untouched. Default behavior is to fail because the git command
                fails.
            auto_lfs_prune (`bool`, *optional*, defaults to `False`):
                Whether to automatically prune files once they have been pushed
                to the remote.
        """
        if clean_ok and self.is_repo_clean():
            logger.info("Repo currently clean.  Ignoring push_to_hub")
            return None
        self.git_add(auto_lfs_track=True)
        self.git_commit(commit_message)
        return self.git_push(
            upstream=f"origin {self.current_branch}",
            blocking=blocking,
            auto_lfs_prune=auto_lfs_prune,
        )

    @contextmanager
    def commit(
        self,
        commit_message: str,
        branch: Optional[str] = None,
        track_large_files: Optional[bool] = True,
        blocking: Optional[bool] = True,
        auto_lfs_prune: Optional[bool] = False,
    ):
        """
        Context manager utility to handle committing to a repository. This
        automatically tracks large files (>10Mb) with git-lfs. Set the
        `track_large_files` argument to `False` if you wish to ignore that
        behavior.

        Args:
            commit_message (`str`):
                Message to use for the commit.
            branch (`str`, *optional*):
                The branch on which the commit will appear. This branch will be
                checked-out before any operation.
            track_large_files (`bool`, *optional*, defaults to `True`):
                Whether to automatically track large files or not. Will do so by
                default.
            blocking (`bool`, *optional*, defaults to `True`):
                Whether the function should return only when the `git push` has
                finished.
            auto_lfs_prune (`bool`, defaults to `True`):
                Whether to automatically prune files once they have been pushed
                to the remote.

        Examples:

        ```python
        >>> with Repository(
        ...     "text-files",
        ...     clone_from="<user>/text-files",
        ...     use_auth_token=True,
        >>> ).commit("My first file :)"):
        ...     with open("file.txt", "w+") as f:
        ...         f.write(json.dumps({"hey": 8}))

        >>> import torch

        >>> model = torch.nn.Transformer()
        >>> with Repository(
        ...     "torch-model",
        ...     clone_from="<user>/torch-model",
        ...     use_auth_token=True,
        >>> ).commit("My cool model :)"):
        ...     torch.save(model.state_dict(), "model.pt")
        ```

        """

        files_to_stage = files_to_be_staged(".", folder=self.local_dir)

        if len(files_to_stage):
            if len(files_to_stage) > 5:
                files_to_stage = str(files_to_stage[:5])[:-1] + ", ...]"

            logger.error(
                "There exists some updated files in the local repository that are not"
                f" committed: {files_to_stage}. This may lead to errors if checking out"
                " a branch. These files and their modifications will be added to the"
                " current commit."
            )

        if branch is not None:
            self.git_checkout(branch, create_branch_ok=True)

        if is_tracked_upstream(self.local_dir):
            logger.warning("Pulling changes ...")
            self.git_pull(rebase=True)
        else:
            logger.warning(
                "The current branch has no upstream branch. Will push to 'origin"
                f" {self.current_branch}'"
            )

        current_working_directory = os.getcwd()
        os.chdir(os.path.join(current_working_directory, self.local_dir))

        try:
            yield self
        finally:
            self.git_add(auto_lfs_track=track_large_files)

            try:
                self.git_commit(commit_message)
            except OSError as e:
                # If no changes are detected, there is nothing to commit.
                if "nothing to commit" not in str(e):
                    raise e

            try:
                self.git_push(
                    upstream=f"origin {self.current_branch}",
                    blocking=blocking,
                    auto_lfs_prune=auto_lfs_prune,
                )
            except OSError as e:
                # If no changes are detected, there is nothing to commit.
                if "could not read Username" in str(e):
                    raise OSError(
                        "Couldn't authenticate user for push. Did you set"
                        " `use_auth_token` to `True`?"
                    ) from e
                else:
                    raise e

            os.chdir(current_working_directory)

    def repocard_metadata_load(self) -> Optional[Dict]:
        filepath = os.path.join(self.local_dir, REPOCARD_NAME)
        if os.path.isfile(filepath):
            return metadata_load(filepath)

    def repocard_metadata_save(self, data: Dict) -> None:
        return metadata_save(os.path.join(self.local_dir, REPOCARD_NAME), data)

    @property
    def commands_failed(self):
        """
        Returns the asynchronous commands that failed.
        """
        return [c for c in self.command_queue if c.status > 0]

    @property
    def commands_in_progress(self):
        """
        Returns the asynchronous commands that are currently in progress.
        """
        return [c for c in self.command_queue if not c.is_done]

    def wait_for_commands(self):
        """
        Blocking method: blocks all subsequent execution until all commands have
        been processed.
        """
        index = 0
        for command_failed in self.commands_failed:
            logger.error(
                f"The {command_failed.title} command with PID"
                f" {command_failed._process.pid} failed."
            )
            logger.error(command_failed.stderr)

        while self.commands_in_progress:
            if index % 10 == 0:
                logger.error(
                    "Waiting for the following commands to finish before shutting"
                    f" down: {self.commands_in_progress}."
                )

            index += 1

            time.sleep(1)


# src/huggingface_hub/snapshot_download.py

# TODO: remove in 0.11

import warnings


warnings.warn(
    "snapshot_download.py has been made private and will no longer be available from"
    " version 0.11. Please use `from huggingface_hub import snapshot_download` to"
    " import the only public function in this module. Other members of the file may be"
    " changed without a deprecation notice.",
    FutureWarning,
)

from ._snapshot_download import *  # noqa
from .constants import REPO_ID_SEPARATOR  # noqa


# src/huggingface_hub/_commit_api.py
"""
Type definitions and utilities for the `create_commit` API
"""

import base64
import io
import os
import sys
from concurrent import futures
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from dataclasses import dataclass, field
from typing import BinaryIO, Dict, Iterable, List, Optional, Tuple, Union


if sys.version_info >= (3, 8):
    from typing import Literal
else:
    from typing_extensions import Literal

import requests

from .constants import ENDPOINT
from .lfs import UploadInfo, _validate_batch_actions, lfs_upload, post_lfs_batch_info
from .utils import logging


logger = logging.get_logger(__name__)


UploadMode = Literal["lfs", "regular"]


@dataclass
class CommitOperationDelete:
    """
    Data structure holding necessary info to delete
    a file from a repository on the Hub

    Args:
        path_in_repo (`str`):
            Relative filepath in the repo, for example:
            `"checkpoints/1fec34a/weights.bin"`
    """

    path_in_repo: str


@dataclass
class CommitOperationAdd:
    """
    Data structure holding necessary info to upload a file
    to a repository on the Hub

    Args:
        path_in_repo (`str`):
            Relative filepath in the repo, for example:
            `"checkpoints/1fec34a/weights.bin"`
        path_or_fileobj (`str`, `bytes`, or `BinaryIO`):
            Either:
            - a path to a local file (as str) to upload
            - a buffer of bytes (`bytes`) holding the content of the file to upload
            - a "file object" (subclass of `io.BufferedIOBase`), typically obtained
                with `open(path, "rb")`. It must support `seek()` and `tell()` methods.
    """

    path_in_repo: str
    path_or_fileobj: Union[str, bytes, BinaryIO]

    __upload_info: Optional[UploadInfo] = field(default=None, init=False)

    def validate(self):
        """
        Ensures `path_or_fileobj` is valid:
        - Ensures it is either a `str`, `bytes` or an instance of `io.BufferedIOBase`
        - If it is a `str`, ensure that it is a file path and that the file exists
        - If it is an instance of `io.BufferedIOBase`, ensures it supports `seek()` and `tell()`

        Raises: `ValueError` if `path_or_fileobj` is not valid
        """
        if isinstance(self.path_or_fileobj, str):
            path_or_fileobj = os.path.normpath(os.path.expanduser(self.path_or_fileobj))
            if not os.path.isfile(path_or_fileobj):
                raise ValueError(
                    f"Provided path: '{path_or_fileobj}' is not a file on the local"
                    " file system"
                )
        elif not isinstance(self.path_or_fileobj, (io.BufferedIOBase, bytes)):
            # ^^ Inspired from: https://stackoverflow.com/questions/44584829/how-to-determine-if-file-is-opened-in-binary-or-text-mode
            raise ValueError(
                "path_or_fileobj must be either an instance of str, bytes or"
                " io.BufferedIOBase. If you passed a file-like object, make sure it is"
                " in binary mode."
            )
        if isinstance(self.path_or_fileobj, io.BufferedIOBase):
            try:
                self.path_or_fileobj.tell()
                self.path_or_fileobj.seek(0, os.SEEK_CUR)
            except (OSError, AttributeError) as exc:
                raise ValueError(
                    "path_or_fileobj is a file-like object but does not implement"
                    " seek() and tell()"
                ) from exc

    def _upload_info(self) -> UploadInfo:
        """
        Computes and caches UploadInfo for the underlying data behind `path_or_fileobj`
        Triggers `self.validate`.

        Raises: `ValueError` if self.validate fails
        """
        self.validate()
        if self.__upload_info is None:
            if isinstance(self.path_or_fileobj, str):
                self.__upload_info = UploadInfo.from_path(self.path_or_fileobj)
            elif isinstance(self.path_or_fileobj, bytes):
                self.__upload_info = UploadInfo.from_bytes(self.path_or_fileobj)
            else:
                self.__upload_info = UploadInfo.from_fileobj(self.path_or_fileobj)
        return self.__upload_info

    @contextmanager
    def as_file(self):
        """
        A context manager that yields a file-like object allowing to read the underlying
        data behind `path_or_fileobj`.

        Triggers `self.validate`.

        Raises: `ValueError` if self.validate fails

        Example:

        ```python
        >>> operation = CommitOperationAdd(
        ...        path_in_repo="remote/dir/weights.h5",
        ...        path_or_fileobj="./local/weights.h5",
        ... )
        CommitOperationAdd(path_in_repo='remote/dir/weights.h5', path_or_fileobj='./local/weights.h5', _upload_info=None)

        >>> with operation.as_file() as file:
        ...     content = file.read()
        ```

        """
        self.validate()
        if isinstance(self.path_or_fileobj, str):
            with open(self.path_or_fileobj, "rb") as file:
                yield file
        elif isinstance(self.path_or_fileobj, bytes):
            yield io.BytesIO(self.path_or_fileobj)
        elif isinstance(self.path_or_fileobj, io.BufferedIOBase):
            prev_pos = self.path_or_fileobj.tell()
            yield self.path_or_fileobj
            self.path_or_fileobj.seek(prev_pos, io.SEEK_SET)

    def b64content(self) -> bytes:
        """
        The base64-encoded content of `path_or_fileobj`

        Returns: `bytes`
        """
        with self.as_file() as file:
            return base64.b64encode(file.read())


CommitOperation = Union[CommitOperationAdd, CommitOperationDelete]


def upload_lfs_files(
    *,
    additions: Iterable[CommitOperationAdd],
    repo_type: str,
    repo_id: str,
    token: str,
    revision: str,
    endpoint: Optional[str] = None,
    num_threads: int = 5,
):
    """
    Uploads the content of `additions` to the Hub using the large file storage protocol.

    Relevant external documentation:
        - LFS Batch API: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md

    Args:
        additions (`Iterable` of `CommitOperationAdd`):
            The files to be uploaded
        repo_type (`str`):
            Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.
        repo_id (`str`):
            A namespace (user or an organization) and a repo name separated
            by a `/`.
        token (`str`):
            An authentication token ( See https://huggingface.co/settings/tokens )
        revision (`str`):
            The git revision to upload the files to. Can be any valid git revision.
        num_threads (`int`, *optional*):
            The number of concurrent threads to use when uploading. Defaults to 5.


    Raises: `RuntimeError` if an upload failed for any reason

    Raises: `ValueError` if the server returns malformed responses

    Raises: `requests.HTTPError` if the LFS batch endpoint returned an HTTP
        error

    """
    # Step 1: retrieve upload instructions from the LFS batch endpoint
    batch_actions, batch_errors = post_lfs_batch_info(
        upload_infos=[op._upload_info() for op in additions],
        token=token,
        repo_id=repo_id,
        repo_type=repo_type,
        revision=revision,
        endpoint=endpoint,
    )
    if batch_errors:
        message = "\n".join(
            [
                f'Encountered error for file with OID {err.get("oid")}:'
                f' `{err.get("error", {}).get("message")}'
                for err in batch_errors
            ]
        )
        raise ValueError(f"LFS batch endpoint returned errors:\n{message}")

    # Step 2: upload files concurrently according to these instructions
    oid2addop = {add_op._upload_info().sha256.hex(): add_op for add_op in additions}
    with ThreadPoolExecutor(max_workers=num_threads) as pool:
        logger.debug(
            f"Uploading {len(batch_actions)} LFS files to the Hub using up to"
            f" {num_threads} threads concurrently"
        )
        # Upload the files concurrently, stopping on the first exception
        futures2operation: Dict[futures.Future, CommitOperationAdd] = {
            pool.submit(
                _upload_lfs_object,
                operation=oid2addop[batch_action["oid"]],
                lfs_batch_action=batch_action,
                token=token,
            ): oid2addop[batch_action["oid"]]
            for batch_action in batch_actions
        }
        completed, pending = futures.wait(
            futures2operation,
            return_when=futures.FIRST_EXCEPTION,
        )

        for pending_future in pending:
            # Cancel pending futures
            # Uploads already in progress can't be stopped unfortunately,
            # as `requests` does not support cancelling / aborting a request
            pending_future.cancel()

        for future in completed:
            operation = futures2operation[future]
            try:
                future.result()
            except Exception as exc:
                # Raise the first exception encountered
                raise RuntimeError(
                    f"Error while uploading {operation.path_in_repo} to the Hub"
                ) from exc


def _upload_lfs_object(
    operation: CommitOperationAdd, lfs_batch_action: dict, token: str
):
    """
    Handles uploading a given object to the Hub with the LFS protocol.

    Defers to [`~huggingface_hub.utils.lfs.lfs_upload`] for the actual upload logic.

    Can be a No-op if the content of the file is already present on the hub
    large file storage.

    Args:
        operation (`CommitOprationAdd`):
            The add operation triggering this upload
        lfs_batch_action (`dict`):
            Upload instructions from the LFS batch endpoint for this object.
            See [`~huggingface_hub.utils.lfs..post_lfs_batch_info`] for more details.
        token (`str`):
            A [user access token](https://hf.co/settings/tokens) to authenticate requests against the Hub

    Raises: `ValueError` if `lfs_batch_action` is improperly formatted
    """
    _validate_batch_actions(lfs_batch_action)
    upload_info = operation._upload_info()
    actions = lfs_batch_action.get("actions")
    if actions is None:
        # The file was already uploaded
        logger.debug(
            f"Content of file {operation.path_in_repo} is already present upstream"
            " - skipping upload"
        )
        return
    upload_action = lfs_batch_action["actions"].get("upload")
    verify_action = lfs_batch_action["actions"].get("verify")

    with operation.as_file() as fileobj:
        lfs_upload(
            fileobj=fileobj,
            upload_action=upload_action,
            verify_action=verify_action,
            upload_info=upload_info,
            token=token,
        )
        logger.debug(f"{operation.path_in_repo}: Upload successful")


def validate_preupload_info(preupload_info: dict):
    files = preupload_info.get("files")
    if not isinstance(files, list):
        raise ValueError("preupload_info is improperly formatted")
    for file_info in files:
        if not (
            isinstance(file_info, dict)
            and isinstance(file_info.get("path"), str)
            and isinstance(file_info.get("uploadMode"), str)
            and (file_info["uploadMode"] in ("lfs", "regular"))
        ):
            raise ValueError("preupload_info is improperly formatted:")
    return preupload_info


def fetch_upload_modes(
    additions: Iterable[CommitOperationAdd],
    repo_type: str,
    repo_id: str,
    token: str,
    revision: str,
    endpoint: Optional[str] = None,
) -> List[Tuple[CommitOperationAdd, UploadMode]]:
    """
    Requests the Hub "preupload" endpoint to determine wether each input file
    should be uploaded as a regular git blob or as git LFS blob.

    Args:
        additions (`Iterable` of :class:`CommitOperationAdd`):
            Iterable of :class:`CommitOperationAdd` describing the files to
            upload to the Hub.
        repo_type (`str`):
            Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.
        repo_id (`str`):
            A namespace (user or an organization) and a repo name separated
            by a `/`.
        token (`str`):
            An authentication token ( See https://huggingface.co/settings/tokens )
        revision (`str`):
            The git revision to upload the files to. Can be any valid git revision.

    Returns:
        list of 2-tuples, the first element being the add operation and
        the second element the associated upload mode

    Raises:
        :class:`requests.HTTPError`:
            If the Hub API returned an error
    """
    endpoint = endpoint if endpoint is not None else ENDPOINT
    headers = {"authorization": f"Bearer {token}"} if token is not None else None
    payload = {
        "files": [
            {
                "path": op.path_in_repo,
                "sample": base64.b64encode(op._upload_info().sample).decode("ascii"),
                "size": op._upload_info().size,
                "sha": op._upload_info().sha256.hex(),
            }
            for op in additions
        ]
    }

    resp = requests.post(
        f"{endpoint}/api/{repo_type}s/{repo_id}/preupload/{revision}",
        json=payload,
        headers=headers,
    )
    resp.raise_for_status()

    preupload_info = validate_preupload_info(resp.json())

    path2mode: Dict[str, UploadMode] = {
        file["path"]: file["uploadMode"] for file in preupload_info["files"]
    }

    return [(op, path2mode[op.path_in_repo]) for op in additions]


def prepare_commit_payload(
    additions: Iterable[Tuple[CommitOperationAdd, UploadMode]],
    deletions: Iterable[CommitOperationDelete],
    commit_message: str,
    commit_description: Optional[str] = None,
):
    """
    Builds the payload to POST to the `/commit` API of the Hub
    """
    commit_description = commit_description if commit_description is not None else ""

    return {
        "summary": commit_message,
        "description": commit_description,
        "files": [
            {
                "path": add_op.path_in_repo,
                "encoding": "base64",
                "content": add_op.b64content().decode(),
            }
            for (add_op, upload_mode) in additions
            if upload_mode == "regular"
        ],
        "lfsFiles": [
            {
                "path": add_op.path_in_repo,
                "algo": "sha256",
                "oid": add_op._upload_info().sha256.hex(),
            }
            for (add_op, upload_mode) in additions
            if upload_mode == "lfs"
        ],
        "deletedFiles": [{"path": del_op.path_in_repo} for del_op in deletions],
    }
