diff --git a/dvc/external_repo.py b/dvc/external_repo.py
--- a/dvc/external_repo.py
+++ b/dvc/external_repo.py
@@ -3,7 +3,7 @@
 import tempfile
 import threading
 from contextlib import contextmanager
-from typing import Dict
+from typing import TYPE_CHECKING, Dict
 
 from funcy import retry, wrap_with
 
@@ -19,6 +19,9 @@
 from dvc.scm import CloneError, map_scm_exception
 from dvc.utils import relpath
 
+if TYPE_CHECKING:
+    from scmrepo import Git
+
 logger = logging.getLogger(__name__)
 
 
@@ -201,14 +204,12 @@ def _clone_default_branch(url, rev, for_write=False):
                     # given repo URL it is easier/safer for us to work with
                     # full clones in this case.
                     logger.debug("erepo: unshallowing clone for '%s'", url)
-                    git.fetch(unshallow=True)
-                    _merge_upstream(git)
+                    _pull(git, unshallow=True)
                     shallow = False
                     CLONES[url] = (clone_path, shallow)
                 else:
                     logger.debug("erepo: git pull '%s'", url)
-                    git.fetch()
-                    _merge_upstream(git)
+                    _pull(git)
         else:
             from dvc.scm import clone
 
@@ -227,7 +228,9 @@ def _clone_default_branch(url, rev, for_write=False):
                             "erepo: using shallow clone for branch '%s'", rev
                         )
                 except CloneError:
-                    pass
+                    git_dir = os.path.join(clone_path, ".git")
+                    if os.path.exists(git_dir):
+                        _remove(git_dir)
             if not git:
                 git = clone(url, clone_path)
                 shallow = False
@@ -239,7 +242,15 @@ def _clone_default_branch(url, rev, for_write=False):
     return clone_path, shallow
 
 
-def _merge_upstream(git):
+def _pull(git: "Git", unshallow: bool = False):
+    from dvc.repo.experiments.utils import fetch_all_exps
+
+    git.fetch(unshallow=unshallow)
+    _merge_upstream(git)
+    fetch_all_exps(git, "origin")
+
+
+def _merge_upstream(git: "Git"):
     from scmrepo.exceptions import SCMError
 
     try:
diff --git a/dvc/repo/experiments/utils.py b/dvc/repo/experiments/utils.py
--- a/dvc/repo/experiments/utils.py
+++ b/dvc/repo/experiments/utils.py
@@ -191,3 +191,16 @@ def check_ref_format(scm: "Git", ref: ExpRefInfo):
             f"Invalid exp name {ref.name}, the exp name must follow rules in "
             "https://git-scm.com/docs/git-check-ref-format"
         )
+
+
+def fetch_all_exps(scm: "Git", url: str, progress: Optional[Callable] = None):
+    refspecs = [
+        f"{ref}:{ref}"
+        for ref in iter_remote_refs(scm, url, base=EXPS_NAMESPACE)
+        if not (ref.startswith(EXEC_NAMESPACE) or ref == EXPS_STASH)
+    ]
+    scm.fetch_refspecs(
+        url,
+        refspecs,
+        progress=progress,
+    )
diff --git a/dvc/scm.py b/dvc/scm.py
--- a/dvc/scm.py
+++ b/dvc/scm.py
@@ -101,9 +101,14 @@ def update_git(self, event: "GitProgressEvent") -> None:
 def clone(url: str, to_path: str, **kwargs):
     from scmrepo.exceptions import CloneError as InternalCloneError
 
+    from dvc.repo.experiments.utils import fetch_all_exps
+
     with TqdmGit(desc="Cloning") as pbar:
         try:
-            return Git.clone(url, to_path, progress=pbar.update_git, **kwargs)
+            git = Git.clone(url, to_path, progress=pbar.update_git, **kwargs)
+            if "shallow_branch" not in kwargs:
+                fetch_all_exps(git, "origin", progress=pbar.update_git)
+            return git
         except InternalCloneError as exc:
             raise CloneError(str(exc))
 

diff --git a/tests/func/experiments/test_remote.py b/tests/func/experiments/test_remote.py
--- a/tests/func/experiments/test_remote.py
+++ b/tests/func/experiments/test_remote.py
@@ -374,3 +374,20 @@ def test_auth_error_push(tmp_dir, scm, dvc, exp_stage, http_auth_patch):
         match=f"HTTP Git authentication is not supported: '{http_auth_patch}'",
     ):
         dvc.experiments.push(http_auth_patch, [ref_info.name])
+
+
+@pytest.mark.parametrize("use_ref", [True, False])
+def test_get(tmp_dir, scm, dvc, exp_stage, erepo_dir, use_ref):
+    from dvc.repo import Repo
+
+    results = dvc.experiments.run(exp_stage.addressing, params=["foo=2"])
+    exp_rev = first(results)
+    exp_ref = first(exp_refs_by_rev(scm, exp_rev))
+
+    with erepo_dir.chdir():
+        Repo.get(
+            str(tmp_dir),
+            "params.yaml",
+            rev=exp_ref.name if use_ref else exp_rev,
+        )
+        assert (erepo_dir / "params.yaml").read_text().strip() == "foo: 2"

get: make it work for experiments and/or improve message
`dvc get --rev exp-name`
`dvc get --rev exp-ref`

should work and I should be able to get artifacts, datasets w/o doing heavy apply.

Now the message:

```
$ dvc get example-get-started/ model.pkl --rev 73c7c57
ERROR: failed to get 'model.pkl' from 'example-get-started/' - '404 Not Found'
```

is confusing. So, even if for some reason we decide it's not needed we have to improve the message and also solve https://github.com/iterative/dvc/issues/5355 

**Use cases:**

- I want to get a few models to compare them side by side now in the workspace
- I want to get a model/checkpoint w/o affecting my workspace
- I want to get a dataset w/o doing full apply

To me it felt very natural to expect, and I though this is a bug.

**More context from Slack:**

@pmrowla 
> exp names are already supposed to be accepted anywhere in DVC that supports --rev, the issue with dvc get is probably that we do not pull exp refs when we make the erepo clone. I don’t think we have a ticket for this right now

So, this can be considered a bug?





tests/func/experiments/test_remote.py::test_get[True]
tests/func/experiments/test_remote.py::test_push[True] tests/func/experiments/test_remote.py::test_push[False] tests/func/experiments/test_remote.py::test_push_args[True-False-True] tests/func/experiments/test_remote.py::test_push_args[False-True-None] tests/func/experiments/test_remote.py::test_push_diverged tests/func/experiments/test_remote.py::test_push_checkpoint tests/func/experiments/test_remote.py::test_push_ambiguous_name tests/func/experiments/test_remote.py::test_list_remote[True] tests/func/experiments/test_remote.py::test_list_remote[False] tests/func/experiments/test_remote.py::test_pull[True] tests/func/experiments/test_remote.py::test_pull[False] tests/func/experiments/test_remote.py::test_pull_args[True-False-True] tests/func/experiments/test_remote.py::test_pull_args[False-True-None] tests/func/experiments/test_remote.py::test_pull_diverged tests/func/experiments/test_remote.py::test_pull_checkpoint tests/func/experiments/test_remote.py::test_pull_ambiguous_name tests/func/experiments/test_remote.py::test_push_pull_cache tests/func/experiments/test_remote.py::test_auth_error_list tests/func/experiments/test_remote.py::test_auth_error_pull tests/func/experiments/test_remote.py::test_auth_error_push tests/func/experiments/test_remote.py::test_get[False]