diff --git a/pydantic/env_settings.py b/pydantic/env_settings.py
--- a/pydantic/env_settings.py
+++ b/pydantic/env_settings.py
@@ -197,7 +197,9 @@ def __call__(self, settings: BaseSettings) -> Dict[str, Any]:
         secrets_path = Path(self.secrets_dir).expanduser()
 
         if not secrets_path.exists():
-            raise SettingsError(f'directory "{secrets_path}" does not exist')
+            warnings.warn(f'directory "{secrets_path}" does not exist')
+            return secrets
+
         if not secrets_path.is_dir():
             raise SettingsError(f'secrets_dir must reference a directory, not a {path_type(secrets_path)}')
 

diff --git a/tests/test_settings.py b/tests/test_settings.py
--- a/tests/test_settings.py
+++ b/tests/test_settings.py
@@ -835,12 +835,10 @@ class Config:
 @pytest.mark.skipif(sys.platform.startswith('win'), reason='windows paths break regex')
 def test_secrets_missing_location(tmp_path):
     class Settings(BaseSettings):
-        foo: str
-
         class Config:
             secrets_dir = tmp_path / 'does_not_exist'
 
-    with pytest.raises(SettingsError, match=f'directory "{tmp_path}/does_not_exist" does not exist'):
+    with pytest.warns(UserWarning, match=f'directory "{tmp_path}/does_not_exist" does not exist'):
         Settings()
 
 

secrets_dir configuration setting raises an error when the directory doesn't exist
### Checks

* [x] I added a descriptive title to this issue
* [x] I have searched (google, github) for similar issues and couldn't find anything
* [x] I have read and followed [the docs](https://pydantic-docs.helpmanual.io/) and still think this is a bug

<!-- Sorry to sound so draconian, but every second saved replying to issues is time spend improving pydantic :-) -->

# Bug

Output of `python -c "import pydantic.utils; print(pydantic.utils.version_info())"`:
```
             pydantic version: 1.7.3
            pydantic compiled: True
                 install path: ~/.pyenv/versions/3.8.5/envs/fetcher/lib/python3.8/site-packages/pydantic
               python version: 3.8.5 (default, Sep  4 2020, 14:39:18)  [GCC 9.3.0]
                     platform: Linux-5.8.0-7630-generic-x86_64-with-glibc2.29
     optional deps. installed: ['typing-extensions', 'devtools']
```

I have declared a settings model that can read values from the environment, a `.env` file, or from Docker secrets as follows.

```py
import pydantic

class Settings(BaseSettings):

    password: SecretStr = Field(..., env="APP_PASSWORD")

    class Config:
        """Configure settings values discovery locations."""

        env_file = ".env"
        secrets_dir = "/run/secrets"
```

[As per the documentation](https://pydantic-docs.helpmanual.io/usage/settings/#field-value-priority), checking for a value from a secret is the fourth attempt pydantic makes. However, when I just work in my development environment where the directory `/run/secrets` doesn't exist, instantiating the model raises an error.

```
pydantic.env_settings.SettingsError: directory "/run/secrets" does not exist
```

This is annoying because it happens even though I have the values in a `.env` file. Thus I have to circumvent this by using `Settings(_secrets_dir=".")` which would mean that I need a special case for my dev environment. I think, when values are provided to the instance, or from the environment, or from `.env` first, it shouldn't matter whether the `secrets_dir` exists or not.

tests/test_settings.py::test_secrets_missing_location
tests/test_settings.py::test_sub_env tests/test_settings.py::test_sub_env_override tests/test_settings.py::test_sub_env_missing tests/test_settings.py::test_other_setting tests/test_settings.py::test_with_prefix tests/test_settings.py::test_nested_env_with_basemodel tests/test_settings.py::test_nested_env_with_dict tests/test_settings.py::test_list tests/test_settings.py::test_set_dict_model tests/test_settings.py::test_invalid_json tests/test_settings.py::test_required_sub_model tests/test_settings.py::test_non_class tests/test_settings.py::test_env_str tests/test_settings.py::test_env_list tests/test_settings.py::test_env_list_field tests/test_settings.py::test_env_list_last tests/test_settings.py::test_env_inheritance tests/test_settings.py::test_env_inheritance_field tests/test_settings.py::test_env_prefix_inheritance_config tests/test_settings.py::test_env_inheritance_config tests/test_settings.py::test_env_invalid tests/test_settings.py::test_env_field tests/test_settings.py::test_aliases_warning tests/test_settings.py::test_aliases_no_warning tests/test_settings.py::test_case_sensitive tests/test_settings.py::test_case_insensitive tests/test_settings.py::test_nested_dataclass tests/test_settings.py::test_env_takes_precedence tests/test_settings.py::test_config_file_settings_nornir tests/test_settings.py::test_env_file_config tests/test_settings.py::test_env_file_config_case_sensitive tests/test_settings.py::test_env_file_export tests/test_settings.py::test_env_file_config_custom_encoding tests/test_settings.py::test_env_file_home_directory tests/test_settings.py::test_env_file_none tests/test_settings.py::test_env_file_override_file tests/test_settings.py::test_env_file_override_none tests/test_settings.py::test_env_file_not_a_file tests/test_settings.py::test_read_env_file_cast_sensitive tests/test_settings.py::test_read_env_file_syntax_wrong tests/test_settings.py::test_env_file_example tests/test_settings.py::test_env_file_custom_encoding tests/test_settings.py::test_alias_set tests/test_settings.py::test_prefix_on_parent tests/test_settings.py::test_frozenset tests/test_settings.py::test_secrets_path tests/test_settings.py::test_secrets_path_url tests/test_settings.py::test_secrets_missing tests/test_settings.py::test_secrets_invalid_secrets_dir tests/test_settings.py::test_secrets_file_is_a_directory tests/test_settings.py::test_secrets_dotenv_precedence tests/test_settings.py::test_external_settings_sources_precedence tests/test_settings.py::test_external_settings_sources_filter_env_vars tests/test_settings.py::test_customise_sources_empty tests/test_settings.py::test_builtins_settings_source_repr