diff --git a/pydantic/fields.py b/pydantic/fields.py
--- a/pydantic/fields.py
+++ b/pydantic/fields.py
@@ -458,17 +458,13 @@ def prepare(self) -> None:
     def _set_default_and_type(self) -> None:
         """
         Set the default value, infer the type if needed and check if `None` value is valid.
-
-        Note: to prevent side effects by calling the `default_factory` for nothing, we only call it
-        when we want to validate the default value i.e. when `validate_all` is set to True.
         """
         if self.default_factory is not None:
             if self.type_ is Undefined:
                 raise errors_.ConfigError(
                     f'you need to set the type of field {self.name!r} when using `default_factory`'
                 )
-            if not self.model_config.validate_all:
-                return
+            return
 
         default_value = self.get_default()
 

I co-discovered this issue. 100% True
Hi and thanks for reporting
`BaseSettings` inherits from `BaseModel` with other defaults for the config including `validate_all` which is set to True by default.
As explained in the [warning section in the doc](https://pydantic-docs.helpmanual.io/usage/models/#field-with-dynamic-default-value), to validate the default value of a field, we actually need to call the `default_factory`.
I guess you could find a workaround in your default factory or set `validate_all = False`

```py
class A(BaseSettings, validate_all=False):
    pass
```

We could also allow people to not validate values with `default_factory`
Hope it helps in the meantime
@PrettyWood 

Thank you, that works for now. It's still kind of strange though, I believe it should be possible to validate all values (including the default ones) on initialization. Might be better to hide this functionality under some option (`validate_on_init`), too.
diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.py
--- a/tests/test_edge_cases.py
+++ b/tests/test_edge_cases.py
@@ -1695,12 +1695,12 @@ class Model(BaseModel):
     Model()
 
 
-def test_default_factory_side_effect():
-    """It may call `default_factory` more than once when `validate_all` is set"""
+def test_default_factory_called_once():
+    """It should never call `default_factory` more than once even when `validate_all` is set"""
 
     v = 0
 
-    def factory():
+    def factory() -> int:
         nonlocal v
         v += 1
         return v
@@ -1712,7 +1712,20 @@ class Config:
             validate_all = True
 
     m1 = MyModel()
-    assert m1.id == 2  # instead of 1
+    assert m1.id == 1
+
+    class MyBadModel(BaseModel):
+        id: List[str] = Field(default_factory=factory)
+
+        class Config:
+            validate_all = True
+
+    with pytest.raises(ValidationError) as exc_info:
+        MyBadModel()
+    assert v == 2  # `factory` has been called to run validation
+    assert exc_info.value.errors() == [
+        {'loc': ('id',), 'msg': 'value is not a valid list', 'type': 'type_error.list'},
+    ]
 
 
 def test_default_factory_validator_child():

BaseSettings descendants call default_factory on import
### Checks

* [x] I added a descriptive title to this issue
* [x] I have searched (google, github) for similar issues and couldn't find anything
* [x] I have read and followed [the docs](https://pydantic-docs.helpmanual.io/) and still think this is a bug

<!-- Sorry to sound so draconian, but every second saved replying to issues is time spend improving pydantic :-) -->

# Bug

When importing `BaseSettings` descendant from another file, `default_factory` functors are always called for uninstantiated models. See the code snippet attached below. `def factory()` is called once on import, even though `B` is not even instantiated. 

Output of `python -c "import pydantic.utils; print(pydantic.utils.version_info())"`:
```
             pydantic version: 1.8.1
            pydantic compiled: True
                 install path: /usr/local/anaconda3/envs/dirty_3.9/lib/python3.9/site-packages/pydantic
               python version: 3.9.2 | packaged by conda-forge | (default, Feb 21 2021, 05:02:20)  [Clang 11.0.1 ]
                     platform: macOS-11.1-x86_64-i386-64bit
     optional deps. installed: ['typing-extensions']
```

<!-- Where possible please include a self-contained code snippet describing your bug: -->

The first file (`kek.py`):

```py
from pydantic import BaseSettings, Field

v = 0

def factory():
    print('factory called')
    global v
    v += 1
    return v


class A(BaseSettings):
    pass


class B(A):
    id: int = Field(default_factory=factory)

        
class C(A):
    pass
```

The second file:

```py
from kek import C
```

Expected output: empty.

Actual output: `factory called`.

It's also worth mentioning that `BaseModel` does not have the described behavior, importing the `BaseModel` descendants doesn't call `default_factory` functors. 

tests/test_edge_cases.py::test_default_factory_called_once
tests/test_edge_cases.py::test_str_bytes tests/test_edge_cases.py::test_str_bytes_none tests/test_edge_cases.py::test_union_int_str tests/test_edge_cases.py::test_union_priority tests/test_edge_cases.py::test_typed_list tests/test_edge_cases.py::test_typed_set tests/test_edge_cases.py::test_dict_dict tests/test_edge_cases.py::test_none_list tests/test_edge_cases.py::test_typed_dict[value0-result0] tests/test_edge_cases.py::test_typed_dict[value1-result1] tests/test_edge_cases.py::test_typed_dict[value2-result2] tests/test_edge_cases.py::test_typed_dict_error[1-errors0] tests/test_edge_cases.py::test_typed_dict_error[value1-errors1] tests/test_edge_cases.py::test_typed_dict_error[value2-errors2] tests/test_edge_cases.py::test_dict_key_error tests/test_edge_cases.py::test_tuple tests/test_edge_cases.py::test_tuple_more tests/test_edge_cases.py::test_tuple_length_error tests/test_edge_cases.py::test_tuple_invalid tests/test_edge_cases.py::test_tuple_value_error tests/test_edge_cases.py::test_recursive_list tests/test_edge_cases.py::test_recursive_list_error tests/test_edge_cases.py::test_list_unions tests/test_edge_cases.py::test_recursive_lists tests/test_edge_cases.py::test_str_enum tests/test_edge_cases.py::test_any_dict tests/test_edge_cases.py::test_success_values_include tests/test_edge_cases.py::test_include_exclude_unset tests/test_edge_cases.py::test_include_exclude_defaults tests/test_edge_cases.py::test_skip_defaults_deprecated tests/test_edge_cases.py::test_advanced_exclude tests/test_edge_cases.py::test_advanced_exclude_by_alias tests/test_edge_cases.py::test_advanced_value_include tests/test_edge_cases.py::test_advanced_value_exclude_include tests/test_edge_cases.py::test_advanced_exclude_nested_lists[exclude0-expected0] tests/test_edge_cases.py::test_advanced_exclude_nested_lists[exclude1-expected1] tests/test_edge_cases.py::test_advanced_exclude_nested_lists[exclude2-expected2] tests/test_edge_cases.py::test_advanced_exclude_nested_lists[exclude3-expected3] tests/test_edge_cases.py::test_advanced_exclude_nested_lists[exclude4-expected4] tests/test_edge_cases.py::test_advanced_exclude_nested_lists[exclude5-expected5] tests/test_edge_cases.py::test_advanced_exclude_nested_lists[exclude6-expected6] tests/test_edge_cases.py::test_advanced_exclude_nested_lists[exclude7-expected7] tests/test_edge_cases.py::test_advanced_exclude_nested_lists[exclude8-expected8] tests/test_edge_cases.py::test_advanced_exclude_nested_lists[exclude9-expected9] tests/test_edge_cases.py::test_advanced_exclude_nested_lists[exclude10-expected10] tests/test_edge_cases.py::test_advanced_exclude_nested_lists[exclude11-expected11] tests/test_edge_cases.py::test_advanced_exclude_nested_lists[exclude12-expected12] tests/test_edge_cases.py::test_advanced_exclude_nested_lists[exclude13-expected13] tests/test_edge_cases.py::test_advanced_exclude_nested_lists[exclude14-expected14] tests/test_edge_cases.py::test_advanced_include_nested_lists[include0-expected0] tests/test_edge_cases.py::test_advanced_include_nested_lists[include1-expected1] tests/test_edge_cases.py::test_advanced_include_nested_lists[include2-expected2] tests/test_edge_cases.py::test_advanced_include_nested_lists[include3-expected3] tests/test_edge_cases.py::test_advanced_include_nested_lists[include4-expected4] tests/test_edge_cases.py::test_advanced_include_nested_lists[include5-expected5] tests/test_edge_cases.py::test_advanced_include_nested_lists[include6-expected6] tests/test_edge_cases.py::test_advanced_include_nested_lists[include7-expected7] tests/test_edge_cases.py::test_advanced_include_nested_lists[include8-expected8] tests/test_edge_cases.py::test_advanced_include_nested_lists[include9-expected9] tests/test_edge_cases.py::test_advanced_include_nested_lists[include10-expected10] tests/test_edge_cases.py::test_advanced_include_nested_lists[include11-expected11] tests/test_edge_cases.py::test_advanced_include_nested_lists[include12-expected12] tests/test_edge_cases.py::test_advanced_include_nested_lists[include13-expected13] tests/test_edge_cases.py::test_advanced_include_nested_lists[include14-expected14] tests/test_edge_cases.py::test_field_set_ignore_extra tests/test_edge_cases.py::test_field_set_allow_extra tests/test_edge_cases.py::test_field_set_field_name tests/test_edge_cases.py::test_values_order tests/test_edge_cases.py::test_inheritance tests/test_edge_cases.py::test_invalid_type tests/test_edge_cases.py::test_valid_string_types[a tests/test_edge_cases.py::test_valid_string_types[some tests/test_edge_cases.py::test_valid_string_types[value2-foobar] tests/test_edge_cases.py::test_valid_string_types[123-123] tests/test_edge_cases.py::test_valid_string_types[123.45-123.45] tests/test_edge_cases.py::test_valid_string_types[value5-12.45] tests/test_edge_cases.py::test_valid_string_types[True-True] tests/test_edge_cases.py::test_valid_string_types[False-False] tests/test_edge_cases.py::test_valid_string_types[a10-a10] tests/test_edge_cases.py::test_valid_string_types[whatever-whatever] tests/test_edge_cases.py::test_invalid_string_types[value0-errors0] tests/test_edge_cases.py::test_invalid_string_types[value1-errors1] tests/test_edge_cases.py::test_inheritance_config tests/test_edge_cases.py::test_partial_inheritance_config tests/test_edge_cases.py::test_annotation_inheritance tests/test_edge_cases.py::test_string_none tests/test_edge_cases.py::test_return_errors_ok tests/test_edge_cases.py::test_return_errors_error tests/test_edge_cases.py::test_optional_required tests/test_edge_cases.py::test_invalid_validator tests/test_edge_cases.py::test_unable_to_infer tests/test_edge_cases.py::test_multiple_errors tests/test_edge_cases.py::test_validate_all tests/test_edge_cases.py::test_force_extra tests/test_edge_cases.py::test_illegal_extra_value tests/test_edge_cases.py::test_multiple_inheritance_config tests/test_edge_cases.py::test_submodel_different_type tests/test_edge_cases.py::test_self tests/test_edge_cases.py::test_self_recursive[BaseModel] tests/test_edge_cases.py::test_self_recursive[BaseSettings] tests/test_edge_cases.py::test_nested_init[BaseModel] tests/test_edge_cases.py::test_nested_init[BaseSettings] tests/test_edge_cases.py::test_init_inspection tests/test_edge_cases.py::test_type_on_annotation tests/test_edge_cases.py::test_assign_type tests/test_edge_cases.py::test_optional_subfields tests/test_edge_cases.py::test_not_optional_subfields tests/test_edge_cases.py::test_optional_field_constraints tests/test_edge_cases.py::test_field_str_shape tests/test_edge_cases.py::test_field_type_display[int-int] tests/test_edge_cases.py::test_field_type_display[type_1-Optional[int]] tests/test_edge_cases.py::test_field_type_display[type_2-Union[NoneType, tests/test_edge_cases.py::test_field_type_display[type_3-Union[int, tests/test_edge_cases.py::test_field_type_display[type_4-List[int]] tests/test_edge_cases.py::test_field_type_display[type_5-Tuple[int, tests/test_edge_cases.py::test_field_type_display[type_6-Union[List[int], tests/test_edge_cases.py::test_field_type_display[type_7-List[Tuple[int, tests/test_edge_cases.py::test_field_type_display[type_8-Mapping[int, tests/test_edge_cases.py::test_field_type_display[type_9-FrozenSet[int]] tests/test_edge_cases.py::test_field_type_display[type_10-Tuple[int, tests/test_edge_cases.py::test_field_type_display[type_11-Optional[List[int]]] tests/test_edge_cases.py::test_field_type_display[dict-dict] tests/test_edge_cases.py::test_field_type_display[type_13-DisplayGen[bool, tests/test_edge_cases.py::test_any_none tests/test_edge_cases.py::test_type_var_any tests/test_edge_cases.py::test_type_var_constraint tests/test_edge_cases.py::test_type_var_bound tests/test_edge_cases.py::test_dict_bare tests/test_edge_cases.py::test_list_bare tests/test_edge_cases.py::test_dict_any tests/test_edge_cases.py::test_modify_fields tests/test_edge_cases.py::test_exclude_none tests/test_edge_cases.py::test_exclude_none_recursive tests/test_edge_cases.py::test_exclude_none_with_extra tests/test_edge_cases.py::test_str_method_inheritance tests/test_edge_cases.py::test_repr_method_inheritance tests/test_edge_cases.py::test_optional_validator tests/test_edge_cases.py::test_required_optional tests/test_edge_cases.py::test_required_any tests/test_edge_cases.py::test_custom_generic_validators tests/test_edge_cases.py::test_custom_generic_arbitrary_allowed tests/test_edge_cases.py::test_custom_generic_disallowed tests/test_edge_cases.py::test_hashable_required tests/test_edge_cases.py::test_hashable_optional[1] tests/test_edge_cases.py::test_hashable_optional[None] tests/test_edge_cases.py::test_default_factory_validator_child tests/test_edge_cases.py::test_cython_function_untouched tests/test_edge_cases.py::test_resolve_annotations_module_missing tests/test_edge_cases.py::test_iter_coverage tests/test_edge_cases.py::test_config_field_info tests/test_edge_cases.py::test_config_field_info_alias tests/test_edge_cases.py::test_config_field_info_merge tests/test_edge_cases.py::test_config_field_info_allow_mutation