diff --git a/HISTORY.rst b/HISTORY.rst
index 6220350..3518afd 100755
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -2,6 +2,8 @@
 History
 =======
 
+#11 Improved error handling.
+
 1.1.2 (2021-11-07)
 ------------------
 
diff --git a/pyembeddedfhir/fhir_runner.py b/pyembeddedfhir/fhir_runner.py
index 46cdef0..44ed6f0 100644
--- a/pyembeddedfhir/fhir_runner.py
+++ b/pyembeddedfhir/fhir_runner.py
@@ -2,6 +2,7 @@ import logging
 from typing import Optional
 
 import docker  # type: ignore[import]
+from docker.client import DockerClient  # type: ignore[import]
 from docker.models.networks import Network  # type: ignore[import]
 from docker.errors import APIError  # type: ignore[import]
 import psutil  # type: ignore[import]
@@ -18,7 +19,7 @@ from .models import Configuration, FHIRFlavor, RunningFHIR
 LOGGER = logging.getLogger(__name__)
 
 
-def _kill_orphaned_containers(docker_client: docker.DockerClient):
+def _kill_orphaned_containers(docker_client: DockerClient):
     containers = docker_client.containers.list(
         filters={
             "label": DOCKER_LABEL_KEY,
@@ -34,7 +35,7 @@ def _kill_orphaned_containers(docker_client: docker.DockerClient):
             container.kill()
 
 
-def _kill_orphaned_networks(docker_client: docker.DockerClient):
+def _kill_orphaned_networks(docker_client: DockerClient):
     networks = docker_client.networks.list(
         filters={
             "label": DOCKER_LABEL_KEY,
@@ -82,7 +83,7 @@ class FHIRRunner(object):
     :type startup_timeout: float, optional
     :param docker_client: A Docker client, will be created
         using ``docker.from_env()`` if not set, defaults to None
-    :type docker_client: Optional[docker.DockerClient], optional
+    :type docker_client: Optional[DockerClient], optional
     :ivar running_fhir: Descriptor of the running FHIR server.
     :vartype running_fhir: RunningFHIR
     :raises NotImplementedError: Selected implementation is not supported.
@@ -104,7 +105,7 @@ class FHIRRunner(object):
         kill_orphans: bool = True,
         network_id: Optional[str] = None,
         startup_timeout: float = 120,
-        docker_client: Optional[docker.DockerClient] = None,
+        docker_client: Optional[DockerClient] = None,
     ) -> None:
         """A constructor of ``RunningFHIR``."""
         self._configuration = Configuration(
@@ -131,7 +132,8 @@ class FHIRRunner(object):
                 _kill_orphaned_containers(docker_client)
                 _kill_orphaned_networks(docker_client)
 
-            if configuration.network_id is None:
+            new_network_created = configuration.network_id is None
+            if new_network_created:
                 network = docker_client.networks.create(
                     name="pyembeddedfhir",
                     driver="bridge",
@@ -141,11 +143,16 @@ class FHIRRunner(object):
                 network = docker_client.networks.get(configuration.network_id)
             self._network = network
 
-            return self._implementation.start(
-                docker_client,
-                configuration,
-                network,
-            )
+            try:
+                return self._implementation.start(
+                    docker_client,
+                    configuration,
+                    network,
+                )
+            except:  # noqa: E722 (intentionally using bare except)
+                if new_network_created:
+                    network.remove()
+                raise
         except APIError as e:
             raise ContainerRuntimeError(e)
 
@@ -167,12 +174,10 @@ class FHIRRunner(object):
         :raises ContainerRuntimeError: An error related to container runtime.
         :raises AlreadyStoppedError: If the runner was already stopped.
         """
-        # TODO: handle errors
         self._stop()
 
     def __enter__(self) -> RunningFHIR:
         return self.running_fhir
 
     def __exit__(self, exc_type, exc_val, exc_tb) -> None:
-        # TODO: handle errors (wrap exc_val if needed)
         self._stop()
diff --git a/pyembeddedfhir/implementations.py b/pyembeddedfhir/implementations.py
index 22eb239..c8008e0 100644
--- a/pyembeddedfhir/implementations.py
+++ b/pyembeddedfhir/implementations.py
@@ -136,6 +136,11 @@ class FHIRImplementation(ABC):
 
 class HAPIFHIRImplementation(FHIRImplementation):
     _CONTAINER_PORT = 8080
+    _containers: List[Container]
+
+    def __init__(self):
+        super().__init__()
+        self._containers = []
 
     def _pull_image(self, docker_client: DockerClient) -> Image:
         LOGGER.info("Pulling HAPI FHIR image...")
@@ -162,7 +167,7 @@ class HAPIFHIRImplementation(FHIRImplementation):
             network=network.id,
             labels={DOCKER_LABEL_KEY: get_docker_label_value()},
         )
-        self._container = container
+        self._containers.append(container)
         container.reload()
         return container
 
@@ -172,35 +177,44 @@ class HAPIFHIRImplementation(FHIRImplementation):
         configuration: Configuration,
         network: Network,
     ) -> RunningFHIR:
-        image = self._pull_image(docker_client)
-        ports_config = _prepare_ports_config(
-            configuration.host_ip, HAPIFHIRImplementation._CONTAINER_PORT
-        )
-        container = self._run_container(
-            docker_client,
-            network,
-            image,
-            ports_config,
-        )
+        try:
+            image = self._pull_image(docker_client)
+            ports_config = _prepare_ports_config(
+                configuration.host_ip, HAPIFHIRImplementation._CONTAINER_PORT
+            )
+            container = self._run_container(
+                docker_client,
+                network,
+                image,
+                ports_config,
+            )
 
-        return _create_running_fhir_from_container(
-            docker_client=docker_client,
-            configuration=configuration,
-            network=network,
-            container=container,
-            base_path="/fhir/",
-            port=HAPIFHIRImplementation._CONTAINER_PORT,
-        )
+            return _create_running_fhir_from_container(
+                docker_client=docker_client,
+                configuration=configuration,
+                network=network,
+                container=container,
+                base_path="/fhir/",
+                port=HAPIFHIRImplementation._CONTAINER_PORT,
+            )
+        except:  # noqa: E722 (intentionally using bare except)
+            self.stop()
+            raise
 
     def stop(self) -> None:
-        self._container.kill()
+        for container in self._containers:
+            container.kill()
 
 
 class MicrosoftFHIRImplemention(FHIRImplementation):
     _SAPASSWORD = "wW89*XK6aedjMSz9s"
     _CONTAINER_PORT = 8080
 
-    _containers: List[Container] = []
+    _containers: List[Container]
+
+    def __init__(self):
+        super().__init__()
+        self._containers = []
 
     def _pull_mssql_image(self, docker_client: DockerClient) -> Image:
         LOGGER.info("Pulling MSSQL image...")
@@ -306,31 +320,43 @@ class MicrosoftFHIRImplemention(FHIRImplementation):
         configuration: Configuration,
         network: Network,
     ) -> RunningFHIR:
-        mssql_image = self._pull_mssql_image(docker_client)
-        mssql_container = self._run_mssql(mssql_image, docker_client, network)
-        self._wait_for_mssql(mssql_container, configuration.startup_timeout)
-        mssql_network_settings = mssql_container.attrs["NetworkSettings"]
-        mssql_network = _select_container_network_by_id(
-            network.id, mssql_network_settings["Networks"].values()
-        )
-        mssql_host = mssql_network["IPAddress"]
+        try:
+            mssql_image = self._pull_mssql_image(docker_client)
+            mssql_container = self._run_mssql(
+                mssql_image,
+                docker_client,
+                network,
+            )
+            self._wait_for_mssql(
+                mssql_container,
+                configuration.startup_timeout,
+            )
+            mssql_network_settings = mssql_container.attrs["NetworkSettings"]
+            mssql_network = _select_container_network_by_id(
+                network.id, mssql_network_settings["Networks"].values()
+            )
+            mssql_host = mssql_network["IPAddress"]
 
-        ports_config = _prepare_ports_config(
-            configuration.host_ip, MicrosoftFHIRImplemention._CONTAINER_PORT
-        )
-        fhir_image = self._pull_fhir_server(docker_client)
-        fhir_container = self._run_fhir_server(
-            fhir_image, docker_client, network, mssql_host, ports_config
-        )
+            ports_config = _prepare_ports_config(
+                configuration.host_ip,
+                MicrosoftFHIRImplemention._CONTAINER_PORT,
+            )
+            fhir_image = self._pull_fhir_server(docker_client)
+            fhir_container = self._run_fhir_server(
+                fhir_image, docker_client, network, mssql_host, ports_config
+            )
 
-        return _create_running_fhir_from_container(
-            docker_client=docker_client,
-            configuration=configuration,
-            network=network,
-            container=fhir_container,
-            base_path="/",
-            port=MicrosoftFHIRImplemention._CONTAINER_PORT,
-        )
+            return _create_running_fhir_from_container(
+                docker_client=docker_client,
+                configuration=configuration,
+                network=network,
+                container=fhir_container,
+                base_path="/",
+                port=MicrosoftFHIRImplemention._CONTAINER_PORT,
+            )
+        except:  # noqa: E722 (intentionally using bare except)
+            self.stop()
+            raise
 
     def stop(self) -> None:
         for container in self._containers:

diff --git a/tests/unit/test_fhir_runner.py b/tests/unit/test_fhir_runner.py
new file mode 100644
index 0000000..ac71256
--- /dev/null
+++ b/tests/unit/test_fhir_runner.py
@@ -0,0 +1,99 @@
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from pyembeddedfhir.fhir_runner import FHIRRunner
+from pyembeddedfhir.models import FHIRFlavor
+
+
+@pytest.fixture
+def network_mock():
+    return MagicMock()
+
+
+@pytest.fixture
+def docker_client_mock(network_mock):
+    mock = MagicMock()
+    mock.networks.create.return_value = network_mock
+    return mock
+
+
+class SomeUnexpectedError(Exception):
+    pass
+
+
+@pytest.fixture
+def implementation_constructor_mock():
+    return MagicMock()
+
+
+@pytest.fixture
+def patched_hapi_fhir_implementation(implementation_constructor_mock):
+    with patch(
+        "pyembeddedfhir.fhir_runner.HAPIFHIRImplementation",
+        implementation_constructor_mock,
+    ):
+        yield implementation_constructor_mock
+
+
+class TestNetworkRemoval:
+    def test_network_removal_when_failure(
+        self,
+        docker_client_mock,
+        patched_hapi_fhir_implementation,
+        network_mock,
+    ):
+        """When faced with any error in the implementation,
+        FHIRRunner should delete created network."""
+        implementation = patched_hapi_fhir_implementation.return_value
+        implementation.start.side_effect = SomeUnexpectedError()
+        with pytest.raises(SomeUnexpectedError):
+            FHIRRunner(
+                FHIRFlavor.HAPI,
+                docker_client=docker_client_mock,
+            )
+        network_mock.remove.assert_called_once()
+
+    def test_when_success(
+        self,
+        docker_client_mock,
+        patched_hapi_fhir_implementation,
+        network_mock,
+    ):
+        """When the implementation succeeds,
+        FHIRRunner should not delete created network."""
+        FHIRRunner(
+            FHIRFlavor.HAPI,
+            docker_client=docker_client_mock,
+        )
+        network_mock.remove.assert_not_called()
+
+    def test_when_context_manager(
+        self,
+        docker_client_mock,
+        patched_hapi_fhir_implementation,
+        network_mock,
+    ):
+        """When FHIRRunner is used as context manager,
+        it should delete created network."""
+        with FHIRRunner(
+            FHIRFlavor.HAPI,
+            docker_client=docker_client_mock,
+        ):
+            pass
+        network_mock.remove.assert_called_once()
+
+    def test_when_stop(
+        self,
+        docker_client_mock,
+        patched_hapi_fhir_implementation,
+        network_mock,
+    ):
+        """When stop() is explicitly called,
+        FHIRRunner should delete created network."""
+        runner = FHIRRunner(
+            FHIRFlavor.HAPI,
+            docker_client=docker_client_mock,
+        )
+        runner.stop()
+        network_mock.remove.assert_called_once()
diff --git a/tests/unit/test_implementations.py b/tests/unit/test_implementations.py
new file mode 100644
index 0000000..c588840
--- /dev/null
+++ b/tests/unit/test_implementations.py
@@ -0,0 +1,113 @@
+from unittest.mock import MagicMock
+import pytest
+
+from pyembeddedfhir.implementations import (
+    HAPIFHIRImplementation,
+    MicrosoftFHIRImplemention,
+)
+from pyembeddedfhir.models import Configuration
+
+
+@pytest.fixture
+def network_mock():
+    return MagicMock()
+
+
+@pytest.fixture
+def container_mock(network_mock):
+    mock = MagicMock()
+    mock.exec_run.return_value = (0, None)
+    mock.attrs = {
+        "NetworkSettings": {
+            "Networks": {
+                "a": {
+                    "NetworkID": network_mock.id,
+                    "IPAddress": "127.0.0.1",
+                },
+            },
+        },
+    }
+    mock.wait.return_value = {"StatusCode": 0}
+    return mock
+
+
+@pytest.fixture
+def docker_client_mock(container_mock):
+    mock = MagicMock()
+    mock.containers.run.return_value = container_mock
+    return mock
+
+
+@pytest.fixture
+def sample_configuration(docker_client_mock):
+    return Configuration(docker_client=docker_client_mock)
+
+
+class SomeError(Exception):
+    pass
+
+
+class TestContainerRemoval:
+    def test_microsoft_kills_both_containers_when_failure(
+        self,
+        docker_client_mock,
+        network_mock,
+        sample_configuration,
+        container_mock,
+    ):
+        container_mock.wait.side_effect = SomeError()
+        implementation = MicrosoftFHIRImplemention()
+        with pytest.raises(SomeError):
+            implementation.start(
+                docker_client_mock,
+                sample_configuration,
+                network_mock,
+            )
+        assert container_mock.kill.call_count == 2
+
+    def test_microsoft_kills_no_containers_when_success(
+        self,
+        docker_client_mock,
+        network_mock,
+        sample_configuration,
+        container_mock,
+    ):
+        implementation = MicrosoftFHIRImplemention()
+        implementation.start(
+            docker_client_mock,
+            sample_configuration,
+            network_mock,
+        )
+        assert container_mock.kill.call_count == 0
+
+    def test_hapi_kills_container_when_failure(
+        self,
+        docker_client_mock,
+        network_mock,
+        sample_configuration,
+        container_mock,
+    ):
+        container_mock.wait.side_effect = SomeError()
+        implementation = HAPIFHIRImplementation()
+        with pytest.raises(SomeError):
+            implementation.start(
+                docker_client_mock,
+                sample_configuration,
+                network_mock,
+            )
+        assert container_mock.kill.call_count == 1
+
+    def test_hapi_kills_no_containers_when_success(
+        self,
+        docker_client_mock,
+        network_mock,
+        sample_configuration,
+        container_mock,
+    ):
+        implementation = HAPIFHIRImplementation()
+        implementation.start(
+            docker_client_mock,
+            sample_configuration,
+            network_mock,
+        )
+        assert container_mock.kill.call_count == 0

Resolve outstanding TODOs related to error handling
tests/unit/test_fhir_runner.py::TestNetworkRemoval::test_network_removal_when_failure tests/unit/test_implementations.py::TestContainerRemoval::test_microsoft_kills_both_containers_when_failure tests/unit/test_implementations.py::TestContainerRemoval::test_hapi_kills_container_when_failure
tests/unit/test_fhir_runner.py::TestNetworkRemoval::test_when_success tests/unit/test_fhir_runner.py::TestNetworkRemoval::test_when_context_manager tests/unit/test_fhir_runner.py::TestNetworkRemoval::test_when_stop tests/unit/test_implementations.py::TestContainerRemoval::test_microsoft_kills_no_containers_when_success tests/unit/test_implementations.py::TestContainerRemoval::test_hapi_kills_no_containers_when_success